From fdb4b7f9ee368c8091090d5dc197235173cdcd3a Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 13 Nov 2025 15:43:50 -0800 Subject: [PATCH 1/4] Handle surrogate pair characters substring --- api/src/org/labkey/api/data/Container.java | 3 +- .../org/labkey/api/data/ExcelCellUtils.java | 3 +- api/src/org/labkey/api/data/ExcelWriter.java | 3 +- api/src/org/labkey/api/data/SQLFragment.java | 2701 +- api/src/org/labkey/api/data/TSVWriter.java | 3 +- .../api/data/dialect/StatementWrapper.java | 6029 ++-- .../org/labkey/api/exp/OntologyManager.java | 7829 ++--- .../org/labkey/api/jsp/LabKeyJspWriter.java | 3 +- api/src/org/labkey/api/util/MemTracker.java | 972 +- .../labkey/core/admin/AdminController.java | 24546 ++++++++-------- .../experiment/api/AbstractRunInput.java | 249 +- .../labkey/experiment/api/ExpDataImpl.java | 1955 +- .../labkey/mothership/MothershipManager.java | 1253 +- .../pipeline/api/WorkDirectoryRemote.java | 1186 +- .../org/labkey/search/SearchController.java | 2395 +- 15 files changed, 24571 insertions(+), 24559 deletions(-) diff --git a/api/src/org/labkey/api/data/Container.java b/api/src/org/labkey/api/data/Container.java index c55c45cd561..b4d69f94270 100644 --- a/api/src/org/labkey/api/data/Container.java +++ b/api/src/org/labkey/api/data/Container.java @@ -61,6 +61,7 @@ import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Path; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; import org.labkey.api.view.FolderTab; @@ -1594,7 +1595,7 @@ public String getContainerNoun(boolean titleCase) String noun = _containerType.getContainerNoun(this); if (titleCase) { - return noun.substring(0, 1).toUpperCase() + noun.substring(1); + return StringUtilsLabKey.leftSurrogatePairFriendly(noun, 1).toUpperCase() + StringUtilsLabKey.rightSurrogatePairFriendly(noun,1); } return noun; diff --git a/api/src/org/labkey/api/data/ExcelCellUtils.java b/api/src/org/labkey/api/data/ExcelCellUtils.java index 3e57a26c2df..bfd0d8a98ab 100644 --- a/api/src/org/labkey/api/data/ExcelCellUtils.java +++ b/api/src/org/labkey/api/data/ExcelCellUtils.java @@ -8,6 +8,7 @@ import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.Workbook; import org.labkey.api.util.DateUtil; +import org.labkey.api.util.StringUtilsLabKey; import java.io.File; import java.math.BigDecimal; @@ -188,7 +189,7 @@ public static void writeCell(Cell cell, CellStyle style, int simpleType, String // Check if the string is too long if (s.length() > 32767) { - s = s.substring(0, 32762) + "..."; + s = StringUtilsLabKey.leftSurrogatePairFriendly(s, 32762) + "..."; } // Ensure the row is tall enough to show the full values when there are newlines int newlines = StringUtils.countMatches(s, '\n'); diff --git a/api/src/org/labkey/api/data/ExcelWriter.java b/api/src/org/labkey/api/data/ExcelWriter.java index 34f2a681b70..e2c66bbfe89 100644 --- a/api/src/org/labkey/api/data/ExcelWriter.java +++ b/api/src/org/labkey/api/data/ExcelWriter.java @@ -36,6 +36,7 @@ import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddress; import org.apache.poi.ss.util.WorkbookUtil; +import org.labkey.api.util.StringUtilsLabKey; import org.apache.poi.xssf.streaming.SXSSFSheet; import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.jetbrains.annotations.NotNull; @@ -375,7 +376,7 @@ public void setSheetName(String sheetName) sheetName = "Sheet"; if (sheetName.length() > 31) - return cleanSheetName(sheetName.substring(0, 31)); + return cleanSheetName(StringUtilsLabKey.leftSurrogatePairFriendly(sheetName, 31)); return WorkbookUtil.createSafeSheetName(sheetName, '_'); } diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index acde96b77c6..79c8f5ce242 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1350 +1,1351 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/** - * Holds both the SQL text and JDBC parameter values to use during invocation. - */ -public class SQLFragment implements Appendable, CharSequence -{ - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - // allow statement separators - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); - } - - - public String toDebugString() - { - return JdbcUtil.format(this); - } - - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - getStringBuilder().append(charseq); - return this; - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - public final SQLFragment appendNowTimestamp() - { - return appendValue(new NowTimestamp()); - } - - // Issue 27534: Stop using {fn now()} in function declarations - // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps - public final SQLFragment appendValue(NowTimestamp now) - { - if (null == now) - return appendNull(); - getStringBuilder().append("CURRENT_TIMESTAMP"); - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. - if (name.contains("'")) - throw new IllegalStateException(); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - public SQLFragment add(Object p, JdbcType type) - { - getMutableParams().add(new Parameter.TypedValue(p, type)); - return this; - } - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = comment.substring(0,1000); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replaceAll("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().get(0)); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Did not expect IllegalArgumentException"); - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replaceAll("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - } - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use on the SQL portion - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, String separator) - { - if (separator.contains("?")) - throw new IllegalStateException("separator must not include a parameter marker"); - - // Join all the SQL statements - String sql = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getSQL) - .collect(Collectors.joining(separator)); - - // Collect all the parameters to a single list - List params = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getParams) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - return new SQLFragment(sql, params); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/** + * Holds both the SQL text and JDBC parameter values to use during invocation. + */ +public class SQLFragment implements Appendable, CharSequence +{ + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + // allow statement separators + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); + } + + + public String toDebugString() + { + return JdbcUtil.format(this); + } + + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + getStringBuilder().append(charseq); + return this; + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + public final SQLFragment appendNowTimestamp() + { + return appendValue(new NowTimestamp()); + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. + if (name.contains("'")) + throw new IllegalStateException(); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + public SQLFragment add(Object p, JdbcType type) + { + getMutableParams().add(new Parameter.TypedValue(p, type)); + return this; + } + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replaceAll("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().get(0)); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Did not expect IllegalArgumentException"); + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replaceAll("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + } + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use on the SQL portion + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, String separator) + { + if (separator.contains("?")) + throw new IllegalStateException("separator must not include a parameter marker"); + + // Join all the SQL statements + String sql = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getSQL) + .collect(Collectors.joining(separator)); + + // Collect all the parameters to a single list + List params = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getParams) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + return new SQLFragment(sql, params); + } +} diff --git a/api/src/org/labkey/api/data/TSVWriter.java b/api/src/org/labkey/api/data/TSVWriter.java index 396cebd9326..a25235e8891 100644 --- a/api/src/org/labkey/api/data/TSVWriter.java +++ b/api/src/org/labkey/api/data/TSVWriter.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.util.FileUtil; +import org.labkey.api.util.StringUtilsLabKey; import java.io.IOException; import java.io.PrintWriter; @@ -109,7 +110,7 @@ public void setFilenamePrefix(String filenamePrefix) _filenamePrefix = badChars.matcher(filenamePrefix).replaceAll("_"); if (_filenamePrefix.length() > 30) - _filenamePrefix = _filenamePrefix.substring(0, 30); + _filenamePrefix = StringUtilsLabKey.leftSurrogatePairFriendly(_filenamePrefix, 30); } public void setDelimiterCharacter(char delimiter) diff --git a/api/src/org/labkey/api/data/dialect/StatementWrapper.java b/api/src/org/labkey/api/data/dialect/StatementWrapper.java index 09cee28c623..94171f65d4f 100644 --- a/api/src/org/labkey/api/data/dialect/StatementWrapper.java +++ b/api/src/org/labkey/api/data/dialect/StatementWrapper.java @@ -1,3014 +1,3015 @@ -/* - * Copyright (c) 2010-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data.dialect; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.collections.OneBasedList; -import org.labkey.api.data.ConnectionWrapper; -import org.labkey.api.data.Container; -import org.labkey.api.data.QueryLogging; -import org.labkey.api.data.ResultSetWrapper; -import org.labkey.api.data.queryprofiler.Query; -import org.labkey.api.data.queryprofiler.QueryProfiler; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.DebugInfoDumper; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.MemTracker; -import org.labkey.api.view.ViewServlet; - -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.Date; -import java.sql.NClob; -import java.sql.ParameterMetaData; -import java.sql.PreparedStatement; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; - -public class StatementWrapper implements Statement, PreparedStatement, CallableStatement -{ - private final ConnectionWrapper _conn; - private final Statement _stmt; - private final Logger _log; - private String _debugSql = ""; - private long _msStart = 0; - private boolean userCancelled = false; - // NOTE: CallableStatement supports getObject(), but PreparedStatement doesn't - private OneBasedList _parameters = null; - private @Nullable StackTraceElement[] _stackTrace = null; - /** Track the place that closed this statement for troubleshooting purposes */ - private @Nullable Throwable _closingStackTrace = null; - private @Nullable Boolean _requestThread = null; - private QueryLogging _queryLogging = QueryLogging.emptyQueryLogging(); - - String _sqlStateTestException = null; - - - public StatementWrapper(ConnectionWrapper conn, Statement stmt) - { - _conn = conn; - _log = conn.getLogger(); - _stmt = stmt; - assert MemTracker.getInstance().put(this); - } - - public StatementWrapper(ConnectionWrapper conn, Statement stmt, String sql) - { - this(conn, stmt); - _debugSql = sql; - } - - public void setStackTrace(@Nullable StackTraceElement[] stackTrace) - { - _stackTrace = stackTrace; - } - - public @NotNull Boolean isRequestThread() - { - return null != _requestThread ? _requestThread : ViewServlet.isRequestThread(); - } - - public @Nullable Throwable getClosingStackTrace() - { - return _closingStackTrace; - } - - public void setRequestThread(@Nullable Boolean requestThread) - { - _requestThread = requestThread; - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - public QueryLogging getQueryLogging() - { - return _queryLogging; - } - - public void setQueryLogging(QueryLogging queryLogging) - { - _queryLogging = queryLogging; - } - - @Override - public boolean wasNull() - throws SQLException - { - try - { - return ((CallableStatement)_stmt).wasNull(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String getString(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getString(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getBoolean(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBoolean(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte getByte(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getByte(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public short getShort(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getShort(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getInt(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getInt(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public long getLong(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getLong(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public float getFloat(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getFloat(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public double getDouble(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDouble(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(int parameterIndex, int scale) - throws SQLException - { - try - { - //noinspection deprecation - return ((CallableStatement)_stmt).getBigDecimal(parameterIndex, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte[] getBytes(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBytes(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBigDecimal(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(int i, Map> map) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(i, map); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Ref getRef(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getRef(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Blob getBlob(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBlob(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Clob getClob(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getClob(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Array getArray(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getArray(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public URL getURL(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getURL(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setURL(String parameterName, URL val) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setURL(parameterName, val); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(String parameterName, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setNull(parameterName, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBoolean(String parameterName, boolean x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBoolean(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setByte(String parameterName, byte x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setByte(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setShort(String parameterName, short x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setShort(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setInt(String parameterName, int x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setInt(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setLong(String parameterName, long x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setLong(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFloat(String parameterName, float x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setFloat(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDouble(String parameterName, double x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDouble(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBigDecimal(String parameterName, BigDecimal x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBigDecimal(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setString(String parameterName, String x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setString(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBytes(String parameterName, byte[] x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBytes(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(String parameterName, Date x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDate(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(String parameterName, Time x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTime(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(String parameterName, Timestamp x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTimestamp(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setAsciiStream(String parameterName, InputStream x, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setAsciiStream(parameterName, x, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBinaryStream(String parameterName, InputStream x, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBinaryStream(parameterName, x, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x, int targetSqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x, int targetSqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setCharacterStream(String parameterName, Reader reader, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setCharacterStream(parameterName, reader, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(String parameterName, Date x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDate(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(String parameterName, Time x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTime(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(String parameterName, Timestamp x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTimestamp(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(String parameterName, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setNull(parameterName, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String getString(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getString(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getBoolean(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBoolean(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte getByte(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getByte(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public short getShort(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getShort(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getInt(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getInt(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public long getLong(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getLong(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public float getFloat(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getFloat(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public double getDouble(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDouble(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte[] getBytes(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBytes(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBigDecimal(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(String parameterName, Map> map) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterName, map); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Ref getRef(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getRef(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Blob getBlob(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBlob(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Clob getClob(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getClob(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Array getArray(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getArray(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public URL getURL(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getURL(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet executeQuery() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - if (null != _sqlStateTestException) - throw new SQLException("Test sql exception", _sqlStateTestException); - - ResultSet rs = ((PreparedStatement)_stmt).executeQuery(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int executeUpdate() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - int rows = -1; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = ((PreparedStatement)_stmt).executeUpdate(); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - private void _set(int i, @Nullable Object o) - { - if (null == _parameters) - _parameters = new OneBasedList<>(10); - while (_parameters.size() < i) - _parameters.add(null); - _parameters.set(i, o); - } - - @Override - public void setNull(int parameterIndex, int sqlType) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType); - _set(parameterIndex, null); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBoolean(int parameterIndex, boolean x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBoolean(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setByte(int parameterIndex, byte x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setByte(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setShort(int parameterIndex, short x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setShort(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setInt(int parameterIndex, int x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setInt(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setLong(int parameterIndex, long x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setLong(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFloat(int parameterIndex, float x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setFloat(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDouble(int parameterIndex, double x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDouble(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBigDecimal(int parameterIndex, BigDecimal x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBigDecimal(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setString(int parameterIndex, String x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setString(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBytes(int parameterIndex, byte[] x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBytes(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(int parameterIndex, Date x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDate(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(int parameterIndex, Time x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTime(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(int parameterIndex, Timestamp x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setAsciiStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setUnicodeStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - //noinspection deprecation - ((PreparedStatement)_stmt).setUnicodeStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBinaryStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearParameters() - throws SQLException - { - try - { - ((PreparedStatement)_stmt).clearParameters(); - if (null != _parameters) _parameters.clear(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x, int targetSqlType, int scale) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType, scale); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x, int targetSqlType) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean execute() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - Boolean ret=null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - ret = ((PreparedStatement)_stmt).execute(); - return ret; - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - int rows = (ret==Boolean.FALSE) ? _stmt.getUpdateCount() : -1; - afterExecute(preQuery, ex, rows); - } - } - - @Override - public void addBatch() - throws SQLException - { - try - { - ((PreparedStatement)_stmt).addBatch(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - // NOTE: We intentionally do not store potentially large parameters (reader, blob, etc.) - - @Override - public void setCharacterStream(int parameterIndex, Reader reader, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setCharacterStream(parameterIndex, reader, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setRef(int i, Ref x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setRef(i, x); - _set(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBlob(int i, Blob x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBlob(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setClob(int i, Clob x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setClob(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setArray(int i, Array x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setArray(i, x); - _set(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSetMetaData getMetaData() - throws SQLException - { - try - { - ResultSetMetaData rs = ((PreparedStatement)_stmt).getMetaData(); - assert MemTracker.getInstance().put(rs); - return rs; - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(int parameterIndex, Date x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDate(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(int parameterIndex, Time x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTime(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(int parameterIndex, int sqlType, String typeName) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType, typeName); - _set(parameterIndex, null); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setURL(int parameterIndex, URL x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setURL(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ParameterMetaData getParameterMetaData() - throws SQLException - { - try - { - return ((PreparedStatement)_stmt).getParameterMetaData(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet executeQuery(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - ResultSet rs = _stmt.executeQuery(sql); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int executeUpdate(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public void close() - throws SQLException - { - try - { - _stmt.close(); - if (AppProps.getInstance().isDevMode() && _closingStackTrace == null) - { - _closingStackTrace = new Throwable("Remembering stack for closing Statement on thread " + Thread.currentThread().getName()); - } - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getMaxFieldSize() - throws SQLException - { - try - { - return _stmt.getMaxFieldSize(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setMaxFieldSize(int max) - throws SQLException - { - try - { - _stmt.setMaxFieldSize(max); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getMaxRows() - throws SQLException - { - try - { - return _stmt.getMaxRows(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setMaxRows(int max) - throws SQLException - { - try - { - _stmt.setMaxRows(max); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setEscapeProcessing(boolean enable) - throws SQLException - { - try - { - _stmt.setEscapeProcessing(enable); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getQueryTimeout() - throws SQLException - { - try - { - return _stmt.getQueryTimeout(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setQueryTimeout(int seconds) - throws SQLException - { - try - { - _stmt.setQueryTimeout(seconds); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void cancel() - throws SQLException - { - try - { - userCancelled = true; - _stmt.cancel(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public SQLWarning getWarnings() - throws SQLException - { - try - { - return _stmt.getWarnings(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearWarnings() - throws SQLException - { - try - { - _stmt.clearWarnings(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setCursorName(String name) - throws SQLException - { - try - { - _stmt.setCursorName(name); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean execute(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public ResultSet getResultSet() - throws SQLException - { - try - { - ResultSet rs = _stmt.getResultSet(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getUpdateCount() - throws SQLException - { - try - { - int updateCount; - updateCount = _stmt.getUpdateCount(); - return updateCount; - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getMoreResults() - throws SQLException - { - try - { - return _stmt.getMoreResults(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFetchDirection(int direction) - throws SQLException - { - try - { - _stmt.setFetchDirection(direction); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getFetchDirection() - throws SQLException - { - try - { - return _stmt.getFetchDirection(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFetchSize(int rows) - throws SQLException - { - try - { - _stmt.setFetchSize(rows); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getFetchSize() - throws SQLException - { - try - { - return _stmt.getFetchSize(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getResultSetConcurrency() - throws SQLException - { - try - { - return _stmt.getResultSetConcurrency(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getResultSetType() - throws SQLException - { - try - { - return _stmt.getResultSetType(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void addBatch(String sql) - throws SQLException - { - try - { - _stmt.addBatch(sql); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearBatch() - throws SQLException - { - try - { - _stmt.clearBatch(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int[] executeBatch() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return _stmt.executeBatch(); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public Connection getConnection() - { - return _conn; - } - - @Override - public boolean getMoreResults(int current) - throws SQLException - { - try - { - return _stmt.getMoreResults(current); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet getGeneratedKeys() - throws SQLException - { - try - { - ResultSet rs = _stmt.getGeneratedKeys(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int executeUpdate(String sql, int autoGeneratedKeys) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, autoGeneratedKeys); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public int executeUpdate(String sql, int[] columnIndexes) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, columnIndexes); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public int executeUpdate(String sql, String[] columnNames) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, columnNames); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public boolean execute(String sql, int autoGeneratedKeys) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, autoGeneratedKeys); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public boolean execute(String sql, int[] columnIndexes) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, columnIndexes); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public boolean execute(String sql, String[] columnNames) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, columnNames); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int getResultSetHoldability() - throws SQLException - { - try - { - return _stmt.getResultSetHoldability(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String toString() - { - return _stmt.toString(); - } - - // TODO: These methods should be properly implemented via delegation. - - @Override - public boolean isWrapperFor(Class iface) - { - throw new UnsupportedOperationException(); - } - - @Override - public T unwrap(Class iface) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isClosed() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isPoolable() - { - throw new UnsupportedOperationException(); - } - - @Override - public void setPoolable(boolean poolable) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException - { - try - { - if (length > Integer.MAX_VALUE) - throw new IllegalArgumentException("File length exceeds " + Integer.MAX_VALUE); - setBinaryStream(parameterIndex, x, (int)length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBlob(int parameterIndex, InputStream inputStream) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(int parameterIndex, InputStream inputStream, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(int parameterIndex, Reader value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(int parameterIndex, Reader value, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, NClob value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNString(int parameterIndex, String value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setRowId(int parameterIndex, RowId x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setSQLXML(int parameterIndex, SQLXML xmlObject) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getCharacterStream(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getCharacterStream(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getNCharacterStream(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getNCharacterStream(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public NClob getNClob(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public NClob getNClob(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public String getNString(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public String getNString(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public RowId getRowId(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public RowId getRowId(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public SQLXML getSQLXML(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public SQLXML getSQLXML(String parameterName) - { - throw new UnsupportedOperationException(); - }//-- - - @Override - public void setAsciiStream(String parameterName, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(String parameterName, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(String parameterName, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(String parameterName, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, InputStream inputStream) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, InputStream inputStream, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, Blob x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Clob x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(String parameterName, Reader value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(String parameterName, Reader value, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, NClob value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNString(String parameterName, String value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setRowId(String parameterName, RowId x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setSQLXML(String parameterName, SQLXML xmlObject) - { - throw new UnsupportedOperationException(); - } - - @Override - public T getObject(int parameterIndex, Class type) - { - throw new UnsupportedOperationException(); - } - - // JDBC 4.1 methods below must be here so we compile on JDK 7; implement once we require JRE 7. - - @Override - public T getObject(String parameterName, Class type) - { - throw new UnsupportedOperationException(); - } - - @Override - public void closeOnCompletion() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isCloseOnCompletion() - { - throw new UnsupportedOperationException(); - } - - private Query beforeExecute(String sql) - { - _debugSql = sql; - // Crawler.java and BaseWebDriverTest.java use "8(" as attempted injection string - if (_debugSql.contains("\"8(\"") && !_debugSql.contains("\"\"8(\"\"")) // 18196 - throw new IllegalArgumentException("SQL injection test failed: " + _debugSql); - _msStart = System.currentTimeMillis(); - - List zeroBasedList = translateParametersForQueryTracking(); - return QueryProfiler.getInstance().preTrack(_conn.getScope(), sql, zeroBasedList, _stackTrace, isRequestThread()); - } - - - private void afterExecute(Query query, @Nullable SQLException x, int rowsAffected) - { - if (null != x) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.DialectSQL, query.getOriginalSql(), true); - if (SqlDialect.isConfigurationException(x)) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - } - } - _logStatement(query.getOriginalSql(), x, rowsAffected, getQueryLogging()); - } - - - private static final Package JAVA_LANG = java.lang.String.class.getPackage(); - - private void _logStatement(String sql, @Nullable SQLException x, int rowsAffected, QueryLogging queryLogging) - { - long elapsed = System.currentTimeMillis() - _msStart; - boolean isAssertEnabled = false; - assert isAssertEnabled = true; - - if (isAssertEnabled && AppProps.getInstance().isDevMode() && isMutatingSql(sql)) - SpringActionController.executingMutatingSql(sql); - - // Hold on to this stack trace so that we can reuse it later (if collection has been enabled) - Query query = QueryProfiler.getInstance().track(_conn.getScope(), sql, translateParametersForQueryTracking(), elapsed, _stackTrace, isRequestThread(), queryLogging); - - if (x != null) - { - _conn.logAndCheckException(x); - } - - //noinspection ConstantConditions - if (!_log.isEnabled(Level.DEBUG) && !isAssertEnabled) - return; - - StringBuilder logEntry = new StringBuilder(sql.length() * 2); - logEntry.append("SQL "); - - Integer sid = _conn.getSPID(); - if (sid != null) - logEntry.append(" [").append(sid).append("]"); - if (_msStart != 0) - logEntry.append(" time ").append(DateUtil.formatDuration(elapsed)); - - if (-1 != rowsAffected) - logEntry.append("\n ").append(rowsAffected).append(" rows affected"); - - logEntry.append("\n "); - logEntry.append(sql.trim().replace("\n", "\n ")); - - if (null != _parameters) - { - for (int i = 1; i <= _parameters.size(); i++) - { - try - { - Object o = _parameters.get(i); - String value; - if (o == null) - value = "NULL"; - else if (o instanceof Container) - value = "'" + ((Container)o).getId() + "' " + ((Container)o).getPath(); - else if (o instanceof String) - value = "'" + escapeSql((String) o) + "'"; - else - value = String.valueOf(o); - if (value.length() > 100) - value = value.substring(0, 100) + ". . ."; - logEntry.append("\n --[").append(i).append("] "); - logEntry.append(value); - Class c = null==o ? null : o.getClass(); - if (null != c && c != String.class && c != Integer.class) - logEntry.append(" :").append(c.getPackage() == JAVA_LANG ? c.getSimpleName() : c.getName()); - } - catch (Exception ex) - { - /* */ - } - } - } - _parameters = null; - - if (userCancelled) - logEntry.append("\n cancelled by user"); - if (null != x) - logEntry.append("\n ").append(x); - _appendTableStackTrace(logEntry, 5, query.getStackTraceElements()); - - final String logString = logEntry.toString(); - _log.log(Level.DEBUG, logString); - - // check for deadlock or transaction related error - if (SqlDialect.isTransactionException(x)) - { - DebugInfoDumper.dumpThreads(_log); - } - } - - private @Nullable List translateParametersForQueryTracking() - { - // Make a copy of the parameters list (it gets modified by callers) and switch to zero-based list (_parameters is a one-based list) - - List zeroBasedList; - - if (null != _parameters) - { - zeroBasedList = new ArrayList<>(_parameters.size()); - - // Translate parameters that can't be cached (for now, just JDBC arrays). I'd rather stash the original parameters and send - // those to the query profiler, but this would require one or more non-standard methods on StatementWrapper. See #24314. - for (Object o : _parameters) - { - if (o instanceof Array a) - { - try - { - o = a.getArray(); - } - catch (Exception e) - { - _log.error("Could not retrieve array", e); - o = null; - } - } - - zeroBasedList.add(o); - } - } - else - { - zeroBasedList = null; - } - return zeroBasedList; - } - - private boolean isMutatingSql(String sql) - { - return new MutatingSqlDetector(sql).isMutating(); - } - - - // Copied from Commons Lang 2.5 StringEscapeUtils. The method has been removed from Commons Lang 3.1 because it's - // simplistic and misleading. But we're only using it for logging. - private static String escapeSql(String str) - { - if (str == null) - { - return null; - } - return StringUtils.replace(str, "'", "''"); - } - - - private void _appendTableStackTrace(StringBuilder sb, int count, @Nullable StackTraceElement[] ste) - { - if (ste != null) - { - int i = 1; // Always skip getStackTrace() call - for (; i < ste.length; i++) - { - String line = ste[i].toString(); - if (!(line.startsWith("org.labkey.api.data.") || line.startsWith("java.lang.Thread"))) - break; - } - int last = Math.min(ste.length, i + count); - for (; i < last; i++) - { - String line = ste[i].toString(); - if (line.startsWith("javax.servlet.http.HttpServlet.service(")) - break; - sb.append("\n ").append(line); - } - } - } - - - public String getDebugSql() - { - return _debugSql; - } - - - ResultSet wrap(ResultSet rs) - { - return new ResultSetWrapper(rs) - { - @Override - public boolean next() throws SQLException - { - try - { - return super.next(); - } - catch (SQLException x) - { - if (SqlDialect.isTransactionException(x)) - _logStatement(_debugSql, x, -1, getQueryLogging()); - throw x; - } - } - }; - } -} +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data.dialect; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.OneBasedList; +import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.Container; +import org.labkey.api.data.QueryLogging; +import org.labkey.api.data.ResultSetWrapper; +import org.labkey.api.data.queryprofiler.Query; +import org.labkey.api.data.queryprofiler.QueryProfiler; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.DebugInfoDumper; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MemTracker; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.view.ViewServlet; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.Date; +import java.sql.NClob; +import java.sql.ParameterMetaData; +import java.sql.PreparedStatement; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Map; + +public class StatementWrapper implements Statement, PreparedStatement, CallableStatement +{ + private final ConnectionWrapper _conn; + private final Statement _stmt; + private final Logger _log; + private String _debugSql = ""; + private long _msStart = 0; + private boolean userCancelled = false; + // NOTE: CallableStatement supports getObject(), but PreparedStatement doesn't + private OneBasedList _parameters = null; + private @Nullable StackTraceElement[] _stackTrace = null; + /** Track the place that closed this statement for troubleshooting purposes */ + private @Nullable Throwable _closingStackTrace = null; + private @Nullable Boolean _requestThread = null; + private QueryLogging _queryLogging = QueryLogging.emptyQueryLogging(); + + String _sqlStateTestException = null; + + + public StatementWrapper(ConnectionWrapper conn, Statement stmt) + { + _conn = conn; + _log = conn.getLogger(); + _stmt = stmt; + assert MemTracker.getInstance().put(this); + } + + public StatementWrapper(ConnectionWrapper conn, Statement stmt, String sql) + { + this(conn, stmt); + _debugSql = sql; + } + + public void setStackTrace(@Nullable StackTraceElement[] stackTrace) + { + _stackTrace = stackTrace; + } + + public @NotNull Boolean isRequestThread() + { + return null != _requestThread ? _requestThread : ViewServlet.isRequestThread(); + } + + public @Nullable Throwable getClosingStackTrace() + { + return _closingStackTrace; + } + + public void setRequestThread(@Nullable Boolean requestThread) + { + _requestThread = requestThread; + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + public QueryLogging getQueryLogging() + { + return _queryLogging; + } + + public void setQueryLogging(QueryLogging queryLogging) + { + _queryLogging = queryLogging; + } + + @Override + public boolean wasNull() + throws SQLException + { + try + { + return ((CallableStatement)_stmt).wasNull(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String getString(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getString(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getBoolean(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBoolean(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte getByte(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getByte(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public short getShort(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getShort(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getInt(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getInt(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public long getLong(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getLong(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public float getFloat(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getFloat(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public double getDouble(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDouble(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(int parameterIndex, int scale) + throws SQLException + { + try + { + //noinspection deprecation + return ((CallableStatement)_stmt).getBigDecimal(parameterIndex, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte[] getBytes(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBytes(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBigDecimal(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(int i, Map> map) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(i, map); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Ref getRef(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getRef(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Blob getBlob(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBlob(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Clob getClob(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getClob(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Array getArray(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getArray(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public URL getURL(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getURL(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setURL(String parameterName, URL val) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setURL(parameterName, val); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(String parameterName, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setNull(parameterName, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBoolean(String parameterName, boolean x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBoolean(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setByte(String parameterName, byte x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setByte(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setShort(String parameterName, short x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setShort(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setInt(String parameterName, int x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setInt(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setLong(String parameterName, long x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setLong(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFloat(String parameterName, float x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setFloat(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDouble(String parameterName, double x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDouble(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBigDecimal(String parameterName, BigDecimal x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBigDecimal(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setString(String parameterName, String x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setString(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBytes(String parameterName, byte[] x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBytes(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(String parameterName, Date x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDate(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(String parameterName, Time x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTime(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(String parameterName, Timestamp x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTimestamp(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setAsciiStream(String parameterName, InputStream x, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setAsciiStream(parameterName, x, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBinaryStream(String parameterName, InputStream x, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBinaryStream(parameterName, x, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x, int targetSqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x, int targetSqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setCharacterStream(String parameterName, Reader reader, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setCharacterStream(parameterName, reader, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(String parameterName, Date x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDate(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(String parameterName, Time x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTime(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(String parameterName, Timestamp x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTimestamp(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(String parameterName, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setNull(parameterName, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String getString(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getString(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getBoolean(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBoolean(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte getByte(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getByte(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public short getShort(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getShort(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getInt(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getInt(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public long getLong(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getLong(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public float getFloat(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getFloat(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public double getDouble(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDouble(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte[] getBytes(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBytes(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBigDecimal(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(String parameterName, Map> map) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterName, map); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Ref getRef(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getRef(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Blob getBlob(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBlob(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Clob getClob(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getClob(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Array getArray(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getArray(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public URL getURL(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getURL(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet executeQuery() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + if (null != _sqlStateTestException) + throw new SQLException("Test sql exception", _sqlStateTestException); + + ResultSet rs = ((PreparedStatement)_stmt).executeQuery(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int executeUpdate() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + int rows = -1; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = ((PreparedStatement)_stmt).executeUpdate(); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + private void _set(int i, @Nullable Object o) + { + if (null == _parameters) + _parameters = new OneBasedList<>(10); + while (_parameters.size() < i) + _parameters.add(null); + _parameters.set(i, o); + } + + @Override + public void setNull(int parameterIndex, int sqlType) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType); + _set(parameterIndex, null); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBoolean(int parameterIndex, boolean x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBoolean(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setByte(int parameterIndex, byte x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setByte(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setShort(int parameterIndex, short x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setShort(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setInt(int parameterIndex, int x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setInt(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setLong(int parameterIndex, long x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setLong(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFloat(int parameterIndex, float x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setFloat(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDouble(int parameterIndex, double x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDouble(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBigDecimal(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setString(int parameterIndex, String x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setString(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBytes(int parameterIndex, byte[] x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBytes(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(int parameterIndex, Date x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDate(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(int parameterIndex, Time x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTime(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setAsciiStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + //noinspection deprecation + ((PreparedStatement)_stmt).setUnicodeStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBinaryStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearParameters() + throws SQLException + { + try + { + ((PreparedStatement)_stmt).clearParameters(); + if (null != _parameters) _parameters.clear(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scale) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType, scale); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean execute() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + Boolean ret=null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + ret = ((PreparedStatement)_stmt).execute(); + return ret; + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + int rows = (ret==Boolean.FALSE) ? _stmt.getUpdateCount() : -1; + afterExecute(preQuery, ex, rows); + } + } + + @Override + public void addBatch() + throws SQLException + { + try + { + ((PreparedStatement)_stmt).addBatch(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + // NOTE: We intentionally do not store potentially large parameters (reader, blob, etc.) + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setCharacterStream(parameterIndex, reader, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setRef(int i, Ref x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setRef(i, x); + _set(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBlob(int i, Blob x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBlob(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setClob(int i, Clob x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setClob(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setArray(int i, Array x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setArray(i, x); + _set(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSetMetaData getMetaData() + throws SQLException + { + try + { + ResultSetMetaData rs = ((PreparedStatement)_stmt).getMetaData(); + assert MemTracker.getInstance().put(rs); + return rs; + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDate(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTime(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType, typeName); + _set(parameterIndex, null); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setURL(int parameterIndex, URL x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setURL(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ParameterMetaData getParameterMetaData() + throws SQLException + { + try + { + return ((PreparedStatement)_stmt).getParameterMetaData(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet executeQuery(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + ResultSet rs = _stmt.executeQuery(sql); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int executeUpdate(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public void close() + throws SQLException + { + try + { + _stmt.close(); + if (AppProps.getInstance().isDevMode() && _closingStackTrace == null) + { + _closingStackTrace = new Throwable("Remembering stack for closing Statement on thread " + Thread.currentThread().getName()); + } + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getMaxFieldSize() + throws SQLException + { + try + { + return _stmt.getMaxFieldSize(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setMaxFieldSize(int max) + throws SQLException + { + try + { + _stmt.setMaxFieldSize(max); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getMaxRows() + throws SQLException + { + try + { + return _stmt.getMaxRows(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setMaxRows(int max) + throws SQLException + { + try + { + _stmt.setMaxRows(max); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setEscapeProcessing(boolean enable) + throws SQLException + { + try + { + _stmt.setEscapeProcessing(enable); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getQueryTimeout() + throws SQLException + { + try + { + return _stmt.getQueryTimeout(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setQueryTimeout(int seconds) + throws SQLException + { + try + { + _stmt.setQueryTimeout(seconds); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void cancel() + throws SQLException + { + try + { + userCancelled = true; + _stmt.cancel(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public SQLWarning getWarnings() + throws SQLException + { + try + { + return _stmt.getWarnings(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearWarnings() + throws SQLException + { + try + { + _stmt.clearWarnings(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setCursorName(String name) + throws SQLException + { + try + { + _stmt.setCursorName(name); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean execute(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public ResultSet getResultSet() + throws SQLException + { + try + { + ResultSet rs = _stmt.getResultSet(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getUpdateCount() + throws SQLException + { + try + { + int updateCount; + updateCount = _stmt.getUpdateCount(); + return updateCount; + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getMoreResults() + throws SQLException + { + try + { + return _stmt.getMoreResults(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFetchDirection(int direction) + throws SQLException + { + try + { + _stmt.setFetchDirection(direction); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getFetchDirection() + throws SQLException + { + try + { + return _stmt.getFetchDirection(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFetchSize(int rows) + throws SQLException + { + try + { + _stmt.setFetchSize(rows); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getFetchSize() + throws SQLException + { + try + { + return _stmt.getFetchSize(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getResultSetConcurrency() + throws SQLException + { + try + { + return _stmt.getResultSetConcurrency(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getResultSetType() + throws SQLException + { + try + { + return _stmt.getResultSetType(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void addBatch(String sql) + throws SQLException + { + try + { + _stmt.addBatch(sql); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearBatch() + throws SQLException + { + try + { + _stmt.clearBatch(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int[] executeBatch() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return _stmt.executeBatch(); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public Connection getConnection() + { + return _conn; + } + + @Override + public boolean getMoreResults(int current) + throws SQLException + { + try + { + return _stmt.getMoreResults(current); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet getGeneratedKeys() + throws SQLException + { + try + { + ResultSet rs = _stmt.getGeneratedKeys(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, autoGeneratedKeys); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, columnIndexes); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public int executeUpdate(String sql, String[] columnNames) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, columnNames); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, autoGeneratedKeys); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public boolean execute(String sql, int[] columnIndexes) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, columnIndexes); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public boolean execute(String sql, String[] columnNames) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, columnNames); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int getResultSetHoldability() + throws SQLException + { + try + { + return _stmt.getResultSetHoldability(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String toString() + { + return _stmt.toString(); + } + + // TODO: These methods should be properly implemented via delegation. + + @Override + public boolean isWrapperFor(Class iface) + { + throw new UnsupportedOperationException(); + } + + @Override + public T unwrap(Class iface) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isClosed() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isPoolable() + { + throw new UnsupportedOperationException(); + } + + @Override + public void setPoolable(boolean poolable) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException + { + try + { + if (length > Integer.MAX_VALUE) + throw new IllegalArgumentException("File length exceeds " + Integer.MAX_VALUE); + setBinaryStream(parameterIndex, x, (int)length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, NClob value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNString(int parameterIndex, String value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setRowId(int parameterIndex, RowId x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getCharacterStream(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getCharacterStream(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getNCharacterStream(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getNCharacterStream(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public NClob getNClob(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public NClob getNClob(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public String getNString(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public String getNString(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public RowId getRowId(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public RowId getRowId(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLXML getSQLXML(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLXML getSQLXML(String parameterName) + { + throw new UnsupportedOperationException(); + }//-- + + @Override + public void setAsciiStream(String parameterName, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(String parameterName, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(String parameterName, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(String parameterName, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, InputStream inputStream) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, InputStream inputStream, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, Blob x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Clob x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(String parameterName, Reader value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(String parameterName, Reader value, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, NClob value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNString(String parameterName, String value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setRowId(String parameterName, RowId x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setSQLXML(String parameterName, SQLXML xmlObject) + { + throw new UnsupportedOperationException(); + } + + @Override + public T getObject(int parameterIndex, Class type) + { + throw new UnsupportedOperationException(); + } + + // JDBC 4.1 methods below must be here so we compile on JDK 7; implement once we require JRE 7. + + @Override + public T getObject(String parameterName, Class type) + { + throw new UnsupportedOperationException(); + } + + @Override + public void closeOnCompletion() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCloseOnCompletion() + { + throw new UnsupportedOperationException(); + } + + private Query beforeExecute(String sql) + { + _debugSql = sql; + // Crawler.java and BaseWebDriverTest.java use "8(" as attempted injection string + if (_debugSql.contains("\"8(\"") && !_debugSql.contains("\"\"8(\"\"")) // 18196 + throw new IllegalArgumentException("SQL injection test failed: " + _debugSql); + _msStart = System.currentTimeMillis(); + + List zeroBasedList = translateParametersForQueryTracking(); + return QueryProfiler.getInstance().preTrack(_conn.getScope(), sql, zeroBasedList, _stackTrace, isRequestThread()); + } + + + private void afterExecute(Query query, @Nullable SQLException x, int rowsAffected) + { + if (null != x) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.DialectSQL, query.getOriginalSql(), true); + if (SqlDialect.isConfigurationException(x)) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + } + } + _logStatement(query.getOriginalSql(), x, rowsAffected, getQueryLogging()); + } + + + private static final Package JAVA_LANG = java.lang.String.class.getPackage(); + + private void _logStatement(String sql, @Nullable SQLException x, int rowsAffected, QueryLogging queryLogging) + { + long elapsed = System.currentTimeMillis() - _msStart; + boolean isAssertEnabled = false; + assert isAssertEnabled = true; + + if (isAssertEnabled && AppProps.getInstance().isDevMode() && isMutatingSql(sql)) + SpringActionController.executingMutatingSql(sql); + + // Hold on to this stack trace so that we can reuse it later (if collection has been enabled) + Query query = QueryProfiler.getInstance().track(_conn.getScope(), sql, translateParametersForQueryTracking(), elapsed, _stackTrace, isRequestThread(), queryLogging); + + if (x != null) + { + _conn.logAndCheckException(x); + } + + //noinspection ConstantConditions + if (!_log.isEnabled(Level.DEBUG) && !isAssertEnabled) + return; + + StringBuilder logEntry = new StringBuilder(sql.length() * 2); + logEntry.append("SQL "); + + Integer sid = _conn.getSPID(); + if (sid != null) + logEntry.append(" [").append(sid).append("]"); + if (_msStart != 0) + logEntry.append(" time ").append(DateUtil.formatDuration(elapsed)); + + if (-1 != rowsAffected) + logEntry.append("\n ").append(rowsAffected).append(" rows affected"); + + logEntry.append("\n "); + logEntry.append(sql.trim().replace("\n", "\n ")); + + if (null != _parameters) + { + for (int i = 1; i <= _parameters.size(); i++) + { + try + { + Object o = _parameters.get(i); + String value; + if (o == null) + value = "NULL"; + else if (o instanceof Container) + value = "'" + ((Container)o).getId() + "' " + ((Container)o).getPath(); + else if (o instanceof String) + value = "'" + escapeSql((String) o) + "'"; + else + value = String.valueOf(o); + if (value.length() > 100) + value = StringUtilsLabKey.leftSurrogatePairFriendly(value, 100) + ". . ."; + logEntry.append("\n --[").append(i).append("] "); + logEntry.append(value); + Class c = null==o ? null : o.getClass(); + if (null != c && c != String.class && c != Integer.class) + logEntry.append(" :").append(c.getPackage() == JAVA_LANG ? c.getSimpleName() : c.getName()); + } + catch (Exception ex) + { + /* */ + } + } + } + _parameters = null; + + if (userCancelled) + logEntry.append("\n cancelled by user"); + if (null != x) + logEntry.append("\n ").append(x); + _appendTableStackTrace(logEntry, 5, query.getStackTraceElements()); + + final String logString = logEntry.toString(); + _log.log(Level.DEBUG, logString); + + // check for deadlock or transaction related error + if (SqlDialect.isTransactionException(x)) + { + DebugInfoDumper.dumpThreads(_log); + } + } + + private @Nullable List translateParametersForQueryTracking() + { + // Make a copy of the parameters list (it gets modified by callers) and switch to zero-based list (_parameters is a one-based list) + + List zeroBasedList; + + if (null != _parameters) + { + zeroBasedList = new ArrayList<>(_parameters.size()); + + // Translate parameters that can't be cached (for now, just JDBC arrays). I'd rather stash the original parameters and send + // those to the query profiler, but this would require one or more non-standard methods on StatementWrapper. See #24314. + for (Object o : _parameters) + { + if (o instanceof Array a) + { + try + { + o = a.getArray(); + } + catch (Exception e) + { + _log.error("Could not retrieve array", e); + o = null; + } + } + + zeroBasedList.add(o); + } + } + else + { + zeroBasedList = null; + } + return zeroBasedList; + } + + private boolean isMutatingSql(String sql) + { + return new MutatingSqlDetector(sql).isMutating(); + } + + + // Copied from Commons Lang 2.5 StringEscapeUtils. The method has been removed from Commons Lang 3.1 because it's + // simplistic and misleading. But we're only using it for logging. + private static String escapeSql(String str) + { + if (str == null) + { + return null; + } + return StringUtils.replace(str, "'", "''"); + } + + + private void _appendTableStackTrace(StringBuilder sb, int count, @Nullable StackTraceElement[] ste) + { + if (ste != null) + { + int i = 1; // Always skip getStackTrace() call + for (; i < ste.length; i++) + { + String line = ste[i].toString(); + if (!(line.startsWith("org.labkey.api.data.") || line.startsWith("java.lang.Thread"))) + break; + } + int last = Math.min(ste.length, i + count); + for (; i < last; i++) + { + String line = ste[i].toString(); + if (line.startsWith("javax.servlet.http.HttpServlet.service(")) + break; + sb.append("\n ").append(line); + } + } + } + + + public String getDebugSql() + { + return _debugSql; + } + + + ResultSet wrap(ResultSet rs) + { + return new ResultSetWrapper(rs) + { + @Override + public boolean next() throws SQLException + { + try + { + return super.next(); + } + catch (SQLException x) + { + if (SqlDialect.isTransactionException(x)) + _logStatement(_debugSql, x, -1, getQueryLogging()); + throw x; + } + } + }; + } +} diff --git a/api/src/org/labkey/api/exp/OntologyManager.java b/api/src/org/labkey/api/exp/OntologyManager.java index 26d1b6d43ec..bd2a07600eb 100644 --- a/api/src/org/labkey/api/exp/OntologyManager.java +++ b/api/src/org/labkey/api/exp/OntologyManager.java @@ -1,3914 +1,3915 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.exp; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.data.*; -import org.labkey.api.data.DbScope.Transaction; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.gwt.client.ui.domain.CancellationException; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.test.TestTimeout; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.view.HttpView; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableCollection; -import static java.util.Collections.unmodifiableList; -import static java.util.Collections.unmodifiableMap; -import static java.util.stream.Collectors.joining; -import static org.labkey.api.util.IntegerUtils.asLong; - -/** - * Lots of static methods for dealing with domains and property descriptors. Tends to operate primarily on the bean-style - * classes like {@link PropertyDescriptor} and {@link DomainDescriptor}. When possible, it's usually preferable to use - * {@link PropertyService}, {@link Domain}, and {@link DomainProperty} instead as they tend to provide higher-level - * abstractions. - */ -public class OntologyManager -{ - private static final Logger _log = LogManager.getLogger(OntologyManager.class); - private static final Cache, Map> PROPERTY_MAP_CACHE = DatabaseCache.get(getExpSchema().getScope(), 100000, "Property maps", new PropertyMapCacheLoader()); - private static final BlockingCache OBJECT_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "ObjectIds", new ObjectIdCacheLoader()); - private static final Cache, PropertyDescriptor> PROP_DESCRIPTOR_CACHE = DatabaseCache.get(getExpSchema().getScope(), 40000, "Property descriptors", new CacheLoader<>() - { - @Override - public PropertyDescriptor load(@NotNull Pair key, @Nullable Object argument) - { - PropertyDescriptor ret = null; - String propertyURI = key.first; - Container c = ContainerManager.getForId(key.second); - if (null != c) - { - Container proj = c.getProject(); - if (null == proj) - proj = c; - _log.debug("Loading a property descriptor for key " + key + " using project " + proj); - String sql = " SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyURI = ? AND Project IN (?,?)"; - List pdArray = new SqlSelector(getExpSchema(), sql, propertyURI, proj, _sharedContainer.getId()).getArrayList(PropertyDescriptor.class); - if (!pdArray.isEmpty()) - { - PropertyDescriptor pd = pdArray.get(0); - - // if someone has explicitly inserted a descriptor with the same URI as an existing one, - // and one of the two is in the shared project, use the project-level descriptor. - if (pdArray.size() > 1) - { - _log.debug("Multiple PropertyDescriptors found for " + propertyURI); - if (pd.getProject().equals(_sharedContainer)) - pd = pdArray.get(1); - } - _log.debug("Loaded property descriptor " + pd); - ret = pd; - } - } - return ret; - } - }); - - /** DomainURI, ContainerEntityId -> DomainDescriptor */ - private static final Cache, DomainDescriptor> DOMAIN_DESCRIPTORS_BY_URI_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, CacheManager.UNLIMITED, "Domain descriptors by URI", (key, argument) -> { - String domainURI = key.first; - Container c = ContainerManager.getForId(key.second); - - if (c == null) - { - return null; - } - - return fetchDomainDescriptorFromDB(domainURI, c); - }); - - @Nullable - private static DomainDescriptor fetchDomainDescriptorFromDB(String domainURI, Container c) - { - return fetchDomainDescriptorFromDB(domainURI, c, false); - } - - /** Goes against the DB, bypassing the cache */ - @Nullable - public static DomainDescriptor fetchDomainDescriptorFromDB(String uriOrName, Container c, boolean isName) - { - Container proj = c.getProject(); - if (null == proj) - proj = c; - - String sql = " SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE " + (isName ? "Name" : "DomainURI") + " = ? AND Project IN (?,?) "; - List ddArray = new SqlSelector(getExpSchema(), sql, uriOrName, - proj, - ContainerManager.getSharedContainer().getId()).getArrayList(DomainDescriptor.class); - DomainDescriptor dd = null; - if (!ddArray.isEmpty()) - { - dd = ddArray.get(0); - - // if someone has explicitly inserted a descriptor with the same URI as an existing one , - // and one of the two is in the shared project, use the project-level descriptor. - if (ddArray.size() > 1) - { - _log.debug("Multiple DomainDescriptors found for " + uriOrName); - if (dd.getProject().equals(ContainerManager.getSharedContainer())) - dd = ddArray.get(0); - } - } - return dd; - } - - private static final BlockingCache DOMAIN_DESC_BY_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(),2000, CacheManager.UNLIMITED,"Domain descriptors by ID", new DomainDescriptorLoader()); - private static final BlockingCache, List>> DOMAIN_PROPERTIES_CACHE = DatabaseCache.get(getExpSchema().getScope(), 5000, CacheManager.UNLIMITED, "Domain properties", new CacheLoader<>() - { - @Override - public List> load(@NotNull Pair key, @Nullable Object argument) - { - String typeURI = key.first; - Container c = ContainerManager.getForId(key.second); - if (null == c) - return Collections.emptyList(); - SQLFragment sql = new SQLFragment("SELECT PropertyURI, Required " + - "FROM " + getTinfoPropertyDescriptor() + " PD\n" + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (PD.PropertyId = PDM.PropertyId)\n" + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)\n" + - "WHERE DD.DomainURI = ? AND DD.Project IN (?, ?) ORDER BY PDM.SortOrder, PD.PropertyId"); - - sql.addAll( - typeURI, - // protect against null project, just double-up shared project - c.isRoot() ? c.getId() : (c.getProject() == null ? _sharedContainer.getProject().getId() : c.getProject().getId()), - _sharedContainer.getProject().getId() - ); - - return new SqlSelector(getExpSchema(), sql).mapStream() - .map(map -> Pair.of((String)map.get("PropertyURI"), (Boolean)map.get("Required"))) - .toList(); - } - }); - private static final Cache> DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "Domain descriptors by container", (c, argument) -> { - String sql = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - - Map dds = new LinkedHashMap<>(); - for (DomainDescriptor dd : new SqlSelector(getExpSchema(), sql, c).getArrayList(DomainDescriptor.class)) - { - dds.putIfAbsent(dd.getDomainURI(), dd); - } - - return unmodifiableMap(dds); - }); - - private static final Container _sharedContainer = ContainerManager.getSharedContainer(); - - public static final String MV_INDICATOR_SUFFIX = "mvindicator"; - - static public String PropertyOrderURI = "urn:exp.labkey.org/#PropertyOrder"; - /** - * A comma-separated list of propertyID that indicates the sort order of the properties attached to an object. - */ - static public SystemProperty PropertyOrder = new SystemProperty(PropertyOrderURI, PropertyType.STRING); - - static - { - BeanObjectFactory.Registry.register(ObjectProperty.class, new ObjectProperty.ObjectPropertyObjectFactory()); - } - - private OntologyManager() - { - } - - /** - * @return map from PropertyURI to value - */ - public static @NotNull Map getProperties(Container container, String parentLSID) - { - Map m = new LinkedHashMap<>(); - Map propVals = getPropertyObjects(container, parentLSID); - if (null != propVals) - { - for (Map.Entry entry : propVals.entrySet()) - { - m.put(entry.getKey(), entry.getValue().value()); - } - } - - return m; - } - - public static final int MAX_PROPS_IN_BATCH = 1000; // Keep this reasonably small so progress indicator is updated regularly - public static final int UPDATE_STATS_BATCH_COUNT = 1000; - - public static void insertTabDelimited(Container c, - User user, - @Nullable Long ownerObjectId, - ImportHelper helper, - Domain domain, - DataIterator rows, - boolean ensureObjects, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - List properties = new ArrayList<>(domain.getProperties().size()); - for (DomainProperty prop : domain.getProperties()) - { - properties.add(prop.getPropertyDescriptor()); - } - insertTabDelimited(c, user, ownerObjectId, helper, properties, rows, ensureObjects, rowCallback); - } - - public interface RowCallback - { - void rowProcessed(Map row, String lsid) throws BatchValidationException; - - default void complete() throws BatchValidationException - {} - - default RowCallback chain(RowCallback other) - { - if (other == NO_OP_ROW_CALLBACK) - { - return this; - } - if (this == NO_OP_ROW_CALLBACK) - { - return other; - } - - RowCallback original = this; - - return new RowCallback() - { - @Override - public void rowProcessed(Map row, String lsid) throws BatchValidationException - { - original.rowProcessed(row, lsid); - other.rowProcessed(row, lsid); - } - - @Override - public void complete() throws BatchValidationException - { - original.complete(); - other.complete(); - } - }; - } - } - - public static final RowCallback NO_OP_ROW_CALLBACK = (row, lsid) -> {}; - - public static void insertTabDelimited(Container c, - User user, - @Nullable Long ownerObjectId, - ImportHelper helper, - List descriptors, - DataIterator rawRows, - boolean ensureObjects, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - MapDataIterator rows = DataIteratorUtil.wrapMap(rawRows, false); - - rowCallback = rowCallback == null ? NO_OP_ROW_CALLBACK : rowCallback; - - CPUTimer total = new CPUTimer("insertTabDelimited"); - CPUTimer before = new CPUTimer("beforeImport"); - CPUTimer ensure = new CPUTimer("ensureObject"); - CPUTimer insert = new CPUTimer("insertProperties"); - - assert total.start(); - assert getExpSchema().getScope().isTransactionActive(); - - // Make sure we have enough rows to handle the overflow of the current row so we don't have to resize the list - List propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); - - ValidatorContext validatorCache = new ValidatorContext(c, user); - - try - { - OntologyObject objInsert = new OntologyObject(); - objInsert.setContainer(c); - if (ownerObjectId != null && ownerObjectId > 0) - objInsert.setOwnerObjectId(ownerObjectId); - - List errors = new ArrayList<>(); - Map> validatorMap = new IntHashMap<>(); - - // cache all the property validators for this upload - for (PropertyDescriptor pd : descriptors) - { - List validators = PropertyService.get().getPropertyValidators(pd); - if (!validators.isEmpty()) - validatorMap.put(pd.getPropertyId(), validators); - } - - int rowCount = 0; - int batchCount = 0; - - while (rows.next()) - { - Map map = rows.getMap(); - // TODO: hack -- should exit and return cancellation status instead of throwing - if (Thread.currentThread().isInterrupted()) - throw new CancellationException(); - - assert before.start(); - - Map modifiableMap = new HashMap<>(map); - String lsid = helper.beforeImportObject(modifiableMap); - map = Collections.unmodifiableMap(modifiableMap); - - if (lsid == null) - { - throw new IllegalStateException("No LSID available"); - } - - assert before.stop(); - - assert ensure.start(); - long objectId; - if (ensureObjects) - objectId = ensureObject(c, lsid, ownerObjectId); - else - { - objInsert.setObjectURI(lsid); - Table.insert(null, getTinfoObject(), objInsert); - objectId = objInsert.getObjectId(); - } - - for (PropertyDescriptor pd : descriptors) - { - Object value = map.get(pd.getPropertyURI()); - if (null == value) - { - if (pd.isRequired()) - throw new BatchValidationException(new ValidationException("Missing value for required property " + pd.getName())); - else - { - continue; - } - } - else - { - if (validatorMap.containsKey(pd.getPropertyId())) - validateProperty(validatorMap.get(pd.getPropertyId()), pd, new ObjectProperty(lsid, c, pd, value), errors, validatorCache); - } - try - { - PropertyRow row = new PropertyRow(objectId, pd, value, pd.getPropertyType()); - propsToInsert.add(row); - } - catch (ConversionException e) - { - throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getPropertyType().getJavaType()))); - } - } - assert ensure.stop(); - - rowCount++; - - if (propsToInsert.size() > MAX_PROPS_IN_BATCH) - { - assert insert.start(); - insertPropertiesBulk(c, propsToInsert, false); - helper.afterBatchInsert(rowCount); - assert insert.stop(); - propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); - - if (++batchCount % UPDATE_STATS_BATCH_COUNT == 0) - { - getExpSchema().getSqlDialect().updateStatistics(getTinfoObject()); - getExpSchema().getSqlDialect().updateStatistics(getTinfoObjectProperty()); - helper.updateStatistics(rowCount); - } - } - - rowCallback.rowProcessed(map, lsid); - } - - if (!errors.isEmpty()) - throw new BatchValidationException(new ValidationException(errors)); - - assert insert.start(); - insertPropertiesBulk(c, propsToInsert, false); - helper.afterBatchInsert(rowCount); - rowCallback.complete(); - assert insert.stop(); - } - catch (SQLException x) - { - SQLException next = x.getNextException(); - if (x instanceof java.sql.BatchUpdateException && null != next) - x = next; - _log.debug("Exception uploading: ", x); - throw x; - } - - assert total.stop(); - _log.debug("\t" + total); - _log.debug("\t" + before); - _log.debug("\t" + ensure); - _log.debug("\t" + insert); - } - - /** - * As an incremental step of QueryUpdateService cleanup, this is a version of insertTabDelimited that works on a - * tableInfo that implements UpdateableTableInfo. Does not support ownerObjectid. - *

- * This code is made complicated by the fact that while we are trying to move toward a TableInfo/ColumnInfo view - * of the world, validators are attached to PropertyDescriptors. Also, missing value handling is attached - * to PropertyDescriptors. - *

- * The original version of this method expects a data to be a map PropertyURI->value. This version will also - * accept Name->value. - *

- * Name->Value is preferred, we are using TableInfo after all. - */ - @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder - public static void insertTabDelimited(TableInfo tableInsert, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator rows, - boolean autoFillDefaultColumns, - Logger logger, - RowCallback rowCallback) - throws SQLException, BatchValidationException - { - saveTabDelimited(tableInsert, c, user, helper, rows, logger, true, autoFillDefaultColumns, rowCallback); - } - - @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder - public static void updateTabDelimited(TableInfo tableInsert, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator rows, - boolean autoFillDefaultColumns, - Logger logger) - throws SQLException, BatchValidationException - { - saveTabDelimited(tableInsert, c, user, helper, rows, logger, false, autoFillDefaultColumns, NO_OP_ROW_CALLBACK); - } - - private static void saveTabDelimited(TableInfo table, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator in, - Logger logger, - boolean insert, - boolean autoFillDefaultColumns, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - if (!(table instanceof UpdateableTableInfo)) - throw new IllegalArgumentException(); - - if (rowCallback == null) - { - rowCallback = NO_OP_ROW_CALLBACK; - } - - DbScope scope = table.getSchema().getScope(); - - assert scope.isTransactionActive(); - - Domain d = table.getDomain(); - List properties = null == d ? Collections.emptyList() : d.getProperties(); - - ValidatorContext validatorCache = new ValidatorContext(c, user); - - Connection conn = null; - ParameterMapStatement parameterMap = null; - - Map currentRow = null; - - MapDataIterator rows = DataIteratorUtil.wrapMap(in, false); - try - { - conn = scope.getConnection(); - if (insert) - { - parameterMap = StatementUtils.insertStatement(conn, table, c, user, true, autoFillDefaultColumns); - } - else - { - parameterMap = StatementUtils.updateStatement(conn, table, c, user, false, autoFillDefaultColumns); - } - List errors = new ArrayList<>(); - - Map> validatorMap = new HashMap<>(); - Map propertiesMap = new HashMap<>(); - - // cache all the property validators for this upload - for (DomainProperty dp : properties) - { - propertiesMap.put(dp.getPropertyURI(), dp); - List validators = dp.getValidators(); - if (!validators.isEmpty()) - validatorMap.put(dp.getPropertyURI(), validators); - } - - List columns = table.getColumns(); - PropertyType[] propertyTypes = new PropertyType[columns.size()]; - for (int i = 0; i < columns.size(); i++) - { - String propertyURI = columns.get(i).getPropertyURI(); - DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); - PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); - if (null != pd) - propertyTypes[i] = pd.getPropertyType(); - } - - int rowCount = 0; - - while (rows.next()) - { - - currentRow = new CaseInsensitiveHashMap<>(rows.getMap()); - - // TODO: hack -- should exit and return cancellation status instead of throwing - if (Thread.currentThread().isInterrupted()) - throw new CancellationException(); - - parameterMap.clearParameters(); - - String lsid = helper.beforeImportObject(currentRow); - currentRow.put("lsid", lsid); - - // - // NOTE we validate based on columninfo/propertydescriptor - // However, we bind by name, and there may be parameters that do not correspond to columninfo - // - - for (int i = 0; i < columns.size(); i++) - { - ColumnInfo col = columns.get(i); - if (col.isMvIndicatorColumn() || col.isRawValueColumn()) //TODO col.isNotUpdatableForSomeReasonSoContinue() - continue; - String propertyURI = col.getPropertyURI(); - DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); - PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); - - Object value = currentRow.get(col.getName()); - if (null == value) - value = currentRow.get(propertyURI); - - if (null == value) - { - // TODO col.isNullable() doesn't seem to work here - if (null != pd && pd.isRequired()) - throw new BatchValidationException(new ValidationException("Missing value for required property " + col.getName())); - } - else - { - if (null != pd) - { - try - { - // Use an ObjectProperty to unwrap MvFieldWrapper, do type conversion, etc - ObjectProperty objectProperty = new ObjectProperty(lsid, c, pd, value); - if (!validateProperty(validatorMap.get(propertyURI), pd, objectProperty, errors, validatorCache)) - { - throw new BatchValidationException(new ValidationException(errors)); - } - } - catch (ConversionException e) - { - throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getJavaClass()))); - } - } - } - - // issue 19391: data from R uses "Inf" to represent infinity - if (JdbcType.DOUBLE.equals(col.getJdbcType())) - { - value = "Inf".equals(value) ? "Infinity" : value; - value = "-Inf".equals(value) ? "-Infinity" : value; - } - - try - { - String key = col.getName(); - if (!parameterMap.containsKey(key)) - key = propertyURI; - if (null == propertyTypes[i]) - { - // some built-in columns won't have parameters (createdby, etc) - if (parameterMap.containsKey(key)) - { - assert !(value instanceof MvFieldWrapper); - // Handle type coercion for these built-in columns as well, though we don't need to - // worry about missing values - value = PropertyType.getFromClass(col.getJavaObjectClass()).convert(value); - parameterMap.put(key, value); - } - } - else - { - Pair p = new Pair<>(value, null); - convertValuePair(pd, propertyTypes[i], p); - parameterMap.put(key, p.first); - if (null != p.second) - { - FieldKey mvName = col.getMvColumnName(); - if (mvName != null) - { - String storageName = table.getColumn(mvName).getMetaDataIdentifier().getId(); - parameterMap.put(storageName, p.second); - } - } - } - } - catch (ConversionException e) - { - throw new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), propertyTypes[i].getJavaType())); - } - } - - helper.bindAdditionalParameters(currentRow, parameterMap); - parameterMap.execute(); - if (insert) - { - long rowId = parameterMap.getRowId(); - currentRow.put("rowId", rowId); - } - lsid = helper.afterImportObject(currentRow); - if (lsid == null) - { - throw new IllegalStateException("No LSID available"); - } - rowCallback.rowProcessed(currentRow, lsid); - rowCount++; - } - - - if (!errors.isEmpty()) - throw new BatchValidationException(new ValidationException(errors)); - - rowCallback.complete(); - - helper.afterBatchInsert(rowCount); - if (logger != null) - logger.debug("inserted row " + rowCount + "."); - } - catch (ValidationException e) - { - throw new BatchValidationException(e); - } - catch (SQLException x) - { - SQLException next = x.getNextException(); - if (x instanceof java.sql.BatchUpdateException && null != next) - x = next; - _log.debug("Exception uploading: ", x); - if (null != currentRow) - _log.debug(currentRow.toString()); - throw x; - } - finally - { - if (null != parameterMap) - parameterMap.close(); - if (null != conn) - scope.releaseConnection(conn); - } - } - - // TODO: Consolidate with ColumnValidator - public static boolean validateProperty(List validators, PropertyDescriptor prop, ObjectProperty objectProperty, - List errors, ValidatorContext validatorCache) - { - boolean ret = true; - - Object value = objectProperty.getObjectValue(); - - if (prop.isRequired() && value == null && objectProperty.getMvIndicator() == null) - { - errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is required", prop.getName())); - ret = false; - } - - // Check if the string is too long. Use either the PropertyDescriptor's scale or VARCHAR(4000) for ontology managed values - int stringLengthLimit = prop.getScale() > 0 ? prop.getScale() : getTinfoObjectProperty().getColumn("StringValue").getScale(); - int stringLength = value == null ? 0 : value.toString().length(); - if (value != null && prop.isStringType() && stringLength > stringLengthLimit) - { - String s = stringLength < 100 ? value.toString() : value.toString().substring(0, 100); - errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is limited to " + stringLengthLimit + " characters, but the value is " + stringLength + " characters. (The value starts with '" + s + "...')", prop.getName())); - ret = false; - } - - // TODO: check date is within postgres date range - - // Don't validate null values, #15683 - if (null != value && validators != null) - { - for (IPropertyValidator validator : validators) - if (!validator.validate(prop, value, errors, validatorCache)) ret = false; - } - return ret; - } - - public interface ImportHelper - { - /** - * may modify map - * - * @return LSID for new or existing Object. Null indicates LSID is still unknown. - */ - String beforeImportObject(Map map) throws SQLException; - - void afterBatchInsert(int currentRow) throws SQLException; - - void updateStatistics(int currentRow) throws SQLException; - } - - - public interface UpdateableTableImportHelper extends ImportHelper - { - /** - * may be used to process attachments, for auditing, etc - * @return the LSID of the inserted row - */ - String afterImportObject(Map map) throws SQLException; - - /** - * may set parameters directly for columns that are not exposed by tableinfo - * e.g. "_key" - *

- * TODO maybe this can be handled declaratively? see UpdateableTableInfo - */ - void bindAdditionalParameters(Map map, ParameterMapStatement target) throws ValidationException; - } - - @NotNull - private static Pair getPropertyMapCacheKey(@Nullable Container container, @NotNull String objectLSID) - { - return Pair.of(container, objectLSID); - } - - /** - * Get ordered map of property values for an object. The order of the properties in the - * Map corresponds to the PropertyOrder property, if present. - * - * @return map from PropertyURI to ObjectProperty - */ - public static Map getPropertyObjects(@Nullable Container container, @NotNull String objectLSID) - { - Pair cacheKey = getPropertyMapCacheKey(container, objectLSID); - return PROPERTY_MAP_CACHE.get(cacheKey); - } - - public static class PropertyMapCacheLoader implements CacheLoader, Map> - { - @Override - public Map load(@NotNull Pair key, @Nullable Object argument) - { - Container container = key.first; - String objectLSID = key.second; - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), objectLSID); - if (container != null) - { - filter.addCondition(FieldKey.fromParts("Container"), container); - } - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getResultSet()) - { - ResultSetUtil.logData(rs); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - List props = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getArrayList(ObjectProperty.class); - - // check for a "PropertyOrder" value - ObjectProperty propertyOrder = props.stream().filter(op -> PropertyOrderURI.equals(op.getPropertyURI())).findFirst().orElse(null); - if (propertyOrder != null) - { - String order = propertyOrder.getStringValue(); - if (order != null) - { - // CONSIDER: Store as a JSONArray of propertyURI instead of propertyId - String[] parts = order.split(","); - try - { - List propertyIds = Arrays.stream(parts).map(s -> ConvertHelper.convert(s, Integer.class)).toList(); - - // Don't include the "PropertyOrder" property - props = new ArrayList<>(props); - props.remove(propertyOrder); - - // Order by the index found in the PropertyOrder list, otherwise just stick it at the end - Comparator comparator = (op1, op2) -> { - int i1 = propertyIds.indexOf(op1.getPropertyId()); - if (i1 == -1) - i1 = propertyIds.size(); - - int i2 = propertyIds.indexOf(op2.getPropertyId()); - if (i2 == -1) - i2 = propertyIds.size(); - return i1 - i2; - }; - props.sort(comparator); - } - catch (ConversionException e) - { - _log.warn("Failed to parse PropertyOrder integer list: " + order); - } - } - } - - Map m = new LinkedHashMap<>(); - for (ObjectProperty value : props) - { - m.put(value.getPropertyURI(), value); - } - - return unmodifiableMap(m); - } - } - - public static void updateObjectPropertyOrder(User user, Container container, String objectLSID, List properties) - throws ValidationException - { - String ids = null; - if (properties != null && !properties.isEmpty()) - ids = properties.stream().map(pd -> Integer.toString(pd.getPropertyId())).collect(joining(",")); - - updateObjectProperty(user, container, PropertyOrder.getPropertyDescriptor(), objectLSID, ids, null, false); - } - - /** - * Moves the properties of an object from one container to another (used when the object is moving) - * @param targetContainer the container to move the properties to - * @param user the user doing the move - * @param objectLSID the LSID of the object to which the properties are attached - * @return number of properties moved - */ - public static int updateContainer(Container targetContainer, User user, @NotNull String objectLSID) - { - return Table.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); - } - - /** - * Get ordered list of the PropertyURI in {@link #PropertyOrder}, if present. - */ - public static List getObjectPropertyOrder(Container c, String objectLSID) - { - Map props = getPropertyObjects(c, objectLSID); - return new ArrayList<>(props.keySet()); - } - - public static long ensureObject(Container container, String objectURI) - { - return ensureObject(container, objectURI, (Long) null); - } - - public static long ensureObject(Container container, String objectURI, String ownerURI) - { - Long ownerId = null; - if (null != ownerURI) - ownerId = ensureObject(container, ownerURI, (Long) null); - return ensureObject(container, objectURI, ownerId); - } - - public static long ensureObject(Container container, String objectURI, Long ownerId) - { - //TODO: (marki) Transact? - Long objId = OBJECT_ID_CACHE.get(objectURI, container); - - if (null == objId) - { - OntologyObject obj = new OntologyObject(); - obj.setContainer(container); - obj.setObjectURI(objectURI); - if (ownerId != null && ownerId > 0) - obj.setOwnerObjectId(ownerId); - obj = Table.insert(null, getTinfoObject(), obj); - objId = obj.getObjectId(); - OBJECT_ID_CACHE.remove(objectURI); - } - - return objId; - } - - private static class ObjectIdCacheLoader implements CacheLoader - { - @Override - public Long load(@NotNull String objectURI, @Nullable Object argument) - { - Container container = (Container)argument; - OntologyObject obj = getOntologyObject(container, objectURI); - - return obj == null ? null : obj.getObjectId(); - } - } - - public static @Nullable OntologyObject getOntologyObject(Container container, String uri) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), uri); - if (container != null) - { - filter.addCondition(FieldKey.fromParts("Container"), container.getId()); - } - return new TableSelector(getTinfoObject(), filter, null).getObject(OntologyObject.class); - } - - // UNDONE: optimize (see deleteOntologyObjects(Integer[]) - public static void deleteOntologyObjects(Container c, String... uris) - { - if (uris.length == 0) - return; - - try - { - DbSchema schema = getExpSchema(); - String sql = getSqlDialect().execute(getExpSchema(), "deleteObject", "?, ?"); - SqlExecutor executor = new SqlExecutor(schema); - - for (String uri : uris) - { - executor.execute(sql, c.getId(), uri); - } - } - finally - { - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - } - } - - public static int deleteOntologyObjects(DbSchema schema, SQLFragment objectUriSql, @Nullable Container c) - { - SQLFragment objectIdSQL = new SQLFragment("SELECT ObjectId FROM ") - .append(getTinfoObject()).append("\n") - .append(" WHERE "); - if (c != null) - { - objectIdSQL.append(" Container = ?").add(c.getId()); - objectIdSQL.append(" AND "); - } - objectIdSQL.append("ObjectUri IN ("); - objectIdSQL.append(objectUriSql); - objectIdSQL.append(")"); - return deleteOntologyObjectsByObjectIdSql(schema, objectIdSQL); - } - - public static int deleteOntologyObjectsByObjectIdSql(DbSchema schema, SQLFragment objectIdSql) - { - if (!schema.getScope().equals(getExpSchema().getScope())) - throw new UnsupportedOperationException("can only use with same DbScope"); - - SQLFragment sqlDeleteProperties = new SQLFragment(); - sqlDeleteProperties.append("DELETE FROM ").append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (\n"); - sqlDeleteProperties.append(objectIdSql); - sqlDeleteProperties.append(")"); - new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); - - SQLFragment sqlDeleteObjects = new SQLFragment(); - sqlDeleteObjects.append("DELETE FROM ").append(getTinfoObject()).append(" WHERE ObjectId IN ("); - sqlDeleteObjects.append(objectIdSql); - sqlDeleteObjects.append(")"); - return new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); - } - - - public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, long... objectIds) - { - deleteOntologyObjects(c, deleteOwnedObjects, true, true, objectIds); - } - - public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, boolean deleteObjectProperties, boolean deleteObjects, long... objectIds) - { - if (objectIds.length == 0) - return; - - try - { - // if it's a long list, split it up - if (objectIds.length > 1000) - { - int countBatches = objectIds.length / 1000; - int lenBatch = 1 + objectIds.length / (countBatches + 1); - - for (int s = 0; s < objectIds.length; s += lenBatch) - { - long[] sub = new long[Math.min(lenBatch, objectIds.length - s)]; - System.arraycopy(objectIds, s, sub, 0, sub.length); - deleteOntologyObjects(c, deleteOwnedObjects, deleteObjectProperties, deleteObjects, sub); - } - - return; - } - - SQLFragment objectIdInClause = new SQLFragment(); - getExpSchema().getSqlDialect().appendInClauseSql(objectIdInClause, Arrays.stream(objectIds).boxed().toList()); - - if (deleteOwnedObjects) - { - // NOTE: owned objects should never be in a different container than the owner, that would be a problem - SQLFragment sqlDeleteOwnedProperties = new SQLFragment("DELETE FROM ") - .append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND OwnerObjectId ") - .add(c) - .append(objectIdInClause) - .append(")"); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedProperties); - - SQLFragment sqlDeleteOwnedObjects = new SQLFragment("DELETE FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND OwnerObjectId ") - .add(c) - .append(objectIdInClause); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedObjects); - } - - if (deleteObjectProperties) - { - deleteProperties(c, objectIdInClause); - } - - if (deleteObjects) - { - SQLFragment sqlDeleteObjects = new SQLFragment("DELETE FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND ObjectId ") - .add(c) - .append(objectIdInClause); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); - } - } - finally - { - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - } - } - - - public static void deleteOntologyObject(String objectURI, Container container, boolean deleteOwnedObjects) - { - OntologyObject ontologyObject = getOntologyObject(container, objectURI); - - if (null != ontologyObject) - { - deleteOntologyObjects(container, deleteOwnedObjects, true, true, ontologyObject.getObjectId()); - } - } - - - public static OntologyObject getOntologyObject(long id) - { - return new TableSelector(getTinfoObject()).getObject(id, OntologyObject.class); - } - - //todo: review this. this doesn't delete the underlying data objects. should it? - public static void deleteObjectsOfType(String domainURI, Container container) - { - DomainDescriptor dd = null; - if (null != domainURI) - dd = getDomainDescriptor(domainURI, container); - if (null == dd) - { - _log.debug("deleteObjectsOfType called on type not found in database: " + domainURI); - return; - } - - try (Transaction t = getExpSchema().getScope().ensureTransaction()) - { - // until we set a domain on objects themselves, we need to create a list of objects to - // delete based on existing entries in ObjectProperties before we delete the objectProperties - // which we need to do before we delete the objects. - // TODO: Doesn't handle the case when PropertyDescriptors are shared across domains - String selectObjectsToDelete = "SELECT DISTINCT O.ObjectId " + - " FROM " + getTinfoObject() + " O " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON(O.ObjectId = OP.ObjectId) " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (OP.PropertyId = PDM.PropertyId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PD.PropertyId = PDM.PropertyId) " + - " WHERE DD.DomainId = " + dd.getDomainId() + - " AND PD.Container = DD.Container"; - Long[] objIdsToDelete = new SqlSelector(getExpSchema(), selectObjectsToDelete).getArray(Long.class); - - String sep; - StringBuilder sqlIN = null; - Long[] ownerObjIds = null; - - if (objIdsToDelete.length > 0) - { - //also need list of owner objects whose subobjects are going to be deleted - // Seems cheaper but less correct to delete the subobjects then cleanup any owner objects with no children - sep = ""; - sqlIN = new StringBuilder(); - for (Long id : objIdsToDelete) - { - sqlIN.append(sep).append(id); - sep = ", "; - } - - String selectOwnerObjects = "SELECT O.ObjectId FROM " + getTinfoObject() + " O " + - " WHERE ObjectId IN " + - " (SELECT DISTINCT SUBO.OwnerObjectId FROM " + getTinfoObject() + " SUBO " + - " WHERE SUBO.ObjectId IN ( " + sqlIN + " ) )"; - - ownerObjIds = new SqlSelector(getExpSchema(), selectOwnerObjects).getArray(Long.class); - } - - String deleteTypePropsSql = "DELETE FROM " + getTinfoObjectProperty() + - " WHERE PropertyId IN " + - " (SELECT PDM.PropertyId FROM " + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PDM.PropertyId = PD.PropertyId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " WHERE DD.DomainId = " + dd.getDomainId() + - " AND PD.Container = DD.Container " + - " ) "; - new SqlExecutor(getExpSchema()).execute(deleteTypePropsSql); - - if (objIdsToDelete.length > 0) - { - // now cleanup the object table entries from the list we made, but make sure they don't have - // other properties attached to them - String deleteObjSql = "DELETE FROM " + getTinfoObject() + - " WHERE ObjectId IN ( " + sqlIN + " ) " + - " AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + - " WHERE OP.ObjectId = " + getTinfoObject() + ".ObjectId)"; - new SqlExecutor(getExpSchema()).execute(deleteObjSql); - - if (ownerObjIds.length > 0) - { - sep = ""; - sqlIN = new StringBuilder(); - for (Long id : ownerObjIds) - { - sqlIN.append(sep).append(id); - sep = ", "; - } - String deleteOwnerSql = "DELETE FROM " + getTinfoObject() + - " WHERE ObjectId IN ( " + sqlIN + " ) " + - " AND NOT EXISTS (SELECT * FROM " + getTinfoObject() + " SUBO " + - " WHERE SUBO.OwnerObjectId = " + getTinfoObject() + ".ObjectId)"; - new SqlExecutor(getExpSchema()).execute(deleteOwnerSql); - } - } - // whew! - clearCaches(); - t.commit(); - } - } - - public static void deleteDomain(String domainURI, Container container) throws DomainNotFoundException - { - DomainDescriptor dd = getDomainDescriptor(domainURI, container); - String msg; - - if (null == dd) - throw new DomainNotFoundException(domainURI); - - if (!dd.getContainer().getId().equals(container.getId())) - { - // this domain was not created in this folder. Allow if in the project-level root - if (!dd.getProject().getId().equals(container.getId())) - { - msg = "DeleteDomain: Domain can only be deleted in original container or from the project root " - + "\nDomain: " + domainURI + " project " + dd.getProject().getName() + " original container " + dd.getContainer().getPath(); - _log.error(msg); - throw new RuntimeException(msg); - } - } - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - String selectPDsToDelete = "SELECT DISTINCT PDM.PropertyId " + - " FROM " + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " WHERE DD.DomainId = ? "; - - Integer[] pdIdsToDelete = new SqlSelector(getExpSchema(), selectPDsToDelete, dd.getDomainId()).getArray(Integer.class); - - String deletePDMs = "DELETE FROM " + getTinfoPropertyDomain() + - " WHERE DomainId = " + - " (SELECT DD.DomainId FROM " + getTinfoDomainDescriptor() + " DD " + - " WHERE DD.DomainId = ? )"; - new SqlExecutor(getExpSchema()).execute(deletePDMs, dd.getDomainId()); - - if (pdIdsToDelete.length > 0) - { - String sep = ""; - StringBuilder sqlIN = new StringBuilder(); - for (Integer id : pdIdsToDelete) - { - PropertyService.get().deleteValidatorsAndFormats(container, id); - - sqlIN.append(sep); - sqlIN.append(id); - sep = ", "; - } - - String deletePDs = "DELETE FROM " + getTinfoPropertyDescriptor() + - " WHERE PropertyId IN ( " + sqlIN + " ) " + - "AND Container = ? " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + - "WHERE OP.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId) " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + - "WHERE PDM.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId)"; - - new SqlExecutor(getExpSchema()).execute(deletePDs, dd.getContainer().getId()); - } - - String deleteDD = "DELETE FROM " + getTinfoDomainDescriptor() + - " WHERE DomainId = ? " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + - "WHERE PDM.DomainId = " + getTinfoDomainDescriptor() + ".DomainId)"; - - new SqlExecutor(getExpSchema()).execute(deleteDD, dd.getDomainId()); - clearCaches(); - - transaction.commit(); - } - } - - - public static void deleteAllObjects(Container c, User user) throws ValidationException - { - Container projectContainer = c.getProject(); - if (null == projectContainer) - projectContainer = c; - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - if (!c.equals(projectContainer)) - { - copyDescriptors(c, projectContainer); - } - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - - // Owned objects should be in same container, so this should work - String deleteObjPropSql = "DELETE FROM " + getTinfoObjectProperty() + " WHERE ObjectId IN (SELECT ObjectId FROM " + getTinfoObject() + " WHERE Container = ?)"; - executor.execute(deleteObjPropSql, c); - String deleteObjSql = "DELETE FROM " + getTinfoObject() + " WHERE Container = ?"; - executor.execute(deleteObjSql, c); - - // delete property validator references on property descriptors - PropertyService.get().deleteValidatorsAndFormats(c); - - // Drop tables directly and allow bulk delete calls below to clean up rows in exp.propertydescriptor, - // exp.domaindescriptor, etc - String selectSQL = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - Collection dds = new SqlSelector(getExpSchema(), selectSQL, c).getCollection(DomainDescriptor.class); - for (DomainDescriptor dd : dds) - { - StorageProvisioner.get().drop(PropertyService.get().getDomain(dd.getDomainId())); - } - - String deletePropDomSqlPD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId IN (SELECT PropertyId FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?)"; - executor.execute(deletePropDomSqlPD, c); - String deletePropDomSqlDD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE DomainId IN (SELECT DomainId FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?)"; - executor.execute(deletePropDomSqlDD, c); - String deleteDomSql = "DELETE FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - executor.execute(deleteDomSql, c); - // now delete the prop descriptors that are referenced in this container only - String deletePropSql = "DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?"; - executor.execute(deletePropSql, c); - - clearCaches(); - transaction.commit(); - } - } - - private static void copyDescriptors(final Container c, final Container project) throws ValidationException - { - _log.debug("OntologyManager.copyDescriptors " + c.getName() + " " + project.getName()); - - // if c is (was) a project, then nothing to do - if (c.getId().equals(project.getId())) - return; - - // check to see if any Properties defined in this folder are used in other folders. - // if so we will make a copy of all PDs and DDs to ensure no orphans - String sql = " SELECT O.ObjectURI, O.Container, PD.PropertyId, PD.PropertyURI " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + - " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + - " WHERE PD.Container = ? " + - " AND O.Container <> PD.Container "; - - final Map mObjsUsingMyProps = new HashMap<>(); - final StringBuilder sqlIn = new StringBuilder(); - final StringBuilder sep = new StringBuilder(); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c).getResultSet()) - { - ResultSetUtil.logData(rs); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - new SqlSelector(getExpSchema(), sql, c).forEach(rs -> { - String objURI = rs.getString(1); - String objContainer = rs.getString(2); - Integer propId = rs.getInt(3); - String propURI = rs.getString(4); - - sqlIn.append(sep).append(propId); - - if (sep.isEmpty()) - sep.append(", "); - - Map mtemp = getPropertyObjects(ContainerManager.getForId(objContainer), objURI); - - if (null != mtemp) - { - for (Map.Entry entry : mtemp.entrySet()) - { - entry.getValue().setPropertyId(0); - if (entry.getValue().getPropertyURI().equals(propURI)) - mObjsUsingMyProps.put(entry.getKey(), entry.getValue()); - } - } - }); - - // For each property that is referenced outside its container, get the - // domains that it belongs to and the other properties in those domains - // so we can make copies of those domains and properties - // Restrict it to properties and domains also in the same container - - if (!mObjsUsingMyProps.isEmpty()) - { - sql = "SELECT PD.PropertyURI, DD.DomainURI " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + - " ON (PD.PropertyId = PDM2.PropertyId) " + - " WHERE PDM.PropertyId IN (" + sqlIn + ") " + - " OR PD.PropertyId IN (" + sqlIn + ") "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - new SqlSelector(getExpSchema(), sql).forEach(rsMyProps -> { - String propUri = rsMyProps.getString(1); - String domUri = rsMyProps.getString(2); - PropertyDescriptor pd = getPropertyDescriptor(propUri, c); - - if (pd.getContainer().getId().equals(c.getId())) - { - _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - DOMAIN_PROPERTIES_CACHE.clear(); - pd.setContainer(project); - pd.setPropertyId(0); - pd = ensurePropertyDescriptor(pd); - } - - if (null != domUri) - { - DomainDescriptor dd = getDomainDescriptor(domUri, c); - if (dd.getContainer().getId().equals(c.getId())) - { - uncache(dd); - dd = dd.edit() - .setContainer(project) - .setDomainId(0) - .build(); - dd = ensureDomainDescriptor(dd); - ensurePropertyDomain(pd, dd); - } - } - }); - - clearCaches(); - - // now unhook the objects that refer to my properties and rehook them to the properties in their own project - for (ObjectProperty op : mObjsUsingMyProps.values()) - { - deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), c); - insertProperties(op.getContainer(), op.getObjectURI(), op); - } - } - } - - private static void uncache(DomainDescriptor dd) - { - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.remove(dd.getContainer()); - } - - - public static void moveContainer(@NotNull final Container c, @NotNull Container oldParent, @NotNull Container newParent) throws SQLException - { - _log.debug("OntologyManager.moveContainer " + c.getName() + " " + oldParent.getName() + "->" + newParent.getName()); - - final Container oldProject = oldParent.getProject(); - Container newProject = newParent.getProject(); - if (null == newProject) // if container is promoted to a project - newProject = c.getProject(); - - if ((null != oldProject) && oldProject.getId().equals(newProject.getId())) - { - //the folder is being moved within the same project. No problems here - return; - } - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - clearCaches(); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - // update project of any descriptors in folder just moved - TableInfo pdTable = getTinfoPropertyDescriptor(); - String sql = "UPDATE " + pdTable + " SET Project = ? WHERE Container = ?"; - - // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. - // Issue 30477: exclude project level properties descriptors (such as Study) that already exist - sql += " AND PropertyUri NOT IN (SELECT PropertyUri FROM " + pdTable + " WHERE Project = ? AND PropertyUri IN (SELECT PropertyUri FROM " + pdTable + " WHERE Container = ?))"; - - new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - TableInfo ddTable = getTinfoDomainDescriptor(); - sql = "UPDATE " + ddTable + " SET Project = ? WHERE Container = ?"; - - // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. - // Issue 30477: exclude project level domain descriptors (such as Study) that already exist - sql += " AND DomainUri NOT IN (SELECT DomainUri FROM " + ddTable + " WHERE Project = ? AND DomainUri IN (SELECT DomainUri FROM " + ddTable + " WHERE Container = ?))"; - - new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); - - if (null == oldProject) // if container was a project & demoted I'm done - { - transaction.commit(); - return; - } - - // this method makes sure I'm not getting rid of descriptors used by another folder - // it is shared by ContainerDelete - copyDescriptors(c, oldProject); - - // if my objects refer to project-scoped properties I need a copy of those properties - sql = " SELECT O.ObjectURI, PD.PropertyURI, PD.PropertyId, PD.Container " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + - " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + - " WHERE O.Container = ? " + - " AND O.Container <> PD.Container " + - " AND PD.Project NOT IN (?,?) "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - - final Map mMyObjsThatRefProjProps = new HashMap<>(); - final StringBuilder sqlIn = new StringBuilder(); - final StringBuilder sep = new StringBuilder(); - - new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).forEach(rs -> { - String objURI = rs.getString(1); - String propURI = rs.getString(2); - Integer propId = rs.getInt(3); - - sqlIn.append(sep).append(propId); - - if (sep.isEmpty()) - sep.append(", "); - - Map mtemp = getPropertyObjects(c, objURI); - - if (null != mtemp) - { - for (Map.Entry entry : mtemp.entrySet()) - { - if (entry.getValue().getPropertyURI().equals(propURI)) - mMyObjsThatRefProjProps.put(entry.getKey(), entry.getValue()); - } - } - }); - - // this sql gets all properties i ref and the domains they belong to and the - // other properties in those domains - //todo what about materialsource ? - if (!mMyObjsThatRefProjProps.isEmpty()) - { - sql = "SELECT PD.PropertyURI, DD.DomainURI, PD.PropertyId " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + - " ON (PD.PropertyId = PDM2.PropertyId) " + - " WHERE PDM.PropertyId IN (" + sqlIn + " ) "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - final Container fNewProject = newProject; - - new SqlSelector(getExpSchema(), sql).forEach(rsPropsRefdByMe -> { - String propUri = rsPropsRefdByMe.getString(1); - String domUri = rsPropsRefdByMe.getString(2); - PropertyDescriptor pd = getPropertyDescriptor(propUri, oldProject); - - if (null != pd) - { - // To prevent iterating over a property descriptor update more than once - // we check to make sure both the container and project are equivalent to the updated - // location - if (!pd.getContainer().equals(c) || !pd.getProject().equals(fNewProject)) - { - pd.setContainer(c); - pd.setPropertyId(0); - } - - pd = ensurePropertyDescriptor(pd); - } - - if (null != domUri) - { - DomainDescriptor dd = getDomainDescriptor(domUri, oldProject); - - // To prevent iterating over a domain descriptor update more than once - // we check to make sure both the container and project are equivalent to the updated - // location - if (!dd.getContainer().equals(c) || !dd.getProject().equals(fNewProject)) - { - dd = dd.edit().setContainer(c).setDomainId(0).build(); - } - - dd = ensureDomainDescriptor(dd); - ensurePropertyDomain(pd, dd); - } - }); - - for (ObjectProperty op : mMyObjsThatRefProjProps.values()) - { - deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), oldProject); - // Treat it as new so it's created in the target container as needed - op.setPropertyId(0); - insertProperties(op.getContainer(), op.getObjectURI(), op); - } - clearCaches(); - } - - transaction.commit(); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - private static PropertyDescriptor ensurePropertyDescriptor(String propertyURI, PropertyType type, String name, Container container) - { - PropertyDescriptor pdNew = new PropertyDescriptor(propertyURI, type, name, container); - return ensurePropertyDescriptor(pdNew); - } - - - private static PropertyDescriptor ensurePropertyDescriptor(PropertyDescriptor pdIn) - { - if (null == pdIn.getContainer()) - { - assert false : "Container should be set on PropertyDescriptor"; - pdIn.setContainer(_sharedContainer); - } - - PropertyDescriptor pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); - if (null == pd) - { - assert pdIn.getPropertyId() == 0; - /* return 1 if inserted 0 if not inserted, uses OUT parameter for new PropertyDescriptor */ - PropertyDescriptor[] out = new PropertyDescriptor[1]; - int rowcount = insertPropertyIfNotExists(null, pdIn, out); - pd = out[0]; - if (1 == rowcount && null != pd) - { - _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - return pd; - } - if (null == pd) - { - throw OptimisticConflictException.create(Table.ERROR_DELETED); - } - } - - if (pd.equals(pdIn)) - { - return pd; - } - else - { - List colDiffs = comparePropertyDescriptors(pdIn, pd); - - if (colDiffs.isEmpty()) - { - // if the descriptor differs by container only and the requested descriptor is in the project fldr - if (!pdIn.getContainer().getId().equals(pd.getContainer().getId()) && - pdIn.getContainer().getId().equals(pdIn.getProject().getId())) - { - pdIn.setPropertyId(pd.getPropertyId()); - pd = updatePropertyDescriptor(pdIn); - } - return pd; - } - - // you are allowed to update if you are coming from the project root, or if you are in the container - // in which the descriptor was created - boolean fUpdateIfExists = false; - if (pdIn.getContainer().getId().equals(pd.getContainer().getId()) - || pdIn.getContainer().getId().equals(pdIn.getProject().getId())) - fUpdateIfExists = true; - - - boolean fMajorDifference = false; - if (colDiffs.toString().contains("RangeURI") || colDiffs.toString().contains("PropertyType")) - fMajorDifference = true; - - String errmsg = "ensurePropertyDescriptor: descriptor In different from Found for " + colDiffs + - "\n\t Descriptor In: " + pdIn + - "\n\t Descriptor Found: " + pd; - - if (fUpdateIfExists) - { - //todo: pass list of cols to update - pdIn.setPropertyId(pd.getPropertyId()); - pd = updatePropertyDescriptor(pdIn); - if (fMajorDifference) - _log.debug(errmsg); - } - else - { - if (fMajorDifference) - _log.error(errmsg); - else - _log.debug(errmsg); - } - } - return pd; - } - - - private static int insertPropertyIfNotExists(User user, PropertyDescriptor pd, PropertyDescriptor[] out) - { - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getInsertStmt(conn, user, t, true)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = f.toMap(pd, null); - stmt.putAll(m); - int rowcount = stmt.execute(); - SQLFragment reselect = new SQLFragment("SELECT * FROM exp.propertydescriptor WHERE propertyuri=? AND container=?", pd.getPropertyURI(), pd.getContainer()); - out[0] = (new SqlSelector(getExpSchema(), reselect).getObject(PropertyDescriptor.class)); - return rowcount; - } - catch(SQLException sqlx) - { - throw ExceptionFramework.Spring.translate(getExpSchema().getScope(), "insertPropertyIfNotExists", sqlx); - } - } - - - private static List comparePropertyDescriptors(PropertyDescriptor pdIn, PropertyDescriptor pd) - { - List colDiffs = new ArrayList<>(); - - // if the returned pd is in a different project, it better be the shared project - if (!pd.getProject().equals(pdIn.getProject()) && !pd.getProject().equals(_sharedContainer)) - colDiffs.add("Project"); - - // check the pd values that can't change - if (!pd.getRangeURI().equals(pdIn.getRangeURI())) - colDiffs.add("RangeURI"); - if (!Objects.equals(pd.getPropertyType(), pdIn.getPropertyType())) - colDiffs.add("PropertyType"); - - if (pdIn.getPropertyId() != 0 && pd.getPropertyId() != pdIn.getPropertyId()) - colDiffs.add("PropertyId"); - - if (!Objects.equals(pdIn.getName(), pd.getName())) - colDiffs.add("Name"); - - if (!Objects.equals(pdIn.getConceptURI(), pd.getConceptURI())) - colDiffs.add("ConceptURI"); - - if (!Objects.equals(pdIn.getDescription(), pd.getDescription())) - colDiffs.add("Description"); - - if (!Objects.equals(pdIn.getFormat(), pd.getFormat())) - colDiffs.add("Format"); - - if (!Objects.equals(pdIn.getLabel(), pd.getLabel())) - colDiffs.add("Label"); - - if (pdIn.isHidden() != pd.isHidden()) - colDiffs.add("IsHidden"); - - if (pdIn.isMvEnabled() != pd.isMvEnabled()) - colDiffs.add("IsMvEnabled"); - - if (!Objects.equals(pdIn.getLookupContainer(), pd.getLookupContainer())) - colDiffs.add("LookupContainer"); - - if (!Objects.equals(pdIn.getLookupSchema(), pd.getLookupSchema())) - colDiffs.add("LookupSchema"); - - if (!Objects.equals(pdIn.getLookupQuery(), pd.getLookupQuery())) - colDiffs.add("LookupQuery"); - - if (!Objects.equals(pdIn.getDerivationDataScope(), pd.getDerivationDataScope())) - colDiffs.add("DerivationDataScope"); - - if (!Objects.equals(pdIn.getSourceOntology(), pd.getSourceOntology())) - colDiffs.add("SourceOntology"); - - if (!Objects.equals(pdIn.getConceptImportColumn(), pd.getConceptImportColumn())) - colDiffs.add("ConceptImportColumn"); - - if (!Objects.equals(pdIn.getConceptLabelColumn(), pd.getConceptLabelColumn())) - colDiffs.add("ConceptLabelColumn"); - - if (!Objects.equals(pdIn.getPrincipalConceptCode(), pd.getPrincipalConceptCode())) - colDiffs.add("PrincipalConceptCode"); - - if (!Objects.equals(pdIn.getConceptSubtree(), pd.getConceptSubtree())) - colDiffs.add("ConceptSubtree"); - - if (pdIn.isScannable() != pd.isScannable()) - colDiffs.add("Scannable"); - - return colDiffs; - } - - public static DomainDescriptor ensureDomainDescriptor(String domainURI, String name, Container container) - { - String trimmedName = StringUtils.trimToNull(name); - if (trimmedName == null) - throw new IllegalArgumentException("Non-blank name is required."); - DomainDescriptor dd = new DomainDescriptor.Builder(domainURI, container).setName(trimmedName).build(); - return ensureDomainDescriptor(dd); - } - - /** Inserts or updates the domain as appropriate */ - @NotNull - public static DomainDescriptor ensureDomainDescriptor(DomainDescriptor ddIn) - { - DomainDescriptor dd = null; - // Try to find the previous version of the domain - if (ddIn.getDomainId() > 0) - { - // Try checking the cache first for a value to compare against - dd = getDomainDescriptor(ddIn.getDomainId()); - - // Since we cache mutable objects, get a fresh copy from the DB if the cache returned the same object that - // was passed in so we can do a diff against what's currently in the DB to see if we need to update - if (dd == ddIn) - { - dd = new TableSelector(getTinfoDomainDescriptor()).getObject(ddIn.getDomainId(), DomainDescriptor.class); - } - } - if (dd == null) - { - dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); - } - - if (null == dd) - { - try - { - DbSchema expSchema = getExpSchema(); - // ensureDomainDescriptor() shouldn't fail if there is a race condition, however Table.insert() will throw if row exists, so can't use that - // also a constraint violation will kill any current transaction - // CONSIDER to generalize add an option to check for existing row to Table.insert(ColumnInfo[] keyCols, Object[] keyValues) - String timestamp = expSchema.getSqlDialect().getSqlTypeName(JdbcType.TIMESTAMP); - String templateJson = null==ddIn.getTemplateInfo() ? null : ddIn.getTemplateInfo().toJSON(); - SQLFragment insert = new SQLFragment( - "INSERT INTO ").append(getTinfoDomainDescriptor()) - .append(" (Name, DomainURI, Description, Container, Project, StorageTableName, StorageSchemaName, ModifiedBy, Modified, TemplateInfo, SystemFieldConfig)\n" + - "SELECT ?,?,?,?,?,?,?,CAST(NULL AS INT),CAST(NULL AS " + timestamp + "),?,?\n") - .addAll(ddIn.getName(), ddIn.getDomainURI(), ddIn.getDescription(), ddIn.getContainer(), ddIn.getProject(), ddIn.getStorageTableName(), ddIn.getStorageSchemaName(), templateJson, ddIn.getSystemFieldConfig()) - .append("WHERE NOT EXISTS (SELECT * FROM ").append(getTinfoDomainDescriptor(),"x").append(" WHERE x.DomainURI=? AND x.Project=?)\n") - .add(ddIn.getDomainURI()).add(ddIn.getProject()); - // belt and suspenders approach to avoiding constraint violation exception - if (expSchema.getSqlDialect().isPostgreSQL()) - insert.append(" ON CONFLICT ON CONSTRAINT uq_domaindescriptor DO NOTHING"); - int count; - try (var tx = expSchema.getScope().ensureTransaction()) - { - count = new SqlExecutor(expSchema.getScope()).execute(insert); - tx.commit(); - } - - // alternately we could reselect rowid and then we wouldn't need this separate round trip - dd = fetchDomainDescriptorFromDB(ddIn.getDomainURI(), ddIn.getContainer()); - if (count > 0) - { - if (null == dd) // don't expect this - throw OptimisticConflictException.create(Table.ERROR_DELETED); - // We may have a cached miss that we need to clear - uncache(dd); - return dd; - } - // fall through to update case() - } - catch (RuntimeSQLException x) - { - // might be an optimistic concurrency problem see 16126 - dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); - if (null == dd) - throw x; - } - } - - if (!dd.deepEquals(ddIn)) - { - DomainDescriptor ddToSave = ddIn.edit().setDomainId(dd.getDomainId()).build(); - dd = Table.update(null, getTinfoDomainDescriptor(), ddToSave, ddToSave.getDomainId()); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(ddIn)); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(ddIn)); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); - } - return dd; - } - - private static void ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd) - { - ensurePropertyDomain(pd, dd, 0); - } - - public static PropertyDescriptor ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) - { - if (null == pd) - throw new IllegalArgumentException("Must supply a PropertyDescriptor"); - if (null == dd) - throw new IllegalArgumentException("Must supply a DomainDescriptor"); - - // Consider: We should check that the pd and dd have been persisted (aka have a non-zero id) - - if (!pd.getContainer().equals(dd.getContainer()) - && !pd.getProject().equals(_sharedContainer)) - throw new IllegalStateException("ensurePropertyDomain: property " + pd.getPropertyURI() + " not in same container as domain " + dd.getDomainURI()); - - SQLFragment sqlInsert = new SQLFragment("INSERT INTO " + getTinfoPropertyDomain() + " ( PropertyId, DomainId, Required, SortOrder ) " + - " SELECT ?, ?, ?, ? WHERE NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + - " WHERE PropertyId=? AND DomainId=?)"); - sqlInsert.add(pd.getPropertyId()); - sqlInsert.add(dd.getDomainId()); - sqlInsert.add(pd.isRequired()); - sqlInsert.add(sortOrder); - sqlInsert.add(pd.getPropertyId()); - sqlInsert.add(dd.getDomainId()); - int count = new SqlExecutor(getExpSchema()).execute(sqlInsert); - // if 0 rows affected, we should do an update to make sure required is correct - if (count == 0) - { - SQLFragment sqlUpdate = new SQLFragment("UPDATE " + getTinfoPropertyDomain() + " SET Required = ?, SortOrder = ? WHERE PropertyId=? AND DomainId= ?"); - sqlUpdate.add(pd.isRequired()); - sqlUpdate.add(sortOrder); - sqlUpdate.add(pd.getPropertyId()); - sqlUpdate.add(dd.getDomainId()); - new SqlExecutor(getExpSchema()).execute(sqlUpdate); - } - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); - return pd; - } - - - private static void insertPropertiesBulk(Container container, List props, boolean insertNullValues) throws SQLException - { - List> floats = new ArrayList<>(); - List> dates = new ArrayList<>(); - List> strings = new ArrayList<>(); - List> mvIndicators = new ArrayList<>(); - - for (PropertyRow property : props) - { - if (null == property) - continue; - - long objectId = property.getObjectId(); - int propertyId = property.getPropertyId(); - String mvIndicator = property.getMvIndicator(); - assert mvIndicator == null || MvUtil.isMvIndicator(mvIndicator, container) : "Attempt to insert an invalid missing value indicator: " + mvIndicator; - - if (null != property.getFloatValue()) - floats.add(Arrays.asList(objectId, propertyId, property.getFloatValue(), mvIndicator)); - else if (null != property.getDateTimeValue()) - dates.add(Arrays.asList(objectId, propertyId, new java.sql.Timestamp(property.getDateTimeValue().getTime()), mvIndicator)); - else if (null != property.getStringValue()) - strings.add(Arrays.asList(objectId, propertyId, property.getStringValue(), mvIndicator)); - else if (null != mvIndicator) - { - mvIndicators.add(Arrays.asList(objectId, propertyId, property.getTypeTag(), mvIndicator)); - } - else if (insertNullValues) - { - strings.add(Arrays.asList(objectId, propertyId, null, null)); - } - } - - assert getExpSchema().getScope().isTransactionActive(); - - if (!dates.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, DateTimeValue, MvIndicator) VALUES (?,?,'d',?, ?)"; - Table.batchExecute(getExpSchema(), sql, dates); - } - - if (!floats.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, FloatValue, MvIndicator) VALUES (?,?,'f',?, ?)"; - Table.batchExecute(getExpSchema(), sql, floats); - } - - if (!strings.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, StringValue, MvIndicator) VALUES (?,?,'s',?, ?)"; - Table.batchExecute(getExpSchema(), sql, strings); - } - - if (!mvIndicators.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, MvIndicator) VALUES (?,?,?,?)"; - Table.batchExecute(getExpSchema(), sql, mvIndicators); - } - - clearPropertyCache(); - } - - public static void deleteProperty(String objectURI, String propertyURI, Container objContainer, Container propContainer) - { - OntologyObject o = getOntologyObject(objContainer, objectURI); - if (o == null) - return; - - PropertyDescriptor pd = getPropertyDescriptor(propertyURI, propContainer); - if (pd == null) - return; - - deleteProperty(o, pd); - } - - public static void deleteProperty(OntologyObject o, PropertyDescriptor pd) - { - deleteProperty(o, pd, true); - } - - public static void deleteProperty(OntologyObject o, PropertyDescriptor pd, boolean deleteCache) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectId"), o.getObjectId()); - filter.addCondition(FieldKey.fromParts("PropertyId"), pd.getPropertyId()); - Table.delete(getTinfoObjectProperty(), filter); - - if (deleteCache) - clearPropertyCache(o.getObjectURI()); - } - - /** - * Delete properties owned by the objects. - */ - public static void deleteProperties(Container objContainer, long objectId) - { - deleteProperties(objContainer, new SQLFragment(" = ?", objectId)); - } - public static void deleteProperties(Container objContainer, SQLFragment objectIdClause) - { - SQLFragment objectUriSql = new SQLFragment("SELECT ObjectURI FROM ") - .append(getTinfoObject(), "o") - .append(" WHERE ObjectId "); - objectUriSql.append(objectIdClause); - - List objectURIs = new SqlSelector(getExpSchema(), objectUriSql).getArrayList(String.class); - - SQLFragment sqlDeleteProperties = new SQLFragment("DELETE FROM ") - .append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND ObjectId ") - .add(objContainer) - .append(objectIdClause) - .append(")"); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); - - for (String uri : objectURIs) - { - clearPropertyCache(uri); - } - } - - /** - * Removes the property from a single domain, and completely deletes it if there are no other references - */ - public static void removePropertyDescriptorFromDomain(DomainProperty domainProp) - { - SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ? AND DomainId = ?", domainProp.getPropertyId(), domainProp.getDomain().getTypeId()); - SqlExecutor executor = new SqlExecutor(getExpSchema()); - DbScope dbScope = getExpSchema().getScope(); - try (Transaction transaction = dbScope.ensureTransaction()) - { - executor.execute(deletePropDomSql); - // Check if there are any other usages - SQLFragment otherUsagesSQL = new SQLFragment("SELECT DomainId FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", domainProp.getPropertyId()); - if (!new SqlSelector(dbScope, otherUsagesSQL).exists()) - { - deletePropertyDescriptor(domainProp.getPropertyDescriptor()); - } - transaction.commit(); - } - } - - /** - * Completely deletes the property from the database - */ - public static void deletePropertyDescriptor(PropertyDescriptor pd) - { - int propId = pd.getPropertyId(); - - SQLFragment deleteObjPropSql = new SQLFragment("DELETE FROM " + getTinfoObjectProperty() + " WHERE PropertyId = ?", propId); - SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", propId); - SQLFragment deletePropSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyId = ?", propId); - - DbScope dbScope = getExpSchema().getScope(); - SqlExecutor executor = new SqlExecutor(getExpSchema()); - try (Transaction transaction = dbScope.ensureTransaction()) - { - executor.execute(deleteObjPropSql); - executor.execute(deletePropDomSql); - executor.execute(deletePropSql); - Pair key = getCacheKey(pd); - _log.debug("Removing property descriptor from cache. Key: " + key + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(key); - DOMAIN_PROPERTIES_CACHE.clear(); - transaction.commit(); - } - } - - /*** - * @deprecated Use {@link #insertProperties(Container, User, String, ObjectProperty...)} so that a user can be - * supplied. - */ - @Deprecated - public static void insertProperties(Container container, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException - { - User user = HttpView.hasCurrentView() ? HttpView.currentContext().getUser() : null; - insertProperties(container, user, ownerObjectLsid, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException - { - insertProperties(container, user, ownerObjectLsid, false, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, ObjectProperty... properties) throws ValidationException - { - insertProperties(container, user, ownerObjectLsid, skipValidation, false, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, boolean insertNullValues, ObjectProperty... properties) throws ValidationException - { - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - Long parentId = ownerObjectLsid == null ? null : ensureObject(container, ownerObjectLsid); - HashMap descriptors = new HashMap<>(); - HashMap objects = new HashMap<>(); - List errors = new ArrayList<>(); - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - for (ObjectProperty property : properties) - { - if (null == property) - continue; - - property.setObjectOwnerId(parentId); - - PropertyDescriptor pd = descriptors.get(property.getPropertyURI()); - if (0 == property.getPropertyId()) - { - if (null == pd) - { - PropertyDescriptor pdIn = new PropertyDescriptor(property.getPropertyURI(), property.getPropertyType(), property.getName(), container); - pdIn.setFormat(property.getFormat()); - pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); - - if (null == pd) - pd = ensurePropertyDescriptor(pdIn); - - descriptors.put(property.getPropertyURI(), pd); - } - property.setPropertyId(pd.getPropertyId()); - } - if (0 == property.getObjectId()) - { - Long objectId = objects.get(property.getObjectURI()); - if (null == objectId) - { - // I'm assuming all properties are in the same container - objectId = ensureObject(property.getContainer(), property.getObjectURI(), property.getObjectOwnerId()); - objects.put(property.getObjectURI(), objectId); - } - property.setObjectId(objectId); - } - if (pd == null) - { - pd = getPropertyDescriptor(property.getPropertyId()); - } - if (!skipValidation) - { - validateProperty(PropertyService.get().getPropertyValidators(pd), pd, property, errors, validatorCache); - } - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - - insertPropertiesBulk(container, List.of(properties), insertNullValues); - - transaction.commit(); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - - public static PropertyDescriptor getPropertyDescriptor(long propertyId) - { - return new TableSelector(getTinfoPropertyDescriptor()).getObject(propertyId, PropertyDescriptor.class); - } - - - public static PropertyDescriptor getPropertyDescriptor(String propertyURI, Container c) - { - // cache lookup by project. if not found at project level, check to see if global - Pair key = getCacheKey(propertyURI, c); - PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(key); - if (null != pd) - return pd; - - key = getCacheKey(propertyURI, _sharedContainer); - return PROP_DESCRIPTOR_CACHE.get(key); - } - - private static TableSelector getPropertyDescriptorTableSelector( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter, - @Nullable String sortColumn) - { - final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); - - // To filter by domain kind, we query the exp.DomainProperty table and filter by domainId. - // To construct a PropertyDescriptor, we will need to traverse the lookup to exp.PropertyDescriptor and select all of its columns. - List fields = new ArrayList<>(); - fields.add(FieldKey.fromParts("domainId")); - for (ColumnInfo col : getTinfoPropertyDescriptor().getColumns()) - { - fields.add(new FieldKey(propertyIdKey, col.getName())); - } - var colMap = QueryService.get().getColumns(getTinfoPropertyDomain(), fields); - - var filter = new SimpleFilter(); - if (propertyFilter != null) - { - filter.addAllClauses(propertyFilter); - } - - filter.addCondition(new FieldKey(propertyIdKey, "container"), c.getId()); - - if (!domains.isEmpty()) - { - filter.addInClause(FieldKey.fromParts("domainId"), domains.stream().map(Domain::getTypeId).collect(Collectors.toSet())); - } - - if (searchTerm != null) - { - // Apply Q filter to only some of the text columns - List searchCols = List.of( - colMap.get(new FieldKey(propertyIdKey, "Name")), - colMap.get(new FieldKey(propertyIdKey, "Label")), - colMap.get(new FieldKey(propertyIdKey, "Description")), - colMap.get(new FieldKey(propertyIdKey, "ImportAliases")) - ); - - var clause = CompareType.Q.createFilterClause(new FieldKey(null, "*"), searchTerm); - clause.setSelectColumns(searchCols); - filter.addCondition(clause); - } - - // use propertyId as the default sort - if (sortColumn == null) - sortColumn = "propertyId"; - Sort sort = new Sort(sortColumn); - - return new TableSelector(getTinfoPropertyDomain(), colMap.values(), filter, sort); - } - - public static Set getDomains( - Container c, User user, - @Nullable Set domainIds, - @Nullable Set domainKinds, - @Nullable Set domainNames) - { - Set domains = new HashSet<>(); - if (domainIds != null && !domainIds.isEmpty()) - { - domains.addAll(domainIds.stream().map(id -> PropertyService.get().getDomain(id)).collect(Collectors.toSet())); - } - - Set kinds = emptySet(); - Set names = emptySet(); - if (domainKinds != null && !domainKinds.isEmpty()) - { - kinds = domainKinds; - } - if (domainNames != null && !domainNames.isEmpty()) - { - names = domainNames; - } - if (!kinds.isEmpty() || !names.isEmpty()) - { - domains.addAll(PropertyService.get().getDomains(c, user, kinds, names, true)); - } - - return domains; - } - - public static List getPropertyDescriptors( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter, - @Nullable String sortColumn, - @Nullable Integer maxRows, - @Nullable Long offset) - { - final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); - - TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, - propertyFilter, sortColumn); - - if (maxRows != null) - ts.setMaxRows(maxRows); - if (offset != null) - ts.setOffset(offset); - - // This is a little annoying. We have to remove the "propertyId" lookup parent from - // the map keys for the ObjectFactory to correctly construct the PropertyDescriptor. - List props = new ArrayList<>(); - try (var results = ts.getResults(true)) - { - ObjectFactory of = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - while (results.next()) - { - Map rowMap = results.getFieldKeyRowMap(); - // remove the "propertyId" part from the FieldKey - Map rekey = new CaseInsensitiveHashMap<>(); - for (Map.Entry pair : rowMap.entrySet()) - { - FieldKey key = pair.getKey(); - if (propertyIdKey.equals(key.getParent())) - { - String name = key.getName(); - rekey.put(name, pair.getValue()); - } - } - props.add(of.fromMap(rekey)); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - return props; - } - - public static long getPropertyDescriptorsRowCount( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter) - { - - TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, - propertyFilter, null); - - return ts.getRowCount(); - } - - public static List getDomainsForPropertyDescriptor(Container container, PropertyDescriptor pd) - { - return PropertyService.get().getDomains(container) - .stream() - .filter(d -> null != d.getPropertyByURI(pd.getPropertyURI())) - .collect(Collectors.toList()); - } - - private static class DomainDescriptorLoader implements CacheLoader - { - @Override - public DomainDescriptor load(@NotNull Integer key, @Nullable Object argument) - { - return new TableSelector(getTinfoDomainDescriptor()).getObject(key, DomainDescriptor.class); - } - } - - public static DomainDescriptor getDomainDescriptor(int id) - { - return getDomainDescriptor(id, false); - } - - public static DomainDescriptor getDomainDescriptor(int id, boolean forUpdate) - { - if (forUpdate) - return new DomainDescriptorLoader().load(id, null); - - return DOMAIN_DESC_BY_ID_CACHE.get(id); - } - - @Nullable - public static DomainDescriptor getDomainDescriptor(String domainURI, Container c) - { - return getDomainDescriptor(domainURI, c, false); - } - - @Nullable - public static DomainDescriptor getDomainDescriptor(String domainURI, Container c, boolean forUpdate) - { - if (c == null) - return null; - - if (forUpdate) - return getDomainDescriptorForUpdate(domainURI, c); - - // cache lookup by project. if not found at project level, check to see if global - Pair key = getCacheKey(domainURI, c); - DomainDescriptor dd = DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); - if (null != dd) - return dd; - - // Try in the /Shared container too - key = getCacheKey(domainURI, _sharedContainer); - return DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); - } - - @Nullable - private static DomainDescriptor getDomainDescriptorForUpdate(String domainURI, Container c) - { - if (c == null) - return null; - - DomainDescriptor dd = fetchDomainDescriptorFromDB(domainURI, c); - if (dd == null) - dd = fetchDomainDescriptorFromDB(domainURI, _sharedContainer); - return dd; - } - - /** - * Get all the domains in the same project as the specified container. They may not be in use in the container directly - */ - public static Collection getDomainDescriptors(Container container) - { - return getDomainDescriptors(container, null, false); - } - - public static Collection getDomainDescriptors(Container container, User user, boolean includeProjectAndShared) - { - if (container == null) - return Collections.emptyList(); - - if (includeProjectAndShared && user == null) - throw new IllegalArgumentException("Can't include data from other containers without a user to check permissions on"); - - Map dds = getCachedDomainDescriptors(container, user); - - if (includeProjectAndShared) - { - dds = new LinkedHashMap<>(dds); - Container project = container.getProject(); - if (project != null) - { - for (Map.Entry entry : getCachedDomainDescriptors(project, user).entrySet()) - { - dds.putIfAbsent(entry.getKey(), entry.getValue()); - } - } - - if (_sharedContainer.hasPermission(user, ReadPermission.class)) - { - for (Map.Entry entry : getCachedDomainDescriptors(_sharedContainer, user).entrySet()) - { - dds.putIfAbsent(entry.getKey(), entry.getValue()); - } - } - } - - return unmodifiableCollection(dds.values()); - } - - @NotNull - private static Map getCachedDomainDescriptors(@NotNull Container c, @Nullable User user) - { - if (user != null && !c.hasPermission(user, ReadPermission.class)) - return Collections.emptyMap(); - - return DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.get(c); - } - - public static Pair getURICacheKey(DomainDescriptor dd) - { - return getCacheKey(dd.getDomainURI(), dd.getContainer()); - } - - - public static Pair getCacheKey(PropertyDescriptor pd) - { - return getCacheKey(pd.getPropertyURI(), pd.getContainer()); - } - - - public static Pair getCacheKey(String uri, Container c) - { - Container proj = c.getProject(); - GUID projId; - - if (null == proj) - projId = c.getEntityId(); - else - projId = proj.getEntityId(); - - return Pair.of(uri, projId); - } - - //TODO: Cache semantics. This loads the cache but does not fetch cause need to get them all together - public static List getPropertiesForType(String typeURI, Container c) - { - List> propertyURIs = DOMAIN_PROPERTIES_CACHE.get(getCacheKey(typeURI, c)); - if (propertyURIs != null) - { - List result = new ArrayList<>(propertyURIs.size()); - for (Pair propertyURI : propertyURIs) - { - PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(getCacheKey(propertyURI.getKey(), c)); - if (pd == null) - { - return null; - } - // NOTE: cached descriptors may have differing values of isRequired() as that is a per-domain setting - // Descriptors returned from this method will have their required bit set as appropriate for this domain - - // Clone so nobody else messes up our copy - pd = pd.clone(); - pd.setRequired(propertyURI.getValue().booleanValue()); - result.add(pd); - } - return unmodifiableList(result); - } - return null; - } - - public static void deleteType(String domainURI, Container c) throws DomainNotFoundException - { - if (null == domainURI) - return; - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - try - { - deleteObjectsOfType(domainURI, c); - deleteDomain(domainURI, c); - } - catch (DomainNotFoundException x) - { - // throw exception but do not kill enclosing transaction - transaction.commit(); - throw x; - } - - transaction.commit(); - } - } - - public static PropertyDescriptor insertOrUpdatePropertyDescriptor(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) - throws ChangePropertyDescriptorException - { - validatePropertyDescriptor(pd); - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - DomainDescriptor dexist = ensureDomainDescriptor(dd); - - if (!dexist.getContainer().equals(pd.getContainer()) - && !pd.getProject().equals(_sharedContainer)) - { - // domain is defined in a different container. - //ToDO define property in the domains container? what security? - throw new ChangePropertyDescriptorException("Attempt to define property for a domain definition that exists in a different folder\n" + - "domain folder = " + dexist.getContainer().getPath() + "\n" + - "property folder = " + pd.getContainer().getPath()); - } - - PropertyDescriptor pexist = ensurePropertyDescriptor(pd); - pexist.setDatabaseDefaultValue(pd.getDatabaseDefaultValue()); - pexist.setNullable(pd.isMvEnabled() || pd.isNullable()); - pexist.setRequired(pd.isRequired()); - - ensurePropertyDomain(pexist, dexist, sortOrder); - - transaction.commit(); - return pexist; - } - } - - - static final String parameters = "propertyuri,name,description,rangeuri,concepturi,label," + - "format,container,project,lookupcontainer,lookupschema,lookupquery,defaultvaluetype,hidden," + - "mvenabled,importaliases,url,shownininsertview,showninupdateview,shownindetailsview,measure,dimension,scale," + - "sourceontology,conceptimportcolumn,conceptlabelcolumn,principalconceptcode,conceptsubtree," + - "recommendedvariable,derivationdatascope,storagecolumnname,facetingbehaviortype,phi,redactedText," + - "excludefromshifting,mvindicatorstoragecolumnname,defaultscale,scannable"; - static final String[] parametersArray = parameters.split(","); - - static ParameterMapStatement getInsertStmt(Connection conn, User user, TableInfo t, boolean ifNotExists) throws SQLException - { - user = null==user ? User.guest : user; - SQLFragment sql = new SQLFragment("INSERT INTO exp.propertydescriptor\n\t\t("); - SQLFragment values = new SQLFragment("\nSELECT\t"); - ColumnInfo c; - String comma = ""; - Parameter container = null; - Parameter propertyuri = null; - for (var p : parametersArray) - { - if (null == (c = t.getColumn(p))) - continue; - sql.append(comma).append(p); - values.append(comma).append("?"); - comma = ","; - Parameter parameter = new Parameter(p, c.getJdbcType()); - values.add(parameter); - if ("container".equals(p)) - container = parameter; - else if ("propertyuri".equals(p)) - propertyuri = parameter; - } - sql.append(", createdby, created, modifiedby, modified)\n"); - values.append(", " + user.getUserId() + ", {fn now()}, " + user.getUserId() + ", {fn now()}"); - sql.append(values); - if (ifNotExists) - { - sql.append("\nWHERE NOT EXISTS (SELECT propertyid FROM exp.propertydescriptor WHERE propertyuri=? AND container=?)\n"); - sql.add(propertyuri).add(container); - } - return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); - } - - static ParameterMapStatement getUpdateStmt(Connection conn, User user, TableInfo t) throws SQLException - { - user = null==user ? User.guest : user; - SQLFragment sql = new SQLFragment("UPDATE exp.propertydescriptor SET "); - ColumnInfo c; - String comma = ""; - for (var p : parametersArray) - { - if (null == (c = t.getColumn(p))) - continue; - sql.append(comma).append(p).append("=?"); - comma = ", "; - sql.add(new Parameter(p, c.getJdbcType())); - } - sql.append(", modifiedby=" + user.getUserId() + ", modified={fn now()}"); - sql.append("\nWHERE propertyid=?"); - sql.add(new Parameter("propertyid", JdbcType.INTEGER)); - return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); - } - - - public static void insertPropertyDescriptors(User user, List pds) throws SQLException - { - if (null == pds || pds.isEmpty()) - return; - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getInsertStmt(conn, user, t, false)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = null; - for (PropertyDescriptor pd : pds) - { - m = f.toMap(pd, m); - stmt.clearParameters(); - stmt.putAll(m); - stmt.addBatch(); - } - stmt.executeBatch(); - } - } - - - public static void updatePropertyDescriptors(User user, List pds) throws SQLException - { - if (null == pds || pds.isEmpty()) - return; - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getUpdateStmt(conn, user, t)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = null; - for (PropertyDescriptor pd : pds) - { - m = f.toMap(pd, m); - stmt.clearParameters(); - stmt.putAll(m); - stmt.addBatch(); - } - stmt.executeBatch(); - } - } - - - public static PropertyDescriptor insertPropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException - { - assert pd.getPropertyId() == 0; - validatePropertyDescriptor(pd); - pd = Table.insert(null, getTinfoPropertyDescriptor(), pd); - _log.debug("Adding property descriptor to cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - return pd; - } - - - //todo: we automatically update a pd to the last one in? - public static PropertyDescriptor updatePropertyDescriptor(PropertyDescriptor pd) - { - assert pd.getPropertyId() != 0; - pd = Table.update(null, getTinfoPropertyDescriptor(), pd, pd.getPropertyId()); - _log.debug("Updating property descriptor in cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - // It's possible that the propertyURI has changed, thus breaking our reference - DOMAIN_PROPERTIES_CACHE.clear(); - return pd; - } - - /** - * Insert or update an object property value. - * - * @param user The user inserting the property - currently only used for validating lookup values. - * @param container Insert the property value into this container. - * @param pd The property descriptor. - * @param lsid The object on which to attach the properties. - * @param value The value to insert. - * @param ownerObjectLsid The "owner" object or "parent" object, which isn't necessarily same as the object. For example, samples use the ExpSampleType as the owner object. - * @param insertNullValues When true, a null value will be inserted if the value is null, otherwise any existing property value will be deleted if the value is null. - * @return The inserted ObjectProperty or null - */ - public static ObjectProperty updateObjectProperty(User user, Container container, PropertyDescriptor pd, String lsid, Object value, @Nullable String ownerObjectLsid, boolean insertNullValues) throws ValidationException - { - ObjectProperty oprop; - RemapCache cache = new RemapCache(); - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), container, pd.getContainer()); - - try - { - oprop = new ObjectProperty(lsid, container, pd, value); - } - catch (ConversionException x) - { - // Issue 43529: Assay run property with large lookup doesn't resolve text input by value - // Attempt to resolve lookups by display value and then try creating the ObjectProperty again - if (pd.getLookup() != null) - { - Object remappedValue = getRemappedValueForLookup(user, container, cache, pd.getLookup(), value); - if (remappedValue != null) - value = remappedValue; - } - oprop = new ObjectProperty(lsid, container, pd, value); - } - - if (value != null || insertNullValues) - { - oprop.setPropertyId(pd.getPropertyId()); - OntologyManager.insertProperties(container, user, ownerObjectLsid, false, insertNullValues, oprop); - } - else - { - // We still need to validate blanks - List errors = new ArrayList<>(); - OntologyManager.validateProperty(PropertyService.get().getPropertyValidators(pd), pd, oprop, errors, new ValidatorContext(pd.getContainer(), user)); - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - transaction.commit(); - } - return oprop; - } - - public static Object getRemappedValueForLookup(User user, Container container, RemapCache cache, Lookup lookup, Object value) - { - Container lkContainer = lookup.getContainer() != null ? lookup.getContainer() : container; - return cache.remap(SchemaKey.fromParts(lookup.getSchemaKey()), lookup.getQueryName(), user, lkContainer, ContainerFilter.Type.CurrentPlusProjectAndShared, String.valueOf(value)); - } - - public static List findPropertyUsages(User user, List propertyIds, int maxUsageCount) - { - List ret = new ArrayList<>(propertyIds.size()); - for (int propertyId : propertyIds) - { - var pd = getPropertyDescriptor(propertyId); - if (pd == null) - throw new IllegalArgumentException("property not found: " + propertyId); - - ret.add(findPropertyUsages(user, pd, maxUsageCount)); - } - - return ret; - } - - public static List findPropertyUsages(User user, Container c, List propertyURIs, int maxUsageCount) - { - List ret = new ArrayList<>(propertyURIs.size()); - for (String propertyURI : propertyURIs) - { - var pd = getPropertyDescriptor(propertyURI, c); - if (pd == null) - throw new IllegalArgumentException("property not found: " + propertyURI); - - ret.add(findPropertyUsages(user, pd, maxUsageCount)); - } - - return ret; - } - - public static PropertyUsages findPropertyUsages(@NotNull User user, @NotNull PropertyDescriptor pd, int maxUsageCount) - { - // query exp.ObjectProperty for usages of the property - FieldKey objectId = FieldKey.fromParts("objectId"); - FieldKey objectId_objectURI = FieldKey.fromParts("objectId", "objectURI"); - FieldKey objectId_container = FieldKey.fromParts("objectId", "container"); - List fields = List.of(objectId, objectId_objectURI, objectId_container); - var colMap = QueryService.get().getColumns(getTinfoObjectProperty(), fields); - - int usageCount; - List objects = new ArrayList<>(maxUsageCount); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("propertyId"), pd.getPropertyId(), CompareType.EQUAL); - filter.addCondition(objectId_objectURI, DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX, CompareType.DOES_NOT_CONTAIN); - - TableSelector ts = new TableSelector(getTinfoObjectProperty(), colMap.values(), filter, new Sort("objectId")); - try (var r = ts.getResults(true)) - { - usageCount = r.getSize(); - - for (int i = 0; i < maxUsageCount && r.next(); i++) - { - var row = r.getFieldKeyRowMap(); - long oid = asLong(row.get(objectId)); - String objectURI = (String) row.get(objectId_objectURI); - String container = (String) row.get(objectId_container); - - Identifiable object = LsidManager.get().getObject(objectURI); - if (object != null) - { - Container c = object.getContainer(); - if (c != null && c.hasPermission(user, ReadPermission.class)) - objects.add(object); - } - else - { - Container c = ContainerManager.getForId(container); - if (c != null && c.hasPermission(user, ReadPermission.class)) - { - OntologyObject oo = new OntologyObject(); - oo.setContainer(c); - oo.setObjectId(oid); - oo.setObjectURI(objectURI); - objects.add(new IdentifiableBase(oo)); - } - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return new PropertyUsages(pd.getPropertyId(), pd.getPropertyURI(), usageCount, objects); - } - - public static class PropertyUsages - { - public final int propertyId; - public final String propertyURI; - public final int usageCount; - public final List objects; - - public PropertyUsages(int propertyId, String propertyURI, int usageCount, List objects) - { - this.propertyId = propertyId; - this.propertyURI = propertyURI; - this.usageCount = usageCount; - this.objects = objects; - } - } - - - public static void invalidateDomain(Domain d) - { - // TODO can we please implement a surgical version of this - clearCaches(); - } - - - public static void clearCaches() - { - _log.debug("Clearing caches"); - ExperimentService.get().clearCaches(); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.clear(); - DOMAIN_DESC_BY_ID_CACHE.clear(); - DOMAIN_PROPERTIES_CACHE.clear(); - PROP_DESCRIPTOR_CACHE.clear(); - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); - } - - public static void clearPropertyCache(String parentObjectURI) - { - PROPERTY_MAP_CACHE.removeUsingFilter(key -> Objects.equals(key.second, parentObjectURI)); - } - - - public static void clearPropertyCache() - { - PROPERTY_MAP_CACHE.clear(); - } - - public static class ImportPropertyDescriptor - { - public final String domainName; - public final String domainURI; - public final PropertyDescriptor pd; - public final List validators; - public final List formats; - public final String defaultValue; - - private ImportPropertyDescriptor(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) - { - this.domainName = domainName; - this.domainURI = domainURI; - this.pd = pd; - this.validators = null != validators ? validators : Collections.emptyList(); - this.formats = null != formats ? formats : Collections.emptyList(); - this.defaultValue = defaultValue; - } - } - - - public static class ImportPropertyDescriptorsList - { - public final ArrayList properties = new ArrayList<>(); - - void add(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) - { - properties.add(new ImportPropertyDescriptor(domainName, domainURI, pd, validators, formats, defaultValue)); - } - } - - /** - * Updates an existing domain property with an import property descriptor generated - * by _propertyDescriptorFromRowMap below. Properties we don't set are explicitly - * called out - */ - public static void updateDomainPropertyFromDescriptor(DomainProperty p, PropertyDescriptor pd) - { - // don't setName - p.setPropertyURI(pd.getPropertyURI()); - p.setLabel(pd.getLabel()); - p.setConceptURI(pd.getConceptURI()); - p.setRangeURI(pd.getRangeURI()); - // don't setContainer - p.setDescription(pd.getDescription()); - p.setURL((pd.getURL() != null) ? pd.getURL().toString() : null); - p.setImportAliasSet(ColumnRenderPropertiesImpl.convertToSet(pd.getImportAliases())); - p.setRequired(pd.isRequired()); - p.setHidden(pd.isHidden()); - p.setShownInInsertView(pd.isShownInInsertView()); - p.setShownInUpdateView(pd.isShownInUpdateView()); - p.setShownInDetailsView(pd.isShownInDetailsView()); - p.setShownInLookupView(pd.isShownInLookupView()); - p.setDimension(pd.isDimension()); - p.setMeasure(pd.isMeasure()); - p.setRecommendedVariable(pd.isRecommendedVariable()); - p.setDefaultScale(pd.getDefaultScale()); - p.setScale(pd.getScale()); - p.setFormat(pd.getFormat()); - p.setMvEnabled(pd.isMvEnabled()); - - Lookup lookup = new Lookup(); - lookup.setQueryName(pd.getLookupQuery()); - lookup.setSchemaName(pd.getLookupSchema()); - String lookupContainerId = pd.getLookupContainer(); - if (lookupContainerId != null) - { - Container container = ContainerManager.getForId(lookupContainerId); - if (container == null) - lookup = null; - else - lookup.setContainer(container); - } - p.setLookup(lookup); - p.setFacetingBehavior(pd.getFacetingBehaviorType()); - p.setPhi(pd.getPHI()); - p.setRedactedText(pd.getRedactedText()); - p.setExcludeFromShifting(pd.isExcludeFromShifting()); - p.setDefaultValueTypeEnum(pd.getDefaultValueTypeEnum()); - p.setScannable(pd.isScannable()); - p.setDerivationDataScope(pd.getDerivationDataScope()); - } - - @TestWhen(TestWhen.When.BVT) - @TestTimeout(120) - public static class TestCase extends Assert - { - @Test - public void testSchema() - { - assertNotNull(getExpSchema()); - assertNotNull(getTinfoPropertyDescriptor()); - assertNotNull(ExperimentService.get().getTinfoSampleType()); - - assertEquals(11, getTinfoPropertyDescriptor().getColumns("PropertyId,PropertyURI,RangeURI,Name,Description,DerivationDataScope,SourceOntology,ConceptImportColumn,ConceptLabelColumn,PrincipalConceptCode,scannable").size()); - assertEquals(4, getTinfoObject().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId").size()); - assertEquals(11, getTinfoObjectPropertiesView().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId,Name,PropertyURI,RangeURI,TypeTag,StringValue,DateTimeValue,FloatValue").size()); - assertEquals(10, ExperimentService.get().getTinfoSampleType().getColumns("RowId,Name,LSID,MaterialLSIDPrefix,Description,Created,CreatedBy,Modified,ModifiedBy,Container").size()); - } - - @Test - public void testBasicPropertiesObject() throws ValidationException - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - User user = TestContext.get().getUser(); - String parentObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - //First delete in case test case failed before - deleteOntologyObjects(c, parentObjectLsid); - assertNull(getOntologyObject(c, parentObjectLsid)); - assertNull(getOntologyObject(c, childObjectLsid)); - ensureObject(c, childObjectLsid, parentObjectLsid); - OntologyObject oParent = getOntologyObject(c, parentObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - assertNull(oParent.getOwnerObjectId()); - assertEquals(oChild.getContainer(), c); - assertEquals(oParent.getContainer(), c); - - String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - PropertyDescriptor strPd = getPropertyDescriptor(strProp, c); - assertEquals(PropertyType.STRING, strPd.getPropertyType()); - - String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - PropertyDescriptor intPd = getPropertyDescriptor(intProp, c); - assertEquals(PropertyType.INTEGER, intPd.getPropertyType()); - - String longProp = new Lsid("Junit", "OntologyManager", "longProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, longProp, 6L)); - PropertyDescriptor longPd = getPropertyDescriptor(longProp, c); - assertEquals(PropertyType.BIGINT, longPd.getPropertyType()); - - Calendar cal = Calendar.getInstance(); - cal.set(Calendar.MILLISECOND, 0); - String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); - PropertyDescriptor datePd = getPropertyDescriptor(dateProp, c); - assertEquals(PropertyType.DATE_TIME, datePd.getPropertyType()); - - Map m = getProperties(c, oChild.getObjectURI()); - assertNotNull(m); - assertEquals(4, m.size()); - assertEquals("The String", m.get(strProp)); - assertEquals(5, m.get(intProp)); - assertEquals(6L, m.get(longProp)); - assertEquals(cal.getTime(), m.get(dateProp)); - - // Set property order: date, str, int. Long property will sort to last since it isn't explicitly included. - List propertyOrder = List.of(datePd, strPd, intPd); - updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); - - Map oProps = getPropertyObjects(c, childObjectLsid); - var iter = oProps.entrySet().iterator(); - assertEquals(cal.getTime(), iter.next().getValue().value()); - assertEquals("The String", iter.next().getValue().value()); - assertEquals(5, iter.next().getValue().value()); - assertEquals(6L, iter.next().getValue().value()); - assertFalse(iter.hasNext()); - - // Update property order: int, date, long, str - propertyOrder = List.of(intPd, datePd, longPd, strPd); - updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); - oProps = getPropertyObjects(c, childObjectLsid); - iter = oProps.entrySet().iterator(); - assertEquals(5, iter.next().getValue().value()); - assertEquals(cal.getTime(), iter.next().getValue().value()); - assertEquals(6L, iter.next().getValue().value()); - assertEquals("The String", iter.next().getValue().value()); - assertFalse(iter.hasNext()); - - deleteOntologyObjects(c, parentObjectLsid); - assertNull(getOntologyObject(c, parentObjectLsid)); - assertNull(getOntologyObject(c, childObjectLsid)); - - m = getProperties(c, oChild.getObjectURI()); - assertEquals(0, m.size()); - } - - @Test - public void testContainerDelete() throws ValidationException - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - - Calendar cal = Calendar.getInstance(); - cal.set(Calendar.MILLISECOND, 0); - String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); - - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - - private void defineCrossFolderProperties(Container fldr1a, Container fldr1b) throws SQLException - { - try - { - String fa = fldr1a.getPath(); - String fb = fldr1b.getPath(); - - //object, prop descriptor in folder being moved - String objP1Fa = new Lsid("OntologyObject", "JUnit", fa.replace('/', '.')).toString(); - ensureObject(fldr1a, objP1Fa); - String propP1Fa = fa + "PD1"; - PropertyDescriptor pd1Fa = ensurePropertyDescriptor(propP1Fa, PropertyType.STRING, "PropertyDescriptor 1" + fa, fldr1a); - insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP1Fa, "same fldr")); - - //object in folder not moving, prop desc in folder moving - String objP2Fb = new Lsid("OntologyObject", "JUnit", fb.replace('/', '.')).toString(); - ensureObject(fldr1b, objP2Fb); - insertProperties(fldr1b, null, new ObjectProperty(objP2Fb, fldr1b, propP1Fa, "object in folder not moving, prop desc in folder moving")); - - //object in folder moving, prop desc in folder not moving - String propP2Fb = fb + "PD1"; - ensurePropertyDescriptor(propP2Fb, PropertyType.STRING, "PropertyDescriptor 1" + fb, fldr1b); - insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP2Fb, "object in folder moving, prop desc in folder not moving")); - - // third prop desc in folder that is moving; shares domain with first prop desc - String propP1Fa3 = fa + "PD3"; - PropertyDescriptor pd1Fa3 = ensurePropertyDescriptor(propP1Fa3, PropertyType.STRING, "PropertyDescriptor 3" + fa, fldr1a); - String domP1Fa = fa + "DD1"; - DomainDescriptor dd1 = ensureDomainDescriptor(domP1Fa, "DomDesc 1" + fa, fldr1a); - ensurePropertyDomain(pd1Fa, dd1); - ensurePropertyDomain(pd1Fa3, dd1); - - //second domain desc in folder that is moving - // second prop desc in folder moving, belongs to 2nd domain - String propP1Fa2 = fa + "PD2"; - PropertyDescriptor pd1Fa2 = ensurePropertyDescriptor(propP1Fa2, PropertyType.STRING, "PropertyDescriptor 2" + fa, fldr1a); - String domP1Fa2 = fa + "DD2"; - DomainDescriptor dd2 = ensureDomainDescriptor(domP1Fa2, "DomDesc 2" + fa, fldr1a); - ensurePropertyDomain(pd1Fa2, dd2); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - @Test - public void testContainerMove() throws Exception - { - deleteMoveTestContainers(); - - Container proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); - Container proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - - proj1 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); - proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - - proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); - proj2 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - } - - private void doMoveTest(Container proj1, Container proj2) throws Exception - { - String p1Path = proj1.getPath() + "/"; - String p2Path = proj2.getPath() + "/"; - if (p1Path.equals("//")) p1Path = "/_ontMgrDemotePromote"; - if (p2Path.equals("//")) p2Path = "/_ontMgrDemotePromote"; - - Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); - Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); - ContainerManager.ensureContainer(p2Path + "Fc", TestContext.get().getUser()); - Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); - Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); - - defineCrossFolderProperties(fldr1a, fldr1b); - //defineCrossFolderProperties(fldr1a, fldr2c); - defineCrossFolderProperties(fldr1aa, fldr1b); - defineCrossFolderProperties(fldr1aaa, fldr1b); - - fldr1a.getProject().getPath(); - String f = fldr1a.getPath(); - String propId = f + "PD1"; - assertNull(getPropertyDescriptor(propId, proj2)); - ContainerManager.move(fldr1a, proj2, TestContext.get().getUser()); - - // if demoting a folder - if (proj1.isRoot()) - { - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD2"; - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj2)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj2)); - - domId = f + "DD2"; - assertNotNull(getDomainDescriptor(domId, proj2)); - } - // if promoting a folder, - else if (proj2.isRoot()) - { - assertNotNull(getPropertyDescriptor(propId, proj1)); - - propId = f + "PD2"; - assertNull(getPropertyDescriptor(propId, proj1)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj1)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj1)); - - domId = f + "DD2"; - assertNull(getDomainDescriptor(domId, proj1)); - } - else - { - assertNotNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD2"; - assertNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj1)); - assertNotNull(getDomainDescriptor(domId, proj2)); - - domId = f + "DD2"; - assertNull(getDomainDescriptor(domId, proj1)); - assertNotNull(getDomainDescriptor(domId, proj2)); - } - } - - @Test - public void testDeleteFoldersWithSharedProps() throws SQLException - { - deleteMoveTestContainers(); - - String projectName = "_ontMgrTestP1"; - Container proj1 = ContainerManager.ensureContainer(projectName, TestContext.get().getUser()); - String p1Path = proj1.getPath() + "/"; - - Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); - Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); - Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); - Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); - - defineCrossFolderProperties(fldr1a, fldr1b); - defineCrossFolderProperties(fldr1aa, fldr1b); - defineCrossFolderProperties(fldr1aaa, fldr1b); - - deleteProjects( projectName); - } - - private void deleteMoveTestContainers() - { - // Remove all projects. Subfolders will be deleted when project is removed. - deleteProjects( - "/_ontMgrTestP1", - "/_ontMgrTestP2", - "/_ontMgrDemotePromoteFa", - "/_ontMgrDemotePromoteFb", - "/_ontMgrDemotePromoteFc", - "/Fa" - ); - } - - private void deleteProjects(String... projectNames) - { - for (String path : projectNames) - { - Container c = ContainerManager.getForPath(path); - - if (null != c) - ContainerManager.deleteAll(c, TestContext.get().getUser()); - } - - for (String path : projectNames) - assertNull("Container " + path + " was not deleted", ContainerManager.getForPath(path)); - } - - @Test - public void testTransactions() throws SQLException - { - try - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - //Create objects in a transaction & make sure they are all gone. - OntologyObject oParent; - OntologyObject oChild; - String strProp; - String intProp; - - try (Transaction ignored = getExpSchema().getScope().beginTransaction()) - { - ensureObject(c, childObjectLsid, ownerObjectLsid); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - } - - assertEquals(0L, getObjectCount(c)); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNull(oParent); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - //Rollback transaction for one new property - try (Transaction ignored = getExpSchema().getScope().beginTransaction()) - { - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - } - - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - Map m = getProperties(c, childObjectLsid); - assertNotNull(m.get(strProp)); - assertNull(m.get(intProp)); - - try (Transaction transaction = getExpSchema().getScope().beginTransaction()) - { - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - transaction.commit(); - } - - m = getProperties(c, childObjectLsid); - assertNotNull(m.get(strProp)); - assertNotNull(m.get(intProp)); - - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - @Test - public void testDomains() throws Exception - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - String child2ObjectLsid = new Lsid("Junit", "OntologyManager", "child2").toString(); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - String domURIa = new Lsid("Junit", "DD", "Domain1").toString(); - String strPropURI = new Lsid("Junit", "PD", "Domain1.stringProp").toString(); - String intPropURI = new Lsid("Junit", "PD", "Domain1.intProp").toString(); - String longPropURI = new Lsid("Junit", "PD", "Domain1.longProp").toString(); - - DomainDescriptor dd = ensureDomainDescriptor(domURIa, "Domain1", c); - assertNotNull(dd); - - PropertyDescriptor pdStr = new PropertyDescriptor(); - pdStr.setPropertyURI(strPropURI); - pdStr.setRangeURI(PropertyType.STRING.getTypeUri()); - pdStr.setContainer(c); - pdStr.setName("Domain1.stringProp"); - - pdStr = ensurePropertyDescriptor(pdStr); - assertNotNull(pdStr); - - PropertyDescriptor pdInt = ensurePropertyDescriptor(intPropURI, PropertyType.INTEGER, "Domain1.intProp", c); - PropertyDescriptor pdLong = ensurePropertyDescriptor(longPropURI, PropertyType.BIGINT, "Domain1.longProp", c); - - ensurePropertyDomain(pdStr, dd); - ensurePropertyDomain(pdInt, dd); - ensurePropertyDomain(pdLong, dd); - - List pds = getPropertiesForType(domURIa, c); - assertEquals(3, pds.size()); - Map mPds = new HashMap<>(); - for (PropertyDescriptor pd1 : pds) - mPds.put(pd1.getPropertyURI(), pd1); - - assertTrue(mPds.containsKey(strPropURI)); - assertTrue(mPds.containsKey(intPropURI)); - assertTrue(mPds.containsKey(longPropURI)); - - ObjectProperty strProp = new ObjectProperty(childObjectLsid, c, strPropURI, "String value"); - ObjectProperty intProp = new ObjectProperty(childObjectLsid, c, intPropURI, 42); - ObjectProperty longProp = new ObjectProperty(childObjectLsid, c, longPropURI, 52L); - insertProperties(c, ownerObjectLsid, strProp); - insertProperties(c, ownerObjectLsid, intProp); - insertProperties(c, ownerObjectLsid, longProp); - - Map m = getProperties(c, oChild.getObjectURI()); - assertNotNull(m); - assertEquals(3, m.size()); - assertEquals("String value", m.get(strPropURI)); - assertEquals(42, m.get(intPropURI)); - assertEquals(52L, m.get(longPropURI)); - - // test insertTabDelimited - List> rows = List.of( - new CaseInsensitiveMapWrapper<>(Map.of( - "lsid", child2ObjectLsid, - strPropURI, "Second value", - intPropURI, 62, - longPropURI, 72L - ) - )); - ImportHelper helper = new ImportHelper() - { - @Override - public String beforeImportObject(Map map) - { - return (String)map.get("lsid"); - } - - @Override - public void afterBatchInsert(int currentRow) - { } - - @Override - public void updateStatistics(int currentRow) - { } - }; - try (Transaction tx = getExpSchema().getScope().ensureTransaction()) - { - insertTabDelimited(c, TestContext.get().getUser(), oParent.getObjectId(), helper, pds, MapDataIterator.of(rows).getDataIterator(new DataIteratorContext()), false, null); - tx.commit(); - } - - m = getProperties(c, child2ObjectLsid); - assertNotNull(m); - assertEquals(3, m.size()); - assertEquals("Second value", m.get(strPropURI)); - assertEquals(62, m.get(intPropURI)); - assertEquals(72L, m.get(longPropURI)); - - deleteType(domURIa, c); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - } - - private static long getObjectCount(Container c) - { - return new TableSelector(getTinfoObject(), SimpleFilter.createContainerFilter(c), null).getRowCount(); - } - - /** - * v.first value IN/OUT parameter - * v.second mvIndicator OUT parameter - */ - public static void convertValuePair(PropertyDescriptor pd, PropertyType pt, Pair v) - { - if (v.first == null) - return; - - // Handle field-level QC - if (v.first instanceof MvFieldWrapper mvWrapper) - { - v.second = mvWrapper.getMvIndicator(); - v.first = mvWrapper.getValue(); - } - else if (pd.isMvEnabled()) - { - // Not all callers will have wrapped an MV value if there isn't also - // a real value - if (MvUtil.isMvIndicator(v.first.toString(), pd.getContainer())) - { - v.second = v.first.toString(); - v.first = null; - } - } - - if (null != v.first && null != pt) - v.first = pt.convert(v.first); - } - - @Deprecated // Fold into ObjectProperty? Eliminate insertTabDelimited() methods, the only usage of PropertyRow. - public static class PropertyRow - { - protected long objectId; - protected int propertyId; - protected char typeTag; - protected Double floatValue; - protected String stringValue; - protected Date dateTimeValue; - protected String mvIndicator; - - public PropertyRow() - { - } - - public PropertyRow(long objectId, PropertyDescriptor pd, Object value, PropertyType pt) - { - this.objectId = objectId; - this.propertyId = pd.getPropertyId(); - this.typeTag = pt.getStorageType(); - - Pair p = new Pair<>(value, null); - convertValuePair(pd, pt, p); - mvIndicator = p.second; - - pt.init(this, p.first); - } - - public long getObjectId() - { - return objectId; - } - - public void setObjectId(long objectId) - { - this.objectId = objectId; - } - - public int getPropertyId() - { - return propertyId; - } - - public void setPropertyId(int propertyId) - { - this.propertyId = propertyId; - } - - public char getTypeTag() - { - return typeTag; - } - - public void setTypeTag(char typeTag) - { - this.typeTag = typeTag; - } - - public Double getFloatValue() - { - return floatValue; - } - - public Boolean getBooleanValue() - { - if (floatValue == null) - { - return null; - } - return floatValue.doubleValue() == 1.0; - } - - public void setFloatValue(Double floatValue) - { - this.floatValue = floatValue; - } - - public String getStringValue() - { - return stringValue; - } - - public void setStringValue(String stringValue) - { - this.stringValue = stringValue; - } - - public Date getDateTimeValue() - { - return dateTimeValue; - } - - public void setDateTimeValue(Date dateTimeValue) - { - this.dateTimeValue = dateTimeValue; - } - - public String getMvIndicator() - { - return mvIndicator; - } - - public void setMvIndicator(String mvIndicator) - { - this.mvIndicator = mvIndicator; - } - - public Object getObjectValue() - { - return stringValue != null ? stringValue : floatValue != null ? floatValue : dateTimeValue; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder(); - sb.append("PropertyRow: "); - - sb.append("objectId=").append(objectId); - sb.append(", propertyId=").append(propertyId); - sb.append(", value="); - - if (stringValue != null) - sb.append(stringValue); - else if (floatValue != null) - sb.append(floatValue); - else if (dateTimeValue != null) - sb.append(dateTimeValue); - else - sb.append("null"); - - if (mvIndicator != null) - sb.append(", mvIndicator=").append(mvIndicator); - - return sb.toString(); - } - } - - public static DbSchema getExpSchema() - { - return DbSchema.get("exp", DbSchemaType.Module); - } - - public static SqlDialect getSqlDialect() - { - return getExpSchema().getSqlDialect(); - } - - public static TableInfo getTinfoPropertyDomain() - { - return getExpSchema().getTable("PropertyDomain"); - } - - public static TableInfo getTinfoObject() - { - return getExpSchema().getTable("Object"); - } - - public static TableInfo getTinfoObjectProperty() - { - return getExpSchema().getTable("ObjectProperty"); - } - - public static TableInfo getTinfoPropertyDescriptor() - { - return getExpSchema().getTable("PropertyDescriptor"); - } - - public static TableInfo getTinfoDomainDescriptor() - { - return getExpSchema().getTable("DomainDescriptor"); - } - - public static TableInfo getTinfoObjectPropertiesView() - { - return getExpSchema().getTable("ObjectPropertiesView"); - } - - public static HtmlString doProjectColumnCheck(boolean bFix) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(); - String descriptorTable = getTinfoPropertyDescriptor().toString(); - String uriColumn = "PropertyURI"; - String idColumn = "PropertyID"; - doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); - - descriptorTable = getTinfoDomainDescriptor().toString(); - uriColumn = "DomainURI"; - idColumn = "DomainID"; - doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); - - return builder.getHtmlString(); - } - - private static void doProjectColumnCheck(final String descriptorTable, final String uriColumn, final String idColumn, final HtmlStringBuilder msgBuilder, final boolean bFix) - { - // get all unique combos of Container, project - - String sql = "SELECT Container, Project FROM " + descriptorTable + " GROUP BY Container, Project"; - - new SqlSelector(getExpSchema(), sql).forEach(rs -> { - String containerId = rs.getString("Container"); - String projectId = rs.getString("Project"); - Container container = ContainerManager.getForId(containerId); - if (null == container) - return; // should be handled by container check - String newProjectId = container.getProject() == null ? container.getId() : container.getProject().getId(); - if (!projectId.equals(newProjectId)) - { - if (bFix) - { - fixProjectColumn(descriptorTable, uriColumn, idColumn, container, projectId, newProjectId); - msgBuilder - .unsafeAppend("
   ") - .append("Fixed inconsistent project ids found for ") - .append(descriptorTable).append(" in folder ") - .append(ContainerManager.getForId(containerId).getPath()); - - } - else - msgBuilder - .unsafeAppend("
   ") - .append("ERROR: Inconsistent project ids found for ") - .append(descriptorTable).append(" in folder ").append(container.getPath()); - } - }); - } - - private static void fixProjectColumn(String descriptorTable, String uriColumn, String idColumn, Container container, String projectId, String newProjId) - { - final SqlExecutor executor = new SqlExecutor(getExpSchema()); - - String sql = "UPDATE " + descriptorTable + " SET Project= ? WHERE Project = ? AND Container=? AND " + uriColumn + " NOT IN " + - "(SELECT " + uriColumn + " FROM " + descriptorTable + " WHERE Project = ?)"; - executor.execute(sql, newProjId, projectId, container.getId(), newProjId); - - // now check to see if there is already an existing descriptor in the target (correct) project. - // this can happen if a folder containing a descriptor is moved to another project - // and the OntologyManager's containerMoved handler fails to fire for some reason. (note not in transaction) - // If this is the case, the descriptor is redundant and it should be deleted, after we move the objects that depend on it. - - sql = " SELECT prev." + idColumn + " AS PrevIdCol, cur." + idColumn + " AS CurIdCol FROM " + descriptorTable + " prev " - + " INNER JOIN " + descriptorTable + " cur ON (prev." + uriColumn + "= cur." + uriColumn + " ) " - + " WHERE cur.Project = ? AND prev.Project= ? AND prev.Container = ? "; - final String updsql1 = " UPDATE " + getTinfoObjectProperty() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; - final String updsql2 = " UPDATE " + getTinfoPropertyDomain() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; - final String delSql = " DELETE FROM " + descriptorTable + " WHERE " + idColumn + " = ? "; - - new SqlSelector(getExpSchema(), sql, newProjId, projectId, container).forEach(rs -> { - int prevPropId = rs.getInt(1); - int curPropId = rs.getInt(2); - executor.execute(updsql1, curPropId, prevPropId); - executor.execute(updsql2, curPropId, prevPropId); - executor.execute(delSql, prevPropId); - }); - } - - public static void validatePropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException - { - String name = pd.getName(); - validateValue(name, "Name", null); - validateValue(pd.getPropertyURI(), "PropertyURI", "Please use a shorter field name. Name = " + name); - validateValue(pd.getLabel(), "Label", null); - validateValue(pd.getImportAliases(), "ImportAliases", null); - validateValue(pd.getURL() != null ? pd.getURL().getSource() : null, "URL", null); - validateValue(pd.getConceptURI(), "ConceptURI", null); - validateValue(pd.getRangeURI(), "RangeURI", null); - - // Issue 15484: adding a column ending in 'mvIndicator' is problematic if another column w/ the same - // root exists, or if you later enable mvIndicators on a column w/ the same root - if (pd.getName() != null && pd.getName().toLowerCase().endsWith(MV_INDICATOR_SUFFIX)) - { - throw new ChangePropertyDescriptorException("Field name cannot end with the suffix 'mvIndicator': " + pd.getName()); - } - - if (null != name) - { - for (char ch : name.toCharArray()) - { - if (Character.isWhitespace(ch) && ' ' != ch) - throw new ChangePropertyDescriptorException("Field name cannot contain whitespace other than ' ' (space)"); - } - } - } - - private static void validateValue(String value, String columnName, String extraMessage) throws ChangePropertyDescriptorException - { - int maxLength = getTinfoPropertyDescriptor().getColumn(columnName).getScale(); - if (value != null && value.length() > maxLength) - { - throw new ChangePropertyDescriptorException(columnName + " cannot exceed " + maxLength + " characters, but was " + value.length() + " characters long. " + (extraMessage == null ? "" : extraMessage)); - } - } - - static public boolean checkObjectExistence(String lsid) - { - return new TableSelector(getTinfoObject(), new SimpleFilter(FieldKey.fromParts("ObjectURI"), lsid), null).exists(); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.exp; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.data.*; +import org.labkey.api.data.DbScope.Transaction; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.ui.domain.CancellationException; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.test.TestTimeout; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.view.HttpView; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableCollection; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.stream.Collectors.joining; +import static org.labkey.api.util.IntegerUtils.asLong; + +/** + * Lots of static methods for dealing with domains and property descriptors. Tends to operate primarily on the bean-style + * classes like {@link PropertyDescriptor} and {@link DomainDescriptor}. When possible, it's usually preferable to use + * {@link PropertyService}, {@link Domain}, and {@link DomainProperty} instead as they tend to provide higher-level + * abstractions. + */ +public class OntologyManager +{ + private static final Logger _log = LogManager.getLogger(OntologyManager.class); + private static final Cache, Map> PROPERTY_MAP_CACHE = DatabaseCache.get(getExpSchema().getScope(), 100000, "Property maps", new PropertyMapCacheLoader()); + private static final BlockingCache OBJECT_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "ObjectIds", new ObjectIdCacheLoader()); + private static final Cache, PropertyDescriptor> PROP_DESCRIPTOR_CACHE = DatabaseCache.get(getExpSchema().getScope(), 40000, "Property descriptors", new CacheLoader<>() + { + @Override + public PropertyDescriptor load(@NotNull Pair key, @Nullable Object argument) + { + PropertyDescriptor ret = null; + String propertyURI = key.first; + Container c = ContainerManager.getForId(key.second); + if (null != c) + { + Container proj = c.getProject(); + if (null == proj) + proj = c; + _log.debug("Loading a property descriptor for key " + key + " using project " + proj); + String sql = " SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyURI = ? AND Project IN (?,?)"; + List pdArray = new SqlSelector(getExpSchema(), sql, propertyURI, proj, _sharedContainer.getId()).getArrayList(PropertyDescriptor.class); + if (!pdArray.isEmpty()) + { + PropertyDescriptor pd = pdArray.get(0); + + // if someone has explicitly inserted a descriptor with the same URI as an existing one, + // and one of the two is in the shared project, use the project-level descriptor. + if (pdArray.size() > 1) + { + _log.debug("Multiple PropertyDescriptors found for " + propertyURI); + if (pd.getProject().equals(_sharedContainer)) + pd = pdArray.get(1); + } + _log.debug("Loaded property descriptor " + pd); + ret = pd; + } + } + return ret; + } + }); + + /** DomainURI, ContainerEntityId -> DomainDescriptor */ + private static final Cache, DomainDescriptor> DOMAIN_DESCRIPTORS_BY_URI_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, CacheManager.UNLIMITED, "Domain descriptors by URI", (key, argument) -> { + String domainURI = key.first; + Container c = ContainerManager.getForId(key.second); + + if (c == null) + { + return null; + } + + return fetchDomainDescriptorFromDB(domainURI, c); + }); + + @Nullable + private static DomainDescriptor fetchDomainDescriptorFromDB(String domainURI, Container c) + { + return fetchDomainDescriptorFromDB(domainURI, c, false); + } + + /** Goes against the DB, bypassing the cache */ + @Nullable + public static DomainDescriptor fetchDomainDescriptorFromDB(String uriOrName, Container c, boolean isName) + { + Container proj = c.getProject(); + if (null == proj) + proj = c; + + String sql = " SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE " + (isName ? "Name" : "DomainURI") + " = ? AND Project IN (?,?) "; + List ddArray = new SqlSelector(getExpSchema(), sql, uriOrName, + proj, + ContainerManager.getSharedContainer().getId()).getArrayList(DomainDescriptor.class); + DomainDescriptor dd = null; + if (!ddArray.isEmpty()) + { + dd = ddArray.get(0); + + // if someone has explicitly inserted a descriptor with the same URI as an existing one , + // and one of the two is in the shared project, use the project-level descriptor. + if (ddArray.size() > 1) + { + _log.debug("Multiple DomainDescriptors found for " + uriOrName); + if (dd.getProject().equals(ContainerManager.getSharedContainer())) + dd = ddArray.get(0); + } + } + return dd; + } + + private static final BlockingCache DOMAIN_DESC_BY_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(),2000, CacheManager.UNLIMITED,"Domain descriptors by ID", new DomainDescriptorLoader()); + private static final BlockingCache, List>> DOMAIN_PROPERTIES_CACHE = DatabaseCache.get(getExpSchema().getScope(), 5000, CacheManager.UNLIMITED, "Domain properties", new CacheLoader<>() + { + @Override + public List> load(@NotNull Pair key, @Nullable Object argument) + { + String typeURI = key.first; + Container c = ContainerManager.getForId(key.second); + if (null == c) + return Collections.emptyList(); + SQLFragment sql = new SQLFragment("SELECT PropertyURI, Required " + + "FROM " + getTinfoPropertyDescriptor() + " PD\n" + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (PD.PropertyId = PDM.PropertyId)\n" + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)\n" + + "WHERE DD.DomainURI = ? AND DD.Project IN (?, ?) ORDER BY PDM.SortOrder, PD.PropertyId"); + + sql.addAll( + typeURI, + // protect against null project, just double-up shared project + c.isRoot() ? c.getId() : (c.getProject() == null ? _sharedContainer.getProject().getId() : c.getProject().getId()), + _sharedContainer.getProject().getId() + ); + + return new SqlSelector(getExpSchema(), sql).mapStream() + .map(map -> Pair.of((String)map.get("PropertyURI"), (Boolean)map.get("Required"))) + .toList(); + } + }); + private static final Cache> DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "Domain descriptors by container", (c, argument) -> { + String sql = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + + Map dds = new LinkedHashMap<>(); + for (DomainDescriptor dd : new SqlSelector(getExpSchema(), sql, c).getArrayList(DomainDescriptor.class)) + { + dds.putIfAbsent(dd.getDomainURI(), dd); + } + + return unmodifiableMap(dds); + }); + + private static final Container _sharedContainer = ContainerManager.getSharedContainer(); + + public static final String MV_INDICATOR_SUFFIX = "mvindicator"; + + static public String PropertyOrderURI = "urn:exp.labkey.org/#PropertyOrder"; + /** + * A comma-separated list of propertyID that indicates the sort order of the properties attached to an object. + */ + static public SystemProperty PropertyOrder = new SystemProperty(PropertyOrderURI, PropertyType.STRING); + + static + { + BeanObjectFactory.Registry.register(ObjectProperty.class, new ObjectProperty.ObjectPropertyObjectFactory()); + } + + private OntologyManager() + { + } + + /** + * @return map from PropertyURI to value + */ + public static @NotNull Map getProperties(Container container, String parentLSID) + { + Map m = new LinkedHashMap<>(); + Map propVals = getPropertyObjects(container, parentLSID); + if (null != propVals) + { + for (Map.Entry entry : propVals.entrySet()) + { + m.put(entry.getKey(), entry.getValue().value()); + } + } + + return m; + } + + public static final int MAX_PROPS_IN_BATCH = 1000; // Keep this reasonably small so progress indicator is updated regularly + public static final int UPDATE_STATS_BATCH_COUNT = 1000; + + public static void insertTabDelimited(Container c, + User user, + @Nullable Long ownerObjectId, + ImportHelper helper, + Domain domain, + DataIterator rows, + boolean ensureObjects, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + List properties = new ArrayList<>(domain.getProperties().size()); + for (DomainProperty prop : domain.getProperties()) + { + properties.add(prop.getPropertyDescriptor()); + } + insertTabDelimited(c, user, ownerObjectId, helper, properties, rows, ensureObjects, rowCallback); + } + + public interface RowCallback + { + void rowProcessed(Map row, String lsid) throws BatchValidationException; + + default void complete() throws BatchValidationException + {} + + default RowCallback chain(RowCallback other) + { + if (other == NO_OP_ROW_CALLBACK) + { + return this; + } + if (this == NO_OP_ROW_CALLBACK) + { + return other; + } + + RowCallback original = this; + + return new RowCallback() + { + @Override + public void rowProcessed(Map row, String lsid) throws BatchValidationException + { + original.rowProcessed(row, lsid); + other.rowProcessed(row, lsid); + } + + @Override + public void complete() throws BatchValidationException + { + original.complete(); + other.complete(); + } + }; + } + } + + public static final RowCallback NO_OP_ROW_CALLBACK = (row, lsid) -> {}; + + public static void insertTabDelimited(Container c, + User user, + @Nullable Long ownerObjectId, + ImportHelper helper, + List descriptors, + DataIterator rawRows, + boolean ensureObjects, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + MapDataIterator rows = DataIteratorUtil.wrapMap(rawRows, false); + + rowCallback = rowCallback == null ? NO_OP_ROW_CALLBACK : rowCallback; + + CPUTimer total = new CPUTimer("insertTabDelimited"); + CPUTimer before = new CPUTimer("beforeImport"); + CPUTimer ensure = new CPUTimer("ensureObject"); + CPUTimer insert = new CPUTimer("insertProperties"); + + assert total.start(); + assert getExpSchema().getScope().isTransactionActive(); + + // Make sure we have enough rows to handle the overflow of the current row so we don't have to resize the list + List propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); + + ValidatorContext validatorCache = new ValidatorContext(c, user); + + try + { + OntologyObject objInsert = new OntologyObject(); + objInsert.setContainer(c); + if (ownerObjectId != null && ownerObjectId > 0) + objInsert.setOwnerObjectId(ownerObjectId); + + List errors = new ArrayList<>(); + Map> validatorMap = new IntHashMap<>(); + + // cache all the property validators for this upload + for (PropertyDescriptor pd : descriptors) + { + List validators = PropertyService.get().getPropertyValidators(pd); + if (!validators.isEmpty()) + validatorMap.put(pd.getPropertyId(), validators); + } + + int rowCount = 0; + int batchCount = 0; + + while (rows.next()) + { + Map map = rows.getMap(); + // TODO: hack -- should exit and return cancellation status instead of throwing + if (Thread.currentThread().isInterrupted()) + throw new CancellationException(); + + assert before.start(); + + Map modifiableMap = new HashMap<>(map); + String lsid = helper.beforeImportObject(modifiableMap); + map = Collections.unmodifiableMap(modifiableMap); + + if (lsid == null) + { + throw new IllegalStateException("No LSID available"); + } + + assert before.stop(); + + assert ensure.start(); + long objectId; + if (ensureObjects) + objectId = ensureObject(c, lsid, ownerObjectId); + else + { + objInsert.setObjectURI(lsid); + Table.insert(null, getTinfoObject(), objInsert); + objectId = objInsert.getObjectId(); + } + + for (PropertyDescriptor pd : descriptors) + { + Object value = map.get(pd.getPropertyURI()); + if (null == value) + { + if (pd.isRequired()) + throw new BatchValidationException(new ValidationException("Missing value for required property " + pd.getName())); + else + { + continue; + } + } + else + { + if (validatorMap.containsKey(pd.getPropertyId())) + validateProperty(validatorMap.get(pd.getPropertyId()), pd, new ObjectProperty(lsid, c, pd, value), errors, validatorCache); + } + try + { + PropertyRow row = new PropertyRow(objectId, pd, value, pd.getPropertyType()); + propsToInsert.add(row); + } + catch (ConversionException e) + { + throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getPropertyType().getJavaType()))); + } + } + assert ensure.stop(); + + rowCount++; + + if (propsToInsert.size() > MAX_PROPS_IN_BATCH) + { + assert insert.start(); + insertPropertiesBulk(c, propsToInsert, false); + helper.afterBatchInsert(rowCount); + assert insert.stop(); + propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); + + if (++batchCount % UPDATE_STATS_BATCH_COUNT == 0) + { + getExpSchema().getSqlDialect().updateStatistics(getTinfoObject()); + getExpSchema().getSqlDialect().updateStatistics(getTinfoObjectProperty()); + helper.updateStatistics(rowCount); + } + } + + rowCallback.rowProcessed(map, lsid); + } + + if (!errors.isEmpty()) + throw new BatchValidationException(new ValidationException(errors)); + + assert insert.start(); + insertPropertiesBulk(c, propsToInsert, false); + helper.afterBatchInsert(rowCount); + rowCallback.complete(); + assert insert.stop(); + } + catch (SQLException x) + { + SQLException next = x.getNextException(); + if (x instanceof java.sql.BatchUpdateException && null != next) + x = next; + _log.debug("Exception uploading: ", x); + throw x; + } + + assert total.stop(); + _log.debug("\t" + total); + _log.debug("\t" + before); + _log.debug("\t" + ensure); + _log.debug("\t" + insert); + } + + /** + * As an incremental step of QueryUpdateService cleanup, this is a version of insertTabDelimited that works on a + * tableInfo that implements UpdateableTableInfo. Does not support ownerObjectid. + *

+ * This code is made complicated by the fact that while we are trying to move toward a TableInfo/ColumnInfo view + * of the world, validators are attached to PropertyDescriptors. Also, missing value handling is attached + * to PropertyDescriptors. + *

+ * The original version of this method expects a data to be a map PropertyURI->value. This version will also + * accept Name->value. + *

+ * Name->Value is preferred, we are using TableInfo after all. + */ + @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder + public static void insertTabDelimited(TableInfo tableInsert, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator rows, + boolean autoFillDefaultColumns, + Logger logger, + RowCallback rowCallback) + throws SQLException, BatchValidationException + { + saveTabDelimited(tableInsert, c, user, helper, rows, logger, true, autoFillDefaultColumns, rowCallback); + } + + @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder + public static void updateTabDelimited(TableInfo tableInsert, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator rows, + boolean autoFillDefaultColumns, + Logger logger) + throws SQLException, BatchValidationException + { + saveTabDelimited(tableInsert, c, user, helper, rows, logger, false, autoFillDefaultColumns, NO_OP_ROW_CALLBACK); + } + + private static void saveTabDelimited(TableInfo table, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator in, + Logger logger, + boolean insert, + boolean autoFillDefaultColumns, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + if (!(table instanceof UpdateableTableInfo)) + throw new IllegalArgumentException(); + + if (rowCallback == null) + { + rowCallback = NO_OP_ROW_CALLBACK; + } + + DbScope scope = table.getSchema().getScope(); + + assert scope.isTransactionActive(); + + Domain d = table.getDomain(); + List properties = null == d ? Collections.emptyList() : d.getProperties(); + + ValidatorContext validatorCache = new ValidatorContext(c, user); + + Connection conn = null; + ParameterMapStatement parameterMap = null; + + Map currentRow = null; + + MapDataIterator rows = DataIteratorUtil.wrapMap(in, false); + try + { + conn = scope.getConnection(); + if (insert) + { + parameterMap = StatementUtils.insertStatement(conn, table, c, user, true, autoFillDefaultColumns); + } + else + { + parameterMap = StatementUtils.updateStatement(conn, table, c, user, false, autoFillDefaultColumns); + } + List errors = new ArrayList<>(); + + Map> validatorMap = new HashMap<>(); + Map propertiesMap = new HashMap<>(); + + // cache all the property validators for this upload + for (DomainProperty dp : properties) + { + propertiesMap.put(dp.getPropertyURI(), dp); + List validators = dp.getValidators(); + if (!validators.isEmpty()) + validatorMap.put(dp.getPropertyURI(), validators); + } + + List columns = table.getColumns(); + PropertyType[] propertyTypes = new PropertyType[columns.size()]; + for (int i = 0; i < columns.size(); i++) + { + String propertyURI = columns.get(i).getPropertyURI(); + DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); + PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); + if (null != pd) + propertyTypes[i] = pd.getPropertyType(); + } + + int rowCount = 0; + + while (rows.next()) + { + + currentRow = new CaseInsensitiveHashMap<>(rows.getMap()); + + // TODO: hack -- should exit and return cancellation status instead of throwing + if (Thread.currentThread().isInterrupted()) + throw new CancellationException(); + + parameterMap.clearParameters(); + + String lsid = helper.beforeImportObject(currentRow); + currentRow.put("lsid", lsid); + + // + // NOTE we validate based on columninfo/propertydescriptor + // However, we bind by name, and there may be parameters that do not correspond to columninfo + // + + for (int i = 0; i < columns.size(); i++) + { + ColumnInfo col = columns.get(i); + if (col.isMvIndicatorColumn() || col.isRawValueColumn()) //TODO col.isNotUpdatableForSomeReasonSoContinue() + continue; + String propertyURI = col.getPropertyURI(); + DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); + PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); + + Object value = currentRow.get(col.getName()); + if (null == value) + value = currentRow.get(propertyURI); + + if (null == value) + { + // TODO col.isNullable() doesn't seem to work here + if (null != pd && pd.isRequired()) + throw new BatchValidationException(new ValidationException("Missing value for required property " + col.getName())); + } + else + { + if (null != pd) + { + try + { + // Use an ObjectProperty to unwrap MvFieldWrapper, do type conversion, etc + ObjectProperty objectProperty = new ObjectProperty(lsid, c, pd, value); + if (!validateProperty(validatorMap.get(propertyURI), pd, objectProperty, errors, validatorCache)) + { + throw new BatchValidationException(new ValidationException(errors)); + } + } + catch (ConversionException e) + { + throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getJavaClass()))); + } + } + } + + // issue 19391: data from R uses "Inf" to represent infinity + if (JdbcType.DOUBLE.equals(col.getJdbcType())) + { + value = "Inf".equals(value) ? "Infinity" : value; + value = "-Inf".equals(value) ? "-Infinity" : value; + } + + try + { + String key = col.getName(); + if (!parameterMap.containsKey(key)) + key = propertyURI; + if (null == propertyTypes[i]) + { + // some built-in columns won't have parameters (createdby, etc) + if (parameterMap.containsKey(key)) + { + assert !(value instanceof MvFieldWrapper); + // Handle type coercion for these built-in columns as well, though we don't need to + // worry about missing values + value = PropertyType.getFromClass(col.getJavaObjectClass()).convert(value); + parameterMap.put(key, value); + } + } + else + { + Pair p = new Pair<>(value, null); + convertValuePair(pd, propertyTypes[i], p); + parameterMap.put(key, p.first); + if (null != p.second) + { + FieldKey mvName = col.getMvColumnName(); + if (mvName != null) + { + String storageName = table.getColumn(mvName).getMetaDataIdentifier().getId(); + parameterMap.put(storageName, p.second); + } + } + } + } + catch (ConversionException e) + { + throw new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), propertyTypes[i].getJavaType())); + } + } + + helper.bindAdditionalParameters(currentRow, parameterMap); + parameterMap.execute(); + if (insert) + { + long rowId = parameterMap.getRowId(); + currentRow.put("rowId", rowId); + } + lsid = helper.afterImportObject(currentRow); + if (lsid == null) + { + throw new IllegalStateException("No LSID available"); + } + rowCallback.rowProcessed(currentRow, lsid); + rowCount++; + } + + + if (!errors.isEmpty()) + throw new BatchValidationException(new ValidationException(errors)); + + rowCallback.complete(); + + helper.afterBatchInsert(rowCount); + if (logger != null) + logger.debug("inserted row " + rowCount + "."); + } + catch (ValidationException e) + { + throw new BatchValidationException(e); + } + catch (SQLException x) + { + SQLException next = x.getNextException(); + if (x instanceof java.sql.BatchUpdateException && null != next) + x = next; + _log.debug("Exception uploading: ", x); + if (null != currentRow) + _log.debug(currentRow.toString()); + throw x; + } + finally + { + if (null != parameterMap) + parameterMap.close(); + if (null != conn) + scope.releaseConnection(conn); + } + } + + // TODO: Consolidate with ColumnValidator + public static boolean validateProperty(List validators, PropertyDescriptor prop, ObjectProperty objectProperty, + List errors, ValidatorContext validatorCache) + { + boolean ret = true; + + Object value = objectProperty.getObjectValue(); + + if (prop.isRequired() && value == null && objectProperty.getMvIndicator() == null) + { + errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is required", prop.getName())); + ret = false; + } + + // Check if the string is too long. Use either the PropertyDescriptor's scale or VARCHAR(4000) for ontology managed values + int stringLengthLimit = prop.getScale() > 0 ? prop.getScale() : getTinfoObjectProperty().getColumn("StringValue").getScale(); + int stringLength = value == null ? 0 : value.toString().length(); + if (value != null && prop.isStringType() && stringLength > stringLengthLimit) + { + String s = stringLength <= 100 ? value.toString() : StringUtilsLabKey.leftSurrogatePairFriendly(value.toString(), 100); + errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is limited to " + stringLengthLimit + " characters, but the value is " + stringLength + " characters. (The value starts with '" + s + "...')", prop.getName())); + ret = false; + } + + // TODO: check date is within postgres date range + + // Don't validate null values, #15683 + if (null != value && validators != null) + { + for (IPropertyValidator validator : validators) + if (!validator.validate(prop, value, errors, validatorCache)) ret = false; + } + return ret; + } + + public interface ImportHelper + { + /** + * may modify map + * + * @return LSID for new or existing Object. Null indicates LSID is still unknown. + */ + String beforeImportObject(Map map) throws SQLException; + + void afterBatchInsert(int currentRow) throws SQLException; + + void updateStatistics(int currentRow) throws SQLException; + } + + + public interface UpdateableTableImportHelper extends ImportHelper + { + /** + * may be used to process attachments, for auditing, etc + * @return the LSID of the inserted row + */ + String afterImportObject(Map map) throws SQLException; + + /** + * may set parameters directly for columns that are not exposed by tableinfo + * e.g. "_key" + *

+ * TODO maybe this can be handled declaratively? see UpdateableTableInfo + */ + void bindAdditionalParameters(Map map, ParameterMapStatement target) throws ValidationException; + } + + @NotNull + private static Pair getPropertyMapCacheKey(@Nullable Container container, @NotNull String objectLSID) + { + return Pair.of(container, objectLSID); + } + + /** + * Get ordered map of property values for an object. The order of the properties in the + * Map corresponds to the PropertyOrder property, if present. + * + * @return map from PropertyURI to ObjectProperty + */ + public static Map getPropertyObjects(@Nullable Container container, @NotNull String objectLSID) + { + Pair cacheKey = getPropertyMapCacheKey(container, objectLSID); + return PROPERTY_MAP_CACHE.get(cacheKey); + } + + public static class PropertyMapCacheLoader implements CacheLoader, Map> + { + @Override + public Map load(@NotNull Pair key, @Nullable Object argument) + { + Container container = key.first; + String objectLSID = key.second; + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), objectLSID); + if (container != null) + { + filter.addCondition(FieldKey.fromParts("Container"), container); + } + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getResultSet()) + { + ResultSetUtil.logData(rs); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + List props = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getArrayList(ObjectProperty.class); + + // check for a "PropertyOrder" value + ObjectProperty propertyOrder = props.stream().filter(op -> PropertyOrderURI.equals(op.getPropertyURI())).findFirst().orElse(null); + if (propertyOrder != null) + { + String order = propertyOrder.getStringValue(); + if (order != null) + { + // CONSIDER: Store as a JSONArray of propertyURI instead of propertyId + String[] parts = order.split(","); + try + { + List propertyIds = Arrays.stream(parts).map(s -> ConvertHelper.convert(s, Integer.class)).toList(); + + // Don't include the "PropertyOrder" property + props = new ArrayList<>(props); + props.remove(propertyOrder); + + // Order by the index found in the PropertyOrder list, otherwise just stick it at the end + Comparator comparator = (op1, op2) -> { + int i1 = propertyIds.indexOf(op1.getPropertyId()); + if (i1 == -1) + i1 = propertyIds.size(); + + int i2 = propertyIds.indexOf(op2.getPropertyId()); + if (i2 == -1) + i2 = propertyIds.size(); + return i1 - i2; + }; + props.sort(comparator); + } + catch (ConversionException e) + { + _log.warn("Failed to parse PropertyOrder integer list: " + order); + } + } + } + + Map m = new LinkedHashMap<>(); + for (ObjectProperty value : props) + { + m.put(value.getPropertyURI(), value); + } + + return unmodifiableMap(m); + } + } + + public static void updateObjectPropertyOrder(User user, Container container, String objectLSID, List properties) + throws ValidationException + { + String ids = null; + if (properties != null && !properties.isEmpty()) + ids = properties.stream().map(pd -> Integer.toString(pd.getPropertyId())).collect(joining(",")); + + updateObjectProperty(user, container, PropertyOrder.getPropertyDescriptor(), objectLSID, ids, null, false); + } + + /** + * Moves the properties of an object from one container to another (used when the object is moving) + * @param targetContainer the container to move the properties to + * @param user the user doing the move + * @param objectLSID the LSID of the object to which the properties are attached + * @return number of properties moved + */ + public static int updateContainer(Container targetContainer, User user, @NotNull String objectLSID) + { + return Table.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); + } + + /** + * Get ordered list of the PropertyURI in {@link #PropertyOrder}, if present. + */ + public static List getObjectPropertyOrder(Container c, String objectLSID) + { + Map props = getPropertyObjects(c, objectLSID); + return new ArrayList<>(props.keySet()); + } + + public static long ensureObject(Container container, String objectURI) + { + return ensureObject(container, objectURI, (Long) null); + } + + public static long ensureObject(Container container, String objectURI, String ownerURI) + { + Long ownerId = null; + if (null != ownerURI) + ownerId = ensureObject(container, ownerURI, (Long) null); + return ensureObject(container, objectURI, ownerId); + } + + public static long ensureObject(Container container, String objectURI, Long ownerId) + { + //TODO: (marki) Transact? + Long objId = OBJECT_ID_CACHE.get(objectURI, container); + + if (null == objId) + { + OntologyObject obj = new OntologyObject(); + obj.setContainer(container); + obj.setObjectURI(objectURI); + if (ownerId != null && ownerId > 0) + obj.setOwnerObjectId(ownerId); + obj = Table.insert(null, getTinfoObject(), obj); + objId = obj.getObjectId(); + OBJECT_ID_CACHE.remove(objectURI); + } + + return objId; + } + + private static class ObjectIdCacheLoader implements CacheLoader + { + @Override + public Long load(@NotNull String objectURI, @Nullable Object argument) + { + Container container = (Container)argument; + OntologyObject obj = getOntologyObject(container, objectURI); + + return obj == null ? null : obj.getObjectId(); + } + } + + public static @Nullable OntologyObject getOntologyObject(Container container, String uri) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), uri); + if (container != null) + { + filter.addCondition(FieldKey.fromParts("Container"), container.getId()); + } + return new TableSelector(getTinfoObject(), filter, null).getObject(OntologyObject.class); + } + + // UNDONE: optimize (see deleteOntologyObjects(Integer[]) + public static void deleteOntologyObjects(Container c, String... uris) + { + if (uris.length == 0) + return; + + try + { + DbSchema schema = getExpSchema(); + String sql = getSqlDialect().execute(getExpSchema(), "deleteObject", "?, ?"); + SqlExecutor executor = new SqlExecutor(schema); + + for (String uri : uris) + { + executor.execute(sql, c.getId(), uri); + } + } + finally + { + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + } + } + + public static int deleteOntologyObjects(DbSchema schema, SQLFragment objectUriSql, @Nullable Container c) + { + SQLFragment objectIdSQL = new SQLFragment("SELECT ObjectId FROM ") + .append(getTinfoObject()).append("\n") + .append(" WHERE "); + if (c != null) + { + objectIdSQL.append(" Container = ?").add(c.getId()); + objectIdSQL.append(" AND "); + } + objectIdSQL.append("ObjectUri IN ("); + objectIdSQL.append(objectUriSql); + objectIdSQL.append(")"); + return deleteOntologyObjectsByObjectIdSql(schema, objectIdSQL); + } + + public static int deleteOntologyObjectsByObjectIdSql(DbSchema schema, SQLFragment objectIdSql) + { + if (!schema.getScope().equals(getExpSchema().getScope())) + throw new UnsupportedOperationException("can only use with same DbScope"); + + SQLFragment sqlDeleteProperties = new SQLFragment(); + sqlDeleteProperties.append("DELETE FROM ").append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (\n"); + sqlDeleteProperties.append(objectIdSql); + sqlDeleteProperties.append(")"); + new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); + + SQLFragment sqlDeleteObjects = new SQLFragment(); + sqlDeleteObjects.append("DELETE FROM ").append(getTinfoObject()).append(" WHERE ObjectId IN ("); + sqlDeleteObjects.append(objectIdSql); + sqlDeleteObjects.append(")"); + return new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); + } + + + public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, long... objectIds) + { + deleteOntologyObjects(c, deleteOwnedObjects, true, true, objectIds); + } + + public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, boolean deleteObjectProperties, boolean deleteObjects, long... objectIds) + { + if (objectIds.length == 0) + return; + + try + { + // if it's a long list, split it up + if (objectIds.length > 1000) + { + int countBatches = objectIds.length / 1000; + int lenBatch = 1 + objectIds.length / (countBatches + 1); + + for (int s = 0; s < objectIds.length; s += lenBatch) + { + long[] sub = new long[Math.min(lenBatch, objectIds.length - s)]; + System.arraycopy(objectIds, s, sub, 0, sub.length); + deleteOntologyObjects(c, deleteOwnedObjects, deleteObjectProperties, deleteObjects, sub); + } + + return; + } + + SQLFragment objectIdInClause = new SQLFragment(); + getExpSchema().getSqlDialect().appendInClauseSql(objectIdInClause, Arrays.stream(objectIds).boxed().toList()); + + if (deleteOwnedObjects) + { + // NOTE: owned objects should never be in a different container than the owner, that would be a problem + SQLFragment sqlDeleteOwnedProperties = new SQLFragment("DELETE FROM ") + .append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND OwnerObjectId ") + .add(c) + .append(objectIdInClause) + .append(")"); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedProperties); + + SQLFragment sqlDeleteOwnedObjects = new SQLFragment("DELETE FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND OwnerObjectId ") + .add(c) + .append(objectIdInClause); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedObjects); + } + + if (deleteObjectProperties) + { + deleteProperties(c, objectIdInClause); + } + + if (deleteObjects) + { + SQLFragment sqlDeleteObjects = new SQLFragment("DELETE FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND ObjectId ") + .add(c) + .append(objectIdInClause); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); + } + } + finally + { + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + } + } + + + public static void deleteOntologyObject(String objectURI, Container container, boolean deleteOwnedObjects) + { + OntologyObject ontologyObject = getOntologyObject(container, objectURI); + + if (null != ontologyObject) + { + deleteOntologyObjects(container, deleteOwnedObjects, true, true, ontologyObject.getObjectId()); + } + } + + + public static OntologyObject getOntologyObject(long id) + { + return new TableSelector(getTinfoObject()).getObject(id, OntologyObject.class); + } + + //todo: review this. this doesn't delete the underlying data objects. should it? + public static void deleteObjectsOfType(String domainURI, Container container) + { + DomainDescriptor dd = null; + if (null != domainURI) + dd = getDomainDescriptor(domainURI, container); + if (null == dd) + { + _log.debug("deleteObjectsOfType called on type not found in database: " + domainURI); + return; + } + + try (Transaction t = getExpSchema().getScope().ensureTransaction()) + { + // until we set a domain on objects themselves, we need to create a list of objects to + // delete based on existing entries in ObjectProperties before we delete the objectProperties + // which we need to do before we delete the objects. + // TODO: Doesn't handle the case when PropertyDescriptors are shared across domains + String selectObjectsToDelete = "SELECT DISTINCT O.ObjectId " + + " FROM " + getTinfoObject() + " O " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON(O.ObjectId = OP.ObjectId) " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (OP.PropertyId = PDM.PropertyId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PD.PropertyId = PDM.PropertyId) " + + " WHERE DD.DomainId = " + dd.getDomainId() + + " AND PD.Container = DD.Container"; + Long[] objIdsToDelete = new SqlSelector(getExpSchema(), selectObjectsToDelete).getArray(Long.class); + + String sep; + StringBuilder sqlIN = null; + Long[] ownerObjIds = null; + + if (objIdsToDelete.length > 0) + { + //also need list of owner objects whose subobjects are going to be deleted + // Seems cheaper but less correct to delete the subobjects then cleanup any owner objects with no children + sep = ""; + sqlIN = new StringBuilder(); + for (Long id : objIdsToDelete) + { + sqlIN.append(sep).append(id); + sep = ", "; + } + + String selectOwnerObjects = "SELECT O.ObjectId FROM " + getTinfoObject() + " O " + + " WHERE ObjectId IN " + + " (SELECT DISTINCT SUBO.OwnerObjectId FROM " + getTinfoObject() + " SUBO " + + " WHERE SUBO.ObjectId IN ( " + sqlIN + " ) )"; + + ownerObjIds = new SqlSelector(getExpSchema(), selectOwnerObjects).getArray(Long.class); + } + + String deleteTypePropsSql = "DELETE FROM " + getTinfoObjectProperty() + + " WHERE PropertyId IN " + + " (SELECT PDM.PropertyId FROM " + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PDM.PropertyId = PD.PropertyId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " WHERE DD.DomainId = " + dd.getDomainId() + + " AND PD.Container = DD.Container " + + " ) "; + new SqlExecutor(getExpSchema()).execute(deleteTypePropsSql); + + if (objIdsToDelete.length > 0) + { + // now cleanup the object table entries from the list we made, but make sure they don't have + // other properties attached to them + String deleteObjSql = "DELETE FROM " + getTinfoObject() + + " WHERE ObjectId IN ( " + sqlIN + " ) " + + " AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + + " WHERE OP.ObjectId = " + getTinfoObject() + ".ObjectId)"; + new SqlExecutor(getExpSchema()).execute(deleteObjSql); + + if (ownerObjIds.length > 0) + { + sep = ""; + sqlIN = new StringBuilder(); + for (Long id : ownerObjIds) + { + sqlIN.append(sep).append(id); + sep = ", "; + } + String deleteOwnerSql = "DELETE FROM " + getTinfoObject() + + " WHERE ObjectId IN ( " + sqlIN + " ) " + + " AND NOT EXISTS (SELECT * FROM " + getTinfoObject() + " SUBO " + + " WHERE SUBO.OwnerObjectId = " + getTinfoObject() + ".ObjectId)"; + new SqlExecutor(getExpSchema()).execute(deleteOwnerSql); + } + } + // whew! + clearCaches(); + t.commit(); + } + } + + public static void deleteDomain(String domainURI, Container container) throws DomainNotFoundException + { + DomainDescriptor dd = getDomainDescriptor(domainURI, container); + String msg; + + if (null == dd) + throw new DomainNotFoundException(domainURI); + + if (!dd.getContainer().getId().equals(container.getId())) + { + // this domain was not created in this folder. Allow if in the project-level root + if (!dd.getProject().getId().equals(container.getId())) + { + msg = "DeleteDomain: Domain can only be deleted in original container or from the project root " + + "\nDomain: " + domainURI + " project " + dd.getProject().getName() + " original container " + dd.getContainer().getPath(); + _log.error(msg); + throw new RuntimeException(msg); + } + } + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + String selectPDsToDelete = "SELECT DISTINCT PDM.PropertyId " + + " FROM " + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " WHERE DD.DomainId = ? "; + + Integer[] pdIdsToDelete = new SqlSelector(getExpSchema(), selectPDsToDelete, dd.getDomainId()).getArray(Integer.class); + + String deletePDMs = "DELETE FROM " + getTinfoPropertyDomain() + + " WHERE DomainId = " + + " (SELECT DD.DomainId FROM " + getTinfoDomainDescriptor() + " DD " + + " WHERE DD.DomainId = ? )"; + new SqlExecutor(getExpSchema()).execute(deletePDMs, dd.getDomainId()); + + if (pdIdsToDelete.length > 0) + { + String sep = ""; + StringBuilder sqlIN = new StringBuilder(); + for (Integer id : pdIdsToDelete) + { + PropertyService.get().deleteValidatorsAndFormats(container, id); + + sqlIN.append(sep); + sqlIN.append(id); + sep = ", "; + } + + String deletePDs = "DELETE FROM " + getTinfoPropertyDescriptor() + + " WHERE PropertyId IN ( " + sqlIN + " ) " + + "AND Container = ? " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + + "WHERE OP.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId) " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + + "WHERE PDM.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId)"; + + new SqlExecutor(getExpSchema()).execute(deletePDs, dd.getContainer().getId()); + } + + String deleteDD = "DELETE FROM " + getTinfoDomainDescriptor() + + " WHERE DomainId = ? " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + + "WHERE PDM.DomainId = " + getTinfoDomainDescriptor() + ".DomainId)"; + + new SqlExecutor(getExpSchema()).execute(deleteDD, dd.getDomainId()); + clearCaches(); + + transaction.commit(); + } + } + + + public static void deleteAllObjects(Container c, User user) throws ValidationException + { + Container projectContainer = c.getProject(); + if (null == projectContainer) + projectContainer = c; + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + if (!c.equals(projectContainer)) + { + copyDescriptors(c, projectContainer); + } + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + + // Owned objects should be in same container, so this should work + String deleteObjPropSql = "DELETE FROM " + getTinfoObjectProperty() + " WHERE ObjectId IN (SELECT ObjectId FROM " + getTinfoObject() + " WHERE Container = ?)"; + executor.execute(deleteObjPropSql, c); + String deleteObjSql = "DELETE FROM " + getTinfoObject() + " WHERE Container = ?"; + executor.execute(deleteObjSql, c); + + // delete property validator references on property descriptors + PropertyService.get().deleteValidatorsAndFormats(c); + + // Drop tables directly and allow bulk delete calls below to clean up rows in exp.propertydescriptor, + // exp.domaindescriptor, etc + String selectSQL = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + Collection dds = new SqlSelector(getExpSchema(), selectSQL, c).getCollection(DomainDescriptor.class); + for (DomainDescriptor dd : dds) + { + StorageProvisioner.get().drop(PropertyService.get().getDomain(dd.getDomainId())); + } + + String deletePropDomSqlPD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId IN (SELECT PropertyId FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?)"; + executor.execute(deletePropDomSqlPD, c); + String deletePropDomSqlDD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE DomainId IN (SELECT DomainId FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?)"; + executor.execute(deletePropDomSqlDD, c); + String deleteDomSql = "DELETE FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + executor.execute(deleteDomSql, c); + // now delete the prop descriptors that are referenced in this container only + String deletePropSql = "DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?"; + executor.execute(deletePropSql, c); + + clearCaches(); + transaction.commit(); + } + } + + private static void copyDescriptors(final Container c, final Container project) throws ValidationException + { + _log.debug("OntologyManager.copyDescriptors " + c.getName() + " " + project.getName()); + + // if c is (was) a project, then nothing to do + if (c.getId().equals(project.getId())) + return; + + // check to see if any Properties defined in this folder are used in other folders. + // if so we will make a copy of all PDs and DDs to ensure no orphans + String sql = " SELECT O.ObjectURI, O.Container, PD.PropertyId, PD.PropertyURI " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + + " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + + " WHERE PD.Container = ? " + + " AND O.Container <> PD.Container "; + + final Map mObjsUsingMyProps = new HashMap<>(); + final StringBuilder sqlIn = new StringBuilder(); + final StringBuilder sep = new StringBuilder(); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c).getResultSet()) + { + ResultSetUtil.logData(rs); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + new SqlSelector(getExpSchema(), sql, c).forEach(rs -> { + String objURI = rs.getString(1); + String objContainer = rs.getString(2); + Integer propId = rs.getInt(3); + String propURI = rs.getString(4); + + sqlIn.append(sep).append(propId); + + if (sep.isEmpty()) + sep.append(", "); + + Map mtemp = getPropertyObjects(ContainerManager.getForId(objContainer), objURI); + + if (null != mtemp) + { + for (Map.Entry entry : mtemp.entrySet()) + { + entry.getValue().setPropertyId(0); + if (entry.getValue().getPropertyURI().equals(propURI)) + mObjsUsingMyProps.put(entry.getKey(), entry.getValue()); + } + } + }); + + // For each property that is referenced outside its container, get the + // domains that it belongs to and the other properties in those domains + // so we can make copies of those domains and properties + // Restrict it to properties and domains also in the same container + + if (!mObjsUsingMyProps.isEmpty()) + { + sql = "SELECT PD.PropertyURI, DD.DomainURI " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + + " ON (PD.PropertyId = PDM2.PropertyId) " + + " WHERE PDM.PropertyId IN (" + sqlIn + ") " + + " OR PD.PropertyId IN (" + sqlIn + ") "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + new SqlSelector(getExpSchema(), sql).forEach(rsMyProps -> { + String propUri = rsMyProps.getString(1); + String domUri = rsMyProps.getString(2); + PropertyDescriptor pd = getPropertyDescriptor(propUri, c); + + if (pd.getContainer().getId().equals(c.getId())) + { + _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + DOMAIN_PROPERTIES_CACHE.clear(); + pd.setContainer(project); + pd.setPropertyId(0); + pd = ensurePropertyDescriptor(pd); + } + + if (null != domUri) + { + DomainDescriptor dd = getDomainDescriptor(domUri, c); + if (dd.getContainer().getId().equals(c.getId())) + { + uncache(dd); + dd = dd.edit() + .setContainer(project) + .setDomainId(0) + .build(); + dd = ensureDomainDescriptor(dd); + ensurePropertyDomain(pd, dd); + } + } + }); + + clearCaches(); + + // now unhook the objects that refer to my properties and rehook them to the properties in their own project + for (ObjectProperty op : mObjsUsingMyProps.values()) + { + deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), c); + insertProperties(op.getContainer(), op.getObjectURI(), op); + } + } + } + + private static void uncache(DomainDescriptor dd) + { + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.remove(dd.getContainer()); + } + + + public static void moveContainer(@NotNull final Container c, @NotNull Container oldParent, @NotNull Container newParent) throws SQLException + { + _log.debug("OntologyManager.moveContainer " + c.getName() + " " + oldParent.getName() + "->" + newParent.getName()); + + final Container oldProject = oldParent.getProject(); + Container newProject = newParent.getProject(); + if (null == newProject) // if container is promoted to a project + newProject = c.getProject(); + + if ((null != oldProject) && oldProject.getId().equals(newProject.getId())) + { + //the folder is being moved within the same project. No problems here + return; + } + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + clearCaches(); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + // update project of any descriptors in folder just moved + TableInfo pdTable = getTinfoPropertyDescriptor(); + String sql = "UPDATE " + pdTable + " SET Project = ? WHERE Container = ?"; + + // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. + // Issue 30477: exclude project level properties descriptors (such as Study) that already exist + sql += " AND PropertyUri NOT IN (SELECT PropertyUri FROM " + pdTable + " WHERE Project = ? AND PropertyUri IN (SELECT PropertyUri FROM " + pdTable + " WHERE Container = ?))"; + + new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + TableInfo ddTable = getTinfoDomainDescriptor(); + sql = "UPDATE " + ddTable + " SET Project = ? WHERE Container = ?"; + + // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. + // Issue 30477: exclude project level domain descriptors (such as Study) that already exist + sql += " AND DomainUri NOT IN (SELECT DomainUri FROM " + ddTable + " WHERE Project = ? AND DomainUri IN (SELECT DomainUri FROM " + ddTable + " WHERE Container = ?))"; + + new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); + + if (null == oldProject) // if container was a project & demoted I'm done + { + transaction.commit(); + return; + } + + // this method makes sure I'm not getting rid of descriptors used by another folder + // it is shared by ContainerDelete + copyDescriptors(c, oldProject); + + // if my objects refer to project-scoped properties I need a copy of those properties + sql = " SELECT O.ObjectURI, PD.PropertyURI, PD.PropertyId, PD.Container " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + + " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + + " WHERE O.Container = ? " + + " AND O.Container <> PD.Container " + + " AND PD.Project NOT IN (?,?) "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + + final Map mMyObjsThatRefProjProps = new HashMap<>(); + final StringBuilder sqlIn = new StringBuilder(); + final StringBuilder sep = new StringBuilder(); + + new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).forEach(rs -> { + String objURI = rs.getString(1); + String propURI = rs.getString(2); + Integer propId = rs.getInt(3); + + sqlIn.append(sep).append(propId); + + if (sep.isEmpty()) + sep.append(", "); + + Map mtemp = getPropertyObjects(c, objURI); + + if (null != mtemp) + { + for (Map.Entry entry : mtemp.entrySet()) + { + if (entry.getValue().getPropertyURI().equals(propURI)) + mMyObjsThatRefProjProps.put(entry.getKey(), entry.getValue()); + } + } + }); + + // this sql gets all properties i ref and the domains they belong to and the + // other properties in those domains + //todo what about materialsource ? + if (!mMyObjsThatRefProjProps.isEmpty()) + { + sql = "SELECT PD.PropertyURI, DD.DomainURI, PD.PropertyId " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + + " ON (PD.PropertyId = PDM2.PropertyId) " + + " WHERE PDM.PropertyId IN (" + sqlIn + " ) "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + final Container fNewProject = newProject; + + new SqlSelector(getExpSchema(), sql).forEach(rsPropsRefdByMe -> { + String propUri = rsPropsRefdByMe.getString(1); + String domUri = rsPropsRefdByMe.getString(2); + PropertyDescriptor pd = getPropertyDescriptor(propUri, oldProject); + + if (null != pd) + { + // To prevent iterating over a property descriptor update more than once + // we check to make sure both the container and project are equivalent to the updated + // location + if (!pd.getContainer().equals(c) || !pd.getProject().equals(fNewProject)) + { + pd.setContainer(c); + pd.setPropertyId(0); + } + + pd = ensurePropertyDescriptor(pd); + } + + if (null != domUri) + { + DomainDescriptor dd = getDomainDescriptor(domUri, oldProject); + + // To prevent iterating over a domain descriptor update more than once + // we check to make sure both the container and project are equivalent to the updated + // location + if (!dd.getContainer().equals(c) || !dd.getProject().equals(fNewProject)) + { + dd = dd.edit().setContainer(c).setDomainId(0).build(); + } + + dd = ensureDomainDescriptor(dd); + ensurePropertyDomain(pd, dd); + } + }); + + for (ObjectProperty op : mMyObjsThatRefProjProps.values()) + { + deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), oldProject); + // Treat it as new so it's created in the target container as needed + op.setPropertyId(0); + insertProperties(op.getContainer(), op.getObjectURI(), op); + } + clearCaches(); + } + + transaction.commit(); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + private static PropertyDescriptor ensurePropertyDescriptor(String propertyURI, PropertyType type, String name, Container container) + { + PropertyDescriptor pdNew = new PropertyDescriptor(propertyURI, type, name, container); + return ensurePropertyDescriptor(pdNew); + } + + + private static PropertyDescriptor ensurePropertyDescriptor(PropertyDescriptor pdIn) + { + if (null == pdIn.getContainer()) + { + assert false : "Container should be set on PropertyDescriptor"; + pdIn.setContainer(_sharedContainer); + } + + PropertyDescriptor pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); + if (null == pd) + { + assert pdIn.getPropertyId() == 0; + /* return 1 if inserted 0 if not inserted, uses OUT parameter for new PropertyDescriptor */ + PropertyDescriptor[] out = new PropertyDescriptor[1]; + int rowcount = insertPropertyIfNotExists(null, pdIn, out); + pd = out[0]; + if (1 == rowcount && null != pd) + { + _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + return pd; + } + if (null == pd) + { + throw OptimisticConflictException.create(Table.ERROR_DELETED); + } + } + + if (pd.equals(pdIn)) + { + return pd; + } + else + { + List colDiffs = comparePropertyDescriptors(pdIn, pd); + + if (colDiffs.isEmpty()) + { + // if the descriptor differs by container only and the requested descriptor is in the project fldr + if (!pdIn.getContainer().getId().equals(pd.getContainer().getId()) && + pdIn.getContainer().getId().equals(pdIn.getProject().getId())) + { + pdIn.setPropertyId(pd.getPropertyId()); + pd = updatePropertyDescriptor(pdIn); + } + return pd; + } + + // you are allowed to update if you are coming from the project root, or if you are in the container + // in which the descriptor was created + boolean fUpdateIfExists = false; + if (pdIn.getContainer().getId().equals(pd.getContainer().getId()) + || pdIn.getContainer().getId().equals(pdIn.getProject().getId())) + fUpdateIfExists = true; + + + boolean fMajorDifference = false; + if (colDiffs.toString().contains("RangeURI") || colDiffs.toString().contains("PropertyType")) + fMajorDifference = true; + + String errmsg = "ensurePropertyDescriptor: descriptor In different from Found for " + colDiffs + + "\n\t Descriptor In: " + pdIn + + "\n\t Descriptor Found: " + pd; + + if (fUpdateIfExists) + { + //todo: pass list of cols to update + pdIn.setPropertyId(pd.getPropertyId()); + pd = updatePropertyDescriptor(pdIn); + if (fMajorDifference) + _log.debug(errmsg); + } + else + { + if (fMajorDifference) + _log.error(errmsg); + else + _log.debug(errmsg); + } + } + return pd; + } + + + private static int insertPropertyIfNotExists(User user, PropertyDescriptor pd, PropertyDescriptor[] out) + { + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getInsertStmt(conn, user, t, true)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = f.toMap(pd, null); + stmt.putAll(m); + int rowcount = stmt.execute(); + SQLFragment reselect = new SQLFragment("SELECT * FROM exp.propertydescriptor WHERE propertyuri=? AND container=?", pd.getPropertyURI(), pd.getContainer()); + out[0] = (new SqlSelector(getExpSchema(), reselect).getObject(PropertyDescriptor.class)); + return rowcount; + } + catch(SQLException sqlx) + { + throw ExceptionFramework.Spring.translate(getExpSchema().getScope(), "insertPropertyIfNotExists", sqlx); + } + } + + + private static List comparePropertyDescriptors(PropertyDescriptor pdIn, PropertyDescriptor pd) + { + List colDiffs = new ArrayList<>(); + + // if the returned pd is in a different project, it better be the shared project + if (!pd.getProject().equals(pdIn.getProject()) && !pd.getProject().equals(_sharedContainer)) + colDiffs.add("Project"); + + // check the pd values that can't change + if (!pd.getRangeURI().equals(pdIn.getRangeURI())) + colDiffs.add("RangeURI"); + if (!Objects.equals(pd.getPropertyType(), pdIn.getPropertyType())) + colDiffs.add("PropertyType"); + + if (pdIn.getPropertyId() != 0 && pd.getPropertyId() != pdIn.getPropertyId()) + colDiffs.add("PropertyId"); + + if (!Objects.equals(pdIn.getName(), pd.getName())) + colDiffs.add("Name"); + + if (!Objects.equals(pdIn.getConceptURI(), pd.getConceptURI())) + colDiffs.add("ConceptURI"); + + if (!Objects.equals(pdIn.getDescription(), pd.getDescription())) + colDiffs.add("Description"); + + if (!Objects.equals(pdIn.getFormat(), pd.getFormat())) + colDiffs.add("Format"); + + if (!Objects.equals(pdIn.getLabel(), pd.getLabel())) + colDiffs.add("Label"); + + if (pdIn.isHidden() != pd.isHidden()) + colDiffs.add("IsHidden"); + + if (pdIn.isMvEnabled() != pd.isMvEnabled()) + colDiffs.add("IsMvEnabled"); + + if (!Objects.equals(pdIn.getLookupContainer(), pd.getLookupContainer())) + colDiffs.add("LookupContainer"); + + if (!Objects.equals(pdIn.getLookupSchema(), pd.getLookupSchema())) + colDiffs.add("LookupSchema"); + + if (!Objects.equals(pdIn.getLookupQuery(), pd.getLookupQuery())) + colDiffs.add("LookupQuery"); + + if (!Objects.equals(pdIn.getDerivationDataScope(), pd.getDerivationDataScope())) + colDiffs.add("DerivationDataScope"); + + if (!Objects.equals(pdIn.getSourceOntology(), pd.getSourceOntology())) + colDiffs.add("SourceOntology"); + + if (!Objects.equals(pdIn.getConceptImportColumn(), pd.getConceptImportColumn())) + colDiffs.add("ConceptImportColumn"); + + if (!Objects.equals(pdIn.getConceptLabelColumn(), pd.getConceptLabelColumn())) + colDiffs.add("ConceptLabelColumn"); + + if (!Objects.equals(pdIn.getPrincipalConceptCode(), pd.getPrincipalConceptCode())) + colDiffs.add("PrincipalConceptCode"); + + if (!Objects.equals(pdIn.getConceptSubtree(), pd.getConceptSubtree())) + colDiffs.add("ConceptSubtree"); + + if (pdIn.isScannable() != pd.isScannable()) + colDiffs.add("Scannable"); + + return colDiffs; + } + + public static DomainDescriptor ensureDomainDescriptor(String domainURI, String name, Container container) + { + String trimmedName = StringUtils.trimToNull(name); + if (trimmedName == null) + throw new IllegalArgumentException("Non-blank name is required."); + DomainDescriptor dd = new DomainDescriptor.Builder(domainURI, container).setName(trimmedName).build(); + return ensureDomainDescriptor(dd); + } + + /** Inserts or updates the domain as appropriate */ + @NotNull + public static DomainDescriptor ensureDomainDescriptor(DomainDescriptor ddIn) + { + DomainDescriptor dd = null; + // Try to find the previous version of the domain + if (ddIn.getDomainId() > 0) + { + // Try checking the cache first for a value to compare against + dd = getDomainDescriptor(ddIn.getDomainId()); + + // Since we cache mutable objects, get a fresh copy from the DB if the cache returned the same object that + // was passed in so we can do a diff against what's currently in the DB to see if we need to update + if (dd == ddIn) + { + dd = new TableSelector(getTinfoDomainDescriptor()).getObject(ddIn.getDomainId(), DomainDescriptor.class); + } + } + if (dd == null) + { + dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); + } + + if (null == dd) + { + try + { + DbSchema expSchema = getExpSchema(); + // ensureDomainDescriptor() shouldn't fail if there is a race condition, however Table.insert() will throw if row exists, so can't use that + // also a constraint violation will kill any current transaction + // CONSIDER to generalize add an option to check for existing row to Table.insert(ColumnInfo[] keyCols, Object[] keyValues) + String timestamp = expSchema.getSqlDialect().getSqlTypeName(JdbcType.TIMESTAMP); + String templateJson = null==ddIn.getTemplateInfo() ? null : ddIn.getTemplateInfo().toJSON(); + SQLFragment insert = new SQLFragment( + "INSERT INTO ").append(getTinfoDomainDescriptor()) + .append(" (Name, DomainURI, Description, Container, Project, StorageTableName, StorageSchemaName, ModifiedBy, Modified, TemplateInfo, SystemFieldConfig)\n" + + "SELECT ?,?,?,?,?,?,?,CAST(NULL AS INT),CAST(NULL AS " + timestamp + "),?,?\n") + .addAll(ddIn.getName(), ddIn.getDomainURI(), ddIn.getDescription(), ddIn.getContainer(), ddIn.getProject(), ddIn.getStorageTableName(), ddIn.getStorageSchemaName(), templateJson, ddIn.getSystemFieldConfig()) + .append("WHERE NOT EXISTS (SELECT * FROM ").append(getTinfoDomainDescriptor(),"x").append(" WHERE x.DomainURI=? AND x.Project=?)\n") + .add(ddIn.getDomainURI()).add(ddIn.getProject()); + // belt and suspenders approach to avoiding constraint violation exception + if (expSchema.getSqlDialect().isPostgreSQL()) + insert.append(" ON CONFLICT ON CONSTRAINT uq_domaindescriptor DO NOTHING"); + int count; + try (var tx = expSchema.getScope().ensureTransaction()) + { + count = new SqlExecutor(expSchema.getScope()).execute(insert); + tx.commit(); + } + + // alternately we could reselect rowid and then we wouldn't need this separate round trip + dd = fetchDomainDescriptorFromDB(ddIn.getDomainURI(), ddIn.getContainer()); + if (count > 0) + { + if (null == dd) // don't expect this + throw OptimisticConflictException.create(Table.ERROR_DELETED); + // We may have a cached miss that we need to clear + uncache(dd); + return dd; + } + // fall through to update case() + } + catch (RuntimeSQLException x) + { + // might be an optimistic concurrency problem see 16126 + dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); + if (null == dd) + throw x; + } + } + + if (!dd.deepEquals(ddIn)) + { + DomainDescriptor ddToSave = ddIn.edit().setDomainId(dd.getDomainId()).build(); + dd = Table.update(null, getTinfoDomainDescriptor(), ddToSave, ddToSave.getDomainId()); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(ddIn)); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(ddIn)); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); + } + return dd; + } + + private static void ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd) + { + ensurePropertyDomain(pd, dd, 0); + } + + public static PropertyDescriptor ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) + { + if (null == pd) + throw new IllegalArgumentException("Must supply a PropertyDescriptor"); + if (null == dd) + throw new IllegalArgumentException("Must supply a DomainDescriptor"); + + // Consider: We should check that the pd and dd have been persisted (aka have a non-zero id) + + if (!pd.getContainer().equals(dd.getContainer()) + && !pd.getProject().equals(_sharedContainer)) + throw new IllegalStateException("ensurePropertyDomain: property " + pd.getPropertyURI() + " not in same container as domain " + dd.getDomainURI()); + + SQLFragment sqlInsert = new SQLFragment("INSERT INTO " + getTinfoPropertyDomain() + " ( PropertyId, DomainId, Required, SortOrder ) " + + " SELECT ?, ?, ?, ? WHERE NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + + " WHERE PropertyId=? AND DomainId=?)"); + sqlInsert.add(pd.getPropertyId()); + sqlInsert.add(dd.getDomainId()); + sqlInsert.add(pd.isRequired()); + sqlInsert.add(sortOrder); + sqlInsert.add(pd.getPropertyId()); + sqlInsert.add(dd.getDomainId()); + int count = new SqlExecutor(getExpSchema()).execute(sqlInsert); + // if 0 rows affected, we should do an update to make sure required is correct + if (count == 0) + { + SQLFragment sqlUpdate = new SQLFragment("UPDATE " + getTinfoPropertyDomain() + " SET Required = ?, SortOrder = ? WHERE PropertyId=? AND DomainId= ?"); + sqlUpdate.add(pd.isRequired()); + sqlUpdate.add(sortOrder); + sqlUpdate.add(pd.getPropertyId()); + sqlUpdate.add(dd.getDomainId()); + new SqlExecutor(getExpSchema()).execute(sqlUpdate); + } + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); + return pd; + } + + + private static void insertPropertiesBulk(Container container, List props, boolean insertNullValues) throws SQLException + { + List> floats = new ArrayList<>(); + List> dates = new ArrayList<>(); + List> strings = new ArrayList<>(); + List> mvIndicators = new ArrayList<>(); + + for (PropertyRow property : props) + { + if (null == property) + continue; + + long objectId = property.getObjectId(); + int propertyId = property.getPropertyId(); + String mvIndicator = property.getMvIndicator(); + assert mvIndicator == null || MvUtil.isMvIndicator(mvIndicator, container) : "Attempt to insert an invalid missing value indicator: " + mvIndicator; + + if (null != property.getFloatValue()) + floats.add(Arrays.asList(objectId, propertyId, property.getFloatValue(), mvIndicator)); + else if (null != property.getDateTimeValue()) + dates.add(Arrays.asList(objectId, propertyId, new java.sql.Timestamp(property.getDateTimeValue().getTime()), mvIndicator)); + else if (null != property.getStringValue()) + strings.add(Arrays.asList(objectId, propertyId, property.getStringValue(), mvIndicator)); + else if (null != mvIndicator) + { + mvIndicators.add(Arrays.asList(objectId, propertyId, property.getTypeTag(), mvIndicator)); + } + else if (insertNullValues) + { + strings.add(Arrays.asList(objectId, propertyId, null, null)); + } + } + + assert getExpSchema().getScope().isTransactionActive(); + + if (!dates.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, DateTimeValue, MvIndicator) VALUES (?,?,'d',?, ?)"; + Table.batchExecute(getExpSchema(), sql, dates); + } + + if (!floats.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, FloatValue, MvIndicator) VALUES (?,?,'f',?, ?)"; + Table.batchExecute(getExpSchema(), sql, floats); + } + + if (!strings.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, StringValue, MvIndicator) VALUES (?,?,'s',?, ?)"; + Table.batchExecute(getExpSchema(), sql, strings); + } + + if (!mvIndicators.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, MvIndicator) VALUES (?,?,?,?)"; + Table.batchExecute(getExpSchema(), sql, mvIndicators); + } + + clearPropertyCache(); + } + + public static void deleteProperty(String objectURI, String propertyURI, Container objContainer, Container propContainer) + { + OntologyObject o = getOntologyObject(objContainer, objectURI); + if (o == null) + return; + + PropertyDescriptor pd = getPropertyDescriptor(propertyURI, propContainer); + if (pd == null) + return; + + deleteProperty(o, pd); + } + + public static void deleteProperty(OntologyObject o, PropertyDescriptor pd) + { + deleteProperty(o, pd, true); + } + + public static void deleteProperty(OntologyObject o, PropertyDescriptor pd, boolean deleteCache) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectId"), o.getObjectId()); + filter.addCondition(FieldKey.fromParts("PropertyId"), pd.getPropertyId()); + Table.delete(getTinfoObjectProperty(), filter); + + if (deleteCache) + clearPropertyCache(o.getObjectURI()); + } + + /** + * Delete properties owned by the objects. + */ + public static void deleteProperties(Container objContainer, long objectId) + { + deleteProperties(objContainer, new SQLFragment(" = ?", objectId)); + } + public static void deleteProperties(Container objContainer, SQLFragment objectIdClause) + { + SQLFragment objectUriSql = new SQLFragment("SELECT ObjectURI FROM ") + .append(getTinfoObject(), "o") + .append(" WHERE ObjectId "); + objectUriSql.append(objectIdClause); + + List objectURIs = new SqlSelector(getExpSchema(), objectUriSql).getArrayList(String.class); + + SQLFragment sqlDeleteProperties = new SQLFragment("DELETE FROM ") + .append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND ObjectId ") + .add(objContainer) + .append(objectIdClause) + .append(")"); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); + + for (String uri : objectURIs) + { + clearPropertyCache(uri); + } + } + + /** + * Removes the property from a single domain, and completely deletes it if there are no other references + */ + public static void removePropertyDescriptorFromDomain(DomainProperty domainProp) + { + SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ? AND DomainId = ?", domainProp.getPropertyId(), domainProp.getDomain().getTypeId()); + SqlExecutor executor = new SqlExecutor(getExpSchema()); + DbScope dbScope = getExpSchema().getScope(); + try (Transaction transaction = dbScope.ensureTransaction()) + { + executor.execute(deletePropDomSql); + // Check if there are any other usages + SQLFragment otherUsagesSQL = new SQLFragment("SELECT DomainId FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", domainProp.getPropertyId()); + if (!new SqlSelector(dbScope, otherUsagesSQL).exists()) + { + deletePropertyDescriptor(domainProp.getPropertyDescriptor()); + } + transaction.commit(); + } + } + + /** + * Completely deletes the property from the database + */ + public static void deletePropertyDescriptor(PropertyDescriptor pd) + { + int propId = pd.getPropertyId(); + + SQLFragment deleteObjPropSql = new SQLFragment("DELETE FROM " + getTinfoObjectProperty() + " WHERE PropertyId = ?", propId); + SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", propId); + SQLFragment deletePropSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyId = ?", propId); + + DbScope dbScope = getExpSchema().getScope(); + SqlExecutor executor = new SqlExecutor(getExpSchema()); + try (Transaction transaction = dbScope.ensureTransaction()) + { + executor.execute(deleteObjPropSql); + executor.execute(deletePropDomSql); + executor.execute(deletePropSql); + Pair key = getCacheKey(pd); + _log.debug("Removing property descriptor from cache. Key: " + key + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(key); + DOMAIN_PROPERTIES_CACHE.clear(); + transaction.commit(); + } + } + + /*** + * @deprecated Use {@link #insertProperties(Container, User, String, ObjectProperty...)} so that a user can be + * supplied. + */ + @Deprecated + public static void insertProperties(Container container, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException + { + User user = HttpView.hasCurrentView() ? HttpView.currentContext().getUser() : null; + insertProperties(container, user, ownerObjectLsid, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException + { + insertProperties(container, user, ownerObjectLsid, false, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, ObjectProperty... properties) throws ValidationException + { + insertProperties(container, user, ownerObjectLsid, skipValidation, false, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, boolean insertNullValues, ObjectProperty... properties) throws ValidationException + { + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + Long parentId = ownerObjectLsid == null ? null : ensureObject(container, ownerObjectLsid); + HashMap descriptors = new HashMap<>(); + HashMap objects = new HashMap<>(); + List errors = new ArrayList<>(); + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + for (ObjectProperty property : properties) + { + if (null == property) + continue; + + property.setObjectOwnerId(parentId); + + PropertyDescriptor pd = descriptors.get(property.getPropertyURI()); + if (0 == property.getPropertyId()) + { + if (null == pd) + { + PropertyDescriptor pdIn = new PropertyDescriptor(property.getPropertyURI(), property.getPropertyType(), property.getName(), container); + pdIn.setFormat(property.getFormat()); + pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); + + if (null == pd) + pd = ensurePropertyDescriptor(pdIn); + + descriptors.put(property.getPropertyURI(), pd); + } + property.setPropertyId(pd.getPropertyId()); + } + if (0 == property.getObjectId()) + { + Long objectId = objects.get(property.getObjectURI()); + if (null == objectId) + { + // I'm assuming all properties are in the same container + objectId = ensureObject(property.getContainer(), property.getObjectURI(), property.getObjectOwnerId()); + objects.put(property.getObjectURI(), objectId); + } + property.setObjectId(objectId); + } + if (pd == null) + { + pd = getPropertyDescriptor(property.getPropertyId()); + } + if (!skipValidation) + { + validateProperty(PropertyService.get().getPropertyValidators(pd), pd, property, errors, validatorCache); + } + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + + insertPropertiesBulk(container, List.of(properties), insertNullValues); + + transaction.commit(); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + + public static PropertyDescriptor getPropertyDescriptor(long propertyId) + { + return new TableSelector(getTinfoPropertyDescriptor()).getObject(propertyId, PropertyDescriptor.class); + } + + + public static PropertyDescriptor getPropertyDescriptor(String propertyURI, Container c) + { + // cache lookup by project. if not found at project level, check to see if global + Pair key = getCacheKey(propertyURI, c); + PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(key); + if (null != pd) + return pd; + + key = getCacheKey(propertyURI, _sharedContainer); + return PROP_DESCRIPTOR_CACHE.get(key); + } + + private static TableSelector getPropertyDescriptorTableSelector( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter, + @Nullable String sortColumn) + { + final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); + + // To filter by domain kind, we query the exp.DomainProperty table and filter by domainId. + // To construct a PropertyDescriptor, we will need to traverse the lookup to exp.PropertyDescriptor and select all of its columns. + List fields = new ArrayList<>(); + fields.add(FieldKey.fromParts("domainId")); + for (ColumnInfo col : getTinfoPropertyDescriptor().getColumns()) + { + fields.add(new FieldKey(propertyIdKey, col.getName())); + } + var colMap = QueryService.get().getColumns(getTinfoPropertyDomain(), fields); + + var filter = new SimpleFilter(); + if (propertyFilter != null) + { + filter.addAllClauses(propertyFilter); + } + + filter.addCondition(new FieldKey(propertyIdKey, "container"), c.getId()); + + if (!domains.isEmpty()) + { + filter.addInClause(FieldKey.fromParts("domainId"), domains.stream().map(Domain::getTypeId).collect(Collectors.toSet())); + } + + if (searchTerm != null) + { + // Apply Q filter to only some of the text columns + List searchCols = List.of( + colMap.get(new FieldKey(propertyIdKey, "Name")), + colMap.get(new FieldKey(propertyIdKey, "Label")), + colMap.get(new FieldKey(propertyIdKey, "Description")), + colMap.get(new FieldKey(propertyIdKey, "ImportAliases")) + ); + + var clause = CompareType.Q.createFilterClause(new FieldKey(null, "*"), searchTerm); + clause.setSelectColumns(searchCols); + filter.addCondition(clause); + } + + // use propertyId as the default sort + if (sortColumn == null) + sortColumn = "propertyId"; + Sort sort = new Sort(sortColumn); + + return new TableSelector(getTinfoPropertyDomain(), colMap.values(), filter, sort); + } + + public static Set getDomains( + Container c, User user, + @Nullable Set domainIds, + @Nullable Set domainKinds, + @Nullable Set domainNames) + { + Set domains = new HashSet<>(); + if (domainIds != null && !domainIds.isEmpty()) + { + domains.addAll(domainIds.stream().map(id -> PropertyService.get().getDomain(id)).collect(Collectors.toSet())); + } + + Set kinds = emptySet(); + Set names = emptySet(); + if (domainKinds != null && !domainKinds.isEmpty()) + { + kinds = domainKinds; + } + if (domainNames != null && !domainNames.isEmpty()) + { + names = domainNames; + } + if (!kinds.isEmpty() || !names.isEmpty()) + { + domains.addAll(PropertyService.get().getDomains(c, user, kinds, names, true)); + } + + return domains; + } + + public static List getPropertyDescriptors( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter, + @Nullable String sortColumn, + @Nullable Integer maxRows, + @Nullable Long offset) + { + final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); + + TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, + propertyFilter, sortColumn); + + if (maxRows != null) + ts.setMaxRows(maxRows); + if (offset != null) + ts.setOffset(offset); + + // This is a little annoying. We have to remove the "propertyId" lookup parent from + // the map keys for the ObjectFactory to correctly construct the PropertyDescriptor. + List props = new ArrayList<>(); + try (var results = ts.getResults(true)) + { + ObjectFactory of = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + while (results.next()) + { + Map rowMap = results.getFieldKeyRowMap(); + // remove the "propertyId" part from the FieldKey + Map rekey = new CaseInsensitiveHashMap<>(); + for (Map.Entry pair : rowMap.entrySet()) + { + FieldKey key = pair.getKey(); + if (propertyIdKey.equals(key.getParent())) + { + String name = key.getName(); + rekey.put(name, pair.getValue()); + } + } + props.add(of.fromMap(rekey)); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + return props; + } + + public static long getPropertyDescriptorsRowCount( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter) + { + + TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, + propertyFilter, null); + + return ts.getRowCount(); + } + + public static List getDomainsForPropertyDescriptor(Container container, PropertyDescriptor pd) + { + return PropertyService.get().getDomains(container) + .stream() + .filter(d -> null != d.getPropertyByURI(pd.getPropertyURI())) + .collect(Collectors.toList()); + } + + private static class DomainDescriptorLoader implements CacheLoader + { + @Override + public DomainDescriptor load(@NotNull Integer key, @Nullable Object argument) + { + return new TableSelector(getTinfoDomainDescriptor()).getObject(key, DomainDescriptor.class); + } + } + + public static DomainDescriptor getDomainDescriptor(int id) + { + return getDomainDescriptor(id, false); + } + + public static DomainDescriptor getDomainDescriptor(int id, boolean forUpdate) + { + if (forUpdate) + return new DomainDescriptorLoader().load(id, null); + + return DOMAIN_DESC_BY_ID_CACHE.get(id); + } + + @Nullable + public static DomainDescriptor getDomainDescriptor(String domainURI, Container c) + { + return getDomainDescriptor(domainURI, c, false); + } + + @Nullable + public static DomainDescriptor getDomainDescriptor(String domainURI, Container c, boolean forUpdate) + { + if (c == null) + return null; + + if (forUpdate) + return getDomainDescriptorForUpdate(domainURI, c); + + // cache lookup by project. if not found at project level, check to see if global + Pair key = getCacheKey(domainURI, c); + DomainDescriptor dd = DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); + if (null != dd) + return dd; + + // Try in the /Shared container too + key = getCacheKey(domainURI, _sharedContainer); + return DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); + } + + @Nullable + private static DomainDescriptor getDomainDescriptorForUpdate(String domainURI, Container c) + { + if (c == null) + return null; + + DomainDescriptor dd = fetchDomainDescriptorFromDB(domainURI, c); + if (dd == null) + dd = fetchDomainDescriptorFromDB(domainURI, _sharedContainer); + return dd; + } + + /** + * Get all the domains in the same project as the specified container. They may not be in use in the container directly + */ + public static Collection getDomainDescriptors(Container container) + { + return getDomainDescriptors(container, null, false); + } + + public static Collection getDomainDescriptors(Container container, User user, boolean includeProjectAndShared) + { + if (container == null) + return Collections.emptyList(); + + if (includeProjectAndShared && user == null) + throw new IllegalArgumentException("Can't include data from other containers without a user to check permissions on"); + + Map dds = getCachedDomainDescriptors(container, user); + + if (includeProjectAndShared) + { + dds = new LinkedHashMap<>(dds); + Container project = container.getProject(); + if (project != null) + { + for (Map.Entry entry : getCachedDomainDescriptors(project, user).entrySet()) + { + dds.putIfAbsent(entry.getKey(), entry.getValue()); + } + } + + if (_sharedContainer.hasPermission(user, ReadPermission.class)) + { + for (Map.Entry entry : getCachedDomainDescriptors(_sharedContainer, user).entrySet()) + { + dds.putIfAbsent(entry.getKey(), entry.getValue()); + } + } + } + + return unmodifiableCollection(dds.values()); + } + + @NotNull + private static Map getCachedDomainDescriptors(@NotNull Container c, @Nullable User user) + { + if (user != null && !c.hasPermission(user, ReadPermission.class)) + return Collections.emptyMap(); + + return DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.get(c); + } + + public static Pair getURICacheKey(DomainDescriptor dd) + { + return getCacheKey(dd.getDomainURI(), dd.getContainer()); + } + + + public static Pair getCacheKey(PropertyDescriptor pd) + { + return getCacheKey(pd.getPropertyURI(), pd.getContainer()); + } + + + public static Pair getCacheKey(String uri, Container c) + { + Container proj = c.getProject(); + GUID projId; + + if (null == proj) + projId = c.getEntityId(); + else + projId = proj.getEntityId(); + + return Pair.of(uri, projId); + } + + //TODO: Cache semantics. This loads the cache but does not fetch cause need to get them all together + public static List getPropertiesForType(String typeURI, Container c) + { + List> propertyURIs = DOMAIN_PROPERTIES_CACHE.get(getCacheKey(typeURI, c)); + if (propertyURIs != null) + { + List result = new ArrayList<>(propertyURIs.size()); + for (Pair propertyURI : propertyURIs) + { + PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(getCacheKey(propertyURI.getKey(), c)); + if (pd == null) + { + return null; + } + // NOTE: cached descriptors may have differing values of isRequired() as that is a per-domain setting + // Descriptors returned from this method will have their required bit set as appropriate for this domain + + // Clone so nobody else messes up our copy + pd = pd.clone(); + pd.setRequired(propertyURI.getValue().booleanValue()); + result.add(pd); + } + return unmodifiableList(result); + } + return null; + } + + public static void deleteType(String domainURI, Container c) throws DomainNotFoundException + { + if (null == domainURI) + return; + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + try + { + deleteObjectsOfType(domainURI, c); + deleteDomain(domainURI, c); + } + catch (DomainNotFoundException x) + { + // throw exception but do not kill enclosing transaction + transaction.commit(); + throw x; + } + + transaction.commit(); + } + } + + public static PropertyDescriptor insertOrUpdatePropertyDescriptor(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) + throws ChangePropertyDescriptorException + { + validatePropertyDescriptor(pd); + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + DomainDescriptor dexist = ensureDomainDescriptor(dd); + + if (!dexist.getContainer().equals(pd.getContainer()) + && !pd.getProject().equals(_sharedContainer)) + { + // domain is defined in a different container. + //ToDO define property in the domains container? what security? + throw new ChangePropertyDescriptorException("Attempt to define property for a domain definition that exists in a different folder\n" + + "domain folder = " + dexist.getContainer().getPath() + "\n" + + "property folder = " + pd.getContainer().getPath()); + } + + PropertyDescriptor pexist = ensurePropertyDescriptor(pd); + pexist.setDatabaseDefaultValue(pd.getDatabaseDefaultValue()); + pexist.setNullable(pd.isMvEnabled() || pd.isNullable()); + pexist.setRequired(pd.isRequired()); + + ensurePropertyDomain(pexist, dexist, sortOrder); + + transaction.commit(); + return pexist; + } + } + + + static final String parameters = "propertyuri,name,description,rangeuri,concepturi,label," + + "format,container,project,lookupcontainer,lookupschema,lookupquery,defaultvaluetype,hidden," + + "mvenabled,importaliases,url,shownininsertview,showninupdateview,shownindetailsview,measure,dimension,scale," + + "sourceontology,conceptimportcolumn,conceptlabelcolumn,principalconceptcode,conceptsubtree," + + "recommendedvariable,derivationdatascope,storagecolumnname,facetingbehaviortype,phi,redactedText," + + "excludefromshifting,mvindicatorstoragecolumnname,defaultscale,scannable"; + static final String[] parametersArray = parameters.split(","); + + static ParameterMapStatement getInsertStmt(Connection conn, User user, TableInfo t, boolean ifNotExists) throws SQLException + { + user = null==user ? User.guest : user; + SQLFragment sql = new SQLFragment("INSERT INTO exp.propertydescriptor\n\t\t("); + SQLFragment values = new SQLFragment("\nSELECT\t"); + ColumnInfo c; + String comma = ""; + Parameter container = null; + Parameter propertyuri = null; + for (var p : parametersArray) + { + if (null == (c = t.getColumn(p))) + continue; + sql.append(comma).append(p); + values.append(comma).append("?"); + comma = ","; + Parameter parameter = new Parameter(p, c.getJdbcType()); + values.add(parameter); + if ("container".equals(p)) + container = parameter; + else if ("propertyuri".equals(p)) + propertyuri = parameter; + } + sql.append(", createdby, created, modifiedby, modified)\n"); + values.append(", " + user.getUserId() + ", {fn now()}, " + user.getUserId() + ", {fn now()}"); + sql.append(values); + if (ifNotExists) + { + sql.append("\nWHERE NOT EXISTS (SELECT propertyid FROM exp.propertydescriptor WHERE propertyuri=? AND container=?)\n"); + sql.add(propertyuri).add(container); + } + return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); + } + + static ParameterMapStatement getUpdateStmt(Connection conn, User user, TableInfo t) throws SQLException + { + user = null==user ? User.guest : user; + SQLFragment sql = new SQLFragment("UPDATE exp.propertydescriptor SET "); + ColumnInfo c; + String comma = ""; + for (var p : parametersArray) + { + if (null == (c = t.getColumn(p))) + continue; + sql.append(comma).append(p).append("=?"); + comma = ", "; + sql.add(new Parameter(p, c.getJdbcType())); + } + sql.append(", modifiedby=" + user.getUserId() + ", modified={fn now()}"); + sql.append("\nWHERE propertyid=?"); + sql.add(new Parameter("propertyid", JdbcType.INTEGER)); + return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); + } + + + public static void insertPropertyDescriptors(User user, List pds) throws SQLException + { + if (null == pds || pds.isEmpty()) + return; + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getInsertStmt(conn, user, t, false)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = null; + for (PropertyDescriptor pd : pds) + { + m = f.toMap(pd, m); + stmt.clearParameters(); + stmt.putAll(m); + stmt.addBatch(); + } + stmt.executeBatch(); + } + } + + + public static void updatePropertyDescriptors(User user, List pds) throws SQLException + { + if (null == pds || pds.isEmpty()) + return; + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getUpdateStmt(conn, user, t)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = null; + for (PropertyDescriptor pd : pds) + { + m = f.toMap(pd, m); + stmt.clearParameters(); + stmt.putAll(m); + stmt.addBatch(); + } + stmt.executeBatch(); + } + } + + + public static PropertyDescriptor insertPropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException + { + assert pd.getPropertyId() == 0; + validatePropertyDescriptor(pd); + pd = Table.insert(null, getTinfoPropertyDescriptor(), pd); + _log.debug("Adding property descriptor to cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + return pd; + } + + + //todo: we automatically update a pd to the last one in? + public static PropertyDescriptor updatePropertyDescriptor(PropertyDescriptor pd) + { + assert pd.getPropertyId() != 0; + pd = Table.update(null, getTinfoPropertyDescriptor(), pd, pd.getPropertyId()); + _log.debug("Updating property descriptor in cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + // It's possible that the propertyURI has changed, thus breaking our reference + DOMAIN_PROPERTIES_CACHE.clear(); + return pd; + } + + /** + * Insert or update an object property value. + * + * @param user The user inserting the property - currently only used for validating lookup values. + * @param container Insert the property value into this container. + * @param pd The property descriptor. + * @param lsid The object on which to attach the properties. + * @param value The value to insert. + * @param ownerObjectLsid The "owner" object or "parent" object, which isn't necessarily same as the object. For example, samples use the ExpSampleType as the owner object. + * @param insertNullValues When true, a null value will be inserted if the value is null, otherwise any existing property value will be deleted if the value is null. + * @return The inserted ObjectProperty or null + */ + public static ObjectProperty updateObjectProperty(User user, Container container, PropertyDescriptor pd, String lsid, Object value, @Nullable String ownerObjectLsid, boolean insertNullValues) throws ValidationException + { + ObjectProperty oprop; + RemapCache cache = new RemapCache(); + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), container, pd.getContainer()); + + try + { + oprop = new ObjectProperty(lsid, container, pd, value); + } + catch (ConversionException x) + { + // Issue 43529: Assay run property with large lookup doesn't resolve text input by value + // Attempt to resolve lookups by display value and then try creating the ObjectProperty again + if (pd.getLookup() != null) + { + Object remappedValue = getRemappedValueForLookup(user, container, cache, pd.getLookup(), value); + if (remappedValue != null) + value = remappedValue; + } + oprop = new ObjectProperty(lsid, container, pd, value); + } + + if (value != null || insertNullValues) + { + oprop.setPropertyId(pd.getPropertyId()); + OntologyManager.insertProperties(container, user, ownerObjectLsid, false, insertNullValues, oprop); + } + else + { + // We still need to validate blanks + List errors = new ArrayList<>(); + OntologyManager.validateProperty(PropertyService.get().getPropertyValidators(pd), pd, oprop, errors, new ValidatorContext(pd.getContainer(), user)); + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + transaction.commit(); + } + return oprop; + } + + public static Object getRemappedValueForLookup(User user, Container container, RemapCache cache, Lookup lookup, Object value) + { + Container lkContainer = lookup.getContainer() != null ? lookup.getContainer() : container; + return cache.remap(SchemaKey.fromParts(lookup.getSchemaKey()), lookup.getQueryName(), user, lkContainer, ContainerFilter.Type.CurrentPlusProjectAndShared, String.valueOf(value)); + } + + public static List findPropertyUsages(User user, List propertyIds, int maxUsageCount) + { + List ret = new ArrayList<>(propertyIds.size()); + for (int propertyId : propertyIds) + { + var pd = getPropertyDescriptor(propertyId); + if (pd == null) + throw new IllegalArgumentException("property not found: " + propertyId); + + ret.add(findPropertyUsages(user, pd, maxUsageCount)); + } + + return ret; + } + + public static List findPropertyUsages(User user, Container c, List propertyURIs, int maxUsageCount) + { + List ret = new ArrayList<>(propertyURIs.size()); + for (String propertyURI : propertyURIs) + { + var pd = getPropertyDescriptor(propertyURI, c); + if (pd == null) + throw new IllegalArgumentException("property not found: " + propertyURI); + + ret.add(findPropertyUsages(user, pd, maxUsageCount)); + } + + return ret; + } + + public static PropertyUsages findPropertyUsages(@NotNull User user, @NotNull PropertyDescriptor pd, int maxUsageCount) + { + // query exp.ObjectProperty for usages of the property + FieldKey objectId = FieldKey.fromParts("objectId"); + FieldKey objectId_objectURI = FieldKey.fromParts("objectId", "objectURI"); + FieldKey objectId_container = FieldKey.fromParts("objectId", "container"); + List fields = List.of(objectId, objectId_objectURI, objectId_container); + var colMap = QueryService.get().getColumns(getTinfoObjectProperty(), fields); + + int usageCount; + List objects = new ArrayList<>(maxUsageCount); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("propertyId"), pd.getPropertyId(), CompareType.EQUAL); + filter.addCondition(objectId_objectURI, DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX, CompareType.DOES_NOT_CONTAIN); + + TableSelector ts = new TableSelector(getTinfoObjectProperty(), colMap.values(), filter, new Sort("objectId")); + try (var r = ts.getResults(true)) + { + usageCount = r.getSize(); + + for (int i = 0; i < maxUsageCount && r.next(); i++) + { + var row = r.getFieldKeyRowMap(); + long oid = asLong(row.get(objectId)); + String objectURI = (String) row.get(objectId_objectURI); + String container = (String) row.get(objectId_container); + + Identifiable object = LsidManager.get().getObject(objectURI); + if (object != null) + { + Container c = object.getContainer(); + if (c != null && c.hasPermission(user, ReadPermission.class)) + objects.add(object); + } + else + { + Container c = ContainerManager.getForId(container); + if (c != null && c.hasPermission(user, ReadPermission.class)) + { + OntologyObject oo = new OntologyObject(); + oo.setContainer(c); + oo.setObjectId(oid); + oo.setObjectURI(objectURI); + objects.add(new IdentifiableBase(oo)); + } + } + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return new PropertyUsages(pd.getPropertyId(), pd.getPropertyURI(), usageCount, objects); + } + + public static class PropertyUsages + { + public final int propertyId; + public final String propertyURI; + public final int usageCount; + public final List objects; + + public PropertyUsages(int propertyId, String propertyURI, int usageCount, List objects) + { + this.propertyId = propertyId; + this.propertyURI = propertyURI; + this.usageCount = usageCount; + this.objects = objects; + } + } + + + public static void invalidateDomain(Domain d) + { + // TODO can we please implement a surgical version of this + clearCaches(); + } + + + public static void clearCaches() + { + _log.debug("Clearing caches"); + ExperimentService.get().clearCaches(); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.clear(); + DOMAIN_DESC_BY_ID_CACHE.clear(); + DOMAIN_PROPERTIES_CACHE.clear(); + PROP_DESCRIPTOR_CACHE.clear(); + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); + } + + public static void clearPropertyCache(String parentObjectURI) + { + PROPERTY_MAP_CACHE.removeUsingFilter(key -> Objects.equals(key.second, parentObjectURI)); + } + + + public static void clearPropertyCache() + { + PROPERTY_MAP_CACHE.clear(); + } + + public static class ImportPropertyDescriptor + { + public final String domainName; + public final String domainURI; + public final PropertyDescriptor pd; + public final List validators; + public final List formats; + public final String defaultValue; + + private ImportPropertyDescriptor(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) + { + this.domainName = domainName; + this.domainURI = domainURI; + this.pd = pd; + this.validators = null != validators ? validators : Collections.emptyList(); + this.formats = null != formats ? formats : Collections.emptyList(); + this.defaultValue = defaultValue; + } + } + + + public static class ImportPropertyDescriptorsList + { + public final ArrayList properties = new ArrayList<>(); + + void add(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) + { + properties.add(new ImportPropertyDescriptor(domainName, domainURI, pd, validators, formats, defaultValue)); + } + } + + /** + * Updates an existing domain property with an import property descriptor generated + * by _propertyDescriptorFromRowMap below. Properties we don't set are explicitly + * called out + */ + public static void updateDomainPropertyFromDescriptor(DomainProperty p, PropertyDescriptor pd) + { + // don't setName + p.setPropertyURI(pd.getPropertyURI()); + p.setLabel(pd.getLabel()); + p.setConceptURI(pd.getConceptURI()); + p.setRangeURI(pd.getRangeURI()); + // don't setContainer + p.setDescription(pd.getDescription()); + p.setURL((pd.getURL() != null) ? pd.getURL().toString() : null); + p.setImportAliasSet(ColumnRenderPropertiesImpl.convertToSet(pd.getImportAliases())); + p.setRequired(pd.isRequired()); + p.setHidden(pd.isHidden()); + p.setShownInInsertView(pd.isShownInInsertView()); + p.setShownInUpdateView(pd.isShownInUpdateView()); + p.setShownInDetailsView(pd.isShownInDetailsView()); + p.setShownInLookupView(pd.isShownInLookupView()); + p.setDimension(pd.isDimension()); + p.setMeasure(pd.isMeasure()); + p.setRecommendedVariable(pd.isRecommendedVariable()); + p.setDefaultScale(pd.getDefaultScale()); + p.setScale(pd.getScale()); + p.setFormat(pd.getFormat()); + p.setMvEnabled(pd.isMvEnabled()); + + Lookup lookup = new Lookup(); + lookup.setQueryName(pd.getLookupQuery()); + lookup.setSchemaName(pd.getLookupSchema()); + String lookupContainerId = pd.getLookupContainer(); + if (lookupContainerId != null) + { + Container container = ContainerManager.getForId(lookupContainerId); + if (container == null) + lookup = null; + else + lookup.setContainer(container); + } + p.setLookup(lookup); + p.setFacetingBehavior(pd.getFacetingBehaviorType()); + p.setPhi(pd.getPHI()); + p.setRedactedText(pd.getRedactedText()); + p.setExcludeFromShifting(pd.isExcludeFromShifting()); + p.setDefaultValueTypeEnum(pd.getDefaultValueTypeEnum()); + p.setScannable(pd.isScannable()); + p.setDerivationDataScope(pd.getDerivationDataScope()); + } + + @TestWhen(TestWhen.When.BVT) + @TestTimeout(120) + public static class TestCase extends Assert + { + @Test + public void testSchema() + { + assertNotNull(getExpSchema()); + assertNotNull(getTinfoPropertyDescriptor()); + assertNotNull(ExperimentService.get().getTinfoSampleType()); + + assertEquals(11, getTinfoPropertyDescriptor().getColumns("PropertyId,PropertyURI,RangeURI,Name,Description,DerivationDataScope,SourceOntology,ConceptImportColumn,ConceptLabelColumn,PrincipalConceptCode,scannable").size()); + assertEquals(4, getTinfoObject().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId").size()); + assertEquals(11, getTinfoObjectPropertiesView().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId,Name,PropertyURI,RangeURI,TypeTag,StringValue,DateTimeValue,FloatValue").size()); + assertEquals(10, ExperimentService.get().getTinfoSampleType().getColumns("RowId,Name,LSID,MaterialLSIDPrefix,Description,Created,CreatedBy,Modified,ModifiedBy,Container").size()); + } + + @Test + public void testBasicPropertiesObject() throws ValidationException + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + User user = TestContext.get().getUser(); + String parentObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + //First delete in case test case failed before + deleteOntologyObjects(c, parentObjectLsid); + assertNull(getOntologyObject(c, parentObjectLsid)); + assertNull(getOntologyObject(c, childObjectLsid)); + ensureObject(c, childObjectLsid, parentObjectLsid); + OntologyObject oParent = getOntologyObject(c, parentObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + assertNull(oParent.getOwnerObjectId()); + assertEquals(oChild.getContainer(), c); + assertEquals(oParent.getContainer(), c); + + String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + PropertyDescriptor strPd = getPropertyDescriptor(strProp, c); + assertEquals(PropertyType.STRING, strPd.getPropertyType()); + + String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + PropertyDescriptor intPd = getPropertyDescriptor(intProp, c); + assertEquals(PropertyType.INTEGER, intPd.getPropertyType()); + + String longProp = new Lsid("Junit", "OntologyManager", "longProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, longProp, 6L)); + PropertyDescriptor longPd = getPropertyDescriptor(longProp, c); + assertEquals(PropertyType.BIGINT, longPd.getPropertyType()); + + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.MILLISECOND, 0); + String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); + PropertyDescriptor datePd = getPropertyDescriptor(dateProp, c); + assertEquals(PropertyType.DATE_TIME, datePd.getPropertyType()); + + Map m = getProperties(c, oChild.getObjectURI()); + assertNotNull(m); + assertEquals(4, m.size()); + assertEquals("The String", m.get(strProp)); + assertEquals(5, m.get(intProp)); + assertEquals(6L, m.get(longProp)); + assertEquals(cal.getTime(), m.get(dateProp)); + + // Set property order: date, str, int. Long property will sort to last since it isn't explicitly included. + List propertyOrder = List.of(datePd, strPd, intPd); + updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); + + Map oProps = getPropertyObjects(c, childObjectLsid); + var iter = oProps.entrySet().iterator(); + assertEquals(cal.getTime(), iter.next().getValue().value()); + assertEquals("The String", iter.next().getValue().value()); + assertEquals(5, iter.next().getValue().value()); + assertEquals(6L, iter.next().getValue().value()); + assertFalse(iter.hasNext()); + + // Update property order: int, date, long, str + propertyOrder = List.of(intPd, datePd, longPd, strPd); + updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); + oProps = getPropertyObjects(c, childObjectLsid); + iter = oProps.entrySet().iterator(); + assertEquals(5, iter.next().getValue().value()); + assertEquals(cal.getTime(), iter.next().getValue().value()); + assertEquals(6L, iter.next().getValue().value()); + assertEquals("The String", iter.next().getValue().value()); + assertFalse(iter.hasNext()); + + deleteOntologyObjects(c, parentObjectLsid); + assertNull(getOntologyObject(c, parentObjectLsid)); + assertNull(getOntologyObject(c, childObjectLsid)); + + m = getProperties(c, oChild.getObjectURI()); + assertEquals(0, m.size()); + } + + @Test + public void testContainerDelete() throws ValidationException + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.MILLISECOND, 0); + String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); + + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + + private void defineCrossFolderProperties(Container fldr1a, Container fldr1b) throws SQLException + { + try + { + String fa = fldr1a.getPath(); + String fb = fldr1b.getPath(); + + //object, prop descriptor in folder being moved + String objP1Fa = new Lsid("OntologyObject", "JUnit", fa.replace('/', '.')).toString(); + ensureObject(fldr1a, objP1Fa); + String propP1Fa = fa + "PD1"; + PropertyDescriptor pd1Fa = ensurePropertyDescriptor(propP1Fa, PropertyType.STRING, "PropertyDescriptor 1" + fa, fldr1a); + insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP1Fa, "same fldr")); + + //object in folder not moving, prop desc in folder moving + String objP2Fb = new Lsid("OntologyObject", "JUnit", fb.replace('/', '.')).toString(); + ensureObject(fldr1b, objP2Fb); + insertProperties(fldr1b, null, new ObjectProperty(objP2Fb, fldr1b, propP1Fa, "object in folder not moving, prop desc in folder moving")); + + //object in folder moving, prop desc in folder not moving + String propP2Fb = fb + "PD1"; + ensurePropertyDescriptor(propP2Fb, PropertyType.STRING, "PropertyDescriptor 1" + fb, fldr1b); + insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP2Fb, "object in folder moving, prop desc in folder not moving")); + + // third prop desc in folder that is moving; shares domain with first prop desc + String propP1Fa3 = fa + "PD3"; + PropertyDescriptor pd1Fa3 = ensurePropertyDescriptor(propP1Fa3, PropertyType.STRING, "PropertyDescriptor 3" + fa, fldr1a); + String domP1Fa = fa + "DD1"; + DomainDescriptor dd1 = ensureDomainDescriptor(domP1Fa, "DomDesc 1" + fa, fldr1a); + ensurePropertyDomain(pd1Fa, dd1); + ensurePropertyDomain(pd1Fa3, dd1); + + //second domain desc in folder that is moving + // second prop desc in folder moving, belongs to 2nd domain + String propP1Fa2 = fa + "PD2"; + PropertyDescriptor pd1Fa2 = ensurePropertyDescriptor(propP1Fa2, PropertyType.STRING, "PropertyDescriptor 2" + fa, fldr1a); + String domP1Fa2 = fa + "DD2"; + DomainDescriptor dd2 = ensureDomainDescriptor(domP1Fa2, "DomDesc 2" + fa, fldr1a); + ensurePropertyDomain(pd1Fa2, dd2); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + @Test + public void testContainerMove() throws Exception + { + deleteMoveTestContainers(); + + Container proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); + Container proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + + proj1 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); + proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + + proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); + proj2 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + } + + private void doMoveTest(Container proj1, Container proj2) throws Exception + { + String p1Path = proj1.getPath() + "/"; + String p2Path = proj2.getPath() + "/"; + if (p1Path.equals("//")) p1Path = "/_ontMgrDemotePromote"; + if (p2Path.equals("//")) p2Path = "/_ontMgrDemotePromote"; + + Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); + Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); + ContainerManager.ensureContainer(p2Path + "Fc", TestContext.get().getUser()); + Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); + Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); + + defineCrossFolderProperties(fldr1a, fldr1b); + //defineCrossFolderProperties(fldr1a, fldr2c); + defineCrossFolderProperties(fldr1aa, fldr1b); + defineCrossFolderProperties(fldr1aaa, fldr1b); + + fldr1a.getProject().getPath(); + String f = fldr1a.getPath(); + String propId = f + "PD1"; + assertNull(getPropertyDescriptor(propId, proj2)); + ContainerManager.move(fldr1a, proj2, TestContext.get().getUser()); + + // if demoting a folder + if (proj1.isRoot()) + { + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD2"; + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj2)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj2)); + + domId = f + "DD2"; + assertNotNull(getDomainDescriptor(domId, proj2)); + } + // if promoting a folder, + else if (proj2.isRoot()) + { + assertNotNull(getPropertyDescriptor(propId, proj1)); + + propId = f + "PD2"; + assertNull(getPropertyDescriptor(propId, proj1)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj1)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj1)); + + domId = f + "DD2"; + assertNull(getDomainDescriptor(domId, proj1)); + } + else + { + assertNotNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD2"; + assertNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj1)); + assertNotNull(getDomainDescriptor(domId, proj2)); + + domId = f + "DD2"; + assertNull(getDomainDescriptor(domId, proj1)); + assertNotNull(getDomainDescriptor(domId, proj2)); + } + } + + @Test + public void testDeleteFoldersWithSharedProps() throws SQLException + { + deleteMoveTestContainers(); + + String projectName = "_ontMgrTestP1"; + Container proj1 = ContainerManager.ensureContainer(projectName, TestContext.get().getUser()); + String p1Path = proj1.getPath() + "/"; + + Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); + Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); + Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); + Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); + + defineCrossFolderProperties(fldr1a, fldr1b); + defineCrossFolderProperties(fldr1aa, fldr1b); + defineCrossFolderProperties(fldr1aaa, fldr1b); + + deleteProjects( projectName); + } + + private void deleteMoveTestContainers() + { + // Remove all projects. Subfolders will be deleted when project is removed. + deleteProjects( + "/_ontMgrTestP1", + "/_ontMgrTestP2", + "/_ontMgrDemotePromoteFa", + "/_ontMgrDemotePromoteFb", + "/_ontMgrDemotePromoteFc", + "/Fa" + ); + } + + private void deleteProjects(String... projectNames) + { + for (String path : projectNames) + { + Container c = ContainerManager.getForPath(path); + + if (null != c) + ContainerManager.deleteAll(c, TestContext.get().getUser()); + } + + for (String path : projectNames) + assertNull("Container " + path + " was not deleted", ContainerManager.getForPath(path)); + } + + @Test + public void testTransactions() throws SQLException + { + try + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + //Create objects in a transaction & make sure they are all gone. + OntologyObject oParent; + OntologyObject oChild; + String strProp; + String intProp; + + try (Transaction ignored = getExpSchema().getScope().beginTransaction()) + { + ensureObject(c, childObjectLsid, ownerObjectLsid); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + } + + assertEquals(0L, getObjectCount(c)); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNull(oParent); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + //Rollback transaction for one new property + try (Transaction ignored = getExpSchema().getScope().beginTransaction()) + { + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + } + + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + Map m = getProperties(c, childObjectLsid); + assertNotNull(m.get(strProp)); + assertNull(m.get(intProp)); + + try (Transaction transaction = getExpSchema().getScope().beginTransaction()) + { + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + transaction.commit(); + } + + m = getProperties(c, childObjectLsid); + assertNotNull(m.get(strProp)); + assertNotNull(m.get(intProp)); + + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + @Test + public void testDomains() throws Exception + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + String child2ObjectLsid = new Lsid("Junit", "OntologyManager", "child2").toString(); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + String domURIa = new Lsid("Junit", "DD", "Domain1").toString(); + String strPropURI = new Lsid("Junit", "PD", "Domain1.stringProp").toString(); + String intPropURI = new Lsid("Junit", "PD", "Domain1.intProp").toString(); + String longPropURI = new Lsid("Junit", "PD", "Domain1.longProp").toString(); + + DomainDescriptor dd = ensureDomainDescriptor(domURIa, "Domain1", c); + assertNotNull(dd); + + PropertyDescriptor pdStr = new PropertyDescriptor(); + pdStr.setPropertyURI(strPropURI); + pdStr.setRangeURI(PropertyType.STRING.getTypeUri()); + pdStr.setContainer(c); + pdStr.setName("Domain1.stringProp"); + + pdStr = ensurePropertyDescriptor(pdStr); + assertNotNull(pdStr); + + PropertyDescriptor pdInt = ensurePropertyDescriptor(intPropURI, PropertyType.INTEGER, "Domain1.intProp", c); + PropertyDescriptor pdLong = ensurePropertyDescriptor(longPropURI, PropertyType.BIGINT, "Domain1.longProp", c); + + ensurePropertyDomain(pdStr, dd); + ensurePropertyDomain(pdInt, dd); + ensurePropertyDomain(pdLong, dd); + + List pds = getPropertiesForType(domURIa, c); + assertEquals(3, pds.size()); + Map mPds = new HashMap<>(); + for (PropertyDescriptor pd1 : pds) + mPds.put(pd1.getPropertyURI(), pd1); + + assertTrue(mPds.containsKey(strPropURI)); + assertTrue(mPds.containsKey(intPropURI)); + assertTrue(mPds.containsKey(longPropURI)); + + ObjectProperty strProp = new ObjectProperty(childObjectLsid, c, strPropURI, "String value"); + ObjectProperty intProp = new ObjectProperty(childObjectLsid, c, intPropURI, 42); + ObjectProperty longProp = new ObjectProperty(childObjectLsid, c, longPropURI, 52L); + insertProperties(c, ownerObjectLsid, strProp); + insertProperties(c, ownerObjectLsid, intProp); + insertProperties(c, ownerObjectLsid, longProp); + + Map m = getProperties(c, oChild.getObjectURI()); + assertNotNull(m); + assertEquals(3, m.size()); + assertEquals("String value", m.get(strPropURI)); + assertEquals(42, m.get(intPropURI)); + assertEquals(52L, m.get(longPropURI)); + + // test insertTabDelimited + List> rows = List.of( + new CaseInsensitiveMapWrapper<>(Map.of( + "lsid", child2ObjectLsid, + strPropURI, "Second value", + intPropURI, 62, + longPropURI, 72L + ) + )); + ImportHelper helper = new ImportHelper() + { + @Override + public String beforeImportObject(Map map) + { + return (String)map.get("lsid"); + } + + @Override + public void afterBatchInsert(int currentRow) + { } + + @Override + public void updateStatistics(int currentRow) + { } + }; + try (Transaction tx = getExpSchema().getScope().ensureTransaction()) + { + insertTabDelimited(c, TestContext.get().getUser(), oParent.getObjectId(), helper, pds, MapDataIterator.of(rows).getDataIterator(new DataIteratorContext()), false, null); + tx.commit(); + } + + m = getProperties(c, child2ObjectLsid); + assertNotNull(m); + assertEquals(3, m.size()); + assertEquals("Second value", m.get(strPropURI)); + assertEquals(62, m.get(intPropURI)); + assertEquals(72L, m.get(longPropURI)); + + deleteType(domURIa, c); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + } + + private static long getObjectCount(Container c) + { + return new TableSelector(getTinfoObject(), SimpleFilter.createContainerFilter(c), null).getRowCount(); + } + + /** + * v.first value IN/OUT parameter + * v.second mvIndicator OUT parameter + */ + public static void convertValuePair(PropertyDescriptor pd, PropertyType pt, Pair v) + { + if (v.first == null) + return; + + // Handle field-level QC + if (v.first instanceof MvFieldWrapper mvWrapper) + { + v.second = mvWrapper.getMvIndicator(); + v.first = mvWrapper.getValue(); + } + else if (pd.isMvEnabled()) + { + // Not all callers will have wrapped an MV value if there isn't also + // a real value + if (MvUtil.isMvIndicator(v.first.toString(), pd.getContainer())) + { + v.second = v.first.toString(); + v.first = null; + } + } + + if (null != v.first && null != pt) + v.first = pt.convert(v.first); + } + + @Deprecated // Fold into ObjectProperty? Eliminate insertTabDelimited() methods, the only usage of PropertyRow. + public static class PropertyRow + { + protected long objectId; + protected int propertyId; + protected char typeTag; + protected Double floatValue; + protected String stringValue; + protected Date dateTimeValue; + protected String mvIndicator; + + public PropertyRow() + { + } + + public PropertyRow(long objectId, PropertyDescriptor pd, Object value, PropertyType pt) + { + this.objectId = objectId; + this.propertyId = pd.getPropertyId(); + this.typeTag = pt.getStorageType(); + + Pair p = new Pair<>(value, null); + convertValuePair(pd, pt, p); + mvIndicator = p.second; + + pt.init(this, p.first); + } + + public long getObjectId() + { + return objectId; + } + + public void setObjectId(long objectId) + { + this.objectId = objectId; + } + + public int getPropertyId() + { + return propertyId; + } + + public void setPropertyId(int propertyId) + { + this.propertyId = propertyId; + } + + public char getTypeTag() + { + return typeTag; + } + + public void setTypeTag(char typeTag) + { + this.typeTag = typeTag; + } + + public Double getFloatValue() + { + return floatValue; + } + + public Boolean getBooleanValue() + { + if (floatValue == null) + { + return null; + } + return floatValue.doubleValue() == 1.0; + } + + public void setFloatValue(Double floatValue) + { + this.floatValue = floatValue; + } + + public String getStringValue() + { + return stringValue; + } + + public void setStringValue(String stringValue) + { + this.stringValue = stringValue; + } + + public Date getDateTimeValue() + { + return dateTimeValue; + } + + public void setDateTimeValue(Date dateTimeValue) + { + this.dateTimeValue = dateTimeValue; + } + + public String getMvIndicator() + { + return mvIndicator; + } + + public void setMvIndicator(String mvIndicator) + { + this.mvIndicator = mvIndicator; + } + + public Object getObjectValue() + { + return stringValue != null ? stringValue : floatValue != null ? floatValue : dateTimeValue; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("PropertyRow: "); + + sb.append("objectId=").append(objectId); + sb.append(", propertyId=").append(propertyId); + sb.append(", value="); + + if (stringValue != null) + sb.append(stringValue); + else if (floatValue != null) + sb.append(floatValue); + else if (dateTimeValue != null) + sb.append(dateTimeValue); + else + sb.append("null"); + + if (mvIndicator != null) + sb.append(", mvIndicator=").append(mvIndicator); + + return sb.toString(); + } + } + + public static DbSchema getExpSchema() + { + return DbSchema.get("exp", DbSchemaType.Module); + } + + public static SqlDialect getSqlDialect() + { + return getExpSchema().getSqlDialect(); + } + + public static TableInfo getTinfoPropertyDomain() + { + return getExpSchema().getTable("PropertyDomain"); + } + + public static TableInfo getTinfoObject() + { + return getExpSchema().getTable("Object"); + } + + public static TableInfo getTinfoObjectProperty() + { + return getExpSchema().getTable("ObjectProperty"); + } + + public static TableInfo getTinfoPropertyDescriptor() + { + return getExpSchema().getTable("PropertyDescriptor"); + } + + public static TableInfo getTinfoDomainDescriptor() + { + return getExpSchema().getTable("DomainDescriptor"); + } + + public static TableInfo getTinfoObjectPropertiesView() + { + return getExpSchema().getTable("ObjectPropertiesView"); + } + + public static HtmlString doProjectColumnCheck(boolean bFix) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(); + String descriptorTable = getTinfoPropertyDescriptor().toString(); + String uriColumn = "PropertyURI"; + String idColumn = "PropertyID"; + doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); + + descriptorTable = getTinfoDomainDescriptor().toString(); + uriColumn = "DomainURI"; + idColumn = "DomainID"; + doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); + + return builder.getHtmlString(); + } + + private static void doProjectColumnCheck(final String descriptorTable, final String uriColumn, final String idColumn, final HtmlStringBuilder msgBuilder, final boolean bFix) + { + // get all unique combos of Container, project + + String sql = "SELECT Container, Project FROM " + descriptorTable + " GROUP BY Container, Project"; + + new SqlSelector(getExpSchema(), sql).forEach(rs -> { + String containerId = rs.getString("Container"); + String projectId = rs.getString("Project"); + Container container = ContainerManager.getForId(containerId); + if (null == container) + return; // should be handled by container check + String newProjectId = container.getProject() == null ? container.getId() : container.getProject().getId(); + if (!projectId.equals(newProjectId)) + { + if (bFix) + { + fixProjectColumn(descriptorTable, uriColumn, idColumn, container, projectId, newProjectId); + msgBuilder + .unsafeAppend("
   ") + .append("Fixed inconsistent project ids found for ") + .append(descriptorTable).append(" in folder ") + .append(ContainerManager.getForId(containerId).getPath()); + + } + else + msgBuilder + .unsafeAppend("
   ") + .append("ERROR: Inconsistent project ids found for ") + .append(descriptorTable).append(" in folder ").append(container.getPath()); + } + }); + } + + private static void fixProjectColumn(String descriptorTable, String uriColumn, String idColumn, Container container, String projectId, String newProjId) + { + final SqlExecutor executor = new SqlExecutor(getExpSchema()); + + String sql = "UPDATE " + descriptorTable + " SET Project= ? WHERE Project = ? AND Container=? AND " + uriColumn + " NOT IN " + + "(SELECT " + uriColumn + " FROM " + descriptorTable + " WHERE Project = ?)"; + executor.execute(sql, newProjId, projectId, container.getId(), newProjId); + + // now check to see if there is already an existing descriptor in the target (correct) project. + // this can happen if a folder containing a descriptor is moved to another project + // and the OntologyManager's containerMoved handler fails to fire for some reason. (note not in transaction) + // If this is the case, the descriptor is redundant and it should be deleted, after we move the objects that depend on it. + + sql = " SELECT prev." + idColumn + " AS PrevIdCol, cur." + idColumn + " AS CurIdCol FROM " + descriptorTable + " prev " + + " INNER JOIN " + descriptorTable + " cur ON (prev." + uriColumn + "= cur." + uriColumn + " ) " + + " WHERE cur.Project = ? AND prev.Project= ? AND prev.Container = ? "; + final String updsql1 = " UPDATE " + getTinfoObjectProperty() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; + final String updsql2 = " UPDATE " + getTinfoPropertyDomain() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; + final String delSql = " DELETE FROM " + descriptorTable + " WHERE " + idColumn + " = ? "; + + new SqlSelector(getExpSchema(), sql, newProjId, projectId, container).forEach(rs -> { + int prevPropId = rs.getInt(1); + int curPropId = rs.getInt(2); + executor.execute(updsql1, curPropId, prevPropId); + executor.execute(updsql2, curPropId, prevPropId); + executor.execute(delSql, prevPropId); + }); + } + + public static void validatePropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException + { + String name = pd.getName(); + validateValue(name, "Name", null); + validateValue(pd.getPropertyURI(), "PropertyURI", "Please use a shorter field name. Name = " + name); + validateValue(pd.getLabel(), "Label", null); + validateValue(pd.getImportAliases(), "ImportAliases", null); + validateValue(pd.getURL() != null ? pd.getURL().getSource() : null, "URL", null); + validateValue(pd.getConceptURI(), "ConceptURI", null); + validateValue(pd.getRangeURI(), "RangeURI", null); + + // Issue 15484: adding a column ending in 'mvIndicator' is problematic if another column w/ the same + // root exists, or if you later enable mvIndicators on a column w/ the same root + if (pd.getName() != null && pd.getName().toLowerCase().endsWith(MV_INDICATOR_SUFFIX)) + { + throw new ChangePropertyDescriptorException("Field name cannot end with the suffix 'mvIndicator': " + pd.getName()); + } + + if (null != name) + { + for (char ch : name.toCharArray()) + { + if (Character.isWhitespace(ch) && ' ' != ch) + throw new ChangePropertyDescriptorException("Field name cannot contain whitespace other than ' ' (space)"); + } + } + } + + private static void validateValue(String value, String columnName, String extraMessage) throws ChangePropertyDescriptorException + { + int maxLength = getTinfoPropertyDescriptor().getColumn(columnName).getScale(); + if (value != null && value.length() > maxLength) + { + throw new ChangePropertyDescriptorException(columnName + " cannot exceed " + maxLength + " characters, but was " + value.length() + " characters long. " + (extraMessage == null ? "" : extraMessage)); + } + } + + static public boolean checkObjectExistence(String lsid) + { + return new TableSelector(getTinfoObject(), new SimpleFilter(FieldKey.fromParts("ObjectURI"), lsid), null).exists(); + } +} diff --git a/api/src/org/labkey/api/jsp/LabKeyJspWriter.java b/api/src/org/labkey/api/jsp/LabKeyJspWriter.java index 5f106580dcc..d3a061dde0e 100644 --- a/api/src/org/labkey/api/jsp/LabKeyJspWriter.java +++ b/api/src/org/labkey/api/jsp/LabKeyJspWriter.java @@ -20,6 +20,7 @@ import org.labkey.api.util.DOM; import org.labkey.api.util.HelpTopic; import org.labkey.api.util.SafeToRender; +import org.labkey.api.util.StringUtilsLabKey; import jakarta.servlet.jsp.JspWriter; import java.io.IOException; @@ -44,7 +45,7 @@ private String truncateAndQuote(String s) { return null; } - return "'" + (s.length() < 50 ? s : (s.substring(0, 50) + "...")) + "'"; + return "'" + (s.length() < 50 ? s : (StringUtilsLabKey.leftSurrogatePairFriendly(s, 50) + "...")) + "'"; } @Override diff --git a/api/src/org/labkey/api/util/MemTracker.java b/api/src/org/labkey/api/util/MemTracker.java index 448a537b058..2ad5d300a0a 100644 --- a/api/src/org/labkey/api/util/MemTracker.java +++ b/api/src/org/labkey/api/util/MemTracker.java @@ -1,486 +1,486 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.util; - -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpSession; -import org.apache.commons.collections4.map.AbstractReferenceMap.ReferenceStrength; -import org.apache.commons.collections4.map.ReferenceIdentityMap; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.RequestInfo; -import org.labkey.api.security.User; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.ValidEmail.InvalidEmailException; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewContext; - -import java.security.Principal; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.IdentityHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * Tracks objects that may be expensive, commonly allocated so that we know that they're not being held and creating - * a memory leak. Will not prevent the tracked objects from being garbage collected. - * User: brittp - * Date: Oct 27, 2005 - */ -public class MemTracker -{ - private static final MemTracker _instance = new MemTracker(); - - private final ThreadLocal _requestTracker = new ThreadLocal<>(); - private final List _recentRequests = new LinkedList<>(); - - private static final String UNVIEWED_KEY = "memtracker-unviewed-requests"; - private static final int MAX_UNVIEWED = 100; - - /** Only keep a short history of allocations for the most recent requests */ - private static final int MAX_TRACKED_REQUESTS = 500; - - public synchronized List getNewRequests(long requestId) - { - return _recentRequests.stream().filter(recentRequest -> recentRequest.getId() > requestId).toList(); - } - - static class AllocationInfo - { - @Nullable - private final StackTraceElement[] _stackTrace; - private final long _threadId; - private final long _allocTime; - - AllocationInfo() - { - this(MiniProfiler.getTroubleshootingStackTrace(), Thread.currentThread().getId(), HeartBeat.currentTimeMillis()); - } - - AllocationInfo(@Nullable StackTraceElement[] stackTrace, long threadId, long allocTime) - { - _stackTrace = stackTrace; - _threadId = threadId; - _allocTime = allocTime; - } - - public HtmlString getHtmlStack() - { - if (_stackTrace == null) - { - return HtmlString.of(MiniProfiler.NO_STACK_TRACE_AVAILABLE); - } - HtmlStringBuilder builder = HtmlStringBuilder.of(); - for (int i = 3; i < _stackTrace.length; i++) - { - String line = _stackTrace[i].toString(); - builder.append(line).append(HtmlString.BR).append("\r\n"); - if (line.contains("org.labkey.api.view.ViewServlet.service")) - break; - } - return builder.getHtmlString(); - } - - public long getThreadId() - { - return _threadId; - } - - public long getAllocationTime() - { - return _allocTime; - } - } - - - public static class HeldReference extends AllocationInfo - { - private final Object _reference; - - private HeldReference(Object held, AllocationInfo allocationInfo) - { - super(allocationInfo._stackTrace, allocationInfo._threadId, allocationInfo._allocTime); - _reference = held; - } - - - public String getClassName() - { - if (_reference instanceof Class) - return ((Class) _reference).getName(); - else - return _reference.getClass().getName(); - } - - public String getObjectSummary() - { - String desc = getObjectDescription(); - return desc.length() > 50 ? desc.substring(0, 50) + "..." : desc; - } - - public boolean hasShortSummary() - { - return getObjectDescription().length() > 50; - } - - public String getObjectDescription() - { - try - { - String toString = _reference.toString(); - if (toString == null) - return "null"; - return toString; - } - catch (Throwable e) - { - return "toString() failed: " + e.getClass().getName() + (e.getMessage() == null ? "" : (" - " + e.getMessage())); - } - } - - public Object getReference() - { - return _reference; - } - } - - public static MemTracker get() - { - return _instance; - } - - public static MemTracker getInstance() - { - return _instance; - } - - /** - * Create new RequestInfo for the current thread and request. - */ - @NotNull - public RequestInfo startProfiler(HttpServletRequest request, @Nullable String name) - { - String url = request.getRequestURI() + (request.getQueryString() == null ? "" : "?" + request.getQueryString()); - HttpSession session = request.getSession(false); - return startProfiler(url, request.getUserPrincipal(), name, session != null ? session.getId() : null); - } - - /** - * Create new RequestInfo for the current thread. - * Used for profiling background requests that will be merged into a parent profiler. - * @see #merge(RequestInfo) - */ - @NotNull - public RequestInfo startProfiler(@Nullable String name) - { - return startProfiler(null, null, name, null); - } - - /** - * Create new RequestInfo for the current thread and request. - */ - @NotNull - public synchronized RequestInfo startProfiler(String url, Principal user, @Nullable String name, @Nullable String sessionId) - { - RequestInfo req = new RequestInfo(url, user, name, sessionId); - if ((user instanceof User) && ((User) user).isSearchUser()) - req.setIgnored(true); - _requestTracker.set(req); - return req; - } - - @Nullable - public RequestInfo current() - { - return _requestTracker.get(); - } - - /** - * Finish the current profiling session and merge its results into the to RequestInfo. - * Unlike requestComplete, the current timing will not be added to the list of recent requests. - */ - public void merge(@NotNull RequestInfo to) - { - RequestInfo requestInfo = _requestTracker.get(); - if (requestInfo != null) - { - requestInfo.getRoot().stop(); - to.merge(requestInfo); - } - _requestTracker.remove(); - } - - /** - * Mark the current profiling session as ignored. Timings won't be collected. - */ - public synchronized void ignore() - { - RequestInfo requestInfo = _requestTracker.get(); - if (requestInfo != null) - requestInfo.setIgnored(true); - } - - /** - * Finish the current profiling session. - */ - public synchronized void requestComplete(RequestInfo req) - { - RequestInfo requestInfo = _requestTracker.get(); - _requestTracker.remove(); - if (req != requestInfo) - _complete(requestInfo); - _complete(req); - } - - private void _complete(RequestInfo requestInfo) - { - boolean shouldTrack = requestInfo != null && !requestInfo.isIgnored(); - if (requestInfo != null) - { - if (shouldTrack) - { - // Now that we're done, move it into the set of recent requests - _recentRequests.add(requestInfo); - trimOlderRequests(); - if (requestInfo.getUser() != null) - addUnviewed(requestInfo.getUser(), requestInfo.getId()); - } - else - { - // Remove it from the list of unviewed requests - if (requestInfo.getUser() != null) - setViewed(requestInfo.getUser(), requestInfo.getId()); - } - } - } - - private void trimOlderRequests() - { - if (_recentRequests.size() > MAX_TRACKED_REQUESTS) - { - List reqs = _recentRequests.subList(0, _recentRequests.size() - MAX_TRACKED_REQUESTS); - for (RequestInfo r : reqs) - r.cancel(); - reqs.clear(); - } - } - - private void addUnviewed(Principal user, long id) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return; - - HttpSession session = context.getSession(); - if (session == null) - return; - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - unviewed.add(id); - if (unviewed.size() > MAX_UNVIEWED) - { - unviewed.subList(0, unviewed.size() - MAX_UNVIEWED).clear(); - } - } - } - - public List getUnviewed(Principal user) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return Collections.emptyList(); - - HttpSession session = context.getSession(); - if (session == null) - return Collections.emptyList(); - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - - return new LongArrayList(unviewed); - } - } - - public void setViewed(Principal user, long id) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return; - - HttpSession session = context.getSession(); - if (session == null) - return; - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - - unviewed.remove(id); - } - } - - @Nullable - public synchronized RequestInfo getRequest(long id) - { - // search recent requests backwards looking for the matching id - for (int i = _recentRequests.size() - 1; i > 0; i--) - { - RequestInfo req = _recentRequests.get(i); - if (req.getId() == id) - return req; - } - - return null; - } - - public boolean put(Object object) - { - assert _put(object); - return true; - } - - public boolean remove(Object object) - { - assert _remove(object); - return true; - } - - public void register(MemTrackerListener generator) - { - assert _listeners.add(generator); - } - - public void unregister(MemTrackerListener queue) - { - assert _listeners.remove(queue); - } - - public Set beforeReport() - { - Set ignorableReferences = Collections.newSetFromMap(new IdentityHashMap<>()); - - for (MemTrackerListener generator : _instance._listeners) - generator.beforeReport(ignorableReferences); - - return ignorableReferences; - } - - // Filters out threads that should be ignored (never displayed as an "Active Thread" on the Memory Usage page) - public interface ThreadFilter - { - // Return true to instruct the Memory Usage page to never display this thread as an "Active Thread" - boolean ignore(Thread thread); - } - - private final List _threadFilters = new CopyOnWriteArrayList<>(); - - public void register(ThreadFilter filter) - { - _threadFilters.add(filter); - } - - public boolean shouldDisplay(Thread thread) - { - return _threadFilters.stream().noneMatch(threadFilter -> threadFilter.ignore(thread)); - } - - // - // reference tracking impl - // - - private final Map _references = new ReferenceIdentityMap<>(ReferenceStrength.WEAK, ReferenceStrength.HARD, true); - private final List _listeners = new CopyOnWriteArrayList<>(); - - private synchronized boolean _put(Object object) - { - if (object != null) - _references.put(object, new AllocationInfo()); - MiniProfiler.addObject(object); - return true; - } - - private synchronized boolean _remove(Object object) - { - if (object != null) - _references.remove(object); - return true; - } - - public synchronized List getReferences() - { - List refs = new ArrayList<>(_references.size()); - for (Map.Entry entry : _references.entrySet()) - { - // get a hard reference so we know that we're placing an actual object into our list: - Object obj = entry.getKey(); - if (obj != null) - refs.add(new HeldReference(entry.getKey(), entry.getValue())); - } - refs.sort(Comparator.comparing(HeldReference::getClassName, String.CASE_INSENSITIVE_ORDER)); - return refs; - } - - public static class TestCase extends Assert - { - @Test - public void testIdentity() throws InvalidEmailException - { - MemTracker t = new MemTracker(); - - // test identity - Object a = "I'm me"; - t._put(a); - assertEquals(1, t.getReferences().size()); - t._put(a); - assertEquals(1, t.getReferences().size()); - - // Test with arbitrary class that implements equals() - Object b = new ValidEmail("test@test.com"); - Object c = new ValidEmail("test@test.com"); - assertNotSame(b, c); - assertEquals(b, c); - t._put(b); - assertEquals(2, t.getReferences().size()); - t._put(c); - assertEquals(3, t.getReferences().size()); - - List list = t.getReferences(); - for (HeldReference o : list) - { - assertTrue(o._reference == a || o._reference == b || o._reference == c); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.util; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpSession; +import org.apache.commons.collections4.map.AbstractReferenceMap.ReferenceStrength; +import org.apache.commons.collections4.map.ReferenceIdentityMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.RequestInfo; +import org.labkey.api.security.User; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.ValidEmail.InvalidEmailException; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; + +import java.security.Principal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Tracks objects that may be expensive, commonly allocated so that we know that they're not being held and creating + * a memory leak. Will not prevent the tracked objects from being garbage collected. + * User: brittp + * Date: Oct 27, 2005 + */ +public class MemTracker +{ + private static final MemTracker _instance = new MemTracker(); + + private final ThreadLocal _requestTracker = new ThreadLocal<>(); + private final List _recentRequests = new LinkedList<>(); + + private static final String UNVIEWED_KEY = "memtracker-unviewed-requests"; + private static final int MAX_UNVIEWED = 100; + + /** Only keep a short history of allocations for the most recent requests */ + private static final int MAX_TRACKED_REQUESTS = 500; + + public synchronized List getNewRequests(long requestId) + { + return _recentRequests.stream().filter(recentRequest -> recentRequest.getId() > requestId).toList(); + } + + static class AllocationInfo + { + @Nullable + private final StackTraceElement[] _stackTrace; + private final long _threadId; + private final long _allocTime; + + AllocationInfo() + { + this(MiniProfiler.getTroubleshootingStackTrace(), Thread.currentThread().getId(), HeartBeat.currentTimeMillis()); + } + + AllocationInfo(@Nullable StackTraceElement[] stackTrace, long threadId, long allocTime) + { + _stackTrace = stackTrace; + _threadId = threadId; + _allocTime = allocTime; + } + + public HtmlString getHtmlStack() + { + if (_stackTrace == null) + { + return HtmlString.of(MiniProfiler.NO_STACK_TRACE_AVAILABLE); + } + HtmlStringBuilder builder = HtmlStringBuilder.of(); + for (int i = 3; i < _stackTrace.length; i++) + { + String line = _stackTrace[i].toString(); + builder.append(line).append(HtmlString.BR).append("\r\n"); + if (line.contains("org.labkey.api.view.ViewServlet.service")) + break; + } + return builder.getHtmlString(); + } + + public long getThreadId() + { + return _threadId; + } + + public long getAllocationTime() + { + return _allocTime; + } + } + + + public static class HeldReference extends AllocationInfo + { + private final Object _reference; + + private HeldReference(Object held, AllocationInfo allocationInfo) + { + super(allocationInfo._stackTrace, allocationInfo._threadId, allocationInfo._allocTime); + _reference = held; + } + + + public String getClassName() + { + if (_reference instanceof Class) + return ((Class) _reference).getName(); + else + return _reference.getClass().getName(); + } + + public String getObjectSummary() + { + String desc = getObjectDescription(); + return desc.length() > 50 ? StringUtilsLabKey.leftSurrogatePairFriendly(desc, 50) + "..." : desc; + } + + public boolean hasShortSummary() + { + return getObjectDescription().length() > 50; + } + + public String getObjectDescription() + { + try + { + String toString = _reference.toString(); + if (toString == null) + return "null"; + return toString; + } + catch (Throwable e) + { + return "toString() failed: " + e.getClass().getName() + (e.getMessage() == null ? "" : (" - " + e.getMessage())); + } + } + + public Object getReference() + { + return _reference; + } + } + + public static MemTracker get() + { + return _instance; + } + + public static MemTracker getInstance() + { + return _instance; + } + + /** + * Create new RequestInfo for the current thread and request. + */ + @NotNull + public RequestInfo startProfiler(HttpServletRequest request, @Nullable String name) + { + String url = request.getRequestURI() + (request.getQueryString() == null ? "" : "?" + request.getQueryString()); + HttpSession session = request.getSession(false); + return startProfiler(url, request.getUserPrincipal(), name, session != null ? session.getId() : null); + } + + /** + * Create new RequestInfo for the current thread. + * Used for profiling background requests that will be merged into a parent profiler. + * @see #merge(RequestInfo) + */ + @NotNull + public RequestInfo startProfiler(@Nullable String name) + { + return startProfiler(null, null, name, null); + } + + /** + * Create new RequestInfo for the current thread and request. + */ + @NotNull + public synchronized RequestInfo startProfiler(String url, Principal user, @Nullable String name, @Nullable String sessionId) + { + RequestInfo req = new RequestInfo(url, user, name, sessionId); + if ((user instanceof User) && ((User) user).isSearchUser()) + req.setIgnored(true); + _requestTracker.set(req); + return req; + } + + @Nullable + public RequestInfo current() + { + return _requestTracker.get(); + } + + /** + * Finish the current profiling session and merge its results into the to RequestInfo. + * Unlike requestComplete, the current timing will not be added to the list of recent requests. + */ + public void merge(@NotNull RequestInfo to) + { + RequestInfo requestInfo = _requestTracker.get(); + if (requestInfo != null) + { + requestInfo.getRoot().stop(); + to.merge(requestInfo); + } + _requestTracker.remove(); + } + + /** + * Mark the current profiling session as ignored. Timings won't be collected. + */ + public synchronized void ignore() + { + RequestInfo requestInfo = _requestTracker.get(); + if (requestInfo != null) + requestInfo.setIgnored(true); + } + + /** + * Finish the current profiling session. + */ + public synchronized void requestComplete(RequestInfo req) + { + RequestInfo requestInfo = _requestTracker.get(); + _requestTracker.remove(); + if (req != requestInfo) + _complete(requestInfo); + _complete(req); + } + + private void _complete(RequestInfo requestInfo) + { + boolean shouldTrack = requestInfo != null && !requestInfo.isIgnored(); + if (requestInfo != null) + { + if (shouldTrack) + { + // Now that we're done, move it into the set of recent requests + _recentRequests.add(requestInfo); + trimOlderRequests(); + if (requestInfo.getUser() != null) + addUnviewed(requestInfo.getUser(), requestInfo.getId()); + } + else + { + // Remove it from the list of unviewed requests + if (requestInfo.getUser() != null) + setViewed(requestInfo.getUser(), requestInfo.getId()); + } + } + } + + private void trimOlderRequests() + { + if (_recentRequests.size() > MAX_TRACKED_REQUESTS) + { + List reqs = _recentRequests.subList(0, _recentRequests.size() - MAX_TRACKED_REQUESTS); + for (RequestInfo r : reqs) + r.cancel(); + reqs.clear(); + } + } + + private void addUnviewed(Principal user, long id) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return; + + HttpSession session = context.getSession(); + if (session == null) + return; + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + unviewed.add(id); + if (unviewed.size() > MAX_UNVIEWED) + { + unviewed.subList(0, unviewed.size() - MAX_UNVIEWED).clear(); + } + } + } + + public List getUnviewed(Principal user) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return Collections.emptyList(); + + HttpSession session = context.getSession(); + if (session == null) + return Collections.emptyList(); + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + + return new LongArrayList(unviewed); + } + } + + public void setViewed(Principal user, long id) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return; + + HttpSession session = context.getSession(); + if (session == null) + return; + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + + unviewed.remove(id); + } + } + + @Nullable + public synchronized RequestInfo getRequest(long id) + { + // search recent requests backwards looking for the matching id + for (int i = _recentRequests.size() - 1; i > 0; i--) + { + RequestInfo req = _recentRequests.get(i); + if (req.getId() == id) + return req; + } + + return null; + } + + public boolean put(Object object) + { + assert _put(object); + return true; + } + + public boolean remove(Object object) + { + assert _remove(object); + return true; + } + + public void register(MemTrackerListener generator) + { + assert _listeners.add(generator); + } + + public void unregister(MemTrackerListener queue) + { + assert _listeners.remove(queue); + } + + public Set beforeReport() + { + Set ignorableReferences = Collections.newSetFromMap(new IdentityHashMap<>()); + + for (MemTrackerListener generator : _instance._listeners) + generator.beforeReport(ignorableReferences); + + return ignorableReferences; + } + + // Filters out threads that should be ignored (never displayed as an "Active Thread" on the Memory Usage page) + public interface ThreadFilter + { + // Return true to instruct the Memory Usage page to never display this thread as an "Active Thread" + boolean ignore(Thread thread); + } + + private final List _threadFilters = new CopyOnWriteArrayList<>(); + + public void register(ThreadFilter filter) + { + _threadFilters.add(filter); + } + + public boolean shouldDisplay(Thread thread) + { + return _threadFilters.stream().noneMatch(threadFilter -> threadFilter.ignore(thread)); + } + + // + // reference tracking impl + // + + private final Map _references = new ReferenceIdentityMap<>(ReferenceStrength.WEAK, ReferenceStrength.HARD, true); + private final List _listeners = new CopyOnWriteArrayList<>(); + + private synchronized boolean _put(Object object) + { + if (object != null) + _references.put(object, new AllocationInfo()); + MiniProfiler.addObject(object); + return true; + } + + private synchronized boolean _remove(Object object) + { + if (object != null) + _references.remove(object); + return true; + } + + public synchronized List getReferences() + { + List refs = new ArrayList<>(_references.size()); + for (Map.Entry entry : _references.entrySet()) + { + // get a hard reference so we know that we're placing an actual object into our list: + Object obj = entry.getKey(); + if (obj != null) + refs.add(new HeldReference(entry.getKey(), entry.getValue())); + } + refs.sort(Comparator.comparing(HeldReference::getClassName, String.CASE_INSENSITIVE_ORDER)); + return refs; + } + + public static class TestCase extends Assert + { + @Test + public void testIdentity() throws InvalidEmailException + { + MemTracker t = new MemTracker(); + + // test identity + Object a = "I'm me"; + t._put(a); + assertEquals(1, t.getReferences().size()); + t._put(a); + assertEquals(1, t.getReferences().size()); + + // Test with arbitrary class that implements equals() + Object b = new ValidEmail("test@test.com"); + Object c = new ValidEmail("test@test.com"); + assertNotSame(b, c); + assertEquals(b, c); + t._put(b); + assertEquals(2, t.getReferences().size()); + t._put(c); + assertEquals(3, t.getReferences().size()); + + List list = t.getReferences(); + for (HeldReference o : list) + { + assertTrue(o._reference == a || o._reference == b || o._reference == c); + } + } + } +} diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java index 91469d21dbe..7e80a2b2af3 100644 --- a/core/src/org/labkey/core/admin/AdminController.java +++ b/core/src/org/labkey/core/admin/AdminController.java @@ -1,12273 +1,12273 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core.admin; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.util.concurrent.UncheckedExecutionException; -import jakarta.mail.MessagingException; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.map.LRUMap; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlOptions; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.jfree.chart.ChartFactory; -import org.jfree.chart.ChartUtilities; -import org.jfree.chart.JFreeChart; -import org.jfree.chart.plot.PlotOrientation; -import org.jfree.data.category.DefaultCategoryDataset; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.BaseApiAction; -import org.labkey.api.action.BaseViewAction; -import org.labkey.api.action.ConfirmAction; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasViewContext; -import org.labkey.api.action.IgnoresAllocationTracking; -import org.labkey.api.action.LabKeyError; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.QueryViewAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AbstractFolderContext.ExportType; -import org.labkey.api.admin.AdminBean; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.FolderWriter; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.HealthCheck; -import org.labkey.api.admin.HealthCheckRegistry; -import org.labkey.api.admin.ImportOptions; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.admin.TableXmlUtils; -import org.labkey.api.admin.sitevalidation.SiteValidationResult; -import org.labkey.api.admin.sitevalidation.SiteValidationResultList; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.cache.CacheStats; -import org.labkey.api.cache.TrackingCache; -import org.labkey.api.cloud.CloudStoreService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.compliance.ComplianceFolderSettings; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.compliance.PhiColumnBehavior; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.ConnectionWrapper; -import org.labkey.api.data.Container; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DatabaseTableType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MenuButton; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.NormalContainerType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TransactionFilter; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; -import org.labkey.api.data.queryprofiler.QueryProfiler; -import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.files.FileContentService; -import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; -import org.labkey.api.message.settings.MessageConfigService.NotificationOption; -import org.labkey.api.message.settings.MessageConfigService.UserPreference; -import org.labkey.api.miniprofiler.RequestInfo; -import org.labkey.api.module.AllowedBeforeInitialUserIsSet; -import org.labkey.api.module.AllowedDuringUpgrade; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.IgnoresForbiddenProjectCheck; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.ModuleLoader.SchemaActions; -import org.labkey.api.module.ModuleLoader.SchemaAndModule; -import org.labkey.api.module.SimpleModule; -import org.labkey.api.moduleeditor.api.ModuleEditorService; -import org.labkey.api.pipeline.DirectoryNotDeletedException; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.PipelineStatusUrls; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.pipeline.view.SetupForm; -import org.labkey.api.products.ProductRegistry; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RuntimeValidationException; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.ExternalScriptEngineDefinition; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.CSRF; -import org.labkey.api.security.Directive; -import org.labkey.api.security.Group; -import org.labkey.api.security.GroupManager; -import org.labkey.api.security.IgnoresTermsOfUse; -import org.labkey.api.security.LoginUrls; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.RequiresLogin; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.RoleAssignment; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicy; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.SecurityUrls; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; -import org.labkey.api.security.impersonation.ImpersonationContext; -import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; -import org.labkey.api.security.impersonation.UserImpersonationContextFactory; -import org.labkey.api.security.permissions.AbstractActionPermissionTest; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.SiteAdminPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.permissions.UploadFileBasedModulePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.security.roles.FolderAdminRole; -import org.labkey.api.security.roles.ProjectAdminRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.security.roles.SharedViewEditorRole; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.ConceptURIProperties; -import org.labkey.api.settings.DateParsingMode; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; -import org.labkey.api.settings.NetworkDriveProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.settings.OptionalFeatureService.FeatureType; -import org.labkey.api.settings.ProductConfiguration; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; -import org.labkey.api.settings.WriteableLookAndFeelProperties; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.DOM; -import org.labkey.api.util.DOM.Renderable; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.DebugInfoDumper; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.FolderDisplayMode; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.HttpsUtil; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.MailHelper; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.MemTracker.HeldReference; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.SafeToRenderEnum; -import org.labkey.api.util.SessionAppender; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; -import org.labkey.api.util.SystemMaintenanceJob; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.Tuple3; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UniqueID; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.util.emailTemplate.EmailTemplate; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.DataView; -import org.labkey.api.view.FolderManagement.FolderManagementViewAction; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; -import org.labkey.api.view.FolderManagement.TYPE; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.Portal; -import org.labkey.api.view.RedirectException; -import org.labkey.api.view.ShortURLRecord; -import org.labkey.api.view.ShortURLService; -import org.labkey.api.view.TabStripView; -import org.labkey.api.view.URLException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.EmptyView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.view.template.PageConfig.Template; -import org.labkey.api.wiki.WikiRendererType; -import org.labkey.api.wiki.WikiRenderingService; -import org.labkey.api.writer.FileSystemFile; -import org.labkey.api.writer.HtmlWriter; -import org.labkey.api.writer.ZipFile; -import org.labkey.api.writer.ZipUtil; -import org.labkey.bootstrap.ExplodedModuleService; -import org.labkey.core.admin.miniprofiler.MiniProfilerController; -import org.labkey.core.admin.sitevalidation.SiteValidationJob; -import org.labkey.core.admin.sql.SqlScriptController; -import org.labkey.core.login.LoginController; -import org.labkey.core.portal.CollaborationFolderType; -import org.labkey.core.portal.ProjectController; -import org.labkey.core.query.CoreQuerySchema; -import org.labkey.core.query.PostgresUserSchema; -import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; -import org.labkey.core.security.AllowedExternalResourceHosts; -import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; -import org.labkey.core.security.BlockListFilter; -import org.labkey.core.security.SecurityController; -import org.labkey.data.xml.TablesDocument; -import org.labkey.filters.ContentSecurityPolicyFilter; -import org.labkey.security.xml.GroupEnumType; -import org.labkey.vfs.FileLike; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.mvc.Controller; - -import java.awt.*; -import java.beans.Introspector; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.lang.management.BufferPoolMXBean; -import java.lang.management.ClassLoadingMXBean; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryType; -import java.lang.management.MemoryUsage; -import java.lang.management.OperatingSystemMXBean; -import java.lang.management.RuntimeMXBean; -import java.lang.management.ThreadMXBean; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.SQLException; -import java.text.DecimalFormat; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; -import static org.labkey.api.util.DOM.A; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.Attribute.method; -import static org.labkey.api.util.DOM.Attribute.name; -import static org.labkey.api.util.DOM.Attribute.style; -import static org.labkey.api.util.DOM.Attribute.title; -import static org.labkey.api.util.DOM.Attribute.type; -import static org.labkey.api.util.DOM.Attribute.value; -import static org.labkey.api.util.DOM.BR; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.LI; -import static org.labkey.api.util.DOM.SPAN; -import static org.labkey.api.util.DOM.STYLE; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.UL; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.api.util.DOM.createHtmlFragment; -import static org.labkey.api.util.HtmlString.NBSP; -import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; -import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; -import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; -import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; -import static org.labkey.api.view.FolderManagement.NOT_ROOT; -import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; -import static org.labkey.api.view.FolderManagement.ROOT; -import static org.labkey.api.view.FolderManagement.addTab; - -public class AdminController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( - AdminController.class, - FileListAction.class, - FilesSiteSettingsAction.class, - UpdateFilePathsAction.class - ); - - private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); - private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); - private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; - - private static long _errorMark = 0; - private static long _primaryLogMark = 0; - - public static void registerAdminConsoleLinks() - { - Container root = ContainerManager.getRoot(); - - // Configuration - AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); - AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); - if (!ProductRegistry.getProducts().isEmpty()) - AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); - // TODO move to FileContentModule - if (ModuleLoader.getInstance().hasModule("FileContent")) - AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); - AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); - AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); - AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); - AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); - - // Diagnostics - AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); - AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); - AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); - AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); - AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); - AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); - - if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); - AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); - AdminConsole.addLink(Diagnostics, "postgres table sizes", new ActionURL(PostgresTableSizesAction.class, root)); - } - - AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); - AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); - AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); - AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); - AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); - AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); - AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); - AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); - } - - public static void registerManagementTabs() - { - addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); - addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); - addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); - addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { - if (!c.isRoot()) - { - // Show module properties tab only if a module w/ properties to set is present for current folder - for (Module m : c.getActiveModules()) - if (!m.getModuleProperties().isEmpty()) - return true; - } - - return false; - }, ModulePropertiesAction.class); - addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { - // Show Concepts tab only if the experiment module is enabled in this container - return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); - }, AdminController.ConceptsAction.class); - // Show Notifications tab only if we have registered notification providers - addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); - addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); - addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); - addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); - addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); - addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); - addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); - - addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); - addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); - addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); - addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); - - addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); - addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); - } - - public AdminController() - { - setActionResolver(_actionResolver); - } - - @RequiresNoPermission - public static class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getShowAdminURL(); - } - } - - private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) - { - addAdminNavTrail(root, childTitle, action, getContainer()); - } - - private static void addAdminNavTrail(NavTree root, @NotNull Container container) - { - if (container.isRoot()) - root.addChild("Admin Console", getShowAdminURL().setFragment("links")); - } - - private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - addAdminNavTrail(root, container); - root.addChild(childTitle, new ActionURL(action, container)); - } - - public static ActionURL getShowAdminURL() - { - return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); - } - - @Override - protected void beforeAction(Controller action) throws ServletException - { - super.beforeAction(action); - if (action instanceof BaseViewAction viewaction) - viewaction.getPageConfig().setRobotsNone(); - } - - @AdminConsoleAction - public static class ShowAdminAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/admin.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); - if (null != returnUrl) - root.addChild("Return to Project", returnUrl); - root.addChild("Admin Console"); - setHelpTopic("siteManagement"); - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ShowModuleErrorsAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Module Errors", this.getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); - } - } - - public static class AdminUrlsImpl implements AdminUrls - { - @Override - public ActionURL getModuleErrorsURL() - { - return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getAdminConsoleURL() - { - return getShowAdminURL(); - } - - @Override - public ActionURL getModuleStatusURL(URLHelper returnUrl) - { - return AdminController.getModuleStatusURL(returnUrl); - } - - @Override - public ActionURL getCustomizeSiteURL() - { - return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) - { - ActionURL url = getCustomizeSiteURL(); - - if (upgradeInProgress) - url.addParameter("upgradeInProgress", "1"); - - return url; - } - - @Override - public ActionURL getProjectSettingsURL(Container c) - { - return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - ActionURL getLookAndFeelResourcesURL(Container c) - { - return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsMenuURL(Container c) - { - return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsFileURL(Container c) - { - return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) - { - return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); - } - - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) - { - ActionURL url = new ActionURL(CustomizeEmailAction.class, c); - if (selectedTemplate != null) - { - url.addParameter("templateClass", selectedTemplate); - } - if (returnUrl != null) - { - url.addReturnUrl(returnUrl); - } - return url; - } - - public ActionURL getResetLookAndFeelPropertiesURL(Container c) - { - return new ActionURL(ResetPropertiesAction.class, c); - } - - @Override - public ActionURL getMaintenanceURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - @Override - public ActionURL getModulesDetailsURL() - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getDeleteModuleURL(String moduleName) - { - return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); - } - - @Override - public ActionURL getManageFoldersURL(Container c) - { - return new ActionURL(ManageFoldersAction.class, c); - } - - @Override - public ActionURL getFolderTypeURL(Container c) - { - return new ActionURL(FolderTypeAction.class, c); - } - - @Override - public ActionURL getExportFolderURL(Container c) - { - return new ActionURL(ExportFolderAction.class, c); - } - - @Override - public ActionURL getImportFolderURL(Container c) - { - return new ActionURL(ImportFolderAction.class, c); - } - - @Override - public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) - { - return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); - } - - @Override - public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) - { - ActionURL result = new ActionURL(CreateFolderAction.class, c); - if (returnUrl != null) - { - result.addReturnUrl(returnUrl); - } - return result; - } - - public ActionURL getSetFolderPermissionsURL(Container c) - { - return new ActionURL(SetFolderPermissionsAction.class, c); - } - - @Override - public void addAdminNavTrail(NavTree root, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, container); - } - - @Override - public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, childTitle, action, container); - } - - @Override - public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) - { - if (container.isRoot()) - addAdminNavTrail(root, "Modules", ModulesAction.class, container); - - root.addChild(childTitle); - } - - @Override - public ActionURL getFileRootsURL(Container c) - { - return new ActionURL(FileRootsAction.class, c); - } - - @Override - public ActionURL getLookAndFeelSettingsURL(Container c) - { - if (c.isRoot()) - return getSiteLookAndFeelSettingsURL(); - else if (c.isProject()) - return getProjectSettingsURL(c); - else - return getFolderSettingsURL(c); - } - - @Override - public ActionURL getSiteLookAndFeelSettingsURL() - { - return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFolderSettingsURL(Container c) - { - return new ActionURL(FolderSettingsAction.class, c); - } - - @Override - public ActionURL getNotificationsURL(Container c) - { - return new ActionURL(NotificationsAction.class, c); - } - - @Override - public ActionURL getModulePropertiesURL(Container c) - { - return new ActionURL(ModulePropertiesAction.class, c); - } - - @Override - public ActionURL getMissingValuesURL(Container c) - { - return new ActionURL(MissingValuesAction.class, c); - } - - public ActionURL getInitialFolderSettingsURL(Container c) - { - return new ActionURL(SetInitialFolderSettingsAction.class, c); - } - - @Override - public ActionURL getMemTrackerURL() - { - return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFilesSiteSettingsURL() - { - return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSessionLoggingURL() - { - return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getTrackedAllocationsViewerURL() - { - return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSystemMaintenanceURL() - { - return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); - } - - public static ActionURL getDeprecatedFeaturesURL() - { - return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); - } - } - - public static class MaintenanceBean - { - public HtmlString content; - public ActionURL loginURL; - } - - /** - * During upgrade, startup, or maintenance mode, the user will be redirected to - * MaintenanceAction and only admin users will be allowed to log into the server. - * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off - * and will redirect to the returnUrl or the loginURL. - * See Issue 18758 for more information. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class MaintenanceAction extends SimpleViewAction - { - private String _title = "Maintenance in progress"; - - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - if (!getUser().hasSiteAdminPermission()) - { - getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); - } - getPageConfig().setTemplate(Template.Dialog); - - boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); - boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); - boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); - - HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); - if (upgradeInProgress) - { - _title = "Upgrade in progress"; - content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (startupInProgress) - { - _title = "Startup in progress"; - content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (maintenanceMode) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); - } - - if (content == null) - content = HtmlString.of(_title); - - ActionURL loginURL = null; - if (getUser().isGuest()) - { - URLHelper returnUrl = form.getReturnUrlHelper(); - if (returnUrl != null) - loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); - else - loginURL = urlProvider(LoginUrls.class).getLoginURL(); - } - - MaintenanceBean bean = new MaintenanceBean(); - bean.content = content; - bean.loginURL = loginURL; - - JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); - view.setTitle(_title); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild(_title); - } - } - - /** - * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class StartupStatusAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); - result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); - - return new ApiSimpleResponse(result); - } - } - - @RequiresSiteAdmin - @IgnoresTermsOfUse - public static class GetPendingRequestCountAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); - - return new ApiSimpleResponse(result); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetModulesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(GetModulesForm form, BindException errors) - { - Container c = ContainerManager.getForPath(getContainer().getPath()); - - ApiSimpleResponse response = new ApiSimpleResponse(); - - List> qinfos = new ArrayList<>(); - - FolderType folderType = c.getFolderType(); - List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); - - //note: this has been altered to use Container.getRequiredModules() instead of FolderType - //this is b/c a parent container must consider child workbooks when determining the set of requiredModules - Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); - Set activeModules = c.getActiveModules(getUser()); - - for (Module m : allModules) - { - Map qinfo = new HashMap<>(); - - qinfo.put("name", m.getName()); - qinfo.put("required", requiredModules.contains(m)); - qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); - qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || - m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); - qinfo.put("tabName", m.getTabName(getViewContext())); - qinfo.put("requireSitePermission", m.getRequireSitePermission()); - qinfos.add(qinfo); - } - - response.put("modules", qinfos); - response.put("folderType", folderType.getName()); - - return response; - } - } - - public static class GetModulesForm - { - } - - @RequiresNoPermission - @AllowedDuringUpgrade - // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress - public static class GuidAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - response.getWriter().write(GUID.makeGUID()); - } - } - - /** - * Preform health checks corresponding to the given categories. - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class HealthCheckAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception - { - if (!ModuleLoader.getInstance().isStartupComplete()) - return new ApiSimpleResponse("healthy", false); - - Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); - HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); - - checkResult.getDetails().put("healthy", checkResult.isHealthy()); - - if (getUser().hasRootAdminPermission()) - { - return new ApiSimpleResponse(checkResult.getDetails()); - } - else - { - if (!checkResult.isHealthy()) - { - try (var writer = createResponseWriter()) - { - writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); - } - return null; - } - - return new ApiSimpleResponse("healthy", checkResult.isHealthy()); - } - } - } - - public static class HealthCheckForm - { - private String _categories; // if null, all categories will be checked. - - public String getCategories() - { - return _categories; - } - - @SuppressWarnings("unused") - public void setCategories(String categories) - { - _categories = categories; - } - } - - // No security checks... anyone (even guests) can view the credits page - @RequiresNoPermission - public class CreditsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - VBox views = new VBox(); - List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); - - addCreditsViews(views, modules, "jars.txt", "JAR"); - addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); - addCreditsViews(views, modules, "source.txt", "Java Source Code"); - addCreditsViews(views, modules, "executables.txt", "Executable"); - - return views; - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Credits", this.getClass()); - } - } - - private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException - { - for (Module module : modules) - { - String wikiSource = getCreditsFile(module, creditsFile); - - if (null != wikiSource) - { - String title = fileType + " Files Distributed with the " + module.getName() + " Module"; - CreditsView credits = new CreditsView(wikiSource, title); - views.addView(credits); - } - } - } - - private static class CreditsView extends WebPartView - { - private Renderable _html; - - CreditsView(@Nullable String wikiSource, String title) - { - super(title); - - wikiSource = StringUtils.trimToEmpty(wikiSource); - - if (StringUtils.isNotEmpty(wikiSource)) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); - _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); - } - } - - @Override - public void renderView(Object model, HtmlWriter out) - { - out.write(_html); - } - } - - private static String getCreditsFile(Module module, String filename) throws IOException - { - // credits files are in /resources/credits - InputStream is = module.getResourceStream("credits/" + filename); - - return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); - } - - private void validateNetworkDrive(NetworkDriveForm form, Errors errors) - { - if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || - isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) - { - errors.reject(ERROR_MSG, "All fields are required"); - } - else if (form.getNetworkDriveLetter().trim().length() > 1) - { - errors.reject(ERROR_MSG, "Network drive letter must be a single character"); - } - else - { - char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); - - if (letter < 'a' || letter > 'z') - { - errors.reject(ERROR_MSG, "Network drive letter must be a letter"); - } - } - } - - public static class ResourceForm - { - private String _resource; - - public String getResource() - { - return _resource; - } - - public void setResource(String resource) - { - _resource = resource; - } - - public ResourceType getResourceType() - { - return ResourceType.valueOf(_resource); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetResourceAction extends FormHandlerAction - { - @Override - public void validateCommand(ResourceForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ResourceForm form, BindException errors) throws Exception - { - form.getResourceType().delete(getContainer(), getUser()); - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - return true; - } - - @Override - public URLHelper getSuccessURL(ResourceForm form) - { - return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetPropertiesAction extends FormHandlerAction - { - private URLHelper _returnUrl; - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - Container c = getContainer(); - boolean folder = !(c.isRoot() || c.isProject()); - boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); - - WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); - props.clear(hasAdminOpsPerm); - props.save(); - // TODO: Audit log? - - AdminUrls urls = new AdminUrlsImpl(); - - // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision - if (!folder) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - _returnUrl = urls.getLookAndFeelSettingsURL(c); - - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return _returnUrl; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class CustomizeSiteAction extends FormViewAction - { - @Override - public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) - { - if (form.isUpgradeInProgress()) - getPageConfig().setTemplate(Template.Dialog); - - SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); - setHelpTopic("configAdmin"); - return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Customize Site", this.getClass()); - } - - @Override - public void validateCommand(SiteSettingsForm form, Errors errors) - { - if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) - { - errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); - } - if (form.getMaxBLOBSize() < 0) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); - } - int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); - if (form.getMaxBLOBSize() > hardCap) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); - } - if (form.getSslPort() < 1 || form.getSslPort() > 65535) - { - errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); - } - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); - } - if (form.getMemoryUsageDumpInterval() < 0) - { - errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); - } - } - - @Override - public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception - { - HttpServletRequest request = getViewContext().getRequest(); - - // We only need to check that SSL is running if the user isn't already using SSL - if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) - { - URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); - Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); - - if (sslResponse != null) - { - errors.reject(ERROR_MSG, sslResponse.first); - return false; - } - } - - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); - } - - WriteableAppProps props = AppProps.getWriteableInstance(); - - props.setPipelineToolsDir(form.getPipelineToolsDirectory()); - props.setNavAccessOpen(form.isNavAccessOpen()); - props.setSSLRequired(form.isSslRequired()); - boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); - props.setSSLPort(form.getSslPort()); - props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); - props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); - props.setMaxBLOBSize(form.getMaxBLOBSize()); - props.setExt3Required(form.isExt3Required()); - props.setExt3APIRequired(form.isExt3APIRequired()); - props.setSelfReportExceptions(form.isSelfReportExceptions()); - - props.setAdminOnlyMessage(form.getAdminOnlyMessage()); - props.setShowRibbonMessage(form.isShowRibbonMessage()); - props.setRibbonMessage(form.getRibbonMessage()); - props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); - - props.setAllowApiKeys(form.isAllowApiKeys()); - props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); - props.setAllowSessionKeys(form.isAllowSessionKeys()); - - try - { - ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); - props.setExceptionReportingLevel(level); - } - catch (IllegalArgumentException ignored) - { - } - - try - { - if (form.getUsageReportingLevel() != null) - { - UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); - props.setUsageReportingLevel(level); - } - } - catch (IllegalArgumentException ignored) - { - } - - props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); - - if (null != form.getBaseServerURL()) - { - if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); - return false; - } - - try - { - props.setBaseServerUrl(form.getBaseServerURL()); - } - catch (URISyntaxException e) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + - "Please enter a valid base URL containing the protocol, hostname, and port if required. " + - "The webapp context path should not be included. " + - "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); - return false; - } - } - - String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); - if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) - { - errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); - return false; - } - props.setXFrameOption(frameOption); - props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); - - props.save(getViewContext().getUser()); - UsageReportingLevel.reportNow(); - if (sslSettingChanged) - ContentSecurityPolicyFilter.regenerateSubstitutionMap(); - - return true; - } - - @Override - public ActionURL getSuccessURL(SiteSettingsForm form) - { - if (form.isUpgradeInProgress()) - { - return AppProps.getInstance().getHomePageActionURL(); - } - else - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - } - - public static class NetworkDriveForm - { - private String _networkDriveLetter; - private String _networkDrivePath; - private String _networkDriveUser; - private String _networkDrivePassword; - - public String getNetworkDriveLetter() - { - return _networkDriveLetter; - } - - public void setNetworkDriveLetter(String networkDriveLetter) - { - _networkDriveLetter = networkDriveLetter; - } - - public String getNetworkDrivePassword() - { - return _networkDrivePassword; - } - - public void setNetworkDrivePassword(String networkDrivePassword) - { - _networkDrivePassword = networkDrivePassword; - } - - public String getNetworkDrivePath() - { - return _networkDrivePath; - } - - public void setNetworkDrivePath(String networkDrivePath) - { - _networkDrivePath = networkDrivePath; - } - - public String getNetworkDriveUser() - { - return _networkDriveUser; - } - - public void setNetworkDriveUser(String networkDriveUser) - { - _networkDriveUser = networkDriveUser; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @AdminConsoleAction - public class MapNetworkDriveAction extends FormViewAction - { - @Override - public void validateCommand(NetworkDriveForm form, Errors errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); - } - - @Override - public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception - { - NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); - NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); - NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); - NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); - - return true; - } - - @Override - public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) - { - return new ActionURL(FilesSiteSettingsAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("setRoots#map"); - addAdminNavTrail(root, "Map Network Drive", this.getClass()); - } - } - - public static class SiteSettingsBean - { - public final boolean _upgradeInProgress; - public final boolean _showSelfReportExceptions; - - private SiteSettingsBean(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); - } - - public HtmlString getSiteSettingsHelpLink(String fragment) - { - return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); - } - } - - public static class SetRibbonMessageForm - { - private Boolean _show = null; - private String _message = null; - - public Boolean isShow() - { - return _show; - } - - public void setShow(Boolean show) - { - _show = show; - } - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class SetRibbonMessageAction extends MutatingApiAction - { - @Override - public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception - { - if (form.isShow() != null || form.getMessage() != null) - { - WriteableAppProps props = AppProps.getWriteableInstance(); - - if (form.isShow() != null) - props.setShowRibbonMessage(form.isShow()); - - if (form.getMessage() != null) - props.setRibbonMessage(form.getMessage()); - - props.save(getViewContext().getUser()); - } - - return null; - } - } - - @RequiresPermission(AdminPermission.class) - public class ConfigureSiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - public static class SiteValidationForm - { - private List _providers; - private boolean _includeSubfolders = false; - private transient Consumer _logger = s -> { - }; // No-op by default - - public List getProviders() - { - return _providers; - } - - public void setProviders(List providers) - { - _providers = providers; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public Consumer getLogger() - { - return _logger; - } - - public void setLogger(Consumer logger) - { - _logger = logger; - } - } - - @RequiresPermission(AdminPermission.class) - public class SiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(SiteValidationForm form, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class SiteValidationBackgroundAction extends FormHandlerAction - { - private ActionURL _redirectUrl; - - @Override - public void validateCommand(SiteValidationForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException - { - ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - SiteValidationJob job = new SiteValidationJob(vbi, root, form); - PipelineService.get().queueJob(job); - String jobGuid = job.getJobGUID(); - - if (null == jobGuid) - throw new NotFoundException("Unable to determine pipeline job GUID"); - - Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - if (null == jobId) - throw new NotFoundException("Unable to determine pipeline job ID"); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _redirectUrl = urls.urlDetails(getContainer(), jobId); - - return true; - } - - @Override - public URLHelper getSuccessURL(SiteValidationForm form) - { - return _redirectUrl; - } - } - - public static class ViewValidationResultsForm - { - private int _rowId; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - } - - @RequiresPermission(AdminPermission.class) - public class ViewValidationResultsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception - { - PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); - if (null == statusFile) - throw new NotFoundException("Status file not found"); - if (!getContainer().equals(statusFile.lookupContainer())) - throw new UnauthorizedException("Wrong container"); - - String logFilePath = statusFile.getFilePath(); - String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; - File htmlFile = new File(htmlFilePath); - - if (!htmlFile.exists()) - throw new NotFoundException("Results file not found"); - return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); - } - } - - public interface FileManagementForm - { - String getFolderRootPath(); - - void setFolderRootPath(String folderRootPath); - - String getFileRootOption(); - - void setFileRootOption(String fileRootOption); - - String getConfirmMessage(); - - void setConfirmMessage(String confirmMessage); - - boolean isDisableFileSharing(); - - boolean hasSiteDefaultRoot(); - - String[] getEnabledCloudStore(); - - @SuppressWarnings("unused") - void setEnabledCloudStore(String[] enabledCloudStore); - - boolean isCloudFileRoot(); - - @Nullable - String getCloudRootName(); - - void setCloudRootName(String cloudRootName); - - void setFileRootChanged(boolean changed); - - void setEnabledCloudStoresChanged(boolean changed); - - String getMigrateFilesOption(); - - void setMigrateFilesOption(String migrateFilesOption); - - default boolean isFolderSetup() - { - return false; - } - } - - public enum MigrateFilesOption implements SafeToRenderEnum - { - leave - { - @Override - public String description() - { - return "Source files not copied or moved"; - } - }, - copy - { - @Override - public String description() - { - return "Copy source files to destination"; - } - }, - move - { - @Override - public String description() - { - return "Move source files to destination"; - } - }; - - public abstract String description(); - } - - public static class ProjectSettingsForm extends FolderSettingsForm - { - // Site-only properties - private String _dateParsingMode; - private String _customWelcome; - - // Site & project properties - private boolean _shouldInherit; // new subfolders should inherit parent permissions - private String _systemDescription; - private boolean _systemDescriptionInherited; - private String _systemShortName; - private boolean _systemShortNameInherited; - private String _themeName; - private boolean _themeNameInherited; - private String _folderDisplayMode; - private boolean _folderDisplayModeInherited; - private String _applicationMenuDisplayMode; - private boolean _applicationMenuDisplayModeInherited; - private boolean _helpMenuEnabled; - private boolean _helpMenuEnabledInherited; - private String _logoHref; - private boolean _logoHrefInherited; - private String _companyName; - private boolean _companyNameInherited; - private String _systemEmailAddress; - private boolean _systemEmailAddressInherited; - private String _reportAProblemPath; - private boolean _reportAProblemPathInherited; - private String _supportEmail; - private boolean _supportEmailInherited; - private String _customLogin; - private boolean _customLoginInherited; - - // Site-only properties - - public String getDateParsingMode() - { - return _dateParsingMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setDateParsingMode(String dateParsingMode) - { - _dateParsingMode = dateParsingMode; - } - - public String getCustomWelcome() - { - return _customWelcome; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomWelcome(String customWelcome) - { - _customWelcome = customWelcome; - } - - // Site & project properties - - public boolean getShouldInherit() - { - return _shouldInherit; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setShouldInherit(boolean b) - { - _shouldInherit = b; - } - - public String getSystemDescription() - { - return _systemDescription; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescription(String systemDescription) - { - _systemDescription = systemDescription; - } - - public boolean isSystemDescriptionInherited() - { - return _systemDescriptionInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescriptionInherited(boolean systemDescriptionInherited) - { - _systemDescriptionInherited = systemDescriptionInherited; - } - - public String getSystemShortName() - { - return _systemShortName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortName(String systemShortName) - { - _systemShortName = systemShortName; - } - - public boolean isSystemShortNameInherited() - { - return _systemShortNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortNameInherited(boolean systemShortNameInherited) - { - _systemShortNameInherited = systemShortNameInherited; - } - - public String getThemeName() - { - return _themeName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeName(String themeName) - { - _themeName = themeName; - } - - public boolean isThemeNameInherited() - { - return _themeNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeNameInherited(boolean themeNameInherited) - { - _themeNameInherited = themeNameInherited; - } - - public String getFolderDisplayMode() - { - return _folderDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayMode(String folderDisplayMode) - { - _folderDisplayMode = folderDisplayMode; - } - - public boolean isFolderDisplayModeInherited() - { - return _folderDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) - { - _folderDisplayModeInherited = folderDisplayModeInherited; - } - - public String getApplicationMenuDisplayMode() - { - return _applicationMenuDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayMode(String displayMode) - { - _applicationMenuDisplayMode = displayMode; - } - - public boolean isApplicationMenuDisplayModeInherited() - { - return _applicationMenuDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) - { - _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; - } - - public boolean isHelpMenuEnabled() - { - return _helpMenuEnabled; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabled(boolean helpMenuEnabled) - { - _helpMenuEnabled = helpMenuEnabled; - } - - public boolean isHelpMenuEnabledInherited() - { - return _helpMenuEnabledInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) - { - _helpMenuEnabledInherited = helpMenuEnabledInherited; - } - - public String getLogoHref() - { - return _logoHref; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHref(String logoHref) - { - _logoHref = logoHref; - } - - public boolean isLogoHrefInherited() - { - return _logoHrefInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHrefInherited(boolean logoHrefInherited) - { - _logoHrefInherited = logoHrefInherited; - } - - public String getReportAProblemPath() - { - return _reportAProblemPath; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPath(String reportAProblemPath) - { - _reportAProblemPath = reportAProblemPath; - } - - public boolean isReportAProblemPathInherited() - { - return _reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) - { - _reportAProblemPathInherited = reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmail(String supportEmail) - { - _supportEmail = supportEmail; - } - - public String getSupportEmail() - { - return _supportEmail; - } - - public boolean isSupportEmailInherited() - { - return _supportEmailInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmailInherited(boolean supportEmailInherited) - { - _supportEmailInherited = supportEmailInherited; - } - - public String getSystemEmailAddress() - { - return _systemEmailAddress; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddress(String systemEmailAddress) - { - _systemEmailAddress = systemEmailAddress; - } - - public boolean isSystemEmailAddressInherited() - { - return _systemEmailAddressInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) - { - _systemEmailAddressInherited = systemEmailAddressInherited; - } - - public String getCompanyName() - { - return _companyName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyName(String companyName) - { - _companyName = companyName; - } - - public boolean isCompanyNameInherited() - { - return _companyNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyNameInherited(boolean companyNameInherited) - { - _companyNameInherited = companyNameInherited; - } - - public String getCustomLogin() - { - return _customLogin; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLogin(String customLogin) - { - _customLogin = customLogin; - } - - public boolean isCustomLoginInherited() - { - return _customLoginInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLoginInherited(boolean customLoginInherited) - { - _customLoginInherited = customLoginInherited; - } - } - - public enum FileRootProp implements SafeToRenderEnum - { - disable, - siteDefault, - folderOverride, - cloudRoot - } - - public static class FilesForm extends SetupForm implements FileManagementForm - { - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _cloudRootName; - private String _migrateFilesOption; - private String[] _enabledCloudStore; - private String _fileRootOption; - private String _folderRootPath; - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - } - - @SuppressWarnings("unused") - public static class SiteSettingsForm - { - private boolean _upgradeInProgress = false; - - private String _pipelineToolsDirectory; - private boolean _sslRequired; - private boolean _adminOnlyMode; - private boolean _showRibbonMessage; - private boolean _ext3Required; - private boolean _ext3APIRequired; - private boolean _selfReportExceptions; - private String _adminOnlyMessage; - private String _ribbonMessage; - private int _sslPort; - private int _memoryUsageDumpInterval; - private int _readOnlyHttpRequestTimeout; - private int _maxBLOBSize; - private String _exceptionReportingLevel; - private String _usageReportingLevel; - private String _administratorContactEmail; - - private String _baseServerURL; - private String _callbackPassword; - private boolean _allowApiKeys; - private int _apiKeyExpirationSeconds; - private boolean _allowSessionKeys; - private boolean _navAccessOpen; - - private String _XFrameOption; - private boolean _includeServerHttpHeader; - - public String getPipelineToolsDirectory() - { - return _pipelineToolsDirectory; - } - - public void setPipelineToolsDirectory(String pipelineToolsDirectory) - { - _pipelineToolsDirectory = pipelineToolsDirectory; - } - - public boolean isNavAccessOpen() - { - return _navAccessOpen; - } - - public void setNavAccessOpen(boolean navAccessOpen) - { - _navAccessOpen = navAccessOpen; - } - - public boolean isSslRequired() - { - return _sslRequired; - } - - public void setSslRequired(boolean sslRequired) - { - _sslRequired = sslRequired; - } - - public boolean isExt3Required() - { - return _ext3Required; - } - - public void setExt3Required(boolean ext3Required) - { - _ext3Required = ext3Required; - } - - public boolean isExt3APIRequired() - { - return _ext3APIRequired; - } - - public void setExt3APIRequired(boolean ext3APIRequired) - { - _ext3APIRequired = ext3APIRequired; - } - - public int getSslPort() - { - return _sslPort; - } - - public void setSslPort(int sslPort) - { - _sslPort = sslPort; - } - - public boolean isAdminOnlyMode() - { - return _adminOnlyMode; - } - - public void setAdminOnlyMode(boolean adminOnlyMode) - { - _adminOnlyMode = adminOnlyMode; - } - - public String getAdminOnlyMessage() - { - return _adminOnlyMessage; - } - - public void setAdminOnlyMessage(String adminOnlyMessage) - { - _adminOnlyMessage = adminOnlyMessage; - } - - public boolean isSelfReportExceptions() - { - return _selfReportExceptions; - } - - public void setSelfReportExceptions(boolean selfReportExceptions) - { - _selfReportExceptions = selfReportExceptions; - } - - public String getExceptionReportingLevel() - { - return _exceptionReportingLevel; - } - - public void setExceptionReportingLevel(String exceptionReportingLevel) - { - _exceptionReportingLevel = exceptionReportingLevel; - } - - public String getUsageReportingLevel() - { - return _usageReportingLevel; - } - - public void setUsageReportingLevel(String usageReportingLevel) - { - _usageReportingLevel = usageReportingLevel; - } - - public String getAdministratorContactEmail() - { - return _administratorContactEmail; - } - - public void setAdministratorContactEmail(String administratorContactEmail) - { - _administratorContactEmail = administratorContactEmail; - } - - public boolean isUpgradeInProgress() - { - return _upgradeInProgress; - } - - public void setUpgradeInProgress(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - } - - public int getMemoryUsageDumpInterval() - { - return _memoryUsageDumpInterval; - } - - public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) - { - _memoryUsageDumpInterval = memoryUsageDumpInterval; - } - - public int getReadOnlyHttpRequestTimeout() - { - return _readOnlyHttpRequestTimeout; - } - - public void setReadOnlyHttpRequestTimeout(int timeout) - { - _readOnlyHttpRequestTimeout = timeout; - } - - public int getMaxBLOBSize() - { - return _maxBLOBSize; - } - - public void setMaxBLOBSize(int maxBLOBSize) - { - _maxBLOBSize = maxBLOBSize; - } - - public String getBaseServerURL() - { - return _baseServerURL; - } - - public void setBaseServerURL(String baseServerURL) - { - _baseServerURL = baseServerURL; - } - - public String getCallbackPassword() - { - return _callbackPassword; - } - - public void setCallbackPassword(String callbackPassword) - { - _callbackPassword = callbackPassword; - } - - public boolean isShowRibbonMessage() - { - return _showRibbonMessage; - } - - public void setShowRibbonMessage(boolean showRibbonMessage) - { - _showRibbonMessage = showRibbonMessage; - } - - public String getRibbonMessage() - { - return _ribbonMessage; - } - - public void setRibbonMessage(String ribbonMessage) - { - _ribbonMessage = ribbonMessage; - } - - public boolean isAllowApiKeys() - { - return _allowApiKeys; - } - - public void setAllowApiKeys(boolean allowApiKeys) - { - _allowApiKeys = allowApiKeys; - } - - public int getApiKeyExpirationSeconds() - { - return _apiKeyExpirationSeconds; - } - - public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) - { - _apiKeyExpirationSeconds = apiKeyExpirationSeconds; - } - - public boolean isAllowSessionKeys() - { - return _allowSessionKeys; - } - - public void setAllowSessionKeys(boolean allowSessionKeys) - { - _allowSessionKeys = allowSessionKeys; - } - - public String getXFrameOption() - { - return _XFrameOption; - } - - public void setXFrameOption(String XFrameOption) - { - _XFrameOption = XFrameOption; - } - - public boolean isIncludeServerHttpHeader() - { - return _includeServerHttpHeader; - } - - public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) - { - _includeServerHttpHeader = includeServerHttpHeader; - } - } - - - @AdminConsoleAction - public class ShowThreadsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Log to labkey.log as well as showing through the browser - DebugInfoDumper.dumpThreads(3); - return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpDebugging#threads"); - addAdminNavTrail(root, "Current Threads", this.getClass()); - } - } - - private abstract class AbstractPostgresAction extends QueryViewAction - { - private final String _queryName; - - protected AbstractPostgresAction(String queryName) - { - super(QueryExportForm.class); - _queryName = queryName; - } - - @Override - protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception - { - if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - throw new NotFoundException("Only available with Postgres as the primary database"); - } - - QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); - QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) - { - @Override - public DataView createDataView() - { - // Troubleshooters don't have normal read access to the root container so grant them special access - // for these queries - DataView view = super.createDataView(); - view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); - return view; - } - }; - result.setTitle(_queryName); - result.setFrame(WebPartView.FrameType.PORTAL); - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("postgresActivity"); - addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); - } - - } - - @AdminConsoleAction - public class PostgresStatActivityAction extends AbstractPostgresAction - { - public PostgresStatActivityAction() - { - super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); - } - } - - @AdminConsoleAction - public class PostgresLocksAction extends AbstractPostgresAction - { - public PostgresLocksAction() - { - super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); - } - } - - @AdminConsoleAction - public class PostgresTableSizesAction extends AbstractPostgresAction - { - public PostgresTableSizesAction() - { - super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME); - } - } - - @AdminConsoleAction - public class DumpHeapAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - File destination = DebugInfoDumper.dumpHeap(); - return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpHeap"); - addAdminNavTrail(root, "Heap dump", getClass()); - } - } - - - public static class ThreadsBean - { - public Map> spids; - public List threads; - public Map stackTraces; - - ThreadsBean() - { - stackTraces = Thread.getAllStackTraces(); - threads = new ArrayList<>(stackTraces.keySet()); - threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); - - spids = new HashMap<>(); - - for (Thread t : threads) - { - spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); - } - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class ShowNetworkDriveTestAction extends SimpleViewAction - { - @Override - public void validate(NetworkDriveForm form, BindException errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, BindException errors) - { - NetworkDrive testDrive = new NetworkDrive(); - testDrive.setPassword(form.getNetworkDrivePassword()); - testDrive.setPath(form.getNetworkDrivePath()); - testDrive.setUser(form.getNetworkDriveUser()); - TestNetworkDriveBean bean = new TestNetworkDriveBean(); - - if (!errors.hasErrors()) - { - char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); - try - { - String mountError = testDrive.mount(driveLetter); - if (mountError != null) - { - errors.reject(ERROR_MSG, mountError); - } - else - { - File f = new File(driveLetter + ":\\"); - if (!f.exists()) - { - errors.reject(ERROR_MSG, "Could not access network drive"); - } - else - { - String[] fileNames = f.list(); - if (fileNames == null) - fileNames = new String[0]; - Arrays.sort(fileNames); - bean.setFiles(fileNames); - } - } - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - try - { - testDrive.unmount(driveLetter); - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - } - - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Test Mapping Network Drive"); - } - } - - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetErrorMarkAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - return HtmlView.of("Are you sure you want to reset the site errors?"); - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - File errorLogFile = getErrorLogFile(); - _errorMark = errorLogFile.length(); - - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull URLHelper getSuccessURL(Object o) - { - return getShowAdminURL(); - } - } - - abstract public static class ShowLogAction extends ExportAction - { - @Override - public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException - { - getPageConfig().setNoIndex(); - export(response); - } - - protected abstract void export(HttpServletResponse response) throws IOException; - } - - @AdminConsoleAction - public class ShowErrorsSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); - } - } - - @AdminConsoleAction - public class ShowAllErrorsAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); - } - } - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetPrimaryLogMarkAction extends MutatingApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - File logFile = getPrimaryLogFile(); - _primaryLogMark = logFile.length(); - return null; - } - } - - @AdminConsoleAction - public class ShowPrimaryLogSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowPrimaryLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowCspReportLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); - } - } - - private File getErrorLogFile() - { - return new File(getLabKeyLogDir(), "labkey-errors.log"); - } - - private File getPrimaryLogFile() - { - return new File(getLabKeyLogDir(), "labkey.log"); - } - - private File getCspReportLogFile() - { - return new File(getLabKeyLogDir(), "csp-report.log"); - } - - private static ActionURL getActionsURL() - { - return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public class ActionsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new ActionsTabStrip(); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("actionsDiagnostics"); - addAdminNavTrail(root, "Actions", this.getClass()); - } - } - - private static class ActionsTabStrip extends TabStripView - { - @Override - public List getTabList() - { - List tabs = new ArrayList<>(3); - - tabs.add(new TabInfo("Summary", "summary", getActionsURL())); - tabs.add(new TabInfo("Details", "details", getActionsURL())); - tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); - - return tabs; - } - - @Override - public HttpView getTabView(String tabId) - { - if ("exceptions".equals(tabId)) - return new ActionsExceptionsView(); - return new ActionsView(!"details".equals(tabId)); - } - } - - @AdminConsoleAction - public static class ExportActionsAction extends ExportAction - { - @Override - public void export(Object form, HttpServletResponse response, BindException errors) throws Exception - { - try (ActionsTsvWriter writer = new ActionsTsvWriter()) - { - writer.write(response); - } - } - } - - private static ActionURL getQueriesURL(@Nullable String statName) - { - ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); - - if (null != statName) - url.addParameter("stat", statName); - - return url; - } - - - @AdminConsoleAction - public class QueriesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueriesForm form, BindException errors) - { - String buttonHTML = ""; - if (getUser().hasRootAdminPermission()) - buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; - buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

"; - - return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, - AdminController::getQueryStackTracesURL); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("queryPerf"); - addAdminNavTrail(root, "Queries", this.getClass()); - } - } - - public static class QueriesForm - { - private String _stat = "Count"; - - public String getStat() - { - return _stat; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setStat(String stat) - { - _stat = stat; - } - } - - - private static ActionURL getQueryStackTracesURL(String sqlHash) - { - ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class QueryStackTracesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces"); - } - } - - - private static ActionURL getExecutionPlanURL(String sqlHash) - { - ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class ExecutionPlanAction extends SimpleViewAction - { - private String _sqlHash; - private ExecutionPlanType _type; - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - _sqlHash = form.getSqlHash(); - _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); - if (null == _type) - throw new NotFoundException("Unknown execution plan type"); - - return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); - root.addChild(_type.getDescription()); - } - } - - - public static class QueryForm - { - private String _sqlHash; - private String _type = "Estimated"; // All dialects support Estimated - private boolean _log = false; - - public String getSqlHash() - { - return _sqlHash; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSqlHash(String sqlHash) - { - _sqlHash = sqlHash; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setType(String type) - { - _type = type; - } - - public boolean isLog() - { - return _log; - } - - public void setLog(boolean log) - { - _log = log; - } - } - - - private ActionURL getExportQueriesURL() - { - return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public static class ExportQueriesAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) - { - writer.setFilenamePrefix("SQL_Queries"); - writer.write(response); - } - } - } - - private static ActionURL getResetQueryStatisticsURL() - { - return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); - } - - - @RequiresPermission(AdminPermission.class) - public static class ResetQueryStatisticsAction extends FormHandlerAction - { - @Override - public void validateCommand(QueriesForm target, Errors errors) - { - } - - @Override - public boolean handlePost(QueriesForm form, BindException errors) throws Exception - { - QueryProfiler.getInstance().resetAllStatistics(); - return true; - } - - @Override - public URLHelper getSuccessURL(QueriesForm form) - { - return getQueriesURL(form.getStat()); - } - } - - - @AdminConsoleAction - public class CachesAction extends SimpleViewAction - { - private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); - private final DecimalFormat percent = new DecimalFormat("0%"); - - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - if (form.isClearCaches()) - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); - throw new RedirectException(redirect); - } - - List> caches = CacheManager.getKnownCaches(); - - if (form.getDebugName() != null) - { - for (TrackingCache cache : caches) - { - if (form.getDebugName().equals(cache.getDebugName())) - { - LOG.info("Purging cache: " + cache.getDebugName()); - cache.clear(); - } - } - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); - throw new RedirectException(redirect); - } - - List cacheStats = new ArrayList<>(); - List transactionStats = new ArrayList<>(); - - for (TrackingCache cache : caches) - { - cacheStats.add(CacheManager.getCacheStats(cache)); - transactionStats.add(CacheManager.getTransactionCacheStats(cache)); - } - - HtmlStringBuilder html = HtmlStringBuilder.of(); - - html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); - html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); - - html.unsafeAppend("

\n"); - appendStats(html, "Caches", cacheStats, false); - - html.unsafeAppend("

\n"); - appendStats(html, "Transaction Caches", transactionStats, true); - - return new HtmlView(html); - } - - private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) - { - List stats = skipUnusedCaches ? - allStats.stream() - .filter(stat->stat.getMaxSize() > 0) - .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : - allStats; - - Collections.sort(stats); - - html.unsafeAppend("

"); - html.append(title); - html.append(" (").append(stats.size()).unsafeAppend(")

\n"); - - html.unsafeAppend("\n"); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - - long size = 0; - long gets = 0; - long misses = 0; - long puts = 0; - long expirations = 0; - long evictions = 0; - long removes = 0; - long clears = 0; - int rowCount = 0; - - for (CacheStats stat : stats) - { - size += stat.getSize(); - gets += stat.getGets(); - misses += stat.getMisses(); - puts += stat.getPuts(); - expirations += stat.getExpirations(); - evictions += stat.getEvictions(); - removes += stat.getRemoves(); - clears += stat.getClears(); - - html.unsafeAppend(""); - - appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); - - Long limit = stat.getLimit(); - long maxSize = stat.getMaxSize(); - - appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); - appendDoubles(html, stat.getMissRatio()); - - html.unsafeAppend("\n"); - - if (null != limit && maxSize >= limit) - html.unsafeAppend(""); - - html.unsafeAppend("\n"); - rowCount++; - } - - double ratio = 0 != gets ? misses / (double)gets : 0; - html.unsafeAppend(""); - - appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); - appendDoubles(html, ratio); - - html.unsafeAppend("\n"); - html.unsafeAppend("
Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
Total
\n"); - } - - private static final List PREFIXES_TO_SKIP = List.of( - "java.base/java.lang.Thread.getStackTrace", - "org.labkey.api.cache.CacheManager", - "org.labkey.api.cache.Throttle", - "org.labkey.api.data.DatabaseCache", - "org.labkey.api.module.ModuleResourceCache" - ); - - private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) - { - StringBuilder sb = new StringBuilder(); - - if (creationStackTrace != null) - { - boolean trimming = true; - for (StackTraceElement element : creationStackTrace) - { - // Skip the first few uninteresting stack trace elements to highlight the caller we care about - if (trimming) - { - if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) - continue; - - trimming = false; - } - sb.append(element); - sb.append("\n"); - } - } - - if (!sb.isEmpty()) - { - String message = PageFlowUtil.jsString(sb); - String id = "id" + UniqueID.getServerSessionScopedUID(); - html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); - HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); - } - } - - private void appendLongs(HtmlStringBuilder html, Long... stats) - { - for (Long stat : stats) - { - if (null == stat) - html.unsafeAppend(" "); - else - html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); - } - } - - private void appendDoubles(HtmlStringBuilder html, double... stats) - { - for (double stat : stats) - html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("cachesDiagnostics"); - addAdminNavTrail(root, "Cache Statistics", this.getClass()); - } - } - - @RequiresSiteAdmin - public class EnvironmentVariablesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Environment Variables", this.getClass()); - } - } - - @RequiresSiteAdmin - public class SystemPropertiesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "System Properties", this.getClass()); - } - } - - - public static class ConfigureSystemMaintenanceForm - { - private String _maintenanceTime; - private Set _enable = Collections.emptySet(); - private boolean _enableSystemMaintenance = true; - - public String getMaintenanceTime() - { - return _maintenanceTime; - } - - @SuppressWarnings("unused") - public void setMaintenanceTime(String maintenanceTime) - { - _maintenanceTime = maintenanceTime; - } - - public Set getEnable() - { - return _enable; - } - - @SuppressWarnings("unused") - public void setEnable(Set enable) - { - _enable = enable; - } - - public boolean isEnableSystemMaintenance() - { - return _enableSystemMaintenance; - } - - @SuppressWarnings("unused") - public void setEnableSystemMaintenance(boolean enableSystemMaintenance) - { - _enableSystemMaintenance = enableSystemMaintenance; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ConfigureSystemMaintenanceAction extends FormViewAction - { - @Override - public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) - { - Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); - - if (null == date) - errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); - } - - @Override - public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) - { - SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); - return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); - } - - @Override - public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) - { - SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); - SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); - - return true; - } - - @Override - public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class ResetSystemMaintenanceAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - SystemMaintenance.clearProperties(); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - - public static class SystemMaintenanceForm - { - private String _taskName; - private boolean _test = false; - - public String getTaskName() - { - return _taskName; - } - - @SuppressWarnings("unused") - public void setTaskName(String taskName) - { - _taskName = taskName; - } - - public boolean isTest() - { - return _test; - } - - public void setTest(boolean test) - { - _test = test; - } - } - - @RequiresSiteAdmin - public class SystemMaintenanceAction extends FormHandlerAction - { - private Long _jobId = null; - private URLHelper _url = null; - - @Override - public void validateCommand(SystemMaintenanceForm form, Errors errors) - { - } - - @Override - public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException - { - // Send the pipeline job details absolute URL back to the test - sendPlainText(_url.getURIString()); - - // Suppress templates, divs, etc. - getPageConfig().setTemplate(Template.None); - return new EmptyView(); - } - - @Override - public boolean handlePost(SystemMaintenanceForm form, BindException errors) - { - String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); - - if (null != jobGuid) - _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SystemMaintenanceForm form) - { - // In the standard case, redirect to the pipeline details URL - // If the test is invoking system maintenance then return the URL instead - return form.isTest() ? null : _url; - } - } - - @AdminConsoleAction - public class AttachmentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getAdminView(getViewContext().getActionURL()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Attachments", getClass()); - } - } - - @AdminConsoleAction - public class FindAttachmentParentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getFindAttachmentParentsView(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Find Attachment Parents", getClass()); - } - } - - public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCachesURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCacheURL(String debugName) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - url.addParameter(MemForm.Params.debugName, debugName); - - return url; - } - - private static volatile String lastCacheMemUsed = null; - - @AdminConsoleAction - public class MemTrackerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - Set objectsToIgnore = MemTracker.getInstance().beforeReport(); - - boolean gc = form.isGc(); - boolean cc = form.isClearCaches(); - - if (getUser().hasRootAdminPermission() && (gc || cc)) - { - // If both are requested then try to determine and record cache memory usage - if (gc && cc) - { - // gc once to get an accurate free memory read - long before = gc(); - clearCaches(); - // gc again now that we cleared caches - long cacheMemoryUsed = before - gc(); - - // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage - String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; - LOG.info("Estimate of cache memory used: " + cacheMemUsed); - lastCacheMemUsed = cacheMemUsed; - } - else if (cc) - { - clearCaches(); - } - else - { - gc(); - } - - LOG.info("Cache clearing and garbage collecting complete"); - } - - return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); - } - - /** @return estimated current memory usage, post-garbage collection */ - private long gc() - { - LOG.info("Garbage collecting"); - System.gc(); - // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will - return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - } - - private void clearCaches() - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - LOG.info("Purging SearchService queues"); - SearchService.get().purgeQueues(); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("memTracker"); - addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); - } - } - - public static class MemForm - { - private enum Params {clearCaches, debugName, gc} - - private boolean _clearCaches = false; - private boolean _gc = false; - private String _debugName; - - public boolean isClearCaches() - { - return _clearCaches; - } - - @SuppressWarnings("unused") - public void setClearCaches(boolean clearCaches) - { - _clearCaches = clearCaches; - } - - public boolean isGc() - { - return _gc; - } - - @SuppressWarnings("unused") - public void setGc(boolean gc) - { - _gc = gc; - } - - public String getDebugName() - { - return _debugName; - } - - @SuppressWarnings("unused") - public void setDebugName(String debugName) - { - _debugName = debugName; - } - } - - public static class MemBean - { - public final List> memoryUsages = new ArrayList<>(); - public final List> systemProperties = new ArrayList<>(); - public final List references; - public final List graphNames = new ArrayList<>(); - public final List activeThreads = new LinkedList<>(); - - public boolean assertsEnabled = false; - - private MemBean(HttpServletRequest request, Set objectsToIgnore) - { - MemTracker memTracker = MemTracker.getInstance(); - List all = memTracker.getReferences(); - long threadId = Thread.currentThread().getId(); - - // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found - for (Thread thread : new ThreadsBean().threads) - { - if (thread.getId() == threadId) - continue; - - Thread.State state = thread.getState(); - - if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) - { - boolean labkeyThread = false; - - if (memTracker.shouldDisplay(thread)) - { - for (StackTraceElement element : thread.getStackTrace()) - { - String className = element.getClassName(); - - if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) - { - labkeyThread = true; - break; - } - } - } - - if (labkeyThread) - { - String threadInfo = thread.getName(); - TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); - if (null != uri) - threadInfo += "; processing URL " + uri.toLogString(); - activeThreads.add(threadInfo); - } - } - } - - // ignore recently allocated - long start = ViewServlet.getRequestStartTime(request) - 2000; - references = new ArrayList<>(all.size()); - - for (HeldReference r : all) - { - if (r.getThreadId() == threadId && r.getAllocationTime() >= start) - continue; - - if (objectsToIgnore.contains(r.getReference())) - continue; - - references.add(r); - } - - // memory: - graphNames.add("Heap"); - graphNames.add("Non Heap"); - - MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); - } - - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); - } - - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.NON_HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) - { - memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); - graphNames.add(pool.getName()); - } - - DecimalFormat commaf0 = new DecimalFormat("#,##0"); - - - // class loader: - ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); - if (classbean != null) - { - systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); - systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); - systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); - } - - // runtime: - RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); - if (runtimeBean != null) - { - systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); - long upTime = runtimeBean.getUptime(); // round to sec - upTime = upTime - (upTime % 1000); - systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); - systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); - systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); - } - - // threads: - ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - if (threadBean != null) - { - systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); - systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); - long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); - systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); - } - - // threads: - List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - for (GarbageCollectorMXBean gcBean : gcBeans) - { - systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); - systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); - } - - String cacheMem = lastCacheMemUsed; - - if (null != cacheMem) - systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); - - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - if (osBean != null) - { - systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); - - DecimalFormat f3 = new DecimalFormat("0.000"); - - if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) - { - systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); - systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); - systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); - systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); - } - } - - //noinspection ConstantConditions - assert assertsEnabled = true; - } - } - - private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) - { - try - { - return getUsage(pool.getUsage()); - } - catch (IllegalArgumentException x) - { - // sometimes we get usage>committed exception with older versions of JRockit - return null; - } - } - - public static class MemoryUsageSummary - { - - public final long _init; - public final long _used; - public final long _committed; - public final long _max; - - public MemoryUsageSummary(MemoryUsage usage) - { - _init = usage.getInit(); - _used = usage.getUsed(); - _committed = usage.getCommitted(); - _max = usage.getMax(); - } - - public MemoryUsageSummary(BufferPoolMXBean pool) - { - _init = -1; - _used = pool.getMemoryUsed(); - _committed = _used; - _max = pool.getTotalCapacity(); - } - } - - private static MemoryUsageSummary getUsage(MemoryUsage usage) - { - if (null == usage) - return null; - - try - { - return new MemoryUsageSummary(usage); - } - catch (IllegalArgumentException x) - { - // sometime we get usage>committed exception with older verions of JRockit - return null; - } - } - - public static class ChartForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - } - - private static class MemoryCategory implements Comparable - { - private final String _type; - private final double _mb; - - public MemoryCategory(String type, double mb) - { - _type = type; - _mb = mb; - } - - @Override - public int compareTo(@NotNull MemoryCategory o) - { - return Double.compare(getMb(), o.getMb()); - } - - public String getType() - { - return _type; - } - - public double getMb() - { - return _mb; - } - } - - @AdminConsoleAction - public static class MemoryChartAction extends ExportAction - { - @Override - public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception - { - MemoryUsage usage = null; - boolean showLegend = false; - String title = form.getType(); - if ("Heap".equals(form.getType())) - { - usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - showLegend = true; - } - else if ("Non Heap".equals(form.getType())) - usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); - else - { - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) - { - MemoryPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - usage = pool.getUsage(); - } - } - - Pair divisor = null; - - List types = new ArrayList<>(4); - - if (usage == null) - { - boolean found = false; - for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) - { - BufferPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - { - long total = pool.getTotalCapacity(); - long used = pool.getMemoryUsed(); - - divisor = getDivisor(total); - - title = "Buffer pool " + title; - - if (total > 0 || used > 0) - { - types.add(new MemoryCategory("Used", used / divisor.first)); - types.add(new MemoryCategory("Max", total / divisor.first)); - } - found = true; - } - } - if (!found) - { - throw new NotFoundException(); - } - } - else - { - if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) - { - divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); - - types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); - types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); - types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); - types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); - } - } - - if (divisor != null) - { - title += " (" + divisor.second + ")"; - } - - DefaultCategoryDataset dataset = new DefaultCategoryDataset(); - - Collections.sort(types); - - for (int i = 0; i < types.size(); i++) - { - double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); - dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); - } - - JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); - chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); - response.setContentType("image/png"); - - ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); - } - - private Pair getDivisor(long l) - { - if (l > 4096L * 1024L * 1024L) - { - return Pair.of(1024L * 1024L * 1024L, "GB"); - } - if (l > 4096L * 1024L) - { - return Pair.of(1024L * 1024L, "MB"); - } - if (l > 4096L) - { - return Pair.of(1024L, "KB"); - } - - return Pair.of(1L, "bytes"); - - } - } - - public static class MemoryStressForm - { - private int _threads = 3; - private int _arraySize = 20_000; - private int _arrayCount = 10_000; - private float _percentChurn = 0.50f; - private int _delay = 20; - private int _iterations = 500; - - public int getThreads() - { - return _threads; - } - - public void setThreads(int threads) - { - _threads = threads; - } - - public int getArraySize() - { - return _arraySize; - } - - public void setArraySize(int arraySize) - { - _arraySize = arraySize; - } - - public int getArrayCount() - { - return _arrayCount; - } - - public void setArrayCount(int arrayCount) - { - _arrayCount = arrayCount; - } - - public float getPercentChurn() - { - return _percentChurn; - } - - public void setPercentChurn(float percentChurn) - { - _percentChurn = percentChurn; - } - - public int getDelay() - { - return _delay; - } - - public void setDelay(int delay) - { - _delay = delay; - } - - public int getIterations() - { - return _iterations; - } - - public void setIterations(int iterations) - { - _iterations = iterations; - } - } - - @RequiresSiteAdmin - public class MemoryStressTestAction extends FormViewAction - { - @Override - public void validateCommand(MemoryStressForm target, Errors errors) - { - - } - - @Override - public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception - { - return new HtmlView( - DOM.LK.FORM(at(method, "POST"), - DOM.LK.ERRORS(errors.getBindingResult()), - DOM.BR(), DOM.BR(), - "This utility action will do a lot of memory allocation to test the memory configuration of the host.", - DOM.BR(), DOM.BR(), - "It spins up threads, all of which allocate a specified number byte arrays of specified length.", - DOM.BR(), - "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", - DOM.BR(), - "They continue for the specified number of allocations.", - DOM.BR(), - "The memory actively held is approximately (threads * array count * array length).", - DOM.BR(), - "The memory turnover is based on the churn percentage, array length, delay, and iterations.", - DOM.BR(), DOM.BR(), - DOM.TABLE( - DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), - DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), - DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), - DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), - DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), - DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) - ), - new ButtonBuilder("Perform stress test").submit(true).build()) - ); - } - - @Override - public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception - { - List threads = new ArrayList<>(); - for (int i = 0; i < memoryStressForm._threads; i++) - { - Thread t = new Thread(() -> - { - Random r = new Random(); - byte[][] arrays = new byte[memoryStressForm._arrayCount][]; - // Initialize the arrays - for (int a = 0; a < arrays.length; a++) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - - for (int iter = 0; iter < memoryStressForm._iterations; iter++) - { - try - { - Thread.sleep(memoryStressForm._delay); - } - catch (InterruptedException ignored) {} - - // Swap the contents based on our desired percent churn - for (int a = 0; a < arrays.length; a++) - { - if (r.nextFloat() <= memoryStressForm._percentChurn) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - } - } - }); - t.setUncaughtExceptionHandler((t2, e) -> { - LOG.error("Stress test exception", e); - errors.reject(null, "Stress test exception: " + e); - }); - t.start(); - threads.add(t); - } - - for (Thread thread : threads) - { - thread.join(); - } - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) - { - return new ActionURL(MemTrackerAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); - root.addChild("Memory Stress Test"); - } - } - - public static ActionURL getModuleStatusURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - public static class ModuleStatusBean - { - public String verb; - public String verbing; - public ActionURL nextURL; - } - - @RequiresPermission(TroubleshooterPermission.class) - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class ModuleStatusAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - ModuleLoader loader = ModuleLoader.getInstance(); - VBox vbox = new VBox(); - ModuleStatusBean bean = new ModuleStatusBean(); - - if (loader.isNewInstall()) - bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); - else if (form.getReturnUrl() != null) - { - try - { - bean.nextURL = form.getReturnActionURL(); - } - catch (URLException x) - { - // might not be an ActionURL e.g. /labkey/_webdav/home - } - } - if (null == bean.nextURL) - bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - - if (loader.isNewInstall()) - bean.verb = "Install"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verb = "Upgrade"; - else - bean.verb = "Start"; - - if (loader.isNewInstall()) - bean.verbing = "Installing"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verbing = "Upgrading"; - else - bean.verbing = "Starting"; - - JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - - getPageConfig().setTemplate(Template.Wizard); - getPageConfig().setTitle(bean.verb + " Modules"); - setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); - - return vbox; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class NewInstallSiteSettingsForm extends FileSettingsForm - { - private String _notificationEmail; - private String _siteName; - - public String getNotificationEmail() - { - return _notificationEmail; - } - - public void setNotificationEmail(String notificationEmail) - { - _notificationEmail = notificationEmail; - } - - public String getSiteName() - { - return _siteName; - } - - public void setSiteName(String siteName) - { - _siteName = siteName; - } - } - - @RequiresSiteAdmin - public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction - { - public NewInstallSiteSettingsAction() - { - super(NewInstallSiteSettingsForm.class); - } - - @Override - public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) - { - super.validateCommand(form, errors); - - if (isBlank(form.getNotificationEmail())) - { - errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); - } - try - { - ValidEmail email = new ValidEmail(form.getNotificationEmail()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - - @Override - public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception - { - boolean success = super.handlePost(form, errors); - if (success) - { - WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); - try - { - lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - lafProps.setSystemShortName(form.getSiteName()); - lafProps.save(); - - // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. - UsageReportingLevel.reportNow(); - - return true; - } - return false; - } - - @Override - public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) - { - if (!reshow) - { - File root = _svc.getSiteDefaultRoot(); - - if (root.exists()) - form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); - form.setSiteName(props.getShortName()); - form.setNotificationEmail(props.getSystemEmailAddress()); - } - - JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Set Defaults"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) - { - return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresSiteAdmin - public static class InstallCompleteAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Complete"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static List getInstallUpgradeWizardSteps() - { - List navTrail = new ArrayList<>(); - if (ModuleLoader.getInstance().isNewInstall()) - { - navTrail.add(new NavTree("Account Setup")); - navTrail.add(new NavTree("Install Modules")); - navTrail.add(new NavTree("Set Defaults")); - } - else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) - { - navTrail.add(new NavTree("Upgrade Modules")); - } - else - { - navTrail.add(new NavTree("Start Modules")); - } - navTrail.add(new NavTree("Complete")); - return navTrail; - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DbCheckerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Check Tools", this.getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DoCheckAction extends SimpleViewAction - { - @Override - public ModelAndView getView(DataCheckForm form, BindException errors) - { - try (var ignore=SpringActionController.ignoreSqlUpdates()) - { - ActionURL currentUrl = getViewContext().cloneActionURL(); - String fixRequested = currentUrl.getParameter("_fix"); - HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
")); - - if (null != fixRequested) - { - HtmlString sqlCheck = HtmlString.EMPTY_STRING; - if (fixRequested.equalsIgnoreCase("container")) - sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); - else if (fixRequested.equalsIgnoreCase("descriptor")) - sqlCheck = OntologyManager.doProjectColumnCheck(true); - contentBuilder.append(sqlCheck); - } - else - { - LOG.info("Starting database check"); // Debugging test timeout - LOG.info("Checking container column references"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking Container Column References..."); - HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "container"); - contentBuilder.unsafeAppend("

    ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); - strTemp = OntologyManager.doProjectColumnCheck(false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "descriptor"); - contentBuilder.unsafeAppend("

    ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

") - .append("Checking Schema consistency with tableXML.") - .unsafeAppend("

"); - Set schemas = DbSchema.getAllSchemasToTest(); - - for (DbSchema schema : schemas) - { - SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); - List results = schemaResult.getResults(null); - if (results.isEmpty()) - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .append(": OK") - .unsafeAppend("
"); - } - else - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .unsafeAppend(""); - for (var r : results) - { - HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); - contentBuilder.unsafeAppend("
  • ") - .append(item) - .unsafeAppend("
    • \n"); - } - contentBuilder.unsafeAppend(""); - } - } - - LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking Consistency of Provisioned Storage...\n"); - StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); - contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); - for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) - { - for (String error : dr.getErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - } - for (String error : pr.getGlobalErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - - LOG.info("Database check complete"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Database Consistency checker complete"); - } - - contentBuilder.unsafeAppend("
    "); - - return new HtmlView(contentBuilder); - } - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Tools", this.getClass()); - } - } - - public static class DataCheckForm - { - private String _dbSchema = ""; - private boolean _full = false; - - public List modules = ModuleLoader.getInstance().getModules(); - public DataCheckForm(){} - - public List getModules() { return modules; } - public String getDbSchema() { return _dbSchema; } - @SuppressWarnings("unused") - public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } - public boolean getFull() { return _full; } - public void setFull(boolean full) { _full = full; } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class GetSchemaXmlDocAction extends ExportAction - { - @Override - public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception - { - String fullyQualifiedSchemaName = form.getDbSchema(); - if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) - { - throw new NotFoundException("Must specify dbSchema parameter"); - } - - boolean bFull = form.getFull(); - - Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); - TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); - StringWriter sw = new StringWriter(); - - XmlOptions xOpt = new XmlOptions(); - xOpt.setSavePrettyPrint(); - xOpt.setUseDefaultNamespace(); - - tdoc.save(sw, xOpt); - - sw.flush(); - PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderInformationAction extends FolderManagementViewAction - { - @Override - protected HtmlView getTabView() - { - Container c = getContainer(); - User currentUser = getUser(); - - User createdBy = UserManager.getUser(c.getCreatedBy()); - Map propValueMap = new LinkedHashMap<>(); - propValueMap.put("Path", c.getPath()); - propValueMap.put("Name", c.getName()); - propValueMap.put("Displayed Title", c.getTitle()); - propValueMap.put("EntityId", c.getId()); - propValueMap.put("RowId", c.getRowId()); - propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); - propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); - propValueMap.put("Folder Type", c.getFolderType().getName()); - propValueMap.put("Description", c.getDescription()); - - return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); - } - } - - public static class MissingValuesForm - { - private boolean _inheritMvIndicators; - private String[] _mvIndicators; - private String[] _mvLabels; - - public boolean isInheritMvIndicators() - { - return _inheritMvIndicators; - } - - public void setInheritMvIndicators(boolean inheritMvIndicators) - { - _inheritMvIndicators = inheritMvIndicators; - } - - public String[] getMvIndicators() - { - return _mvIndicators; - } - - public void setMvIndicators(String[] mvIndicators) - { - _mvIndicators = mvIndicators; - } - - public String[] getMvLabels() - { - return _mvLabels; - } - - public void setMvLabels(String[] mvLabels) - { - _mvLabels = mvLabels; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MissingValuesAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); - } - - @Override - public void validateCommand(MissingValuesForm form, Errors errors) - { - } - - @Override - public boolean handlePost(MissingValuesForm form, BindException errors) - { - if (form.isInheritMvIndicators()) - { - MvUtil.inheritMvIndicators(getContainer()); - return true; - } - else - { - // Javascript should have enforced any constraints - MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); - return true; - } - } - } - - @SuppressWarnings("unused") - public static class RConfigForm - { - private Integer _reportEngine; - private Integer _pipelineEngine; - private boolean _overrideDefault; - - public Integer getReportEngine() - { - return _reportEngine; - } - - public void setReportEngine(Integer reportEngine) - { - _reportEngine = reportEngine; - } - - public Integer getPipelineEngine() - { - return _pipelineEngine; - } - - public void setPipelineEngine(Integer pipelineEngine) - { - _pipelineEngine = pipelineEngine; - } - - public boolean getOverrideDefault() - { - return _overrideDefault; - } - - public void setOverrideDefault(String overrideDefault) - { - _overrideDefault = "override".equals(overrideDefault); - } - } - - @RequiresPermission(AdminPermission.class) - public static class RConfigurationAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); - } - - @Override - public void validateCommand(RConfigForm form, Errors errors) - { - if (form.getOverrideDefault()) - { - if (form.getReportEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); - if (form.getPipelineEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); - } - } - - @Override - public URLHelper getSuccessURL(RConfigForm rConfigForm) - { - return getContainer().getStartURL(getUser()); - } - - @Override - public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception - { - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - if (null != mgr) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - if (rConfigForm.getOverrideDefault()) - { - ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); - ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); - - if (reportEngine != null) - mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - else - { - // need to clear the current scope (if any) - ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); - ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); - - if (reportEngine != null) - mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - transaction.commit(); - } - return true; - } - return false; - } - } - - @SuppressWarnings("unused") - public static class ExportFolderForm - { - private String[] _types; - private int _location; - private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. - private String _exportType; - private boolean _includeSubfolders; - private PHI _exportPhiLevel; // Input: max level when viewing form - private boolean _shiftDates; - private boolean _alternateIds; - private boolean _maskClinic; - - public String[] getTypes() - { - return _types; - } - - public void setTypes(String[] types) - { - _types = types; - } - - public int getLocation() - { - return _location; - } - - public void setLocation(int location) - { - _location = location; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - - public ExportType getExportType() - { - if ("study".equals(_exportType)) - return ExportType.STUDY; - else - return ExportType.ALL; - } - - public void setExportType(String exportType) - { - _exportType = exportType; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public PHI getExportPhiLevel() - { - return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - public boolean isShiftDates() - { - return _shiftDates; - } - - public void setShiftDates(boolean shiftDates) - { - _shiftDates = shiftDates; - } - - public boolean isAlternateIds() - { - return _alternateIds; - } - - public void setAlternateIds(boolean alternateIds) - { - _alternateIds = alternateIds; - } - - public boolean isMaskClinic() - { - return _maskClinic; - } - - public void setMaskClinic(boolean maskClinic) - { - _maskClinic = maskClinic; - } - } - - public enum ExportOption - { - PipelineRootAsFiles("file root as multiple files") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - else if (root.isCloudRoot()) - { - errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); - } - else - { - File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); - try - { - writer.write(container, ctx, new FileSystemFile(exportDir)); - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - return null; - } - }, - - PipelineRootAsZip("file root as a single zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); - FileUtil.createDirectories(exportDir); - exportFolderToFile(exportDir, container, writer, ctx, errors); - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - }, - DownloadAsZip("browser download as a zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - try - { - // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 - // Same pattern as ExportListArchiveAction - Path tempDir = FileUtil.getTempDirectory().toPath(); - Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); - - // No exceptions, so stream the resulting zip file to the browser and delete it - try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) - { - Files.copy(tempZipFile, os); - } - finally - { - Files.delete(tempZipFile); - } - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return null; - } - }; - - private final String _description; - - ExportOption(String description) - { - _description = description; - } - - public String getDescription() - { - return _description; - } - - public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; - - Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception - { - String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); - - try (ZipFile zip = new ZipFile(exportDir, filename)) - { - writer.write(container, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return exportDir.resolve(filename); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ExportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception - { - // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so - // rendering a view would throw. - return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); - } - - @Override - protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) - { - form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); - - ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); - PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); - PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); - form.setExportPhiLevel(maxAllowedPhiForExport); - - return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ExportFolderForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ExportOption exportOption = null; - if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) - { - exportOption = ExportOption.values()[form.getLocation()]; - } - if (exportOption == null) - { - throw new NotFoundException("Invalid export location: " + form.getLocation()); - } - ContainerManager.checkContainerValidity(container); - - FolderWriterImpl writer = new FolderWriterImpl(); - FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), - form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), - form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); - - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); - AuditLogService.get().addEvent(getUser(), event); - - _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) - { - return _successURL; - } - } - - public static class ImportFolderForm - { - private boolean _createSharedDatasets; - private boolean _validateQueries; - private boolean _failForUndefinedVisits; - private String _sourceTemplateFolder; - private String _sourceTemplateFolderId; - private String _origin; - - public boolean isCreateSharedDatasets() - { - return _createSharedDatasets; - } - - public void setCreateSharedDatasets(boolean createSharedDatasets) - { - _createSharedDatasets = createSharedDatasets; - } - - public boolean isValidateQueries() - { - return _validateQueries; - } - - public boolean isFailForUndefinedVisits() - { - return _failForUndefinedVisits; - } - - public void setFailForUndefinedVisits(boolean failForUndefinedVisits) - { - _failForUndefinedVisits = failForUndefinedVisits; - } - - public void setValidateQueries(boolean validateQueries) - { - _validateQueries = validateQueries; - } - - public String getSourceTemplateFolder() - { - return _sourceTemplateFolder; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolder(String sourceTemplateFolder) - { - _sourceTemplateFolder = sourceTemplateFolder; - } - - public String getSourceTemplateFolderId() - { - return _sourceTemplateFolderId; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolderId(String sourceTemplateFolderId) - { - _sourceTemplateFolderId = sourceTemplateFolderId; - } - - public String getOrigin() - { - return _origin; - } - - public void setOrigin(String origin) - { - _origin = origin; - } - - public Container getSourceTemplateFolderContainer() - { - if (null == getSourceTemplateFolderId()) - return null; - return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ImportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL; - - @Override - protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) - { - // default the createSharedDatasets and validateQueries to true if this is not a form error reshow - if (!errors.hasErrors()) - { - form.setCreateSharedDatasets(true); - form.setValidateQueries(true); - } - - return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ImportFolderForm form, Errors errors) - { - // don't allow import into the root container - if (getContainer().isRoot()) - { - throw new NotFoundException(); - } - } - - @Override - public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception - { - ViewContext context = getViewContext(); - ActionURL url = context.getActionURL(); - User user = getUser(); - Container container = getContainer(); - PipeRoot pipelineRoot; - FileLike pipelineUnzipDir; // Should be local & writable - PipelineUrls pipelineUrlProvider; - - if (form.getOrigin() == null) - { - form.setOrigin("Folder"); - } - - // make sure we have a pipeline url provider to use for the success URL redirect - pipelineUrlProvider = urlProvider(PipelineUrls.class); - if (pipelineUrlProvider == null) - { - errors.reject("folderImport", "Pipeline url provider does not exist."); - return false; - } - - // make sure that the pipeline root is valid for this container - pipelineRoot = PipelineService.get().findPipelineRoot(container); - if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) - { - errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); - return false; - } - - // make sure we are able to delete any existing unzip dir in the pipeline root - try - { - pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); - } - catch (DirectoryNotDeletedException e) - { - errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); - return false; - } - - FolderImportConfig fiConfig; - if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) - { - fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); - } - else - { - fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); - if (fiConfig == null || errors.hasErrors()) - { - return false; - } - } - - // get the folder.xml file from the unzipped import archive - FileLike archiveXml = pipelineUnzipDir.resolveChild("folder.xml"); - if (!archiveXml.exists()) - { - errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); - return false; - } - - ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); - options.setSkipQueryValidation(!form.isValidateQueries()); - options.setCreateSharedDatasets(form.isCreateSharedDatasets()); - options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); - options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); - - // finally, create the study or folder import pipeline job - _successURL = pipelineUrlProvider.urlBegin(container); - PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); - - return !errors.hasErrors(); - } - - private @Nullable FolderImportConfig getFolderFromZipArchive(FileLike pipelineUnzipDir, BindException errors) - { - // user chose to import from a zip file - Map map = getFileMap(); - - // make sure we have a single file selected for import - if (map.size() != 1) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // make sure the file is not empty and that it has a .zip extension - MultipartFile zipFile = map.values().iterator().next(); - String originalFilename = zipFile.getOriginalFilename(); - if (0 == zipFile.getSize() || isBlank(originalFilename) || !originalFilename.toLowerCase().endsWith(".zip")) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // copy and unzip the uploaded import archive zip file to the pipeline unzip dir - try - { - FileLike pipelineUnzipFile = pipelineUnzipDir.resolveFile(org.labkey.api.util.Path.parse(originalFilename)); - // Check that the resolved file is under the pipelineUnzipDir - if (!pipelineUnzipFile.toNioPathForRead().normalize().startsWith(pipelineUnzipDir.toNioPathForRead().normalize())) - { - errors.reject("folderImport", "Invalid file path - must be within the unzip directory"); - return null; - } - - FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) - FileUtil.createNewFile(pipelineUnzipFile, true); - try (OutputStream os = pipelineUnzipFile.openOutputStream()) - { - FileUtil.copyData(zipFile.getInputStream(), os); - } - ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); - - return new FolderImportConfig( - false, - originalFilename, - pipelineUnzipFile, - pipelineUnzipFile - ); - } - catch (FileNotFoundException e) - { - LOG.debug("Failed to import '" + originalFilename + "'.", e); - errors.reject("folderImport", "File not found."); - return null; - } - catch (IOException e) - { - LOG.debug("Failed to import '" + originalFilename + "'.", e); - errors.reject("folderImport", "Unable to unzip folder archive."); - return null; - } - } - - private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final FileLike pipelineUnzipDir, final BindException errors) throws Exception - { - // user choose to import from a template source folder - Container sourceContainer = form.getSourceTemplateFolderContainer(); - - // In order to support the Advanced import options to import into multiple target folders we need to zip - // the source template folder so that the zip file can be passed to the pipeline processes. - FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, - getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, - PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - FolderWriterImpl writer = new FolderWriterImpl(); - String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); - FileLike implicitZipFile = pipelineUnzipDir.resolveChild(zipFileName); - if (!pipelineUnzipDir.isDirectory()) - pipelineUnzipDir.mkdirs(); - implicitZipFile.createFile(); - try (OutputStream out = implicitZipFile.openOutputStream(); - ZipFile zip = new ZipFile(out, false)) - { - writer.write(sourceContainer, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container - ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); - - return new FolderImportConfig( - StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), - implicitZipFile.getName(), - implicitZipFile, - null - ); - } - - private static class FolderImportConfig { - FileLike pipelineUnzipFile; - String originalFileName; - FileLike archiveFile; - boolean fromTemplateSourceFolder; - - public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, FileLike archiveFile, @Nullable FileLike pipelineUnzipFile) - { - this.originalFileName = originalFileName; - this.archiveFile = archiveFile; - this.fromTemplateSourceFolder = fromTemplateSourceFolder; - this.pipelineUnzipFile = pipelineUnzipFile; - } - } - - @Override - public URLHelper getSuccessURL(ImportFolderForm importFolderForm) - { - return _successURL; - } - } - - private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) - { - // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of - // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers - // to apply during the implicit export. - Set registeredFolderWriters = new HashSet<>(); - FolderSerializationRegistry registry = FolderSerializationRegistry.get(); - if (null == registry) - { - throw new RuntimeException(); - } - Collection registeredWriters = registry.getRegisteredFolderWriters(); - for (FolderWriter writer : registeredWriters) - { - String dataType = writer.getDataType(); - boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); - boolean excludeForTemplate = !writer.includeWithTemplate(); - - if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) - { - registeredFolderWriters.add(dataType); - - // for each Writer also determine if there are related children Writers, if so include them also - Collection> childWriters = writer.getChildren(true, true); - if (!childWriters.isEmpty()) - { - for (org.labkey.api.writer.Writer child : childWriters) - { - dataType = child.getDataType(); - if (dataType != null) - registeredFolderWriters.add(dataType); - } - } - } - } - return registeredFolderWriters; - } - - public static class FolderSettingsForm - { - private String _defaultDateFormat; - private boolean _defaultDateFormatInherited; - private String _defaultDateTimeFormat; - private boolean _defaultDateTimeFormatInherited; - private String _defaultTimeFormat; - private boolean _defaultTimeFormatInherited; - private String _defaultNumberFormat; - private boolean _defaultNumberFormatInherited; - private boolean _restrictedColumnsEnabled; - private boolean _restrictedColumnsEnabledInherited; - - public String getDefaultDateFormat() - { - return _defaultDateFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormat(String defaultDateFormat) - { - _defaultDateFormat = defaultDateFormat; - } - - public boolean isDefaultDateFormatInherited() - { - return _defaultDateFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) - { - _defaultDateFormatInherited = defaultDateFormatInherited; - } - - public String getDefaultDateTimeFormat() - { - return _defaultDateTimeFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormat(String defaultDateTimeFormat) - { - _defaultDateTimeFormat = defaultDateTimeFormat; - } - - public boolean isDefaultDateTimeFormatInherited() - { - return _defaultDateTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) - { - _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; - } - - public String getDefaultTimeFormat() - { - return _defaultTimeFormat; - } - - @SuppressWarnings("UnusedDeclaration") - public void setDefaultTimeFormat(String defaultTimeFormat) - { - _defaultTimeFormat = defaultTimeFormat; - } - - public boolean isDefaultTimeFormatInherited() - { - return _defaultTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) - { - _defaultTimeFormatInherited = defaultTimeFormatInherited; - } - - public String getDefaultNumberFormat() - { - return _defaultNumberFormat; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormat(String defaultNumberFormat) - { - _defaultNumberFormat = defaultNumberFormat; - } - - public boolean isDefaultNumberFormatInherited() - { - return _defaultNumberFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) - { - _defaultNumberFormatInherited = defaultNumberFormatInherited; - } - - public boolean areRestrictedColumnsEnabled() - { - return _restrictedColumnsEnabled; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) - { - _restrictedColumnsEnabled = restrictedColumnsEnabled; - } - - public boolean isRestrictedColumnsEnabledInherited() - { - return _restrictedColumnsEnabledInherited; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) - { - _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderSettingsAction extends FolderManagementViewPostAction - { - @Override - protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(FolderSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(FolderSettingsForm form, BindException errors) - { - return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - } - } - - // Validate and populate the folder settings; save & log all changes - private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) - { - validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); - validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); - validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); - validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); - - setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); - - if (!errors.hasErrors()) - { - props.save(); - - //write an audit log event - props.writeAuditLogEvent(c, user); - } - - return !errors.hasErrors(); - } - - private interface FormatSaver - { - void save(String format) throws IllegalArgumentException; - } - - private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) - { - String defaultFormat = StringUtils.trimToNull(format); - if (inherited) - { - clearer.run(); - } - else - { - try - { - saver.save(defaultFormat); - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class ModulePropertiesAction extends FolderManagementViewAction - { - @Override - protected JspView getTabView() - { - return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); - } - } - - @SuppressWarnings("unused") - public static class FolderTypeForm - { - private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; - private String _defaultModule; - private String _folderType; - private boolean _wizard; - - public String[] getActiveModules() - { - return _activeModules; - } - - public void setActiveModules(String[] activeModules) - { - _activeModules = activeModules; - } - - public String getDefaultModule() - { - return _defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - _defaultModule = defaultModule; - } - - public String getFolderType() - { - return _folderType; - } - - public void setFolderType(String folderType) - { - _folderType = folderType; - } - - public boolean isWizard() - { - return _wizard; - } - - public void setWizard(boolean wizard) - { - _wizard = wizard; - } - } - - @RequiresPermission(AdminPermission.class) - @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments - public static class FolderTypeAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); - } - - @Override - public void validateCommand(FolderTypeForm form, Errors errors) - { - boolean fEmpty = true; - for (String module : form._activeModules) - { - if (module != null) - { - fEmpty = false; - break; - } - } - if (fEmpty && "None".equals(form.getFolderType())) - { - errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); - } - } - - @Override - public boolean handlePost(FolderTypeForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - String[] modules = form.getActiveModules(); - - if (modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) - { - container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - container.setDefaultModule(defaultModule); - } - else - { - FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); - if (container.isContainerTab() && folderType.hasContainerTabs()) - errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); - else - container.setFolderType(folderType, getUser(), errors, activeModules); - } - if (errors.hasErrors()) - return false; - - if (form.isWizard()) - { - _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - } - else - _successURL = container.getFolderType().getStartURL(container, getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) - { - return _successURL; - } - } - - @SuppressWarnings("unused") - public static class FileRootsForm extends SetupForm implements FileManagementForm - { - private String _folderRootPath; - private String _fileRootOption; - private String _cloudRootName; - private boolean _isFolderSetup; - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _migrateFilesOption; - - // cloud settings - private String[] _enabledCloudStore; - //file management - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public boolean isFolderSetup() - { - return _isFolderSetup; - } - - public void setFolderSetup(boolean folderSetup) - { - _isFolderSetup = folderSetup; - } - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsStandAloneAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - view.setFrame(WebPartView.FrameType.NONE); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); - getPageConfig().setTemplate(PageConfig.Template.Wizard); - getPageConfig().setTitle("Change File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) - .addParameter("folderSetup", true) - .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - /** - * This standalone file root management action can be used on folder types that do not support - * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL - * navigation when a workbook needs it. - */ - @RequiresPermission(AdminPermission.class) - public class ManageFileRootAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - getPageConfig().setTitle("Manage File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = getContainer().getStartURL(getUser()); - - if (getViewContext().getActionURL().getReturnUrl() != null) - { - url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - } - - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) - { - return getFileRootsView(form, errors, getReshow()); - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - } - - private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) - { - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - view.setFrame(WebPartView.FrameType.DIV); - try - { - if (!reshow) - setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return view; - } - - private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception - { - if (form.isPipelineRootForm()) - { - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - } - else - { - setFileRootFromForm(getViewContext(), form, errors); - setEnabledCloudStores(getViewContext(), form, errors); - return !errors.hasErrors(); - } - } - - public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) - { - FileContentService service = FileContentService.get(); - if (null != service) - { - boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); - String cloudRootName = form.getCloudRootName(); - if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) - { - Path defaultRootPath = service.getDefaultRootPath(container, false); - cloudRootName = service.getDefaultRootInfo(container).getCloudName(); - isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); - } - - if (isOrDefaultsToCloudRoot && null != cloudRootName) - { - if (null != form.getEnabledCloudStore()) - { - for (String storeName : form.getEnabledCloudStore()) - { - if (Strings.CI.equals(cloudRootName, storeName)) - return; - } - } - // Didn't find cloud root in enabled list - errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); - } - } - } - - public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) - { - boolean changed = false; - boolean shouldCopyMove = false; - FileContentService service = FileContentService.get(); - if (null != service) - { - // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. - // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. - MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? - MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : - MigrateFilesOption.leave; - List> sourceInfos = - ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? - Collections.emptyList() : - getCopySourceInfo(service, ctx.getContainer()); - - if (form.isDisableFileSharing()) - { - if (!service.isFileRootDisabled(ctx.getContainer())) - { - service.disableFileRoot(ctx.getContainer()); - changed = true; - } - } - else if (form.hasSiteDefaultRoot()) - { - if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), true); - changed = true; - shouldCopyMove = true; - } - } - else if (form.isCloudFileRoot()) - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String cloudRootName = form.getCloudRootName(); - if (null != cloudRootName && - (!service.isCloudRoot(ctx.getContainer()) || - !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setCloudRoot(ctx.getContainer(), cloudRootName); - try - { - PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); - if (form.isFolderSetup() && !sourceInfos.isEmpty()) - { - // File root was set to cloud storage, remove folder created - Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded - if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) - { - try - { - Files.deleteIfExists(fromPath.getParent()); - } - catch (IOException e) - { - LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); - } - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - changed = true; - shouldCopyMove = true; - } - } - else - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - URI uri = FileUtil.createUri(root, false); // root is unencoded - Path path = FileUtil.getPath(ctx.getContainer(), uri); - if (null == path || !Files.exists(path)) - { - errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); - } - else - { - Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setFileRootPath(ctx.getContainer(), root); - changed = true; - shouldCopyMove = true; - } - } - } - else - { - service.setFileRootPath(ctx.getContainer(), null); - changed = true; - } - } - - if (!errors.hasErrors()) - { - if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) - { - // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); - if (null != pipeRoot) - { - try - { - initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); - } - catch (PipelineValidationException e) - { - throw new RuntimeValidationException(e); - } - } - else - { - LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); - } - } - - form.setFileRootChanged(changed); - if (changed && null != ctx.getUser()) - { - setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); - String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); - AuditLogService.get().addEvent(ctx.getUser(), event); - } - } - } - } - - private static List> getCopySourceInfo(FileContentService service, Container container) - { - - List> sourceInfo = new ArrayList<>(); - addCopySourceInfo(service, container, sourceInfo, true); - return sourceInfo; - } - - private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) - { - if (isRoot || service.isUseDefaultRoot(container)) - { - Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); - if (null != sourceFileRootDir) - { - String pathStr = FileUtil.pathToString(sourceFileRootDir); - if (null != pathStr) - sourceInfo.add(new Pair<>(container, pathStr)); - else - throw new RuntimeValidationException("Unexpected error converting path to string"); - } - } - for (Container childContainer : container.getChildren()) - addCopySourceInfo(service, childContainer, sourceInfo, false); - } - - private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, - MigrateFilesOption migrateFilesOption) throws PipelineValidationException - { - CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); - PipelineService.get().queueJob(job); - } - - private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) - { - // test permissions. only site admins are able to turn on a custom file root for a folder - // this is only relevant if the folder is either being switched to a custom file root, - // or if the file root is changed. - if (!service.isUseDefaultRoot(ctx.getContainer())) - { - Path fileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null != fileRootPath) - { - String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); - if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) - { - if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) - throw new UnauthorizedException("Only site admins can change file roots"); - } - } - } - } - - public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) - { - String[] enabledCloudStores = form.getEnabledCloudStore(); - CloudStoreService cloud = CloudStoreService.get(); - if (cloud != null) - { - Set enabled = Collections.emptySet(); - if (enabledCloudStores != null) - enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); - - try - { - // Check if anything changed - boolean changed = false; - Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); - if (enabled.size() != storeNames.size()) - changed = true; - else - if (!enabled.containsAll(storeNames)) - changed = true; - if (changed) - cloud.setEnabledCloudStores(ctx.getContainer(), enabled); - form.setEnabledCloudStoresChanged(changed); - } - catch (UncheckedExecutionException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException - // is what BlobStore hands us if bucket (S3 container) does not exist - if (null != e.getCause()) - errors.reject(ERROR_MSG, e.getCause().getMessage()); - else - throw e; - } - catch (RuntimeException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - errors.reject(ERROR_MSG, e.getMessage()); - } - } - } - - - public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException - { - String rootSetParam = ctx.getActionURL().getParameter("rootSet"); - boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); - String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); - boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); - setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); - } - - public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, - String migrateFilesOption) throws IllegalArgumentException - { - FileContentService service = FileContentService.get(); - String confirmMessage = null; - - String migrateFilesMessage = ""; - if (fileRootChanged && !form.isFolderSetup()) - { - if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) - migrateFilesMessage = ". Existing files not copied or moved."; - else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files copied."; - form.setMigrateFilesOption(migrateFilesOption); - } - else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files moved."; - form.setMigrateFilesOption(migrateFilesOption); - } - } - - if (service != null) - { - if (service.isFileRootDisabled(container)) - { - form.setFileRootOption(FileRootProp.disable.name()); - if (fileRootChanged) - confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); - } - else if (service.isUseDefaultRoot(container)) - { - form.setFileRootOption(FileRootProp.siteDefault.name()); - Path root = service.getFileRootPath(container); - if (root != null && Files.exists(root) && fileRootChanged) - confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; - } - else if (!service.isCloudRoot(container)) - { - Path root = service.getFileRootPath(container); - - form.setFileRootOption(FileRootProp.folderOverride.name()); - if (root != null) - { - String absolutePath = FileUtil.getAbsolutePath(container, root); - form.setFolderRootPath(absolutePath); - if (Files.exists(root)) - { - if (fileRootChanged) - confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; - } - } - } - else - { - form.setFileRootOption(FileRootProp.cloudRoot.name()); - form.setCloudRootName(service.getCloudRootName(container)); - Path root = service.getFileRootPath(container); - if (root != null && fileRootChanged) - { - confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; - } - } - } - - if (fileRootChanged && confirmMessage != null) - form.setConfirmMessage(confirmMessage); - else if (enabledCloudChanged) - form.setConfirmMessage("The enabled cloud stores changed."); - } - - @RequiresPermission(AdminPermission.class) - public static class ManageFoldersAction extends FolderManagementViewAction - { - @Override - protected HttpView getTabView() - { - return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); - } - } - - public static class NotificationsForm - { - private String _provider; - - public String getProvider() - { - return _provider; - } - - public void setProvider(String provider) - { - _provider = provider; - } - } - - private static final String DATA_REGION_NAME = "Users"; - - @RequiresPermission(AdminPermission.class) - public static class NotificationsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) - { - final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); - DataRegionSelection.clearAll(getViewContext(), key); - - QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); - settings.setAllowChooseView(true); - settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); - - UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); - QueryView queryView = new QueryView(schema, settings, errors) - { - @Override - public List getDisplayColumns() - { - List columns = new ArrayList<>(); - SecurityPolicy policy = getContainer().getPolicy(); - Set assignmentSet = new HashSet<>(); - - for (RoleAssignment assignment : policy.getAssignments()) - { - Group g = SecurityManager.getGroup(assignment.getUserId()); - if (g != null) - assignmentSet.add(g.getName()); - } - - for (DisplayColumn col : super.getDisplayColumns()) - { - if (col.getName().equalsIgnoreCase("Groups")) - columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); - else - columns.add(col); - } - return columns; - } - - @Override - protected void populateButtonBar(DataView dataView, ButtonBar bar) - { - try - { - // add the provider configuration menu items to the admin panel button - MenuButton adminButton = new MenuButton("Update user settings"); - adminButton.setRequiresSelection(true); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); - - bar.add(adminButton); - super.populateButtonBar(dataView, bar); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }; - queryView.setShadeAlternatingRows(true); - queryView.setShowBorders(true); - queryView.setShowDetailsColumn(false); - queryView.setShowRecordSelectors(true); - queryView.setFrame(WebPartView.FrameType.NONE); - queryView.disableContainerFilterSelection(); - queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - - VBox defaultsView = new VBox( - HtmlView.unsafe( - "
    Default settings
    " + - "You can change this folder's default settings for email notifications here.") - ); - - PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - { - defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); - } - - return new VBox( - new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), - defaultsView, - new VBox( - HtmlView.unsafe( - "
    User settings
    " + - "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + - "notification setting is visible in the appropriately named column.

    " + - "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), - queryView - ) - ); - } - - @Override - public void validateCommand(NotificationsForm form, Errors errors) - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - provider.validateCommand(getViewContext(), errors); - } - - @Override - public boolean handlePost(NotificationsForm form, BindException errors) throws Exception - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - { - return provider.handlePost(getViewContext(), errors); - } - errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); - return false; - } - } - - public static class NotifyOptionsForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public ConfigTypeProvider getProvider() - { - return MessageConfigService.get().getConfigType(getType()); - } - } - - /** - * Action to populate an Ext store with email notification options for admin settings - */ - @RequiresPermission(AdminPermission.class) - public static class GetEmailOptionsAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(NotifyOptionsForm form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - ConfigTypeProvider provider = form.getProvider(); - if (provider != null) - { - List options = new ArrayList<>(); - - // if the list of options is not for the folder default, add an option to use the folder default - if (getViewContext().get("isDefault") == null) - options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); - - for (NotificationOption option : provider.getOptions()) - { - options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); - } - resp.put("success", true); - if (!options.isEmpty()) - resp.put("options", options); - } - else - resp.put("success", false); - - return resp; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetBulkEmailOptionsAction extends MutatingApiAction - { - @Override - public ApiResponse execute(EmailConfigFormImpl form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - ConfigTypeProvider provider = form.getProvider(); - String srcIdentifier = getContainer().getId(); - - Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); - - if (!selections.isEmpty() && provider != null) - { - int newOption = form.getIndividualEmailOption(); - - for (String user : selections) - { - User projectUser = UserManager.getUser(Integer.parseInt(user)); - UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); - - int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; - - //has this projectUser's option changed? if so, update - //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default - if (currentEmailOption != newOption) - { - provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); - } - } - resp.put("success", true); - } - else - { - resp.put("success", false); - resp.put("message", "There were no users selected"); - } - return resp; - } - } - - /** Renders only the groups that are assigned roles in this container */ - private static class FolderGroupColumn extends DataColumn - { - private final Set _assignmentSet; - - public FolderGroupColumn(Set assignmentSet, ColumnInfo col) - { - super(col); - _assignmentSet = assignmentSet; - } - - @Override - public void renderGridCellContents(RenderContext ctx, HtmlWriter out) - { - String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); - - if (value != null) - { - out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) - .filter(_assignmentSet::contains) - .map(HtmlString::of) - .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); - } - } - } - - private static class PanelConfig implements MessageConfigService.PanelInfo - { - private final ActionURL _returnUrl; - private final String _dataRegionSelectionKey; - - public PanelConfig(ActionURL returnUrl, String selectionKey) - { - _returnUrl = returnUrl; - _dataRegionSelectionKey = selectionKey; - } - - @Override - public ActionURL getReturnUrl() - { - return _returnUrl; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - } - - public static class ConceptsForm - { - private String _conceptURI; - private String _containerId; - private String _schemaName; - private String _queryName; - - public String getConceptURI() - { - return _conceptURI; - } - - public void setConceptURI(String conceptURI) - { - _conceptURI = conceptURI; - } - - public String getContainerId() - { - return _containerId; - } - - public void setContainerId(String containerId) - { - _containerId = containerId; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConceptsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); - } - - @Override - public void validateCommand(ConceptsForm form, Errors errors) - { - // validate that the required input fields are provided - String missingRequired = "", sep = ""; - if (form.getConceptURI() == null) - { - missingRequired += "conceptURI"; - sep = ", "; - } - if (form.getSchemaName() == null) - { - missingRequired += sep + "schemaName"; - sep = ", "; - } - if (form.getQueryName() == null) - missingRequired += sep + "queryName"; - if (!missingRequired.isEmpty()) - errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); - - // validate that, if provided, the containerId matches an existing container - Container postContainer = null; - if (form.getContainerId() != null) - { - postContainer = ContainerManager.getForId(form.getContainerId()); - if (postContainer == null) - errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); - } - - // validate that the schema and query names provided exist - if (form.getSchemaName() != null && form.getQueryName() != null) - { - Container c = postContainer != null ? postContainer : getContainer(); - UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); - if (schema == null) - errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); - else if (schema.getTable(form.getQueryName()) == null) - errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); - } - } - - @Override - public boolean handlePost(ConceptsForm form, BindException errors) - { - Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); - ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); - - return true; - } - } - - @RequiresPermission(AdminPermission.class) - public class FolderAliasesAction extends FormViewAction - { - @Override - public void validateCommand(FolderAliasesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/folderAliases.jsp"); - } - - @Override - public boolean handlePost(FolderAliasesForm form, BindException errors) - { - List aliases = new ArrayList<>(); - if (form.getAliases() != null) - { - StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); - while (st.hasMoreTokens()) - { - String alias = st.nextToken().trim(); - if (!alias.startsWith("/")) - { - alias = "/" + alias; - } - while (alias.endsWith("/")) - { - alias = alias.substring(0, alias.lastIndexOf('/')); - } - aliases.add(alias); - } - } - ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); - - return true; - } - - @Override - public ActionURL getSuccessURL(FolderAliasesForm form) - { - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); - } - } - - public static class FolderAliasesForm - { - private String _aliases; - - public String getAliases() - { - return _aliases; - } - - @SuppressWarnings("unused") - public void setAliases(String aliases) - { - _aliases = aliases; - } - } - - @RequiresPermission(AdminPermission.class) - public class CustomizeEmailAction extends FormViewAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) - { - JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); - result.setTitle("Email Template"); - return result; - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - - template.setSubject(form.getEmailSubject()); - template.setSenderName(form.getEmailSender()); - template.setReplyToEmail(form.getEmailReplyTo()); - template.setBody(form.getEmailMessage()); - - String[] errorStrings = new String[1]; - if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. - EmailTemplateService.get().saveEmailTemplate(template, getContainer()); - else - errors.reject(ERROR_MSG, errorStrings[0]); - } - - return !errors.hasErrors(); - } - - @Override - public ActionURL getSuccessURL(CustomEmailForm form) - { - ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); - result.replaceParameter("templateClass", form.getTemplateClass()); - if (form.getReturnActionURL() != null) - { - result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - } - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("customEmail"); - addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class DeleteCustomEmailAction extends FormHandlerAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - template.setSubject(form.getEmailSubject()); - template.setBody(form.getEmailMessage()); - - EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(CustomEmailForm form) - { - return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); - } - } - - @SuppressWarnings("unused") - public static class CustomEmailForm extends ReturnUrlForm - { - private String _templateClass; - private String _emailSubject; - private String _emailSender; - private String _emailReplyTo; - private String _emailMessage; - private String _templateDescription; - - public void setTemplateClass(String name){_templateClass = name;} - public String getTemplateClass(){return _templateClass;} - public void setEmailSubject(String subject){_emailSubject = subject;} - public String getEmailSubject(){return _emailSubject;} - public void setEmailSender(String sender){_emailSender = sender;} - public String getEmailSender(){return _emailSender;} - public void setEmailMessage(String body){_emailMessage = body;} - public String getEmailMessage(){return _emailMessage;} - public String getEmailReplyTo(){return _emailReplyTo;} - public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} - - public String getTemplateDescription() - { - return _templateDescription; - } - - public void setTemplateDescription(String templateDescription) - { - _templateDescription = templateDescription; - } - } - - private ActionURL getManageFoldersURL() - { - return new AdminUrlsImpl().getManageFoldersURL(getContainer()); - } - - public static class ManageFoldersForm extends ReturnUrlForm - { - private String name; - private String title; - private boolean titleSameAsName; - private String folder; - private String target; - private String folderType; - private String defaultModule; - private String[] activeModules; - private boolean hasLoaded = false; - private boolean showAll; - private boolean confirmed = false; - private boolean addAlias = false; - private String templateSourceId; - private String[] templateWriterTypes; - private boolean templateIncludeSubfolders = false; - private String[] targets; - private PHI _exportPhiLevel = PHI.NotPHI; - - public boolean getHasLoaded() - { - return hasLoaded; - } - - public void setHasLoaded(boolean hasLoaded) - { - this.hasLoaded = hasLoaded; - } - - public String[] getActiveModules() - { - return activeModules; - } - - public void setActiveModules(String[] activeModules) - { - this.activeModules = activeModules; - } - - public String getDefaultModule() - { - return defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - this.defaultModule = defaultModule; - } - - public boolean isShowAll() - { - return showAll; - } - - public void setShowAll(boolean showAll) - { - this.showAll = showAll; - } - - public String getFolder() - { - return folder; - } - - public void setFolder(String folder) - { - this.folder = folder; - } - - public String getName() - { - return name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isTitleSameAsName() - { - return titleSameAsName; - } - - public void setTitleSameAsName(boolean updateTitle) - { - this.titleSameAsName = updateTitle; - } - public void setName(String name) - { - this.name = name; - } - - public boolean isConfirmed() - { - return confirmed; - } - - public void setConfirmed(boolean confirmed) - { - this.confirmed = confirmed; - } - - public String getFolderType() - { - return folderType; - } - - public void setFolderType(String folderType) - { - this.folderType = folderType; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - - public String getTarget() - { - return target; - } - - public void setTarget(String target) - { - this.target = target; - } - - public void setTemplateSourceId(String templateSourceId) - { - this.templateSourceId = templateSourceId; - } - - public String getTemplateSourceId() - { - return templateSourceId; - } - - public Container getTemplateSourceContainer() - { - if (null == getTemplateSourceId()) - return null; - return ContainerManager.getForId(getTemplateSourceId()); - } - - public String[] getTemplateWriterTypes() - { - return templateWriterTypes; - } - - public void setTemplateWriterTypes(String[] templateWriterTypes) - { - this.templateWriterTypes = templateWriterTypes; - } - - public boolean getTemplateIncludeSubfolders() - { - return templateIncludeSubfolders; - } - - public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) - { - this.templateIncludeSubfolders = templateIncludeSubfolders; - } - - public String[] getTargets() - { - return targets; - } - - public void setTargets(String[] targets) - { - this.targets = targets; - } - - public PHI getExportPhiLevel() - { - return _exportPhiLevel; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - /** - * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, - * but it will work for non-workbook children as well. - */ - public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException - { - if (getTargets() != null) - { - final List targets = new ArrayList<>(); - final List directChildren = ContainerManager.getChildren(currentContainer); - - Arrays.stream(getTargets()).forEach(x -> { - Container c = ContainerManager.getForId(x); - if (c == null) - { - try - { - Integer rowId = ConvertHelper.convert(x, Integer.class); - if (rowId > 0) - c = ContainerManager.getForRowId(rowId); - } - catch (ConversionException e) - { - //ignore - } - } - - if (c != null) - { - if (!c.equals(currentContainer)) - { - if (!directChildren.contains(c)) - { - throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); - } - - if (c.getContainerType().canHaveChildren()) - { - throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); - } - } - - targets.add(c); - } - else - { - throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); - } - }); - - return targets; - } - else - { - return Collections.singletonList(currentContainer); - } - } - } - - public static class RenameContainerForm - { - private String name; - private String title; - private boolean addAlias = true; - - public String getName() - { - return name; - } - - public void setName(String name) - { - this.name = name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - } - - // Note that validation checks occur in ContainerManager.rename() - @RequiresPermission(AdminPermission.class) - public static class RenameContainerAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RenameContainerForm form, BindException errors) - { - Container container = getContainer(); - String name = StringUtils.trimToNull(form.getName()); - String title = StringUtils.trimToNull(form.getTitle()); - - String nameValue = name; - String titleValue = title; - if (name == null && title == null) - { - errors.reject(ERROR_MSG, "Please specify a name or a title."); - return new ApiSimpleResponse("success", false); - } - else if (name != null && title == null) - { - titleValue = name; - } - else if (name == null) - { - nameValue = container.getName(); - } - - boolean addAlias = form.isAddAlias(); - - try - { - Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); - return new ApiSimpleResponse(c.toJSON(getUser())); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - return new ApiSimpleResponse("success", false); - } - } - } - - @RequiresPermission(AdminPermission.class) - public class RenameFolderAction extends FormViewAction - { - private ActionURL _returnUrl; - - @Override - public void validateCommand(ManageFoldersForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - try - { - String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); - Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); - _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); - return true; - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - } - - return false; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - return _returnUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - String containerType = getContainer().isProject() ? "Project" : "Folder"; - addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); - } - } - - public static class MoveFolderTreeView extends JspView - { - private MoveFolderTreeView(ManageFoldersForm form, BindException errors) - { - super("/org/labkey/core/admin/moveFolder.jsp", form, errors); - } - } - - @RequiresPermission(AdminPermission.class) - @ActionNames("ShowMoveFolderTree,MoveFolder") - public class MoveFolderAction extends FormViewAction - { - boolean showConfirmPage = false; - boolean moveFailed = false; - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - Container c = getContainer(); - - if (c.isRoot()) - throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root - - if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); - - Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); - if (null == newParent) - { - errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); - } - else if (!newParent.hasPermission(getUser(), AdminPermission.class)) - { - throw new UnauthorizedException(); - } - else if (newParent.hasChild(c.getName())) - { - errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception - { - if (showConfirmPage) - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - if (moveFailed) - return new SimpleErrorView(errors); - else - return new MoveFolderTreeView(form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception - { - Container c = getContainer(); - Container newParent = ContainerManager.getForPath(form.getTarget()); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) - { - showConfirmPage = true; - return false; // reshow - } - - try - { - ContainerManager.move(c, newParent, getUser()); - } - catch (ValidationException e) - { - moveFailed = true; - getPageConfig().setTemplate(Template.Dialog); - for (ValidationError validationError : e.getErrors()) - { - errors.addError(new LabKeyError(validationError.getMessage())); - } - if (!errors.hasErrors()) - errors.addError(new LabKeyError("Move failed")); - return false; - } - - if (form.isAddAlias()) - { - List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); - newAliases.add(c.getPath()); - ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) - { - Container c = getContainer(); - c = ContainerManager.getForId(c.getId()); // Reload container to populate new location - return new AdminUrlsImpl().getManageFoldersURL(c); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Folder Management", getManageFoldersURL()); - root.addChild("Move Folder"); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConfirmProjectMoveAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ManageFoldersForm form, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm Project Move"); - } - } - - private static abstract class AbstractCreateFolderAction
    extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FORM target, Errors errors) - { - } - - @Override - public ModelAndView getView(FORM form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - if (!reshow) - { - FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); - if (null != folderType) - { - // If a default folder type has been configured by a site admin set that as the default folder type choice - form.setFolderType(folderType.getName()); - } - form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); - } - JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); - vbox.addView(statusView); - - Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder - - setHelpTopic("createProject"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); - getPageConfig().setTemplate(Template.Wizard); - - if (c.isRoot()) - getPageConfig().setTitle("Create Project"); - else - { - String title = "Create Folder"; - - title += " in /"; - if (c == ContainerManager.getHomeContainer()) - title += "Home"; - else - title += c.getName(); - - getPageConfig().setTitle(title); - } - - return vbox; - } - - @Override - public boolean handlePost(FORM form, BindException errors) throws Exception - { - Container parent = getViewContext().getContainerNoTab(); - String folderName = StringUtils.trimToNull(form.getName()); - String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); - StringBuilder error = new StringBuilder(); - Consumer afterCreateHandler = getAfterCreateHandler(form); - - Container container; - - if (Container.isLegalName(folderName, parent.isRoot(), error)) - { - if (parent.hasChild(folderName)) - { - if (parent.isRoot()) - { - error.append("The server already has a project with this name."); - } - else - { - error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); - } - } - else - { - String folderType = form.getFolderType(); - - if (null == folderType) - { - errors.reject(null, "Folder type must be specified"); - return false; - } - - if ("Template".equals(folderType)) // Create folder from selected template - { - Container sourceContainer = form.getTemplateSourceContainer(); - if (null == sourceContainer) - { - errors.reject(null, "Source template folder not selected"); - return false; - } - else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) - { - errors.reject(null, "User does not have administrator permissions to the source container"); - return false; - } - else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) - { - errors.reject(null, "The source folder has a restricted module for which you do not have permission."); - return false; - } - - FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", - form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, - new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - - container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); - } - else - { - FolderType type = FolderTypeManager.get().getFolderType(folderType); - - if (type == null) - { - errors.reject(null, "Folder type not recognized"); - return false; - } - - String[] modules = form.getActiveModules(); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - if (null == modules || modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - } - - // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. - Consumer configureContainer = (newContainer) -> - { - afterCreateHandler.accept(newContainer); - newContainer.setFolderType(type, getUser(), errors); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - newContainer.setDefaultModule(defaultModule); - } - }; - container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); - } - - _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - } - - errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); - return false; - } - - /** - * Return a Consumer that provides post-creation handling on the new Container - */ - abstract public Consumer getAfterCreateHandler(FORM form); - - @Override - protected String getCommandClassMethodName() - { - return "getAfterCreateHandler"; - } - - @Override - public ActionURL getSuccessURL(FORM form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public static class CreateFolderAction extends AbstractCreateFolderAction - { - @Override - public Consumer getAfterCreateHandler(ManageFoldersForm form) - { - // No special handling - return container -> {}; - } - } - - public static class CreateProjectForm extends ManageFoldersForm - { - private boolean _assignProjectAdmin = false; - - public boolean isAssignProjectAdmin() - { - return _assignProjectAdmin; - } - - @SuppressWarnings("unused") - public void setAssignProjectAdmin(boolean assignProjectAdmin) - { - _assignProjectAdmin = assignProjectAdmin; - } - } - - @RequiresPermission(CreateProjectPermission.class) - public static class CreateProjectAction extends AbstractCreateFolderAction - { - @Override - public void validateCommand(CreateProjectForm target, Errors errors) - { - super.validateCommand(target, errors); - if (!getContainer().isRoot()) - errors.reject(ERROR_MSG, "Must be invoked from the root"); - } - - @Override - public Consumer getAfterCreateHandler(CreateProjectForm form) - { - if (form.isAssignProjectAdmin()) - { - return c -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); - policy.addRoleAssignment(getUser(), ProjectAdminRole.class); - User savePolicyUser = getUser(); - if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - SecurityPolicyManager.savePolicy(policy, savePolicyUser); - }; - } - else - { - return c -> {}; - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetFolderPermissionsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(SetFolderPermissionsForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); - vbox.addView(statusView); - - Container c = getContainer(); - getPageConfig().setTitle("Users / Permissions"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - setHelpTopic("createProject"); - - return vbox; - } - - @Override - public boolean handlePost(SetFolderPermissionsForm form, BindException errors) - { - Container c = getContainer(); - String permissionType = form.getPermissionType(); - - if(c.isProject()){ - _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); - } - else - { - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - if (form.isAdvanced()) - { - _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); - } - else - { - _successURL = getContainer().getStartURL(getUser()); - } - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - } - - if(permissionType == null){ - errors.reject(ERROR_MSG, "You must select one of the options for permissions."); - return false; - } - - switch (permissionType) - { - case "CurrentUser" -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); - policy.addRoleAssignment(getUser(), role); - SecurityPolicyManager.savePolicy(policy, getUser()); - } - case "Inherit" -> SecurityManager.setInheritPermissions(c); - case "CopyExistingProject" -> { - String targetProject = form.getTargetProject(); - if (targetProject == null) - { - errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); - return false; - } - Container source = ContainerManager.getForId(targetProject); - if (source == null) - { - source = ContainerManager.getForPath(targetProject); - } - if (source == null) - { - throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); - } - Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); - - //copy role assignments - SecurityPolicy op = SecurityPolicyManager.getPolicy(source); - MutableSecurityPolicy np = new MutableSecurityPolicy(c); - for (RoleAssignment assignment : op.getAssignments()) - { - int userId = assignment.getUserId(); - UserPrincipal p = SecurityManager.getPrincipal(userId); - Role r = assignment.getRole(); - - if (p instanceof Group g) - { - if (!g.isProjectGroup()) - { - np.addRoleAssignment(p, r, false); - } - else - { - np.addRoleAssignment(groupMap.get(p), r, false); - } - } - else - { - np.addRoleAssignment(p, r, false); - } - } - SecurityPolicyManager.savePolicy(np, getUser()); - } - default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); - } - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - - @Override - public ActionURL getSuccessURL(SetFolderPermissionsForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - } - } - - public static class SetFolderPermissionsForm - { - private String targetProject; - private String permissionType; - private boolean advanced; - - public String getPermissionType() - { - return permissionType; - } - - @SuppressWarnings("unused") - public void setPermissionType(String permissionType) - { - this.permissionType = permissionType; - } - - public String getTargetProject() - { - return targetProject; - } - - @SuppressWarnings("unused") - public void setTargetProject(String targetProject) - { - this.targetProject = targetProject; - } - - public boolean isAdvanced() - { - return advanced; - } - - @SuppressWarnings("unused") - public void setAdvanced(boolean advanced) - { - this.advanced = advanced; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetInitialFolderSettingsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FilesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - Container c = getContainer(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - - String noun = c.isProject() ? "Project": "Folder"; - getPageConfig().setTitle(noun + " Settings"); - - return vbox; - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) - { - Container c = getContainer(); - String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); - String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; - - if(folderRootPath == null && !fileRootOption.equals("default")) - { - errors.reject(ERROR_MSG, "Error: Must supply a default file location."); - return false; - } - - FileContentService service = FileContentService.get(); - if(fileRootOption.equals("default")) - { - service.setIsUseDefaultRoot(c, true); - } - // Requires AdminOperationsPermission to set file root - else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) - { - if (!service.isValidProjectRoot(folderRootPath)) - { - errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - return false; - } - - service.setIsUseDefaultRoot(c.getProject(), false); - service.setFileRootPath(c.getProject(), folderRootPath); - } - - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - _successURL = getContainer().getStartURL(getUser()); - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - - return true; - } - - @Override - public ActionURL getSuccessURL(FilesForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - setHelpTopic("createProject"); - } - } - - @RequiresPermission(DeletePermission.class) - public static class DeleteWorkbooksAction extends SimpleRedirectAction - { - public void validateCommand(ReturnUrlForm target, Errors errors) - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - if (ids.isEmpty()) - { - errors.reject(ERROR_MSG, "No IDs provided"); - } - } - - @Override - public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - - ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); - ids.forEach(id -> ret.addParameter("targets", id)); - - ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - - return ret; - } - } - - //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent - @RequiresPermission(DeletePermission.class) - public static class DeleteFolderAction extends FormViewAction - { - private final List _deleted = new ArrayList<>(); - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - try - { - List targets = form.getTargetContainers(getContainer()); - for (Container target : targets) - { - if (!ContainerManager.isDeletable(target)) - errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); - - if (target.isProject() && !getUser().hasRootAdminPermission()) - { - throw new UnauthorizedException(); - } - - Class permClass = target.getPermissionNeededToDelete(); - if (!target.hasPermission(getUser(), permClass)) - { - Permission perm = RoleManager.getPermission(permClass); - throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); - } - - if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) - { - throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); - } - - if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); - } - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - List targets = form.getTargetContainers(getContainer()); - - // Must be site/app admin to delete a project - for (Container c : targets) - { - ContainerManager.deleteAll(c, getUser()); - } - - _deleted.addAll(targets); - - return true; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: - // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders - if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) - { - Container c = getContainer(); - if (c.isProject()) - return AppProps.getInstance().getHomePageActionURL(); - else - return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); - } - else - { - if (form.getReturnUrl() != null) - { - return form.getReturnActionURL(); - } - else - { - return getContainer().getStartURL(getUser()); - } - } - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersAction extends FormViewAction - { - @Override - public void validateCommand(FolderReorderForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); - } - - @Override - public boolean handlePost(FolderReorderForm form, BindException errors) - { - return ReorderFolders(form, errors); - } - - @Override - public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) - { - if (getContainer().isRoot()) - return getShowAdminURL(); - else - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); - addAdminNavTrail(root, title, this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersApiAction extends MutatingApiAction - { - @Override - public ApiResponse execute(FolderReorderForm form, BindException errors) - { - return new ApiSimpleResponse("success", ReorderFolders(form, errors)); - } - } - - private boolean ReorderFolders(FolderReorderForm form, BindException errors) - { - Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); - if (form.isResetToAlphabetical()) - ContainerManager.setChildOrderToAlphabetical(parent); - else if (form.getOrder() != null) - { - List children = parent.getChildren(); - String[] order = form.getOrder().split(";"); - Map nameToContainer = new HashMap<>(); - for (Container child : children) - nameToContainer.put(child.getName(), child); - List sorted = new ArrayList<>(children.size()); - for (String childName : order) - { - Container child = nameToContainer.get(childName); - sorted.add(child); - } - - try - { - ContainerManager.setChildOrder(parent, sorted); - } - catch (ContainerException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - } - - return true; - } - - public static class FolderReorderForm - { - private String _order; - private boolean _resetToAlphabetical; - - public String getOrder() - { - return _order; - } - - @SuppressWarnings("unused") - public void setOrder(String order) - { - _order = order; - } - - public boolean isResetToAlphabetical() - { - return _resetToAlphabetical; - } - - @SuppressWarnings("unused") - public void setResetToAlphabetical(boolean resetToAlphabetical) - { - _resetToAlphabetical = resetToAlphabetical; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RevertFolderAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RevertFolderForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - - boolean success = false; - Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); - if (null != revertContainer) - { - if (revertContainer.isContainerTab()) - { - FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - revertContainer.setFolderType(origFolderType, getUser(), errors); - if (!errors.hasErrors()) - success = true; - } - } - } - else if (revertContainer.getFolderType().hasContainerTabs()) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - List children = revertContainer.getChildren(); - for (Container container : children) - { - if (container.isContainerTab()) - { - FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - container.setFolderType(origFolderType, getUser(), errors); - } - } - } - } - if (!errors.hasErrors()) - { - transaction.commit(); - success = true; - } - } - } - } - return new ApiSimpleResponse("success", success); - } - } - - public static class RevertFolderForm - { - private String _containerPath; - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - } - - public static class EmailTestForm - { - private String _to; - private String _body; - private ConfigurationException _exception; - - public String getTo() - { - return _to; - } - - public void setTo(String to) - { - _to = to; - } - - public String getBody() - { - return _body; - } - - public void setBody(String body) - { - _body = body; - } - - public ConfigurationException getException() - { - return _exception; - } - - public void setException(ConfigurationException exception) - { - _exception = exception; - } - - public String getFrom(Container c) - { - LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); - return props.getSystemEmailAddress(); - } - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class EmailTestAction extends FormViewAction - { - @Override - public void validateCommand(EmailTestForm form, Errors errors) - { - if(null == form.getTo() || form.getTo().isEmpty()) - { - errors.reject(ERROR_MSG, "To field cannot be blank."); - form.setException(new ConfigurationException("To field cannot be blank")); - return; - } - - try - { - ValidEmail email = new ValidEmail(form.getTo()); - } - catch(ValidEmail.InvalidEmailException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - form.setException(new ConfigurationException(e.getMessage())); - } - } - - @Override - public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) - { - JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); - testView.setTitle("Send a Test Email"); - - if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) - { - JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); - emailPropsView.setTitle("Current Email Settings"); - - return new VBox(emailPropsView, testView); - } - else - return testView; - } - - @Override - public boolean handlePost(EmailTestForm form, BindException errors) throws Exception - { - if (errors.hasErrors()) - { - return false; - } - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); - try - { - MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); - msg.setSubject("Test email message sent from " + props.getShortName()); - msg.setText(PageFlowUtil.filter(form.getBody())); - - try - { - MailHelper.send(msg, getUser(), getContainer()); - } - catch (ConfigurationException e) - { - form.setException(e); - return false; - } - catch (Exception e) - { - form.setException(new ConfigurationException(e.getMessage())); - return false; - } - } - catch (MessagingException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - return true; - } - - @Override - public URLHelper getSuccessURL(EmailTestForm emailTestForm) - { - return new ActionURL(EmailTestAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Test Email Configuration", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class RecreateViewsAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - getPageConfig().setShowHeader(false); - getPageConfig().setTitle("Recreate Views?"); - return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - ModuleLoader.getInstance().recreateViews(); - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull ActionURL getSuccessURL(Object o) - { - return AppProps.getInstance().getHomePageActionURL(); - } - } - - static public class LoggingForm - { - public boolean isLogging() - { - return logging; - } - - public void setLogging(boolean logging) - { - this.logging = logging; - } - - public boolean logging = false; - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class GetSessionLogEventsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - Integer eventId = null; - try - { - String s = getViewContext().getRequest().getParameter("eventId"); - if (null != s) - eventId = Integer.parseInt(s); - } - catch (NumberFormatException ignored) {} - ApiSimpleResponse res = new ApiSimpleResponse(); - res.put("success", true); - res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); - return res; - } - } - - @RequiresLogin - @AllowedBeforeInitialUserIsSet - @AllowedDuringUpgrade - @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ - public static class GetTrackedAllocationsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - long requestId = 0; - try - { - String s = getViewContext().getRequest().getParameter("requestId"); - if (null != s) - requestId = Long.parseLong(s); - } - catch (NumberFormatException ignored) {} - List requests = MemTracker.getInstance().getNewRequests(requestId); - List> jsonRequests = new ArrayList<>(requests.size()); - for (RequestInfo requestInfo : requests) - { - Map m = new HashMap<>(); - m.put("requestId", requestInfo.getId()); - m.put("url", requestInfo.getUrl()); - m.put("date", requestInfo.getDate()); - - - List> sortedObjects = sortByCounts(requestInfo); - - List> jsonObjects = new ArrayList<>(sortedObjects.size()); - for (Map.Entry entry : sortedObjects) - { - Map jsonObject = new HashMap<>(); - jsonObject.put("name", entry.getKey()); - jsonObject.put("count", entry.getValue()); - jsonObjects.add(jsonObject); - } - m.put("objects", jsonObjects); - jsonRequests.add(m); - } - return new ApiSimpleResponse("requests", jsonRequests); - } - - private List> sortByCounts(RequestInfo requestInfo) - { - List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); - objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); - return objects; - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class TrackedAllocationsViewerAction extends SimpleViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - getPageConfig().setTemplate(Template.Print); - return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class SessionLoggingAction extends FormViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) - throw new UnauthorizedException(); - } - - @Override - public boolean handlePost(LoggingForm form, BindException errors) - { - boolean on = SessionAppender.isLogging(getViewContext().getRequest()); - if (form.logging != on) - { - if (!form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging OFF"); - SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); - if (form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging ON"); - } - return true; - } - - @Override - public void validateCommand(LoggingForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) - { - SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); - getPageConfig().setTemplate(Template.Print); - return new LoggingView(); - } - - @Override - public ActionURL getSuccessURL(LoggingForm o) - { - return new ActionURL(SessionLoggingAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); - root.addChild("View Event Log"); - } - } - - static class LoggingView extends JspView - { - LoggingView() - { - super("/org/labkey/core/admin/logging.jsp", null); - } - } - - public static class LogForm - { - private String _message; - private String _level; - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = level; - } - } - - - // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when - // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. - @RequiresLogin - public static class LogAction extends MutatingApiAction - { - @Override - public ApiResponse execute(LogForm logForm, BindException errors) - { - // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate - // \n to newlines to allow them at the beginning or end of the message as well. - StringBuilder message = new StringBuilder(); - message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); - - Level level = Level.toLevel(logForm.getLevel(), Level.INFO); - CLIENT_LOG.log(level, message); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class ValidateDomainsAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file - for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) - { - PipeRoot root = PipelineService.get().findPipelineRoot(project); - if (root != null && root.isValid()) - { - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipelineJob job = new ValidateDomainsPipelineJob(info, root); - PipelineService.get().queueJob(job); - return true; - } - } - return false; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); - } - } - - public static class ModulesForm - { - private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) - private boolean _managedOnly = false; - private boolean _unmanagedOnly = false; - - public double[] getIgnore() - { - return _ignore; - } - - public void setIgnore(double[] ignore) - { - _ignore = ignore; - } - - private Set getIgnoreSet() - { - return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); - } - - public boolean isManagedOnly() - { - return _managedOnly; - } - - @SuppressWarnings("unused") - public void setManagedOnly(boolean managedOnly) - { - _managedOnly = managedOnly; - } - - public boolean isUnmanagedOnly() - { - return _unmanagedOnly; - } - - @SuppressWarnings("unused") - public void setUnmanagedOnly(boolean unmanagedOnly) - { - _unmanagedOnly = unmanagedOnly; - } - } - - public enum ManageFilter - { - ManagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && module.shouldManageVersion(); - } - }, - UnmanagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && !module.shouldManageVersion(); - } - }, - All - { - @Override - public boolean accept(Module module) - { - return true; - } - }; - - public abstract boolean accept(Module module); - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ModulesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModulesForm form, BindException errors) - { - ModuleLoader ml = ModuleLoader.getInstance(); - boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); - - Collection unknownModules = ml.getUnknownModuleContexts().values(); - Collection knownModules = ml.getAllModuleContexts(); - knownModules.removeAll(unknownModules); - - Set ignoreSet = form.getIgnoreSet(); - HtmlString managedLink = HtmlString.EMPTY_STRING; - HtmlString unmanagedLink = HtmlString.EMPTY_STRING; - - // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release - // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. - if (AppProps.getInstance().isDevMode()) - { - if (ignoreSet.isEmpty() && !form.isManagedOnly()) - { - String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("ignore", "0.00," + lowestSchemaVersion); - url.addParameter("managedOnly", true); - managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); - } - else - { - List ignore = ignoreSet - .stream() - .map(ModuleContext::formatVersion) - .collect(Collectors.toCollection(LinkedList::new)); - - String ignoreString = ignore.isEmpty() ? null : ignore.toString(); - String unmanaged = form.isManagedOnly() ? "unmanaged" : null; - - managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); - } - - if (!form.isUnmanagedOnly()) - { - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("unmanagedOnly", true); - unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); - } - else - { - unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); - } - } - - ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); - - HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); - if (hasAdminOpsPerm) - { - deleteInstructions.unsafeAppend("

    ").append( - "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + - "Module files are typically deployed in /modules and /externalModules.") - .unsafeAppend("

    ").append( - LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); - } - - HtmlStringBuilder docLink = HtmlStringBuilder.of(); - docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); - - HtmlStringBuilder knownDescription = HtmlStringBuilder.of() - .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); - HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); - - HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() - .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + - "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + - "is stored is not accessible, or the module file is corrupted.") - .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); - HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); - HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); - - return new VBox(known, unknown); - } - - private class ModulesView extends WebPartView - { - private final Collection _contexts; - private final HtmlString _descriptionHtml; - private final HtmlString _noModulesDescriptionHtml; - private final Set _ignoreVersions; - private final ManageFilter _manageFilter; - - private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) - { - super(FrameType.PORTAL); - List sorted = new ArrayList<>(contexts); - sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); - - _contexts = sorted; - _descriptionHtml = descriptionHtml; - _noModulesDescriptionHtml = noModulesDescriptionHtml; - _ignoreVersions = ignoreVersions; - _manageFilter = manageFilter; - setTitle(type + " Modules"); - } - - @Override - protected void renderView(Object model, HtmlWriter out) - { - boolean isDevMode = AppProps.getInstance().isDevMode(); - boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); - boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); - final AtomicInteger rowCount = new AtomicInteger(); - ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); - final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); - final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); - - if (_contexts.isEmpty()) - { - out.write(_noModulesDescriptionHtml); - } - else - { - DIV( - DIV(_descriptionHtml), - TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), - TR( - TD(cl("labkey-column-header"),"Name"), - TD(cl("labkey-column-header"),"Release Version"), - TD(cl("labkey-column-header"),"Schema Version"), - TD(cl("labkey-column-header"),"Class"), - TD(cl("labkey-column-header"),"Location"), - TD(cl("labkey-column-header"),"Schemas"), - !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions - null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions - !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions - ), - _contexts.stream() - .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) - .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) - .filter(pair -> _manageFilter.accept(pair.getValue())) - .map(pair -> - { - ModuleContext moduleContext = pair.getKey(); - Module module = pair.getValue(); - List schemas = moduleContext.getSchemaList(); - Double schemaVersion = moduleContext.getSchemaVersion(); - boolean replaceableModule = false; - if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) - { - File zip = module.getZippedPath(); - if (null != zip && zip.getParentFile().equals(externalModulesDir)) - replaceableModule = true; - } - boolean deleteableModule = replaceableModule || null == module; - String className = StringUtils.trimToEmpty(moduleContext.getClassName()); - String fullPathToModule = ""; - String shortPathToModule = ""; - if (null != module) - { - Path p = module.getExplodedPath().toPath(); - if (null != module.getZippedPath()) - p = module.getZippedPath().toPath(); - if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) - if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) - p = Paths.get(module.getSourcePath()); - fullPathToModule = p.toString(); - shortPathToModule = fullPathToModule; - Path rel = relativeRoot.relativize(p); - if (!rel.startsWith("..")) - shortPathToModule = rel.toString(); - } - ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); - - return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), - TD(moduleContext.getName()), - TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), - TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), - TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), - TD(SPAN(at(title,fullPathToModule),shortPathToModule)), - TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), - !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), - null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), - !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) - ); - }) - ) - ).appendTo(out); - } - } - } - - private ActionURL getDeleteURL(String name) - { - ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); - if (null != url) - return url; - url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getUpdateURL(String name) - { - ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); - if (null != url) - return url; - url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getModuleEditorURL(String name) - { - return ModuleEditorService.get().getModuleEditorURL(name); - } - - private ActionURL getCreateURL() - { - ActionURL url = ModuleEditorService.get().getCreateModuleURL(); - if (null != url) - return url; - url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("defaultModules"); - addAdminNavTrail(root, "Modules", getClass()); - } - } - - public static class SchemaVersionTestCase extends Assert - { - @Test - public void verifyMinimumSchemaVersion() - { - List modulesTooLow = ModuleLoader.getInstance().getModules().stream() - .filter(ManageFilter.ManagedOnly::accept) - .filter(m -> null != m.getSchemaVersion()) - .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) - .toList(); - - if (!modulesTooLow.isEmpty()) - fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); - } - - @Test - public void modulesWithSchemaVersionButNoScripts() - { - // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. - List moduleNames = ModuleLoader.getInstance().getModules().stream() - .filter(m -> m.getSchemaVersion() != null) - .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) - .map(m -> m.getName() + ": " + m.getSchemaVersion()) - .toList(); - - if (!moduleNames.isEmpty()) - fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); - } - } - - public static class ModuleForm - { - private String _name; - - public String getName() - { - return _name; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setName(String name) - { - _name = name; - } - - @NotNull - private ModuleContext getModuleContext() - { - ModuleLoader ml = ModuleLoader.getInstance(); - ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); - - if (null == ctx) - throw new NotFoundException("Module not found"); - - return ctx; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DeleteModuleAction extends ConfirmAction - { - @Override - public void validateCommand(ModuleForm form, Errors errors) - { - } - - @Override - public ModelAndView getConfirmView(ModuleForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Module"); - - ModuleContext ctx = form.getModuleContext(); - Module module = ModuleLoader.getInstance().getModule(ctx.getName()); - boolean hasSchemas = !ctx.getSchemaList().isEmpty(); - boolean hasFiles = false; - if (null != module) - hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); - - HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); - HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); - if (hasSchemas) - { - SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); - List deleteList = schemaActions.deleteList(); - List skipList = schemaActions.skipList(); - - // List all the schemas that will be deleted - if (!deleteList.isEmpty()) - { - description.append(" and delete all data in "); - description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); - } - - // For unknown modules, also list the schemas that won't be deleted - if (!skipList.isEmpty()) - { - skippedSchemas.append(HtmlString.BR); - skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) - .append("Note: Schema \"") - .append(sam.schema()) - .append("\" will not be deleted because it's in use by module \"") - .append(sam.module()) - .append("\"")); - } - } - - return new HtmlView(DIV( - !hasFiles ? null : DIV(cl("labkey-warning-messages"), - "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", - null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), - null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) - ), - BR(), - "Are you sure you want to remove the ", description, "? ", - "This operation cannot be undone!", - skippedSchemas, - BR(), - !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." - )); - } - - @Override - public boolean handlePost(ModuleForm form, BindException errors) - { - ModuleLoader.getInstance().removeModule(form.getModuleContext()); - - return true; - } - - @Override - public @NotNull URLHelper getSuccessURL(ModuleForm form) - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class UpdateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class CreateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class OptionalFeatureForm - { - private String feature; - private boolean enabled; - - public String getFeature() - { - return feature; - } - - public void setFeature(String feature) - { - this.feature = feature; - } - - public boolean isEnabled() - { - return enabled; - } - - public void setEnabled(boolean enabled) - { - this.enabled = enabled; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @ActionNames("OptionalFeature, ExperimentalFeature") - public static class OptionalFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(OptionalFeatureForm form, BindException errors) - { - String feature = StringUtils.trimToNull(form.getFeature()); - if (feature == null) - throw new ApiUsageException("feature is required"); - - OptionalFeatureService svc = OptionalFeatureService.get(); - if (svc == null) - throw new IllegalStateException(); - - Map ret = new HashMap<>(); - ret.put("feature", feature); - - if (isPost()) - { - ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); - svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); - } - - ret.put("enabled", svc.isFeatureEnabled(feature)); - return new ApiSimpleResponse(ret); - } - } - - public static class OptionalFeaturesForm - { - private String _type; - private boolean _showHidden; - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - public @NotNull FeatureType getTypeEnum() - { - return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); - } - - public boolean isShowHidden() - { - return _showHidden; - } - - @SuppressWarnings("unused") - public void setShowHidden(boolean showHidden) - { - _showHidden = showHidden; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class OptionalFeaturesAction extends SimpleViewAction - { - private FeatureType _type; - - @Override - public ModelAndView getView(OptionalFeaturesForm form, BindException errors) - { - _type = form.getTypeEnum(); - JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("experimental"); - addAdminNavTrail(root, _type.name() + " Features", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ProductFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(ProductConfigForm form, BindException errors) - { - String productKey = StringUtils.trimToNull(form.getProductKey()); - - Map ret = new HashMap<>(); - - if (isPost()) - { - ProductConfiguration.setProductKey(productKey); - } - - ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); - return new ApiSimpleResponse(ret); - } - } - - public static class ProductConfigForm - { - private String productKey; - - public String getProductKey() - { - return productKey; - } - - public void setProductKey(String productKey) - { - this.productKey = productKey; - } - - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class ProductConfigurationAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Product Configuration", getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - } - - - public static class FolderTypesBean - { - private final Collection _allFolderTypes; - private final Collection _enabledFolderTypes; - private final FolderType _defaultFolderType; - - public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) - { - _allFolderTypes = allFolderTypes; - _enabledFolderTypes = enabledFolderTypes; - _defaultFolderType = defaultFolderType; - } - - public Collection getAllFolderTypes() - { - return _allFolderTypes; - } - - public Collection getEnabledFolderTypes() - { - return _enabledFolderTypes; - } - - public FolderType getDefaultFolderType() - { - return _defaultFolderType; - } - } - - @AdminConsoleAction - @RequiresPermission(AdminPermission.class) - public class FolderTypesAction extends FormViewAction - { - @Override - public void validateCommand(Object form, Errors errors) - { - } - - @Override - public ModelAndView getView(Object form, boolean reshow, BindException errors) - { - FolderTypesBean bean; - if (reshow) - { - bean = getOptionsFromRequest(); - } - else - { - FolderTypeManager manager = FolderTypeManager.get(); - var defaultFolderType = manager.getDefaultFolderType(); - // If a default folder type has not yet been configuration use "Collaboration" folder type as the default - defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); - boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); - bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); - } - - return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); - } - - @Override - public boolean handlePost(Object form, BindException errors) - { - FolderTypesBean bean = getOptionsFromRequest(); - var defaultFolderType = bean.getDefaultFolderType(); - if (defaultFolderType == null) - { - errors.reject(ERROR_MSG, "Please select a default folder type."); - return false; - } - var enabledFolderTypes = bean.getEnabledFolderTypes(); - if (!enabledFolderTypes.contains(defaultFolderType)) - { - errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); - return false; - } - - FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); - return true; - } - - private FolderTypesBean getOptionsFromRequest() - { - var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); - List enabledFolderTypes = new ArrayList<>(); - FolderType defaultFolderType = null; - String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); - - for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) - { - boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); - if (enabled) - { - enabledFolderTypes.add(folderType); - } - if (folderType.getName().equals(defaultFolderTypeParam)) - { - defaultFolderType = folderType; - } - } - return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); - } - - @Override - public URLHelper getSuccessURL(Object form) - { - return getShowAdminURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Types", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class CustomizeMenuAction extends MutatingApiAction - { - @Override - public ApiResponse execute(CustomizeMenuForm form, BindException errors) - { - if (null != form.getUrl()) - { - String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); - if (null != errorMessage) - { - errors.reject(ERROR_MSG, errorMessage); - return new ApiSimpleResponse("success", false); - } - } - - setCustomizeMenuForm(form, getContainer(), getUser()); - return new ApiSimpleResponse("success", true); - } - } - - protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; - protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; - protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; - protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; - protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; - protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; - protected static final String CUSTOMMENU_URL = "customMenuUrl"; - protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; - protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; - protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; - protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; - protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; - - public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) - { - CustomizeMenuForm form = new CustomizeMenuForm(); - Map menuProps = webPart.getPropertyMap(); - - String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); - String queryName = menuProps.get(CUSTOMMENU_QUERY); - String columnName = menuProps.get(CUSTOMMENU_COLUMN); - String viewName = menuProps.get(CUSTOMMENU_VIEW); - String folderName = menuProps.get(CUSTOMMENU_FOLDER); - String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; - String urlBottom = menuProps.get(CUSTOMMENU_URL); - String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); - String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); - String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); - boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); - String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); - boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); - String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); - boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); - - form.setSchemaName(schemaName); - form.setQueryName(queryName); - form.setColumnName(columnName); - form.setViewName(viewName); - form.setFolderName(folderName); - form.setTitle(title); - form.setUrl(urlBottom); - form.setRootFolder(rootFolder); - form.setFolderTypes(folderTypes); - form.setChoiceListQuery(choiceListQuery); - form.setIncludeAllDescendants(includeAllDescendants); - form.setCurrentProjectOnly(currentProjectOnly); - - form.setWebPartIndex(webPart.getIndex()); - form.setPageId(webPart.getPageId()); - return form; - } - - private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) - { - Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); - if (null == webPart) - throw new NotFoundException(); - Map menuProps = webPart.getPropertyMap(); - - menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); - menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); - menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); - menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); - menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); - menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); - menuProps.put(CUSTOMMENU_URL, form.getUrl()); - - // If root folder not specified, set as current container - menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); - menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); - menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); - menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); - menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); - - Portal.updatePart(user, webPart); - } - - @RequiresPermission(AdminPermission.class) - public static class AddTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - if(tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived - // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something - // longer than 64 characters. - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - if (name.length() > 50) - name = name.substring(0, 50).trim(); - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); - - for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) - { - folderTabMap.put(tab.getName(), tab); - } - - if (pages.containsKey(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (page.getCaption() != null && page.getCaption().equals(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - else if (folderTabMap.containsKey(page.getPageId())) - { - if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - validateCommand(form, errors); - - if(errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - String name = form.getTabName(); - String caption = form.getTabName(); - - // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the - // name, and is editable, is allowed to be 64 characters. - if (name.length() > 50) - name = name.substring(0, 50).trim(); - - Portal.saveParts(container, name); - Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); - - if (!name.equals(caption)) - { - // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(name); - // Get a mutable copy - page = page.copy(); - page.setCaption(caption); - Portal.updatePortalPage(container, page); - } - - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); - tabURL.addParameter("pageId", name); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ShowTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); - - if (form.getTabPageId() == null) - { - errors.reject(ERROR_MSG, "PageId cannot be blank."); - } - - if (!pages.containsKey(form.getTabPageId())) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - validateCommand(form, errors); - if (errors.hasErrors()) - return response; - - Portal.showPage(tabContainer, form.getTabPageId()); - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); - tabURL.addParameter("pageId", form.getTabPageId()); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - - public static class TabActionForm extends ReturnUrlForm - { - // This class is used for tab related actions (add, rename, show, etc.) - String _tabName; - String _tabPageId; - - public String getTabName() - { - return _tabName; - } - - public void setTabName(String name) - { - _tabName = name; - } - - public String getTabPageId() - { - return _tabPageId; - } - - public void setTabPageId(String tabPageId) - { - _tabPageId = tabPageId; - } - } - - @RequiresPermission(AdminPermission.class) - public class MoveTabAction extends MutatingApiAction - { - @Override - public ApiResponse execute(MoveTabForm form, BindException errors) - { - final Map properties = new HashMap<>(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage tab = pages.get(form.getPageId()); - - if (null != tab) - { - int oldIndex = tab.getIndex(); - Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); - - if (null != pageToSwap) - { - properties.put("oldIndex", oldIndex); - properties.put("newIndex", tab.getIndex()); - properties.put("pageId", tab.getPageId()); - properties.put("pageIdToSwap", pageToSwap.getPageId()); - } - else - { - properties.put("error", "Unable to move tab."); - } - } - else - { - properties.put("error", "Requested tab does not exist."); - } - - return new ApiSimpleResponse(properties); - } - } - - public static class MoveTabForm implements HasViewContext - { - private int _direction; - private String _pageId; - private ViewContext _viewContext; - - public int getDirection() - { - // 0 moves left, 1 moves right. - return _direction; - } - - public void setDirection(int direction) - { - _direction = direction; - } - - public String getPageId() - { - return _pageId; - } - - public void setPageId(String pageId) - { - _pageId = pageId; - } - - @Override - public ViewContext getViewContext() - { - return _viewContext; - } - - @Override - public void setViewContext(ViewContext viewContext) - { - _viewContext = viewContext; - } - } - - private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) - { - Map pageMap = new CaseInsensitiveHashMap<>(); - for (Portal.PortalPage pp : Portal.getTabPages(c, true)) - pageMap.put(pp.getPageId(), pp); - - for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) - { - if (pageMap.containsKey(folderTab.getName())) - { - // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to - // be excluded when adjusting the relative indexes. - if (!folderTab.isVisible(c, user)) - pageMap.remove(folderTab.getName()); - } - } - List pagesList = new ArrayList<>(pageMap.values()); - pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); - - int visibleIndex; - for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) - { - if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) - { - break; - } - } - - if (visibleIndex == pagesList.size()) - { - return null; - } - - if (direction == Portal.MOVE_DOWN) - { - if (visibleIndex == pagesList.size() - 1) - { - return page; - } - - Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); - - if (null == nextPage) - return null; - Portal.swapPageIndexes(c, page, nextPage); - return nextPage; - } - else - { - if (visibleIndex < 1) - { - return page; - } - - Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); - - if (null == prevPage) - return null; - Portal.swapPageIndexes(c, page, prevPage); - return prevPage; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RenameTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - if (tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); - if (null == pageToChange) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (!page.equals(pageToChange)) - { - if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - if (page.getPageId().equalsIgnoreCase(name)) - { - if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - else - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - } - } - - List folderTabs = tabContainer.getFolderType().getDefaultTabs(); - for (FolderTab folderTab : folderTabs) - { - String folderTabCaption = folderTab.getCaption(getViewContext()); - if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - return; - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - validateCommand(form, errors); - - if (errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(form.getTabPageId()); - page = page.copy(); - page.setCaption(form.getTabName()); - // Update the page the caption is saved. - Portal.updatePortalPage(container, page); - - response.put("success", true); - return response; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ClearDeletedTabFoldersAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeletedFoldersForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - Container container = ContainerManager.getForPath(form.getContainerPath()); - for (String tabName : form.getResurrectFolders()) - { - ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); - } - return new ApiSimpleResponse("success", true); - } - } - - @SuppressWarnings("unused") - public static class DeletedFoldersForm - { - private String _containerPath; - private String _newFolderType; - private List _resurrectFolders; - - public List getResurrectFolders() - { - return _resurrectFolders; - } - - public void setResurrectFolders(List resurrectFolders) - { - _resurrectFolders = resurrectFolders; - } - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - - public String getNewFolderType() - { - return _newFolderType; - } - - public void setNewFolderType(String newFolderType) - { - _newFolderType = newFolderType; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetFolderTabsAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object form, BindException errors) throws Exception - { - var data = getContainer() - .getFolderType() - .getAppBar(getViewContext(), getPageConfig()) - .getButtons() - .stream() - .map(this::getProperties) - .toList(); - - return success(data); - } - - private Map getProperties(NavTree navTree) - { - Map props = new HashMap<>(); - props.put("id", navTree.getId()); - props.put("text", navTree.getText()); - props.put("href", navTree.getHref()); - props.put("disabled", navTree.isDisabled()); - return props; - } - } - - @SuppressWarnings("unused") - public static class ShortURLForm - { - private String _shortURL; - private String _fullURL; - private boolean _delete; - - private List _savedShortURLs; - - public void setShortURL(String shortURL) - { - _shortURL = shortURL; - } - - public void setFullURL(String fullURL) - { - _fullURL = fullURL; - } - - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getShortURL() - { - return _shortURL; - } - - public String getFullURL() - { - return _fullURL; - } - - public boolean isDelete() - { - return _delete; - } - } - - public abstract static class AbstractShortURLAdminAction extends FormViewAction - { - @Override - public void validateCommand(ShortURLForm target, Errors errors) {} - - @Override - public boolean handlePost(ShortURLForm form, BindException errors) throws Exception - { - String shortURL = StringUtils.trimToEmpty(form.getShortURL()); - if (StringUtils.isEmpty(shortURL)) - { - errors.addError(new LabKeyError("Short URL must not be blank")); - } - if (shortURL.endsWith(".url")) - shortURL = shortURL.substring(0,shortURL.length()-".url".length()); - if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) - { - errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); - } - URLHelper fullURL = null; - if (!form.isDelete()) - { - String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); - if (trimmedFullURL == null) - { - errors.addError(new LabKeyError("Target URL must not be blank")); - } - else - { - try - { - fullURL = new URLHelper(trimmedFullURL); - } - catch (URISyntaxException e) - { - errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); - } - } - } - if (errors.getErrorCount() > 0) - { - return false; - } - - ShortURLService service = ShortURLService.get(); - if (form.isDelete()) - { - ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); - if (shortURLRecord == null) - { - throw new NotFoundException("No such short URL: " + shortURL); - } - try - { - service.deleteShortURL(shortURLRecord, getUser()); - } - catch (ValidationException e) - { - errors.addError(new LabKeyError("Error deleting short URL:")); - for(ValidationError error: e.getErrors()) - { - errors.addError(new LabKeyError(error.getMessage())); - } - } - - if (errors.getErrorCount() > 0) - { - return false; - } - } - else - { - ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); - MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); - // Add a role assignment to let another group manage the URL. This grants permission to the journal - // to change where the URL redirects you to after they copy the data - SecurityPolicyManager.savePolicy(policy, getUser()); - } - return true; - } - } - - @AdminConsoleAction - public class ShortURLAdminAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); - boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); - newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); - newView.setFrame(WebPartView.FrameType.PORTAL); - - QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); - qSettings.setBaseSort(new Sort("-Created")); - QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); - if (!isAppAdmin) - { - existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - } - existingView.setTitle("Existing Short URLs"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Short URL Admin", getClass()); - } - } - - @RequiresPermission(ApplicationAdminPermission.class) - public class UpdateShortURLAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); - if (shortUrlRecord == null) - { - errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); - return new SimpleErrorView(errors); - } - form.setFullURL(shortUrlRecord.getFullURL()); - - JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); - view.setTitle("Update Short URL"); - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Update Short URL", getClass()); - } - } - - // API for reporting client-side exceptions. - // UNDONE: Throttle by IP to avoid DOS from buggy clients. - @Marshal(Marshaller.Jackson) - @SuppressWarnings("UnusedDeclaration") - @RequiresLogin // Issue 52520: Prevent bots from submitting reports - @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root - public static class LogClientExceptionAction extends MutatingApiAction - { - @Override - public Object execute(ExceptionForm form, BindException errors) - { - String errorCode = ExceptionUtil.logClientExceptionToMothership( - form.getStackTrace(), - form.getExceptionMessage(), - form.getBrowser(), - null, - form.getRequestURL(), - form.getReferrerURL(), - form.getUsername() - ); - - Map results = new HashMap<>(); - results.put("errorCode", errorCode); - results.put("loggedToMothership", errorCode != null); - - return success(results); - } - } - - @SuppressWarnings("unused") - public static class ExceptionForm - { - private String _exceptionMessage; - private String _stackTrace; - private String _requestURL; - private String _browser; - private String _username; - private String _referrerURL; - private String _file; - private String _line; - private String _platform; - - public String getExceptionMessage() - { - return _exceptionMessage; - } - - public void setExceptionMessage(String exceptionMessage) - { - _exceptionMessage = exceptionMessage; - } - - public String getUsername() - { - return _username; - } - - public void setUsername(String username) - { - _username = username; - } - - public String getStackTrace() - { - return _stackTrace; - } - - public void setStackTrace(String stackTrace) - { - _stackTrace = stackTrace; - } - - public String getRequestURL() - { - return _requestURL; - } - - public void setRequestURL(String requestURL) - { - _requestURL = requestURL; - } - - public String getBrowser() - { - return _browser; - } - - public void setBrowser(String browser) - { - _browser = browser; - } - - public String getReferrerURL() - { - return _referrerURL; - } - - public void setReferrerURL(String referrerURL) - { - _referrerURL = referrerURL; - } - - public String getFile() - { - return _file; - } - - public void setFile(String file) - { - _file = file; - } - - public String getLine() - { - return _line; - } - - public void setLine(String line) - { - _line = line; - } - - public String getPlatform() - { - return _platform; - } - - public void setPlatform(String platform) - { - _platform = platform; - } - } - - - /** generate URLS to seed web-site scanner */ - @SuppressWarnings("UnusedDeclaration") - @RequiresSiteAdmin - public static class SpiderAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Spider Initialization"); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - List urls = new ArrayList<>(1000); - - if (getContainer().equals(ContainerManager.getRoot())) - { - for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) - { - urls.add(c.getStartURL(getUser()).toString()); - urls.add(new ActionURL(SpiderAction.class, c).toString()); - } - - Container home = ContainerManager.getHomeContainer(); - for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) - { - ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); - urls.add(url.toString()); - } - } - else - { - DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); - def.getSchemaNames().forEach(name -> - { - QuerySchema q = def.getSchema(name); - if (null == q) - return; - var tableNames = q.getTableNames(); - if (null == tableNames) - return; - tableNames.forEach(table -> - { - try - { - var t = q.getTable(table); - if (null != t) - { - ActionURL grid = t.getGridURL(getContainer()); - if (null != grid) - urls.add(grid.toString()); - else - urls.add(new ActionURL("query", "executeQuery.view", getContainer()) - .addParameter("schemaName", q.getSchemaName()) - .addParameter("query.queryName", t.getName()) - .toString()); - } - } - catch (Exception x) - { - // pass - } - }); - }); - - ModuleLoader.getInstance().getModules().forEach(m -> - { - ActionURL url = m.getTabURL(getContainer(), getUser()); - if (null != url) - urls.add(url.toString()); - }); - } - - return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); - } - } - - @SuppressWarnings("UnusedDeclaration") - @RequiresPermission(TroubleshooterPermission.class) - public static class TestMothershipReportAction extends ReadOnlyApiAction - { - @Override - public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception - { - MothershipReport report; - MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; - if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) - { - report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); - } - else - { - report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), - new SQLException("Intentional exception for testing purposes", "400"), - (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), - target, - ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); - } - - final Map params; - if (report == null) - { - params = new LinkedHashMap<>(); - } - else - { - params = report.getJsonFriendlyParams(); - if (form.isSubmit()) - { - report.setForwardedFor(form.getForwardedFor()); - report.run(); - if (null != report.getUpgradeMessage()) - params.put("upgradeMessage", report.getUpgradeMessage()); - } - } - if (form.isDownload()) - { - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); - } - return new ApiSimpleResponse(params); - } - } - - - static class MothershipReportSelectionForm - { - private String _type = MothershipReport.Type.CheckForUpdates.toString(); - private String _level = UsageReportingLevel.ON.toString(); - private boolean _submit = false; - private boolean _download = false; - private String _forwardedFor = null; - // indicates action is being invoked for dev/test - private boolean _testMode = false; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = StringUtils.upperCase(level); - } - - public boolean isSubmit() - { - return _submit; - } - - public void setSubmit(boolean submit) - { - _submit = submit; - } - - public String getForwardedFor() - { - return _forwardedFor; - } - - public void setForwardedFor(String forwardedFor) - { - _forwardedFor = forwardedFor; - } - - public boolean isTestMode() - { - return _testMode; - } - - public void setTestMode(boolean testMode) - { - _testMode = testMode; - } - - public boolean isDownload() - { - return _download; - } - - public void setDownload(boolean download) - { - _download = download; - } - } - - - @RequiresPermission(TroubleshooterPermission.class) - public class SuspiciousAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - Collection list = BlockListFilter.reportSuspicious(); - HtmlStringBuilder html = HtmlStringBuilder.of(); - if (list.isEmpty()) - { - html.append("No suspicious activity.\n"); - } - else - { - html.unsafeAppend("") - .unsafeAppend("\n"); - for (BlockListFilter.Suspicious s : list) - { - html.unsafeAppend("\n"); - } - html.unsafeAppend("
    host (user)user-agentcount
    ") - .append(s.host); - if (!isBlank(s.user)) - html.append(HtmlString.NBSP).append("(" + s.user + ")"); - html.unsafeAppend("") - .append(s.userAgent) - .unsafeAppend("") - .append(s.count) - .unsafeAppend("
    "); - } - return new HtmlView(html); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); - } - } - - /** This is a very crude API right now, mostly using default serialization of pre-existing objects - * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedBeforeInitialUserIsSet - public static class ConfigurationSummaryAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) - { - if (!getContainer().isRoot()) - throw new NotFoundException("Must be invoked in the root"); - - // requires site-admin, unless there are no users - if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) - throw new UnauthorizedException(); - - return getConfigurationJson(); - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - ObjectMapper result = JsonUtil.createDefaultMapper(); - result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); - return result; - } - - /* returns a jackson serializable object that reports superset of information returned in admin console */ - private JSONObject getConfigurationJson() - { - JSONObject res = new JSONObject(); - - res.put("server", AdminBean.getPropertyMap()); - - final Map> sets = new TreeMap<>(); - new SqlSelector(CoreSchema.getInstance().getScope(), - new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + - "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> - { - String category = (String)m.get("category"); - String name = (String)m.get("name"); - Object value = m.get("value"); - if (!sets.containsKey(category)) - sets.put(category, new TreeMap<>()); - sets.get(category).put(name,value); - } - ); - res.put("siteSettings", sets); - - HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); - res.put("health", result); - - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - res.put("scriptEngines", mgr.getEngineDefinitions()); - - return res; - } - } - - @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) - private static class IgnorePasswordMixIn - { - } - - @AdminConsoleAction() - public class AllowListAction extends FormViewAction - { - private AllowListType _type; - - @Override - public void validateCommand(AllowListForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) - { - _type = form.getTypeEnum(); - - form.setExistingValuesList(form.getTypeEnum().getValues()); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); - newView.setTitle("Register New " + form.getTypeEnum().getTitle()); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); - existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - AllowListType allowListType = form.getTypeEnum(); - //handle delete of existing value - if (form.isDelete()) - { - String urlToDelete = form.getExistingValue(); - List values = new ArrayList<>(allowListType.getValues()); - for (String value : values) - { - if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) - { - values.remove(value); - allowListType.setValues(values, getUser()); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated urls - else if (form.isSaveAll()) - { - Set validatedValues = form.validateValues(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(validatedValues.stream().toList(), getUser()); - } - //save new external value - else if (form.isSaveNew()) - { - Set valueSet = form.validateNewValue(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(valueSet, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic(_type.getHelpTopic()); - addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); - } - } - - public static class AllowListForm - { - private String _newValue; - private String _existingValue; - private boolean _delete; - private String _existingValues; - private boolean _saveAll; - private boolean _saveNew; - private String _type; - - private List _existingValuesList; - - public String getNewValue() - { - return _newValue; - } - - @SuppressWarnings("unused") - public void setNewValue(String newValue) - { - _newValue = newValue; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getExistingValues() - { - return _existingValues; - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public List getExistingValuesList() - { - //for updated urls that comes in as String values from the jsp/html form - if (null != getExistingValues()) - { - // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. - return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); - } - return _existingValuesList; - } - - public void setExistingValuesList(List valuesList) - { - _existingValuesList = valuesList; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - @NotNull - public AllowListType getTypeEnum() - { - return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); - } - - @JsonIgnore - public Set validateNewValue(BindException errors) - { - String value = StringUtils.trimToEmpty(getNewValue()); - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - return null; - - Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); - checkDuplicatesByAddition(value, valueSet, errors); - return valueSet; - } - - @JsonIgnore - public Set validateValues(BindException errors) - { - List values = getExistingValuesList(); //get values from the form, this includes updated values - Set valueSet = new CaseInsensitiveHashSet(); - - if (null != values && !values.isEmpty()) - { - for (String value : values) - { - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - continue; - - checkDuplicatesByAddition(value, valueSet, errors); - } - } - - return valueSet; - } - - /** - * Adds value to value set unless it is a duplicate, in which case it adds an error - * @param value to check - * @param valueSet of existing values - * @param errors collections of errors observed - */ - @JsonIgnore - private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) - { - String trimValue = StringUtils.trimToEmpty(value); - if (!valueSet.add(trimValue)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); - } - } - - @AdminConsoleAction - public static class DeleteAllValuesAction extends FormHandlerAction - { - @Override - public void validateCommand(AllowListForm form, Errors errors) - { - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - form.getTypeEnum().setValues(Collections.emptyList(), getUser()); - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - } - - public static class ExternalSourcesForm - { - private boolean _delete; - private boolean _saveNew; - private boolean _saveAll; - - private String _newDirective; - private String _newHost; - private String _existingValue; - private String _existingValues; - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public String getNewDirective() - { - return _newDirective; - } - - @SuppressWarnings("unused") - public void setNewDirective(String newDirective) - { - _newDirective = newDirective; - } - - public String getNewHost() - { - return _newHost; - } - - @SuppressWarnings("unused") - public void setNewHost(String newHost) - { - _newHost = newHost; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public List getExistingValues() - { - return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .toList(); - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - private AllowedHost getExistingAllowedHost(BindException errors) - { - return getAllowedHost(getExistingValue(), errors); - } - - private AllowedHost getAllowedHost(String value, BindException errors) - { - String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts - if (parts.length != 2) - { - errors.addError(new LabKeyError("Can't parse allowed host.")); - return null; - } - return validateHost(parts[0], parts[1], errors); - } - - private List getExistingAllowedHosts(BindException errors) - { - List existing = getExistingValues().stream() - .map(value-> getAllowedHost(value, errors)) - .toList(); - - if (errors.hasErrors()) - return null; - - return checkDuplicates(existing, errors); - } - - private List validateNewAllowedHost(BindException errors) throws JsonProcessingException - { - AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); - - if (errors.hasErrors()) - return null; - - List hosts = getSavedAllowedHosts(); - hosts.add(newAllowedHost); - - return checkDuplicates(hosts, errors); - } - - // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons - public static AllowedHost validateHost(String directiveString, String host, BindException errors) - { - AllowedHost ret = null; - - if (StringUtils.isEmpty(directiveString)) - { - errors.addError(new LabKeyError("Directive must not be blank")); - } - else if (StringUtils.isEmpty(host)) - { - errors.addError(new LabKeyError("Host must not be blank")); - } - else if (host.contains(";")) - { - errors.addError(new LabKeyError("Semicolons are not allowed in host names")); - } - else - { - Directive directive = EnumUtils.getEnum(Directive.class, directiveString); - - if (null == directive) - { - errors.addError(new LabKeyError("Unknown directive: " + directiveString)); - } - else - { - ret = new AllowedHost(directive, host.trim()); - } - } - - return ret; - } - - /** - * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons - - * @param hosts a list of AllowedHost objects to check for duplicates - * @param errors errors to populate - * @return hosts if there are no duplicates, otherwise {@code null} - */ - public static @Nullable List checkDuplicates(List hosts, BindException errors) - { - // Not a simple Set check since we want host check to be case-insensitive - MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); - - hosts.forEach(allowedHost -> { - String host = allowedHost.host().trim(); - if (!map.put(allowedHost.directive(), host)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); - }); - - return errors.hasErrors() ? null : hosts; - } - - // Returns a mutable list - public List getSavedAllowedHosts() throws JsonProcessingException - { - return AllowedExternalResourceHosts.readAllowedHosts(); - } - } - - @AdminConsoleAction() - public class ExternalSourcesAction extends FormViewAction - { - @Override - public void validateCommand(ExternalSourcesForm form, Errors errors) - { - } - - @Override - public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) - { - boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); - newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); - existingView.setTitle("Existing External Resource Hosts"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - private static final Object HOST_LOCK = new Object(); - - @Override - public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception - { - List allowedHosts = null; - - // Multiple requests could access this in parallel, so synchronize access, Issue 53457 - synchronized (HOST_LOCK) - { - //handle delete of an existing value - if (form.isDelete()) - { - AllowedHost subToDelete = form.getExistingAllowedHost(errors); - if (errors.hasErrors()) - return false; - allowedHosts = form.getSavedAllowedHosts(); - var iter = allowedHosts.listIterator(); - while (iter.hasNext()) - { - AllowedHost sub = iter.next(); - if (sub.equals(subToDelete)) - { - iter.remove(); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated hosts - else if (form.isSaveAll()) - { - allowedHosts = form.getExistingAllowedHosts(errors); - if (errors.hasErrors()) - return false; - } - //save new external value - else if (form.isSaveNew()) - { - allowedHosts = form.validateNewAllowedHost(errors); - } - - if (errors.hasErrors()) - return false; - - AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(ExternalSourcesForm form) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("externalHosts"); - addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ProjectSettingsAction extends ProjectSettingsViewPostAction - { - @Override - protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(ProjectSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception - { - return saveProjectSettings(getContainer(), getUser(), form, errors); - } - } - - private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) - { - WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); - boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); - - // Site-only properties - - if (c.isRoot()) - { - DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); - props.setDateParsingMode(dateParsingMode); - - if (hasAdminOpsPerm) - { - String customWelcome = form.getCustomWelcome(); - String welcomeUrl = StringUtils.trimToNull(customWelcome); - if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); - } - else - { - props.setCustomWelcome(welcomeUrl); - } - } - } - - // Site & project properties - - boolean shouldInherit = form.getShouldInherit(); - if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) - { - SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); - } - - setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); - setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); - setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); - setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); - setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); - setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); - - // a few properties on this page should be restricted to operational permissions (i.e. site admin) - if (hasAdminOpsPerm) - { - setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { - String systemEmailAddress = form.getSystemEmailAddress(); - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(systemEmailAddress); - props.setSystemEmailAddress(email); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - }); - - setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { - String customLogin = form.getCustomLogin(); - if (props.isValidUrl(customLogin)) - { - props.setCustomLogin(customLogin); - } - else - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); - } - }); - } - - setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); - setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); - setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); - setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { - String supportEmail = form.getSupportEmail(); - - if (!isBlank(supportEmail)) - { - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(supportEmail); - props.setSupportEmail(email.toString()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - } - else - { - // This stores a blank value, not null (which would mean inherit) - props.setSupportEmail(null); - } - }); - - boolean noErrors = !saveFolderSettings(c, user, props, form, errors); - - if (noErrors) - { - // Bump the look & feel revision so browsers retrieve the new theme stylesheet - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - } - - return noErrors; - } - - private static void setProperty(boolean inherited, Runnable clear, Runnable set) - { - if (inherited) - clear.run(); - else - set.run(); - } - - // Same as ProjectSettingsAction, but provides special admin console permissions handling - @AdminConsoleAction(ApplicationAdminPermission.class) - public static class LookAndFeelSettingsAction extends ProjectSettingsAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class UpdateContainerSettingsAction extends MutatingApiAction - { - @Override - public Object execute(FolderSettingsForm form, BindException errors) - { - boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("success", saved && !errors.hasErrors()); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResourcesAction extends ProjectSettingsViewPostAction - { - @Override - protected JspView getTabView(Object o, boolean reshow, BindException errors) - { - LookAndFeelBean bean = new LookAndFeelBean(); - return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); - } - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - Container c = getContainer(); - Map fileMap = getFileMap(); - - for (ResourceType type : ResourceType.values()) - { - MultipartFile file = fileMap.get(type.name()); - - if (file != null && !file.isEmpty()) - { - try - { - type.save(file, c, getUser()); - } - catch (Exception e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - } - } - - // Note that audit logging happens via the attachment code, so we don't log separately here - - // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - return true; - } - } - - // Same as ResourcesAction, but provides special admin console permissions handling - @AdminConsoleAction - public static class AdminConsoleResourcesAction extends ResourcesAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MenuBarAction extends ProjectSettingsViewAction - { - @Override - protected HttpView getTabView() - { - if (getContainer().isRoot()) - return HtmlView.err("Menu bar must be configured for each project separately."); - - WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); - v.setView("menubar", new VBox()); - Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); - - return v; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FilesAction extends ProjectSettingsViewPostAction - { - @Override - protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) - { - Container c = getContainer(); - - if (c.isRoot()) - return HtmlView.err("Files must be configured for each project separately."); - - if (!reshow || form.isPipelineRootForm()) - { - try - { - AdminController.setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - VBox box = new VBox(); - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - box.addView(view); - - // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root - if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) - { - SetupForm setupForm = SetupForm.init(c); - setupForm.setShowAdditionalOptionsLink(true); - setupForm.setErrors(errors); - PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); - - if (pipeRoot != null) - { - for (String errorMessage : pipeRoot.validate()) - errors.addError(new LabKeyError(errorMessage)); - } - JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); - pipelineView.setTitle("Configure Data Processing Pipeline"); - box.addView(pipelineView); - } - - return box; - } - - @Override - public void validateCommand(FilesForm form, Errors errors) - { - if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) - { - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - File f = new File(root); - if (!f.exists() || !f.isDirectory()) - { - errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - } - } - else - errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); - } - else if (form.isCloudFileRoot()) - { - AdminController.validateCloudFileRoot(form, getContainer(), errors); - } - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) throws Exception - { - FileContentService service = FileContentService.get(); - if (service != null) - { - if (form.isPipelineRootForm()) - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - else - { - AdminController.setFileRootFromForm(getViewContext(), form, errors); - } - } - - // Cloud settings - AdminController.setEnabledCloudStores(getViewContext(), form, errors); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(FilesForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); - if (form.isPipelineRootForm()) - { - url.addParameter("piperootSet", true); - } - else - { - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - } - return url; - } - } - - public static class LookAndFeelView extends JspView - { - LookAndFeelView(BindException errors) - { - super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); - } - } - - - public static class LookAndFeelBean - { - public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); - } - - @RequiresPermission(AdminPermission.class) - public static class AdjustSystemTimestampsAction extends FormViewAction - { - @Override - public void addNavTrail(NavTree root) - { - } - - @Override - public void validateCommand(AdjustTimestampsForm form, Errors errors) - { - if (form.getHourDelta() == null || form.getHourDelta() == 0) - errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); - } - - @Override - public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); - } - - private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) - { - SQLFragment sql = new SQLFragment(); - DbSchema schema = tInfo.getSchema(); - String comma = ""; - List updating = new ArrayList<>(); - - for (String fieldName: fieldNames) - { - ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); - if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) - { - updating.add(fieldName); - if (sql.isEmpty()) - sql.append("UPDATE ").append(tInfo, "").append(" SET "); - sql.append(comma) - .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); - comma = ", "; - } - } - - if (!sql.isEmpty()) - { - logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); - logger.debug(sql.toDebugString()); - int numRows = new SqlExecutor(schema).execute(sql); - logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); - } - } - - @Override - public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception - { - List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); - logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - DbScope scope = DbScope.getLabKeyScope(); - try (DbScope.Transaction t = scope.ensureTransaction()) - { - ModuleLoader.getInstance().getModules().forEach(module -> { - logger.info("==> Beginning update of timestamps for module: " + module.getName()); - module.getSchemaNames().stream().sorted().forEach(schemaName -> { - DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); - scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from - schema.getTableNames().forEach(tableName -> { - TableInfo tInfo = schema.getTable(tableName); - if (tInfo.getTableType() == DatabaseTableType.TABLE) - { - updateFields(tInfo, toUpdate, form.getHourDelta()); - } - }); - }); - logger.info("<== DONE updating timestamps for module: " + module.getName()); - }); - t.commit(); - } - logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - return true; - } - - @Override - public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) - { - return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); - } - } - - public static class AdjustTimestampsForm - { - private Integer hourDelta; - - public Integer getHourDelta() - { - return hourDelta; - } - - public void setHourDelta(Integer hourDelta) - { - this.hourDelta = hourDelta; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ViewUsageStatistics extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Usage Statistics", this.getClass()); - } - } - - private static final URI LABKEY_ORG_REPORT_ACTION; - - static - { - LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); - } - - @RequiresNoPermission - @CSRF(CSRF.Method.NONE) - public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction - { - private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); - - // recent reports, to help avoid log spam - private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); - - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - var ret = new JSONObject().put("success", true); - - // fail fast - if (!_log.isWarnEnabled()) - return ret; - - var request = getViewContext().getRequest(); - assert null != request; - - var userAgent = request.getHeader("User-Agent"); - if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) - return ret; - - // NOTE User may be "guest", and will always be guest if being relayed to labkey.org - var jsonObj = form.getJsonObject(); - if (null != jsonObj) - { - JSONObject cspReport = jsonObj.optJSONObject("csp-report"); - if (cspReport != null) - { - String blockedUri = cspReport.optString("blocked-uri", null); - - // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org - if (blockedUri != null && - blockedUri.startsWith("https://labkey.org%2C") && - blockedUri.endsWith("undefined") && - !_log.isDebugEnabled()) - { - return ret; - } - - String urlString = cspReport.optString("document-uri", null); - if (urlString != null) - { - String path = new URLHelper(urlString).deleteParameters().getURIString(); - if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) - { - // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. - boolean forwarded = jsonObj.optBoolean("forwarded", false); - if (!forwarded) - { - User user = getUser(); - String email = null; - // If the user is not logged in, we may still be able to snag the email address from our cookie - if (user.isGuest()) - email = LoginController.getEmailFromCookie(getViewContext().getRequest()); - if (null == email) - email = user.getEmail(); - jsonObj.put("user", email); - String ipAddress = request.getHeader("X-FORWARDED-FOR"); - if (ipAddress == null) - ipAddress = request.getRemoteAddr(); - jsonObj.put("ip", ipAddress); - if (isNotBlank(userAgent)) - jsonObj.put("user-agent", userAgent); - String labkeyVersion = request.getParameter("labkeyVersion"); - if (null != labkeyVersion) - jsonObj.put("labkeyVersion", labkeyVersion); - String cspVersion = request.getParameter("cspVersion"); - if (null != cspVersion) - jsonObj.put("cspVersion", cspVersion); - } - - var jsonStr = jsonObj.toString(2); - _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); - - if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) - { - jsonObj.put("forwarded", true); - - // Create an HttpClient - HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(10)) - .build(); - - // Create the POST request - HttpRequest remoteRequest = HttpRequest.newBuilder() - .uri(LABKEY_ORG_REPORT_ACTION) - .header("Content-Type", request.getContentType()) // Use whatever the browser set - .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) - .build(); - - // Send the request and get the response - HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() != 200) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); - } - else - { - JSONObject jsonResponse = new JSONObject(response.body()); - boolean success = jsonResponse.optBoolean("success", false); - if (!success) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); - } - } - } - } - } - } - } - return ret; - } - } - - - public static class TestCase extends AbstractActionPermissionTest - { - @Override - @Test - public void testActionPermissions() - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - AdminController controller = new AdminController(); - - // @RequiresPermission(ReadPermission.class) - assertForReadPermission(user, false, - new GetModulesAction(), - new GetFolderTabsAction(), - new ClearDeletedTabFoldersAction() - ); - - // @RequiresPermission(DeletePermission.class) - assertForUpdateOrDeletePermission(user, - new DeleteFolderAction() - ); - - // @RequiresPermission(AdminPermission.class) - assertForAdminPermission(user, - controller.new CustomizeEmailAction(), - controller.new FolderAliasesAction(), - controller.new MoveFolderAction(), - controller.new MoveTabAction(), - controller.new RenameFolderAction(), - controller.new ReorderFoldersAction(), - controller.new ReorderFoldersApiAction(), - controller.new SiteValidationAction(), - new AddTabAction(), - new ConfirmProjectMoveAction(), - new CreateFolderAction(), - new CustomizeMenuAction(), - new DeleteCustomEmailAction(), - new FilesAction(), - new MenuBarAction(), - new ProjectSettingsAction(), - new RenameContainerAction(), - new RenameTabAction(), - new ResetPropertiesAction(), - new ResetQueryStatisticsAction(), - new ResetResourceAction(), - new ResourcesAction(), - new RevertFolderAction(), - new SetFolderPermissionsAction(), - new SetInitialFolderSettingsAction(), - new ShowTabAction() - ); - - //TODO @RequiresPermission(AdminReadPermission.class) - //controller.new TestMothershipReportAction() - - // @RequiresPermission(AdminOperationsPermission.class) - assertForAdminOperationsPermission(ContainerManager.getRoot(), user, - controller.new DbCheckerAction(), - controller.new DeleteModuleAction(), - controller.new DoCheckAction(), - controller.new EmailTestAction(), - controller.new ShowNetworkDriveTestAction(), - controller.new ValidateDomainsAction(), - new OptionalFeatureAction(), - new GetSchemaXmlDocAction(), - new RecreateViewsAction() - ); - - // @AdminConsoleAction - assertForAdminPermission(ContainerManager.getRoot(), user, - controller.new ActionsAction(), - controller.new CachesAction(), - controller.new ConfigureSystemMaintenanceAction(), - controller.new CustomizeSiteAction(), - controller.new DumpHeapAction(), - controller.new ExecutionPlanAction(), - controller.new FolderTypesAction(), - controller.new MemTrackerAction(), - controller.new ModulesAction(), - controller.new QueriesAction(), - controller.new QueryStackTracesAction(), - controller.new ResetErrorMarkAction(), - controller.new ShortURLAdminAction(), - controller.new ShowAllErrorsAction(), - controller.new ShowErrorsSinceMarkAction(), - controller.new ShowPrimaryLogAction(), - controller.new ShowCspReportLogAction(), - controller.new ShowThreadsAction(), - new ExportActionsAction(), - new ExportQueriesAction(), - new MemoryChartAction(), - new ShowAdminAction() - ); - - // @RequiresSiteAdmin - assertForRequiresSiteAdmin(user, - controller.new EnvironmentVariablesAction(), - controller.new SystemMaintenanceAction(), - controller.new SystemPropertiesAction(), - new GetPendingRequestCountAction(), - new InstallCompleteAction(), - new NewInstallSiteSettingsAction() - ); - - assertForTroubleshooterPermission(ContainerManager.getRoot(), user, - controller.new OptionalFeaturesAction(), - controller.new ShowModuleErrorsAction(), - new ModuleStatusAction() - ); - } - } - - public static class SerializationTest extends PipelineJob.TestSerialization - { - static class TestJob extends PipelineJob - { - ImpersonationContext _impersonationContext; - ImpersonationContext _impersonationContext1; - ImpersonationContext _impersonationContext2; - - @Override - public URLHelper getStatusHref() - { - return null; - } - - @Override - public String getDescription() - { - return "Test Job"; - } - } - - @Test - public void testSerialization() - { - TestJob job = new TestJob(); - TestContext ctx = TestContext.get(); - ViewContext viewContext = new ViewContext(); - viewContext.setContainer(ContainerManager.getSharedContainer()); - viewContext.setUser(ctx.getUser()); - RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( - viewContext.getContainer(), viewContext.getUser(), - Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); - job._impersonationContext = factory.getImpersonationContext(); - - try - { - UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - UserManager.getGuestUser(), null); - job._impersonationContext1 = factory1.getImpersonationContext(); - } - catch (Exception e) - { - LOG.error("Invalid user email for impersonating."); - } - - GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); - job._impersonationContext2 = factory2.getImpersonationContext(); - testSerialize(job, LOG); - } - } - - public static class WorkbookDeleteTestCase extends Assert - { - private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; - private static final String TEST_EMAIL = "testDelete@myDomain.com"; - - @Test - public void testWorkbookDelete() throws Exception - { - doCleanup(); - - Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); - Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); - - ValidEmail email = new ValidEmail(TEST_EMAIL); - SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); - User nonAdminUser = newUserStatus.getUser(); - MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, ReaderRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - // User lacks any permission, throw unauthorized for parent and workbook: - HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Grant permission, should be able to delete the workbook but not parent: - policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, EditorRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Hitting delete action results in a redirect: - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); - - doCleanup(); - } - - protected static void doCleanup() throws Exception - { - Container project = ContainerManager.getForPath(FOLDER_NAME); - if (project != null) - { - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) - { - User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); - UserManager.deleteUser(u.getUserId()); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.admin; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.util.concurrent.UncheckedExecutionException; +import jakarta.mail.MessagingException; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.map.LRUMap; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jfree.chart.ChartFactory; +import org.jfree.chart.ChartUtilities; +import org.jfree.chart.JFreeChart; +import org.jfree.chart.plot.PlotOrientation; +import org.jfree.data.category.DefaultCategoryDataset; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.BaseApiAction; +import org.labkey.api.action.BaseViewAction; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasViewContext; +import org.labkey.api.action.IgnoresAllocationTracking; +import org.labkey.api.action.LabKeyError; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.QueryViewAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AbstractFolderContext.ExportType; +import org.labkey.api.admin.AdminBean; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.FolderWriter; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.HealthCheck; +import org.labkey.api.admin.HealthCheckRegistry; +import org.labkey.api.admin.ImportOptions; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.admin.TableXmlUtils; +import org.labkey.api.admin.sitevalidation.SiteValidationResult; +import org.labkey.api.admin.sitevalidation.SiteValidationResultList; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.cache.CacheStats; +import org.labkey.api.cache.TrackingCache; +import org.labkey.api.cloud.CloudStoreService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.compliance.ComplianceFolderSettings; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.compliance.PhiColumnBehavior; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.Container; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MenuButton; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.NormalContainerType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.RenderContext; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TransactionFilter; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; +import org.labkey.api.data.queryprofiler.QueryProfiler; +import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.files.FileContentService; +import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; +import org.labkey.api.message.settings.MessageConfigService.NotificationOption; +import org.labkey.api.message.settings.MessageConfigService.UserPreference; +import org.labkey.api.miniprofiler.RequestInfo; +import org.labkey.api.module.AllowedBeforeInitialUserIsSet; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.IgnoresForbiddenProjectCheck; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleLoader.SchemaActions; +import org.labkey.api.module.ModuleLoader.SchemaAndModule; +import org.labkey.api.module.SimpleModule; +import org.labkey.api.moduleeditor.api.ModuleEditorService; +import org.labkey.api.pipeline.DirectoryNotDeletedException; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.pipeline.view.SetupForm; +import org.labkey.api.products.ProductRegistry; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RuntimeValidationException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.ExternalScriptEngineDefinition; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.CSRF; +import org.labkey.api.security.Directive; +import org.labkey.api.security.Group; +import org.labkey.api.security.GroupManager; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.LoginUrls; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresLogin; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.RoleAssignment; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicy; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.SecurityUrls; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; +import org.labkey.api.security.impersonation.ImpersonationContext; +import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; +import org.labkey.api.security.impersonation.UserImpersonationContextFactory; +import org.labkey.api.security.permissions.AbstractActionPermissionTest; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.SiteAdminPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.permissions.UploadFileBasedModulePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.security.roles.FolderAdminRole; +import org.labkey.api.security.roles.ProjectAdminRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.security.roles.SharedViewEditorRole; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.ConceptURIProperties; +import org.labkey.api.settings.DateParsingMode; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; +import org.labkey.api.settings.NetworkDriveProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.settings.OptionalFeatureService.FeatureType; +import org.labkey.api.settings.ProductConfiguration; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; +import org.labkey.api.settings.WriteableLookAndFeelProperties; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.DOM; +import org.labkey.api.util.DOM.Renderable; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.DebugInfoDumper; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.FolderDisplayMode; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.HttpsUtil; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.MemTracker; +import org.labkey.api.util.MemTracker.HeldReference; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.SafeToRenderEnum; +import org.labkey.api.util.SessionAppender; +import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; +import org.labkey.api.util.SystemMaintenanceJob; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.Tuple3; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UniqueID; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.util.emailTemplate.EmailTemplate; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.DataView; +import org.labkey.api.view.FolderManagement.FolderManagementViewAction; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; +import org.labkey.api.view.FolderManagement.TYPE; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.Portal; +import org.labkey.api.view.RedirectException; +import org.labkey.api.view.ShortURLRecord; +import org.labkey.api.view.ShortURLService; +import org.labkey.api.view.TabStripView; +import org.labkey.api.view.URLException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.EmptyView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.view.template.PageConfig.Template; +import org.labkey.api.wiki.WikiRendererType; +import org.labkey.api.wiki.WikiRenderingService; +import org.labkey.api.writer.FileSystemFile; +import org.labkey.api.writer.HtmlWriter; +import org.labkey.api.writer.ZipFile; +import org.labkey.api.writer.ZipUtil; +import org.labkey.bootstrap.ExplodedModuleService; +import org.labkey.core.admin.miniprofiler.MiniProfilerController; +import org.labkey.core.admin.sitevalidation.SiteValidationJob; +import org.labkey.core.admin.sql.SqlScriptController; +import org.labkey.core.login.LoginController; +import org.labkey.core.portal.CollaborationFolderType; +import org.labkey.core.portal.ProjectController; +import org.labkey.core.query.CoreQuerySchema; +import org.labkey.core.query.PostgresUserSchema; +import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; +import org.labkey.core.security.AllowedExternalResourceHosts; +import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; +import org.labkey.core.security.BlockListFilter; +import org.labkey.core.security.SecurityController; +import org.labkey.data.xml.TablesDocument; +import org.labkey.filters.ContentSecurityPolicyFilter; +import org.labkey.security.xml.GroupEnumType; +import org.labkey.vfs.FileLike; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.Controller; + +import java.awt.*; +import java.beans.Introspector; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; +import static org.labkey.api.util.DOM.A; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.Attribute.name; +import static org.labkey.api.util.DOM.Attribute.style; +import static org.labkey.api.util.DOM.Attribute.title; +import static org.labkey.api.util.DOM.Attribute.type; +import static org.labkey.api.util.DOM.Attribute.value; +import static org.labkey.api.util.DOM.BR; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.LI; +import static org.labkey.api.util.DOM.SPAN; +import static org.labkey.api.util.DOM.STYLE; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.UL; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.api.util.DOM.createHtmlFragment; +import static org.labkey.api.util.HtmlString.NBSP; +import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; +import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; +import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; +import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; +import static org.labkey.api.view.FolderManagement.NOT_ROOT; +import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; +import static org.labkey.api.view.FolderManagement.ROOT; +import static org.labkey.api.view.FolderManagement.addTab; + +public class AdminController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( + AdminController.class, + FileListAction.class, + FilesSiteSettingsAction.class, + UpdateFilePathsAction.class + ); + + private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); + private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); + private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; + + private static long _errorMark = 0; + private static long _primaryLogMark = 0; + + public static void registerAdminConsoleLinks() + { + Container root = ContainerManager.getRoot(); + + // Configuration + AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); + AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); + if (!ProductRegistry.getProducts().isEmpty()) + AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); + // TODO move to FileContentModule + if (ModuleLoader.getInstance().hasModule("FileContent")) + AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); + AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); + AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); + AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); + AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); + + // Diagnostics + AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); + AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); + AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); + AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); + AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); + AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); + AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); + AdminConsole.addLink(Diagnostics, "postgres table sizes", new ActionURL(PostgresTableSizesAction.class, root)); + } + + AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); + AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); + AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); + AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); + AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); + AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); + AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); + AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); + } + + public static void registerManagementTabs() + { + addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); + addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); + addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); + addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { + if (!c.isRoot()) + { + // Show module properties tab only if a module w/ properties to set is present for current folder + for (Module m : c.getActiveModules()) + if (!m.getModuleProperties().isEmpty()) + return true; + } + + return false; + }, ModulePropertiesAction.class); + addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { + // Show Concepts tab only if the experiment module is enabled in this container + return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); + }, AdminController.ConceptsAction.class); + // Show Notifications tab only if we have registered notification providers + addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); + addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); + addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); + addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); + addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); + addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); + addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); + + addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); + addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); + addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); + addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); + + addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); + addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); + } + + public AdminController() + { + setActionResolver(_actionResolver); + } + + @RequiresNoPermission + public static class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getShowAdminURL(); + } + } + + private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) + { + addAdminNavTrail(root, childTitle, action, getContainer()); + } + + private static void addAdminNavTrail(NavTree root, @NotNull Container container) + { + if (container.isRoot()) + root.addChild("Admin Console", getShowAdminURL().setFragment("links")); + } + + private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + addAdminNavTrail(root, container); + root.addChild(childTitle, new ActionURL(action, container)); + } + + public static ActionURL getShowAdminURL() + { + return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); + } + + @Override + protected void beforeAction(Controller action) throws ServletException + { + super.beforeAction(action); + if (action instanceof BaseViewAction viewaction) + viewaction.getPageConfig().setRobotsNone(); + } + + @AdminConsoleAction + public static class ShowAdminAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/admin.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); + if (null != returnUrl) + root.addChild("Return to Project", returnUrl); + root.addChild("Admin Console"); + setHelpTopic("siteManagement"); + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ShowModuleErrorsAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Module Errors", this.getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); + } + } + + public static class AdminUrlsImpl implements AdminUrls + { + @Override + public ActionURL getModuleErrorsURL() + { + return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getAdminConsoleURL() + { + return getShowAdminURL(); + } + + @Override + public ActionURL getModuleStatusURL(URLHelper returnUrl) + { + return AdminController.getModuleStatusURL(returnUrl); + } + + @Override + public ActionURL getCustomizeSiteURL() + { + return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) + { + ActionURL url = getCustomizeSiteURL(); + + if (upgradeInProgress) + url.addParameter("upgradeInProgress", "1"); + + return url; + } + + @Override + public ActionURL getProjectSettingsURL(Container c) + { + return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + ActionURL getLookAndFeelResourcesURL(Container c) + { + return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsMenuURL(Container c) + { + return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsFileURL(Container c) + { + return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) + { + return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); + } + + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) + { + ActionURL url = new ActionURL(CustomizeEmailAction.class, c); + if (selectedTemplate != null) + { + url.addParameter("templateClass", selectedTemplate); + } + if (returnUrl != null) + { + url.addReturnUrl(returnUrl); + } + return url; + } + + public ActionURL getResetLookAndFeelPropertiesURL(Container c) + { + return new ActionURL(ResetPropertiesAction.class, c); + } + + @Override + public ActionURL getMaintenanceURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + @Override + public ActionURL getModulesDetailsURL() + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getDeleteModuleURL(String moduleName) + { + return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); + } + + @Override + public ActionURL getManageFoldersURL(Container c) + { + return new ActionURL(ManageFoldersAction.class, c); + } + + @Override + public ActionURL getFolderTypeURL(Container c) + { + return new ActionURL(FolderTypeAction.class, c); + } + + @Override + public ActionURL getExportFolderURL(Container c) + { + return new ActionURL(ExportFolderAction.class, c); + } + + @Override + public ActionURL getImportFolderURL(Container c) + { + return new ActionURL(ImportFolderAction.class, c); + } + + @Override + public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) + { + return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); + } + + @Override + public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) + { + ActionURL result = new ActionURL(CreateFolderAction.class, c); + if (returnUrl != null) + { + result.addReturnUrl(returnUrl); + } + return result; + } + + public ActionURL getSetFolderPermissionsURL(Container c) + { + return new ActionURL(SetFolderPermissionsAction.class, c); + } + + @Override + public void addAdminNavTrail(NavTree root, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, container); + } + + @Override + public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, childTitle, action, container); + } + + @Override + public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) + { + if (container.isRoot()) + addAdminNavTrail(root, "Modules", ModulesAction.class, container); + + root.addChild(childTitle); + } + + @Override + public ActionURL getFileRootsURL(Container c) + { + return new ActionURL(FileRootsAction.class, c); + } + + @Override + public ActionURL getLookAndFeelSettingsURL(Container c) + { + if (c.isRoot()) + return getSiteLookAndFeelSettingsURL(); + else if (c.isProject()) + return getProjectSettingsURL(c); + else + return getFolderSettingsURL(c); + } + + @Override + public ActionURL getSiteLookAndFeelSettingsURL() + { + return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFolderSettingsURL(Container c) + { + return new ActionURL(FolderSettingsAction.class, c); + } + + @Override + public ActionURL getNotificationsURL(Container c) + { + return new ActionURL(NotificationsAction.class, c); + } + + @Override + public ActionURL getModulePropertiesURL(Container c) + { + return new ActionURL(ModulePropertiesAction.class, c); + } + + @Override + public ActionURL getMissingValuesURL(Container c) + { + return new ActionURL(MissingValuesAction.class, c); + } + + public ActionURL getInitialFolderSettingsURL(Container c) + { + return new ActionURL(SetInitialFolderSettingsAction.class, c); + } + + @Override + public ActionURL getMemTrackerURL() + { + return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFilesSiteSettingsURL() + { + return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSessionLoggingURL() + { + return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getTrackedAllocationsViewerURL() + { + return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSystemMaintenanceURL() + { + return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); + } + + public static ActionURL getDeprecatedFeaturesURL() + { + return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); + } + } + + public static class MaintenanceBean + { + public HtmlString content; + public ActionURL loginURL; + } + + /** + * During upgrade, startup, or maintenance mode, the user will be redirected to + * MaintenanceAction and only admin users will be allowed to log into the server. + * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off + * and will redirect to the returnUrl or the loginURL. + * See Issue 18758 for more information. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class MaintenanceAction extends SimpleViewAction + { + private String _title = "Maintenance in progress"; + + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + if (!getUser().hasSiteAdminPermission()) + { + getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); + } + getPageConfig().setTemplate(Template.Dialog); + + boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); + boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); + boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); + + HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); + if (upgradeInProgress) + { + _title = "Upgrade in progress"; + content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (startupInProgress) + { + _title = "Startup in progress"; + content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (maintenanceMode) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); + } + + if (content == null) + content = HtmlString.of(_title); + + ActionURL loginURL = null; + if (getUser().isGuest()) + { + URLHelper returnUrl = form.getReturnUrlHelper(); + if (returnUrl != null) + loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); + else + loginURL = urlProvider(LoginUrls.class).getLoginURL(); + } + + MaintenanceBean bean = new MaintenanceBean(); + bean.content = content; + bean.loginURL = loginURL; + + JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); + view.setTitle(_title); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild(_title); + } + } + + /** + * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class StartupStatusAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); + result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); + + return new ApiSimpleResponse(result); + } + } + + @RequiresSiteAdmin + @IgnoresTermsOfUse + public static class GetPendingRequestCountAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); + + return new ApiSimpleResponse(result); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetModulesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(GetModulesForm form, BindException errors) + { + Container c = ContainerManager.getForPath(getContainer().getPath()); + + ApiSimpleResponse response = new ApiSimpleResponse(); + + List> qinfos = new ArrayList<>(); + + FolderType folderType = c.getFolderType(); + List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); + + //note: this has been altered to use Container.getRequiredModules() instead of FolderType + //this is b/c a parent container must consider child workbooks when determining the set of requiredModules + Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); + Set activeModules = c.getActiveModules(getUser()); + + for (Module m : allModules) + { + Map qinfo = new HashMap<>(); + + qinfo.put("name", m.getName()); + qinfo.put("required", requiredModules.contains(m)); + qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); + qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || + m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); + qinfo.put("tabName", m.getTabName(getViewContext())); + qinfo.put("requireSitePermission", m.getRequireSitePermission()); + qinfos.add(qinfo); + } + + response.put("modules", qinfos); + response.put("folderType", folderType.getName()); + + return response; + } + } + + public static class GetModulesForm + { + } + + @RequiresNoPermission + @AllowedDuringUpgrade + // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress + public static class GuidAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + response.getWriter().write(GUID.makeGUID()); + } + } + + /** + * Preform health checks corresponding to the given categories. + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class HealthCheckAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception + { + if (!ModuleLoader.getInstance().isStartupComplete()) + return new ApiSimpleResponse("healthy", false); + + Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); + HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); + + checkResult.getDetails().put("healthy", checkResult.isHealthy()); + + if (getUser().hasRootAdminPermission()) + { + return new ApiSimpleResponse(checkResult.getDetails()); + } + else + { + if (!checkResult.isHealthy()) + { + try (var writer = createResponseWriter()) + { + writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); + } + return null; + } + + return new ApiSimpleResponse("healthy", checkResult.isHealthy()); + } + } + } + + public static class HealthCheckForm + { + private String _categories; // if null, all categories will be checked. + + public String getCategories() + { + return _categories; + } + + @SuppressWarnings("unused") + public void setCategories(String categories) + { + _categories = categories; + } + } + + // No security checks... anyone (even guests) can view the credits page + @RequiresNoPermission + public class CreditsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + VBox views = new VBox(); + List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); + + addCreditsViews(views, modules, "jars.txt", "JAR"); + addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); + addCreditsViews(views, modules, "source.txt", "Java Source Code"); + addCreditsViews(views, modules, "executables.txt", "Executable"); + + return views; + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Credits", this.getClass()); + } + } + + private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException + { + for (Module module : modules) + { + String wikiSource = getCreditsFile(module, creditsFile); + + if (null != wikiSource) + { + String title = fileType + " Files Distributed with the " + module.getName() + " Module"; + CreditsView credits = new CreditsView(wikiSource, title); + views.addView(credits); + } + } + } + + private static class CreditsView extends WebPartView + { + private Renderable _html; + + CreditsView(@Nullable String wikiSource, String title) + { + super(title); + + wikiSource = StringUtils.trimToEmpty(wikiSource); + + if (StringUtils.isNotEmpty(wikiSource)) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); + _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); + } + } + + @Override + public void renderView(Object model, HtmlWriter out) + { + out.write(_html); + } + } + + private static String getCreditsFile(Module module, String filename) throws IOException + { + // credits files are in /resources/credits + InputStream is = module.getResourceStream("credits/" + filename); + + return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); + } + + private void validateNetworkDrive(NetworkDriveForm form, Errors errors) + { + if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || + isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) + { + errors.reject(ERROR_MSG, "All fields are required"); + } + else if (form.getNetworkDriveLetter().trim().length() > 1) + { + errors.reject(ERROR_MSG, "Network drive letter must be a single character"); + } + else + { + char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); + + if (letter < 'a' || letter > 'z') + { + errors.reject(ERROR_MSG, "Network drive letter must be a letter"); + } + } + } + + public static class ResourceForm + { + private String _resource; + + public String getResource() + { + return _resource; + } + + public void setResource(String resource) + { + _resource = resource; + } + + public ResourceType getResourceType() + { + return ResourceType.valueOf(_resource); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetResourceAction extends FormHandlerAction + { + @Override + public void validateCommand(ResourceForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ResourceForm form, BindException errors) throws Exception + { + form.getResourceType().delete(getContainer(), getUser()); + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + return true; + } + + @Override + public URLHelper getSuccessURL(ResourceForm form) + { + return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetPropertiesAction extends FormHandlerAction + { + private URLHelper _returnUrl; + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Container c = getContainer(); + boolean folder = !(c.isRoot() || c.isProject()); + boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); + + WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); + props.clear(hasAdminOpsPerm); + props.save(); + // TODO: Audit log? + + AdminUrls urls = new AdminUrlsImpl(); + + // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision + if (!folder) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + _returnUrl = urls.getLookAndFeelSettingsURL(c); + + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return _returnUrl; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class CustomizeSiteAction extends FormViewAction + { + @Override + public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) + { + if (form.isUpgradeInProgress()) + getPageConfig().setTemplate(Template.Dialog); + + SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); + setHelpTopic("configAdmin"); + return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Customize Site", this.getClass()); + } + + @Override + public void validateCommand(SiteSettingsForm form, Errors errors) + { + if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) + { + errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); + } + if (form.getMaxBLOBSize() < 0) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); + } + int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); + if (form.getMaxBLOBSize() > hardCap) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); + } + if (form.getSslPort() < 1 || form.getSslPort() > 65535) + { + errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); + } + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); + } + if (form.getMemoryUsageDumpInterval() < 0) + { + errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); + } + } + + @Override + public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception + { + HttpServletRequest request = getViewContext().getRequest(); + + // We only need to check that SSL is running if the user isn't already using SSL + if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) + { + URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); + Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); + + if (sslResponse != null) + { + errors.reject(ERROR_MSG, sslResponse.first); + return false; + } + } + + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); + } + + WriteableAppProps props = AppProps.getWriteableInstance(); + + props.setPipelineToolsDir(form.getPipelineToolsDirectory()); + props.setNavAccessOpen(form.isNavAccessOpen()); + props.setSSLRequired(form.isSslRequired()); + boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); + props.setSSLPort(form.getSslPort()); + props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); + props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); + props.setMaxBLOBSize(form.getMaxBLOBSize()); + props.setExt3Required(form.isExt3Required()); + props.setExt3APIRequired(form.isExt3APIRequired()); + props.setSelfReportExceptions(form.isSelfReportExceptions()); + + props.setAdminOnlyMessage(form.getAdminOnlyMessage()); + props.setShowRibbonMessage(form.isShowRibbonMessage()); + props.setRibbonMessage(form.getRibbonMessage()); + props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); + + props.setAllowApiKeys(form.isAllowApiKeys()); + props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); + props.setAllowSessionKeys(form.isAllowSessionKeys()); + + try + { + ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); + props.setExceptionReportingLevel(level); + } + catch (IllegalArgumentException ignored) + { + } + + try + { + if (form.getUsageReportingLevel() != null) + { + UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); + props.setUsageReportingLevel(level); + } + } + catch (IllegalArgumentException ignored) + { + } + + props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); + + if (null != form.getBaseServerURL()) + { + if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); + return false; + } + + try + { + props.setBaseServerUrl(form.getBaseServerURL()); + } + catch (URISyntaxException e) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + + "Please enter a valid base URL containing the protocol, hostname, and port if required. " + + "The webapp context path should not be included. " + + "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); + return false; + } + } + + String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); + if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) + { + errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); + return false; + } + props.setXFrameOption(frameOption); + props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); + + props.save(getViewContext().getUser()); + UsageReportingLevel.reportNow(); + if (sslSettingChanged) + ContentSecurityPolicyFilter.regenerateSubstitutionMap(); + + return true; + } + + @Override + public ActionURL getSuccessURL(SiteSettingsForm form) + { + if (form.isUpgradeInProgress()) + { + return AppProps.getInstance().getHomePageActionURL(); + } + else + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + } + + public static class NetworkDriveForm + { + private String _networkDriveLetter; + private String _networkDrivePath; + private String _networkDriveUser; + private String _networkDrivePassword; + + public String getNetworkDriveLetter() + { + return _networkDriveLetter; + } + + public void setNetworkDriveLetter(String networkDriveLetter) + { + _networkDriveLetter = networkDriveLetter; + } + + public String getNetworkDrivePassword() + { + return _networkDrivePassword; + } + + public void setNetworkDrivePassword(String networkDrivePassword) + { + _networkDrivePassword = networkDrivePassword; + } + + public String getNetworkDrivePath() + { + return _networkDrivePath; + } + + public void setNetworkDrivePath(String networkDrivePath) + { + _networkDrivePath = networkDrivePath; + } + + public String getNetworkDriveUser() + { + return _networkDriveUser; + } + + public void setNetworkDriveUser(String networkDriveUser) + { + _networkDriveUser = networkDriveUser; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @AdminConsoleAction + public class MapNetworkDriveAction extends FormViewAction + { + @Override + public void validateCommand(NetworkDriveForm form, Errors errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); + } + + @Override + public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception + { + NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); + NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); + NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); + NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); + + return true; + } + + @Override + public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) + { + return new ActionURL(FilesSiteSettingsAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("setRoots#map"); + addAdminNavTrail(root, "Map Network Drive", this.getClass()); + } + } + + public static class SiteSettingsBean + { + public final boolean _upgradeInProgress; + public final boolean _showSelfReportExceptions; + + private SiteSettingsBean(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); + } + + public HtmlString getSiteSettingsHelpLink(String fragment) + { + return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); + } + } + + public static class SetRibbonMessageForm + { + private Boolean _show = null; + private String _message = null; + + public Boolean isShow() + { + return _show; + } + + public void setShow(Boolean show) + { + _show = show; + } + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class SetRibbonMessageAction extends MutatingApiAction + { + @Override + public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception + { + if (form.isShow() != null || form.getMessage() != null) + { + WriteableAppProps props = AppProps.getWriteableInstance(); + + if (form.isShow() != null) + props.setShowRibbonMessage(form.isShow()); + + if (form.getMessage() != null) + props.setRibbonMessage(form.getMessage()); + + props.save(getViewContext().getUser()); + } + + return null; + } + } + + @RequiresPermission(AdminPermission.class) + public class ConfigureSiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + public static class SiteValidationForm + { + private List _providers; + private boolean _includeSubfolders = false; + private transient Consumer _logger = s -> { + }; // No-op by default + + public List getProviders() + { + return _providers; + } + + public void setProviders(List providers) + { + _providers = providers; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public Consumer getLogger() + { + return _logger; + } + + public void setLogger(Consumer logger) + { + _logger = logger; + } + } + + @RequiresPermission(AdminPermission.class) + public class SiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(SiteValidationForm form, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class SiteValidationBackgroundAction extends FormHandlerAction + { + private ActionURL _redirectUrl; + + @Override + public void validateCommand(SiteValidationForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException + { + ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + SiteValidationJob job = new SiteValidationJob(vbi, root, form); + PipelineService.get().queueJob(job); + String jobGuid = job.getJobGUID(); + + if (null == jobGuid) + throw new NotFoundException("Unable to determine pipeline job GUID"); + + Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + if (null == jobId) + throw new NotFoundException("Unable to determine pipeline job ID"); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _redirectUrl = urls.urlDetails(getContainer(), jobId); + + return true; + } + + @Override + public URLHelper getSuccessURL(SiteValidationForm form) + { + return _redirectUrl; + } + } + + public static class ViewValidationResultsForm + { + private int _rowId; + + public int getRowId() + { + return _rowId; + } + + public void setRowId(int rowId) + { + _rowId = rowId; + } + } + + @RequiresPermission(AdminPermission.class) + public class ViewValidationResultsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception + { + PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); + if (null == statusFile) + throw new NotFoundException("Status file not found"); + if (!getContainer().equals(statusFile.lookupContainer())) + throw new UnauthorizedException("Wrong container"); + + String logFilePath = statusFile.getFilePath(); + String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; + File htmlFile = new File(htmlFilePath); + + if (!htmlFile.exists()) + throw new NotFoundException("Results file not found"); + return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); + } + } + + public interface FileManagementForm + { + String getFolderRootPath(); + + void setFolderRootPath(String folderRootPath); + + String getFileRootOption(); + + void setFileRootOption(String fileRootOption); + + String getConfirmMessage(); + + void setConfirmMessage(String confirmMessage); + + boolean isDisableFileSharing(); + + boolean hasSiteDefaultRoot(); + + String[] getEnabledCloudStore(); + + @SuppressWarnings("unused") + void setEnabledCloudStore(String[] enabledCloudStore); + + boolean isCloudFileRoot(); + + @Nullable + String getCloudRootName(); + + void setCloudRootName(String cloudRootName); + + void setFileRootChanged(boolean changed); + + void setEnabledCloudStoresChanged(boolean changed); + + String getMigrateFilesOption(); + + void setMigrateFilesOption(String migrateFilesOption); + + default boolean isFolderSetup() + { + return false; + } + } + + public enum MigrateFilesOption implements SafeToRenderEnum + { + leave + { + @Override + public String description() + { + return "Source files not copied or moved"; + } + }, + copy + { + @Override + public String description() + { + return "Copy source files to destination"; + } + }, + move + { + @Override + public String description() + { + return "Move source files to destination"; + } + }; + + public abstract String description(); + } + + public static class ProjectSettingsForm extends FolderSettingsForm + { + // Site-only properties + private String _dateParsingMode; + private String _customWelcome; + + // Site & project properties + private boolean _shouldInherit; // new subfolders should inherit parent permissions + private String _systemDescription; + private boolean _systemDescriptionInherited; + private String _systemShortName; + private boolean _systemShortNameInherited; + private String _themeName; + private boolean _themeNameInherited; + private String _folderDisplayMode; + private boolean _folderDisplayModeInherited; + private String _applicationMenuDisplayMode; + private boolean _applicationMenuDisplayModeInherited; + private boolean _helpMenuEnabled; + private boolean _helpMenuEnabledInherited; + private String _logoHref; + private boolean _logoHrefInherited; + private String _companyName; + private boolean _companyNameInherited; + private String _systemEmailAddress; + private boolean _systemEmailAddressInherited; + private String _reportAProblemPath; + private boolean _reportAProblemPathInherited; + private String _supportEmail; + private boolean _supportEmailInherited; + private String _customLogin; + private boolean _customLoginInherited; + + // Site-only properties + + public String getDateParsingMode() + { + return _dateParsingMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setDateParsingMode(String dateParsingMode) + { + _dateParsingMode = dateParsingMode; + } + + public String getCustomWelcome() + { + return _customWelcome; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomWelcome(String customWelcome) + { + _customWelcome = customWelcome; + } + + // Site & project properties + + public boolean getShouldInherit() + { + return _shouldInherit; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setShouldInherit(boolean b) + { + _shouldInherit = b; + } + + public String getSystemDescription() + { + return _systemDescription; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescription(String systemDescription) + { + _systemDescription = systemDescription; + } + + public boolean isSystemDescriptionInherited() + { + return _systemDescriptionInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescriptionInherited(boolean systemDescriptionInherited) + { + _systemDescriptionInherited = systemDescriptionInherited; + } + + public String getSystemShortName() + { + return _systemShortName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortName(String systemShortName) + { + _systemShortName = systemShortName; + } + + public boolean isSystemShortNameInherited() + { + return _systemShortNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortNameInherited(boolean systemShortNameInherited) + { + _systemShortNameInherited = systemShortNameInherited; + } + + public String getThemeName() + { + return _themeName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeName(String themeName) + { + _themeName = themeName; + } + + public boolean isThemeNameInherited() + { + return _themeNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeNameInherited(boolean themeNameInherited) + { + _themeNameInherited = themeNameInherited; + } + + public String getFolderDisplayMode() + { + return _folderDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayMode(String folderDisplayMode) + { + _folderDisplayMode = folderDisplayMode; + } + + public boolean isFolderDisplayModeInherited() + { + return _folderDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) + { + _folderDisplayModeInherited = folderDisplayModeInherited; + } + + public String getApplicationMenuDisplayMode() + { + return _applicationMenuDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayMode(String displayMode) + { + _applicationMenuDisplayMode = displayMode; + } + + public boolean isApplicationMenuDisplayModeInherited() + { + return _applicationMenuDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) + { + _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; + } + + public boolean isHelpMenuEnabled() + { + return _helpMenuEnabled; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabled(boolean helpMenuEnabled) + { + _helpMenuEnabled = helpMenuEnabled; + } + + public boolean isHelpMenuEnabledInherited() + { + return _helpMenuEnabledInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) + { + _helpMenuEnabledInherited = helpMenuEnabledInherited; + } + + public String getLogoHref() + { + return _logoHref; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHref(String logoHref) + { + _logoHref = logoHref; + } + + public boolean isLogoHrefInherited() + { + return _logoHrefInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHrefInherited(boolean logoHrefInherited) + { + _logoHrefInherited = logoHrefInherited; + } + + public String getReportAProblemPath() + { + return _reportAProblemPath; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPath(String reportAProblemPath) + { + _reportAProblemPath = reportAProblemPath; + } + + public boolean isReportAProblemPathInherited() + { + return _reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) + { + _reportAProblemPathInherited = reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmail(String supportEmail) + { + _supportEmail = supportEmail; + } + + public String getSupportEmail() + { + return _supportEmail; + } + + public boolean isSupportEmailInherited() + { + return _supportEmailInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmailInherited(boolean supportEmailInherited) + { + _supportEmailInherited = supportEmailInherited; + } + + public String getSystemEmailAddress() + { + return _systemEmailAddress; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddress(String systemEmailAddress) + { + _systemEmailAddress = systemEmailAddress; + } + + public boolean isSystemEmailAddressInherited() + { + return _systemEmailAddressInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) + { + _systemEmailAddressInherited = systemEmailAddressInherited; + } + + public String getCompanyName() + { + return _companyName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyName(String companyName) + { + _companyName = companyName; + } + + public boolean isCompanyNameInherited() + { + return _companyNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyNameInherited(boolean companyNameInherited) + { + _companyNameInherited = companyNameInherited; + } + + public String getCustomLogin() + { + return _customLogin; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLogin(String customLogin) + { + _customLogin = customLogin; + } + + public boolean isCustomLoginInherited() + { + return _customLoginInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLoginInherited(boolean customLoginInherited) + { + _customLoginInherited = customLoginInherited; + } + } + + public enum FileRootProp implements SafeToRenderEnum + { + disable, + siteDefault, + folderOverride, + cloudRoot + } + + public static class FilesForm extends SetupForm implements FileManagementForm + { + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _cloudRootName; + private String _migrateFilesOption; + private String[] _enabledCloudStore; + private String _fileRootOption; + private String _folderRootPath; + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + } + + @SuppressWarnings("unused") + public static class SiteSettingsForm + { + private boolean _upgradeInProgress = false; + + private String _pipelineToolsDirectory; + private boolean _sslRequired; + private boolean _adminOnlyMode; + private boolean _showRibbonMessage; + private boolean _ext3Required; + private boolean _ext3APIRequired; + private boolean _selfReportExceptions; + private String _adminOnlyMessage; + private String _ribbonMessage; + private int _sslPort; + private int _memoryUsageDumpInterval; + private int _readOnlyHttpRequestTimeout; + private int _maxBLOBSize; + private String _exceptionReportingLevel; + private String _usageReportingLevel; + private String _administratorContactEmail; + + private String _baseServerURL; + private String _callbackPassword; + private boolean _allowApiKeys; + private int _apiKeyExpirationSeconds; + private boolean _allowSessionKeys; + private boolean _navAccessOpen; + + private String _XFrameOption; + private boolean _includeServerHttpHeader; + + public String getPipelineToolsDirectory() + { + return _pipelineToolsDirectory; + } + + public void setPipelineToolsDirectory(String pipelineToolsDirectory) + { + _pipelineToolsDirectory = pipelineToolsDirectory; + } + + public boolean isNavAccessOpen() + { + return _navAccessOpen; + } + + public void setNavAccessOpen(boolean navAccessOpen) + { + _navAccessOpen = navAccessOpen; + } + + public boolean isSslRequired() + { + return _sslRequired; + } + + public void setSslRequired(boolean sslRequired) + { + _sslRequired = sslRequired; + } + + public boolean isExt3Required() + { + return _ext3Required; + } + + public void setExt3Required(boolean ext3Required) + { + _ext3Required = ext3Required; + } + + public boolean isExt3APIRequired() + { + return _ext3APIRequired; + } + + public void setExt3APIRequired(boolean ext3APIRequired) + { + _ext3APIRequired = ext3APIRequired; + } + + public int getSslPort() + { + return _sslPort; + } + + public void setSslPort(int sslPort) + { + _sslPort = sslPort; + } + + public boolean isAdminOnlyMode() + { + return _adminOnlyMode; + } + + public void setAdminOnlyMode(boolean adminOnlyMode) + { + _adminOnlyMode = adminOnlyMode; + } + + public String getAdminOnlyMessage() + { + return _adminOnlyMessage; + } + + public void setAdminOnlyMessage(String adminOnlyMessage) + { + _adminOnlyMessage = adminOnlyMessage; + } + + public boolean isSelfReportExceptions() + { + return _selfReportExceptions; + } + + public void setSelfReportExceptions(boolean selfReportExceptions) + { + _selfReportExceptions = selfReportExceptions; + } + + public String getExceptionReportingLevel() + { + return _exceptionReportingLevel; + } + + public void setExceptionReportingLevel(String exceptionReportingLevel) + { + _exceptionReportingLevel = exceptionReportingLevel; + } + + public String getUsageReportingLevel() + { + return _usageReportingLevel; + } + + public void setUsageReportingLevel(String usageReportingLevel) + { + _usageReportingLevel = usageReportingLevel; + } + + public String getAdministratorContactEmail() + { + return _administratorContactEmail; + } + + public void setAdministratorContactEmail(String administratorContactEmail) + { + _administratorContactEmail = administratorContactEmail; + } + + public boolean isUpgradeInProgress() + { + return _upgradeInProgress; + } + + public void setUpgradeInProgress(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + } + + public int getMemoryUsageDumpInterval() + { + return _memoryUsageDumpInterval; + } + + public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) + { + _memoryUsageDumpInterval = memoryUsageDumpInterval; + } + + public int getReadOnlyHttpRequestTimeout() + { + return _readOnlyHttpRequestTimeout; + } + + public void setReadOnlyHttpRequestTimeout(int timeout) + { + _readOnlyHttpRequestTimeout = timeout; + } + + public int getMaxBLOBSize() + { + return _maxBLOBSize; + } + + public void setMaxBLOBSize(int maxBLOBSize) + { + _maxBLOBSize = maxBLOBSize; + } + + public String getBaseServerURL() + { + return _baseServerURL; + } + + public void setBaseServerURL(String baseServerURL) + { + _baseServerURL = baseServerURL; + } + + public String getCallbackPassword() + { + return _callbackPassword; + } + + public void setCallbackPassword(String callbackPassword) + { + _callbackPassword = callbackPassword; + } + + public boolean isShowRibbonMessage() + { + return _showRibbonMessage; + } + + public void setShowRibbonMessage(boolean showRibbonMessage) + { + _showRibbonMessage = showRibbonMessage; + } + + public String getRibbonMessage() + { + return _ribbonMessage; + } + + public void setRibbonMessage(String ribbonMessage) + { + _ribbonMessage = ribbonMessage; + } + + public boolean isAllowApiKeys() + { + return _allowApiKeys; + } + + public void setAllowApiKeys(boolean allowApiKeys) + { + _allowApiKeys = allowApiKeys; + } + + public int getApiKeyExpirationSeconds() + { + return _apiKeyExpirationSeconds; + } + + public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) + { + _apiKeyExpirationSeconds = apiKeyExpirationSeconds; + } + + public boolean isAllowSessionKeys() + { + return _allowSessionKeys; + } + + public void setAllowSessionKeys(boolean allowSessionKeys) + { + _allowSessionKeys = allowSessionKeys; + } + + public String getXFrameOption() + { + return _XFrameOption; + } + + public void setXFrameOption(String XFrameOption) + { + _XFrameOption = XFrameOption; + } + + public boolean isIncludeServerHttpHeader() + { + return _includeServerHttpHeader; + } + + public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) + { + _includeServerHttpHeader = includeServerHttpHeader; + } + } + + + @AdminConsoleAction + public class ShowThreadsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Log to labkey.log as well as showing through the browser + DebugInfoDumper.dumpThreads(3); + return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpDebugging#threads"); + addAdminNavTrail(root, "Current Threads", this.getClass()); + } + } + + private abstract class AbstractPostgresAction extends QueryViewAction + { + private final String _queryName; + + protected AbstractPostgresAction(String queryName) + { + super(QueryExportForm.class); + _queryName = queryName; + } + + @Override + protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception + { + if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + throw new NotFoundException("Only available with Postgres as the primary database"); + } + + QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); + QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) + { + @Override + public DataView createDataView() + { + // Troubleshooters don't have normal read access to the root container so grant them special access + // for these queries + DataView view = super.createDataView(); + view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); + return view; + } + }; + result.setTitle(_queryName); + result.setFrame(WebPartView.FrameType.PORTAL); + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("postgresActivity"); + addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); + } + + } + + @AdminConsoleAction + public class PostgresStatActivityAction extends AbstractPostgresAction + { + public PostgresStatActivityAction() + { + super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); + } + } + + @AdminConsoleAction + public class PostgresLocksAction extends AbstractPostgresAction + { + public PostgresLocksAction() + { + super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); + } + } + + @AdminConsoleAction + public class PostgresTableSizesAction extends AbstractPostgresAction + { + public PostgresTableSizesAction() + { + super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME); + } + } + + @AdminConsoleAction + public class DumpHeapAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + File destination = DebugInfoDumper.dumpHeap(); + return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpHeap"); + addAdminNavTrail(root, "Heap dump", getClass()); + } + } + + + public static class ThreadsBean + { + public Map> spids; + public List threads; + public Map stackTraces; + + ThreadsBean() + { + stackTraces = Thread.getAllStackTraces(); + threads = new ArrayList<>(stackTraces.keySet()); + threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); + + spids = new HashMap<>(); + + for (Thread t : threads) + { + spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); + } + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class ShowNetworkDriveTestAction extends SimpleViewAction + { + @Override + public void validate(NetworkDriveForm form, BindException errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, BindException errors) + { + NetworkDrive testDrive = new NetworkDrive(); + testDrive.setPassword(form.getNetworkDrivePassword()); + testDrive.setPath(form.getNetworkDrivePath()); + testDrive.setUser(form.getNetworkDriveUser()); + TestNetworkDriveBean bean = new TestNetworkDriveBean(); + + if (!errors.hasErrors()) + { + char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); + try + { + String mountError = testDrive.mount(driveLetter); + if (mountError != null) + { + errors.reject(ERROR_MSG, mountError); + } + else + { + File f = new File(driveLetter + ":\\"); + if (!f.exists()) + { + errors.reject(ERROR_MSG, "Could not access network drive"); + } + else + { + String[] fileNames = f.list(); + if (fileNames == null) + fileNames = new String[0]; + Arrays.sort(fileNames); + bean.setFiles(fileNames); + } + } + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + try + { + testDrive.unmount(driveLetter); + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + } + + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Test Mapping Network Drive"); + } + } + + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetErrorMarkAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + return HtmlView.of("Are you sure you want to reset the site errors?"); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + File errorLogFile = getErrorLogFile(); + _errorMark = errorLogFile.length(); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return getShowAdminURL(); + } + } + + abstract public static class ShowLogAction extends ExportAction + { + @Override + public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException + { + getPageConfig().setNoIndex(); + export(response); + } + + protected abstract void export(HttpServletResponse response) throws IOException; + } + + @AdminConsoleAction + public class ShowErrorsSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); + } + } + + @AdminConsoleAction + public class ShowAllErrorsAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); + } + } + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetPrimaryLogMarkAction extends MutatingApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + File logFile = getPrimaryLogFile(); + _primaryLogMark = logFile.length(); + return null; + } + } + + @AdminConsoleAction + public class ShowPrimaryLogSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowPrimaryLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowCspReportLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); + } + } + + private File getErrorLogFile() + { + return new File(getLabKeyLogDir(), "labkey-errors.log"); + } + + private File getPrimaryLogFile() + { + return new File(getLabKeyLogDir(), "labkey.log"); + } + + private File getCspReportLogFile() + { + return new File(getLabKeyLogDir(), "csp-report.log"); + } + + private static ActionURL getActionsURL() + { + return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public class ActionsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new ActionsTabStrip(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("actionsDiagnostics"); + addAdminNavTrail(root, "Actions", this.getClass()); + } + } + + private static class ActionsTabStrip extends TabStripView + { + @Override + public List getTabList() + { + List tabs = new ArrayList<>(3); + + tabs.add(new TabInfo("Summary", "summary", getActionsURL())); + tabs.add(new TabInfo("Details", "details", getActionsURL())); + tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); + + return tabs; + } + + @Override + public HttpView getTabView(String tabId) + { + if ("exceptions".equals(tabId)) + return new ActionsExceptionsView(); + return new ActionsView(!"details".equals(tabId)); + } + } + + @AdminConsoleAction + public static class ExportActionsAction extends ExportAction + { + @Override + public void export(Object form, HttpServletResponse response, BindException errors) throws Exception + { + try (ActionsTsvWriter writer = new ActionsTsvWriter()) + { + writer.write(response); + } + } + } + + private static ActionURL getQueriesURL(@Nullable String statName) + { + ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); + + if (null != statName) + url.addParameter("stat", statName); + + return url; + } + + + @AdminConsoleAction + public class QueriesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueriesForm form, BindException errors) + { + String buttonHTML = ""; + if (getUser().hasRootAdminPermission()) + buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; + buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

    "; + + return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, + AdminController::getQueryStackTracesURL); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("queryPerf"); + addAdminNavTrail(root, "Queries", this.getClass()); + } + } + + public static class QueriesForm + { + private String _stat = "Count"; + + public String getStat() + { + return _stat; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setStat(String stat) + { + _stat = stat; + } + } + + + private static ActionURL getQueryStackTracesURL(String sqlHash) + { + ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class QueryStackTracesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces"); + } + } + + + private static ActionURL getExecutionPlanURL(String sqlHash) + { + ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class ExecutionPlanAction extends SimpleViewAction + { + private String _sqlHash; + private ExecutionPlanType _type; + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + _sqlHash = form.getSqlHash(); + _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); + if (null == _type) + throw new NotFoundException("Unknown execution plan type"); + + return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); + root.addChild(_type.getDescription()); + } + } + + + public static class QueryForm + { + private String _sqlHash; + private String _type = "Estimated"; // All dialects support Estimated + private boolean _log = false; + + public String getSqlHash() + { + return _sqlHash; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSqlHash(String sqlHash) + { + _sqlHash = sqlHash; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setType(String type) + { + _type = type; + } + + public boolean isLog() + { + return _log; + } + + public void setLog(boolean log) + { + _log = log; + } + } + + + private ActionURL getExportQueriesURL() + { + return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public static class ExportQueriesAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) + { + writer.setFilenamePrefix("SQL_Queries"); + writer.write(response); + } + } + } + + private static ActionURL getResetQueryStatisticsURL() + { + return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); + } + + + @RequiresPermission(AdminPermission.class) + public static class ResetQueryStatisticsAction extends FormHandlerAction + { + @Override + public void validateCommand(QueriesForm target, Errors errors) + { + } + + @Override + public boolean handlePost(QueriesForm form, BindException errors) throws Exception + { + QueryProfiler.getInstance().resetAllStatistics(); + return true; + } + + @Override + public URLHelper getSuccessURL(QueriesForm form) + { + return getQueriesURL(form.getStat()); + } + } + + + @AdminConsoleAction + public class CachesAction extends SimpleViewAction + { + private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); + private final DecimalFormat percent = new DecimalFormat("0%"); + + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + if (form.isClearCaches()) + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); + throw new RedirectException(redirect); + } + + List> caches = CacheManager.getKnownCaches(); + + if (form.getDebugName() != null) + { + for (TrackingCache cache : caches) + { + if (form.getDebugName().equals(cache.getDebugName())) + { + LOG.info("Purging cache: " + cache.getDebugName()); + cache.clear(); + } + } + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); + throw new RedirectException(redirect); + } + + List cacheStats = new ArrayList<>(); + List transactionStats = new ArrayList<>(); + + for (TrackingCache cache : caches) + { + cacheStats.add(CacheManager.getCacheStats(cache)); + transactionStats.add(CacheManager.getTransactionCacheStats(cache)); + } + + HtmlStringBuilder html = HtmlStringBuilder.of(); + + html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); + html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); + + html.unsafeAppend("

    \n"); + appendStats(html, "Caches", cacheStats, false); + + html.unsafeAppend("

    \n"); + appendStats(html, "Transaction Caches", transactionStats, true); + + return new HtmlView(html); + } + + private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) + { + List stats = skipUnusedCaches ? + allStats.stream() + .filter(stat->stat.getMaxSize() > 0) + .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : + allStats; + + Collections.sort(stats); + + html.unsafeAppend("

    "); + html.append(title); + html.append(" (").append(stats.size()).unsafeAppend(")

    \n"); + + html.unsafeAppend("\n"); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + + long size = 0; + long gets = 0; + long misses = 0; + long puts = 0; + long expirations = 0; + long evictions = 0; + long removes = 0; + long clears = 0; + int rowCount = 0; + + for (CacheStats stat : stats) + { + size += stat.getSize(); + gets += stat.getGets(); + misses += stat.getMisses(); + puts += stat.getPuts(); + expirations += stat.getExpirations(); + evictions += stat.getEvictions(); + removes += stat.getRemoves(); + clears += stat.getClears(); + + html.unsafeAppend(""); + + appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); + + Long limit = stat.getLimit(); + long maxSize = stat.getMaxSize(); + + appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); + appendDoubles(html, stat.getMissRatio()); + + html.unsafeAppend("\n"); + + if (null != limit && maxSize >= limit) + html.unsafeAppend(""); + + html.unsafeAppend("\n"); + rowCount++; + } + + double ratio = 0 != gets ? misses / (double)gets : 0; + html.unsafeAppend(""); + + appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); + appendDoubles(html, ratio); + + html.unsafeAppend("\n"); + html.unsafeAppend("
    Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
    ").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
    Total
    \n"); + } + + private static final List PREFIXES_TO_SKIP = List.of( + "java.base/java.lang.Thread.getStackTrace", + "org.labkey.api.cache.CacheManager", + "org.labkey.api.cache.Throttle", + "org.labkey.api.data.DatabaseCache", + "org.labkey.api.module.ModuleResourceCache" + ); + + private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) + { + StringBuilder sb = new StringBuilder(); + + if (creationStackTrace != null) + { + boolean trimming = true; + for (StackTraceElement element : creationStackTrace) + { + // Skip the first few uninteresting stack trace elements to highlight the caller we care about + if (trimming) + { + if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) + continue; + + trimming = false; + } + sb.append(element); + sb.append("\n"); + } + } + + if (!sb.isEmpty()) + { + String message = PageFlowUtil.jsString(sb); + String id = "id" + UniqueID.getServerSessionScopedUID(); + html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); + HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); + } + } + + private void appendLongs(HtmlStringBuilder html, Long... stats) + { + for (Long stat : stats) + { + if (null == stat) + html.unsafeAppend(" "); + else + html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); + } + } + + private void appendDoubles(HtmlStringBuilder html, double... stats) + { + for (double stat : stats) + html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("cachesDiagnostics"); + addAdminNavTrail(root, "Cache Statistics", this.getClass()); + } + } + + @RequiresSiteAdmin + public class EnvironmentVariablesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Environment Variables", this.getClass()); + } + } + + @RequiresSiteAdmin + public class SystemPropertiesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "System Properties", this.getClass()); + } + } + + + public static class ConfigureSystemMaintenanceForm + { + private String _maintenanceTime; + private Set _enable = Collections.emptySet(); + private boolean _enableSystemMaintenance = true; + + public String getMaintenanceTime() + { + return _maintenanceTime; + } + + @SuppressWarnings("unused") + public void setMaintenanceTime(String maintenanceTime) + { + _maintenanceTime = maintenanceTime; + } + + public Set getEnable() + { + return _enable; + } + + @SuppressWarnings("unused") + public void setEnable(Set enable) + { + _enable = enable; + } + + public boolean isEnableSystemMaintenance() + { + return _enableSystemMaintenance; + } + + @SuppressWarnings("unused") + public void setEnableSystemMaintenance(boolean enableSystemMaintenance) + { + _enableSystemMaintenance = enableSystemMaintenance; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ConfigureSystemMaintenanceAction extends FormViewAction + { + @Override + public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) + { + Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); + + if (null == date) + errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); + } + + @Override + public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) + { + SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); + return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); + } + + @Override + public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) + { + SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); + SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); + + return true; + } + + @Override + public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class ResetSystemMaintenanceAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + SystemMaintenance.clearProperties(); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + + public static class SystemMaintenanceForm + { + private String _taskName; + private boolean _test = false; + + public String getTaskName() + { + return _taskName; + } + + @SuppressWarnings("unused") + public void setTaskName(String taskName) + { + _taskName = taskName; + } + + public boolean isTest() + { + return _test; + } + + public void setTest(boolean test) + { + _test = test; + } + } + + @RequiresSiteAdmin + public class SystemMaintenanceAction extends FormHandlerAction + { + private Long _jobId = null; + private URLHelper _url = null; + + @Override + public void validateCommand(SystemMaintenanceForm form, Errors errors) + { + } + + @Override + public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException + { + // Send the pipeline job details absolute URL back to the test + sendPlainText(_url.getURIString()); + + // Suppress templates, divs, etc. + getPageConfig().setTemplate(Template.None); + return new EmptyView(); + } + + @Override + public boolean handlePost(SystemMaintenanceForm form, BindException errors) + { + String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); + + if (null != jobGuid) + _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SystemMaintenanceForm form) + { + // In the standard case, redirect to the pipeline details URL + // If the test is invoking system maintenance then return the URL instead + return form.isTest() ? null : _url; + } + } + + @AdminConsoleAction + public class AttachmentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getAdminView(getViewContext().getActionURL()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Attachments", getClass()); + } + } + + @AdminConsoleAction + public class FindAttachmentParentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getFindAttachmentParentsView(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Find Attachment Parents", getClass()); + } + } + + public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCachesURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCacheURL(String debugName) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + url.addParameter(MemForm.Params.debugName, debugName); + + return url; + } + + private static volatile String lastCacheMemUsed = null; + + @AdminConsoleAction + public class MemTrackerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + Set objectsToIgnore = MemTracker.getInstance().beforeReport(); + + boolean gc = form.isGc(); + boolean cc = form.isClearCaches(); + + if (getUser().hasRootAdminPermission() && (gc || cc)) + { + // If both are requested then try to determine and record cache memory usage + if (gc && cc) + { + // gc once to get an accurate free memory read + long before = gc(); + clearCaches(); + // gc again now that we cleared caches + long cacheMemoryUsed = before - gc(); + + // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage + String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; + LOG.info("Estimate of cache memory used: " + cacheMemUsed); + lastCacheMemUsed = cacheMemUsed; + } + else if (cc) + { + clearCaches(); + } + else + { + gc(); + } + + LOG.info("Cache clearing and garbage collecting complete"); + } + + return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); + } + + /** @return estimated current memory usage, post-garbage collection */ + private long gc() + { + LOG.info("Garbage collecting"); + System.gc(); + // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } + + private void clearCaches() + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + LOG.info("Purging SearchService queues"); + SearchService.get().purgeQueues(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("memTracker"); + addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); + } + } + + public static class MemForm + { + private enum Params {clearCaches, debugName, gc} + + private boolean _clearCaches = false; + private boolean _gc = false; + private String _debugName; + + public boolean isClearCaches() + { + return _clearCaches; + } + + @SuppressWarnings("unused") + public void setClearCaches(boolean clearCaches) + { + _clearCaches = clearCaches; + } + + public boolean isGc() + { + return _gc; + } + + @SuppressWarnings("unused") + public void setGc(boolean gc) + { + _gc = gc; + } + + public String getDebugName() + { + return _debugName; + } + + @SuppressWarnings("unused") + public void setDebugName(String debugName) + { + _debugName = debugName; + } + } + + public static class MemBean + { + public final List> memoryUsages = new ArrayList<>(); + public final List> systemProperties = new ArrayList<>(); + public final List references; + public final List graphNames = new ArrayList<>(); + public final List activeThreads = new LinkedList<>(); + + public boolean assertsEnabled = false; + + private MemBean(HttpServletRequest request, Set objectsToIgnore) + { + MemTracker memTracker = MemTracker.getInstance(); + List all = memTracker.getReferences(); + long threadId = Thread.currentThread().getId(); + + // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found + for (Thread thread : new ThreadsBean().threads) + { + if (thread.getId() == threadId) + continue; + + Thread.State state = thread.getState(); + + if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) + { + boolean labkeyThread = false; + + if (memTracker.shouldDisplay(thread)) + { + for (StackTraceElement element : thread.getStackTrace()) + { + String className = element.getClassName(); + + if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) + { + labkeyThread = true; + break; + } + } + } + + if (labkeyThread) + { + String threadInfo = thread.getName(); + TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); + if (null != uri) + threadInfo += "; processing URL " + uri.toLogString(); + activeThreads.add(threadInfo); + } + } + } + + // ignore recently allocated + long start = ViewServlet.getRequestStartTime(request) - 2000; + references = new ArrayList<>(all.size()); + + for (HeldReference r : all) + { + if (r.getThreadId() == threadId && r.getAllocationTime() >= start) + continue; + + if (objectsToIgnore.contains(r.getReference())) + continue; + + references.add(r); + } + + // memory: + graphNames.add("Heap"); + graphNames.add("Non Heap"); + + MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); + } + + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); + } + + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.NON_HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) + { + memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); + graphNames.add(pool.getName()); + } + + DecimalFormat commaf0 = new DecimalFormat("#,##0"); + + + // class loader: + ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); + if (classbean != null) + { + systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); + systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); + systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); + } + + // runtime: + RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + if (runtimeBean != null) + { + systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); + long upTime = runtimeBean.getUptime(); // round to sec + upTime = upTime - (upTime % 1000); + systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); + systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); + systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); + } + + // threads: + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + if (threadBean != null) + { + systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); + systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); + long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); + systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); + } + + // threads: + List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + for (GarbageCollectorMXBean gcBean : gcBeans) + { + systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); + systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); + } + + String cacheMem = lastCacheMemUsed; + + if (null != cacheMem) + systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); + + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + if (osBean != null) + { + systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); + + DecimalFormat f3 = new DecimalFormat("0.000"); + + if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) + { + systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); + systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); + systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); + systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); + } + } + + //noinspection ConstantConditions + assert assertsEnabled = true; + } + } + + private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) + { + try + { + return getUsage(pool.getUsage()); + } + catch (IllegalArgumentException x) + { + // sometimes we get usage>committed exception with older versions of JRockit + return null; + } + } + + public static class MemoryUsageSummary + { + + public final long _init; + public final long _used; + public final long _committed; + public final long _max; + + public MemoryUsageSummary(MemoryUsage usage) + { + _init = usage.getInit(); + _used = usage.getUsed(); + _committed = usage.getCommitted(); + _max = usage.getMax(); + } + + public MemoryUsageSummary(BufferPoolMXBean pool) + { + _init = -1; + _used = pool.getMemoryUsed(); + _committed = _used; + _max = pool.getTotalCapacity(); + } + } + + private static MemoryUsageSummary getUsage(MemoryUsage usage) + { + if (null == usage) + return null; + + try + { + return new MemoryUsageSummary(usage); + } + catch (IllegalArgumentException x) + { + // sometime we get usage>committed exception with older verions of JRockit + return null; + } + } + + public static class ChartForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + } + + private static class MemoryCategory implements Comparable + { + private final String _type; + private final double _mb; + + public MemoryCategory(String type, double mb) + { + _type = type; + _mb = mb; + } + + @Override + public int compareTo(@NotNull MemoryCategory o) + { + return Double.compare(getMb(), o.getMb()); + } + + public String getType() + { + return _type; + } + + public double getMb() + { + return _mb; + } + } + + @AdminConsoleAction + public static class MemoryChartAction extends ExportAction + { + @Override + public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception + { + MemoryUsage usage = null; + boolean showLegend = false; + String title = form.getType(); + if ("Heap".equals(form.getType())) + { + usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + showLegend = true; + } + else if ("Non Heap".equals(form.getType())) + usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); + else + { + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) + { + MemoryPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + usage = pool.getUsage(); + } + } + + Pair divisor = null; + + List types = new ArrayList<>(4); + + if (usage == null) + { + boolean found = false; + for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) + { + BufferPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + { + long total = pool.getTotalCapacity(); + long used = pool.getMemoryUsed(); + + divisor = getDivisor(total); + + title = "Buffer pool " + title; + + if (total > 0 || used > 0) + { + types.add(new MemoryCategory("Used", used / divisor.first)); + types.add(new MemoryCategory("Max", total / divisor.first)); + } + found = true; + } + } + if (!found) + { + throw new NotFoundException(); + } + } + else + { + if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) + { + divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); + + types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); + types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); + types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); + types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); + } + } + + if (divisor != null) + { + title += " (" + divisor.second + ")"; + } + + DefaultCategoryDataset dataset = new DefaultCategoryDataset(); + + Collections.sort(types); + + for (int i = 0; i < types.size(); i++) + { + double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); + dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); + } + + JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); + chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); + response.setContentType("image/png"); + + ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); + } + + private Pair getDivisor(long l) + { + if (l > 4096L * 1024L * 1024L) + { + return Pair.of(1024L * 1024L * 1024L, "GB"); + } + if (l > 4096L * 1024L) + { + return Pair.of(1024L * 1024L, "MB"); + } + if (l > 4096L) + { + return Pair.of(1024L, "KB"); + } + + return Pair.of(1L, "bytes"); + + } + } + + public static class MemoryStressForm + { + private int _threads = 3; + private int _arraySize = 20_000; + private int _arrayCount = 10_000; + private float _percentChurn = 0.50f; + private int _delay = 20; + private int _iterations = 500; + + public int getThreads() + { + return _threads; + } + + public void setThreads(int threads) + { + _threads = threads; + } + + public int getArraySize() + { + return _arraySize; + } + + public void setArraySize(int arraySize) + { + _arraySize = arraySize; + } + + public int getArrayCount() + { + return _arrayCount; + } + + public void setArrayCount(int arrayCount) + { + _arrayCount = arrayCount; + } + + public float getPercentChurn() + { + return _percentChurn; + } + + public void setPercentChurn(float percentChurn) + { + _percentChurn = percentChurn; + } + + public int getDelay() + { + return _delay; + } + + public void setDelay(int delay) + { + _delay = delay; + } + + public int getIterations() + { + return _iterations; + } + + public void setIterations(int iterations) + { + _iterations = iterations; + } + } + + @RequiresSiteAdmin + public class MemoryStressTestAction extends FormViewAction + { + @Override + public void validateCommand(MemoryStressForm target, Errors errors) + { + + } + + @Override + public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception + { + return new HtmlView( + DOM.LK.FORM(at(method, "POST"), + DOM.LK.ERRORS(errors.getBindingResult()), + DOM.BR(), DOM.BR(), + "This utility action will do a lot of memory allocation to test the memory configuration of the host.", + DOM.BR(), DOM.BR(), + "It spins up threads, all of which allocate a specified number byte arrays of specified length.", + DOM.BR(), + "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", + DOM.BR(), + "They continue for the specified number of allocations.", + DOM.BR(), + "The memory actively held is approximately (threads * array count * array length).", + DOM.BR(), + "The memory turnover is based on the churn percentage, array length, delay, and iterations.", + DOM.BR(), DOM.BR(), + DOM.TABLE( + DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), + DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), + DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), + DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), + DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), + DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) + ), + new ButtonBuilder("Perform stress test").submit(true).build()) + ); + } + + @Override + public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception + { + List threads = new ArrayList<>(); + for (int i = 0; i < memoryStressForm._threads; i++) + { + Thread t = new Thread(() -> + { + Random r = new Random(); + byte[][] arrays = new byte[memoryStressForm._arrayCount][]; + // Initialize the arrays + for (int a = 0; a < arrays.length; a++) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + + for (int iter = 0; iter < memoryStressForm._iterations; iter++) + { + try + { + Thread.sleep(memoryStressForm._delay); + } + catch (InterruptedException ignored) {} + + // Swap the contents based on our desired percent churn + for (int a = 0; a < arrays.length; a++) + { + if (r.nextFloat() <= memoryStressForm._percentChurn) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + } + } + }); + t.setUncaughtExceptionHandler((t2, e) -> { + LOG.error("Stress test exception", e); + errors.reject(null, "Stress test exception: " + e); + }); + t.start(); + threads.add(t); + } + + for (Thread thread : threads) + { + thread.join(); + } + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) + { + return new ActionURL(MemTrackerAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); + root.addChild("Memory Stress Test"); + } + } + + public static ActionURL getModuleStatusURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + public static class ModuleStatusBean + { + public String verb; + public String verbing; + public ActionURL nextURL; + } + + @RequiresPermission(TroubleshooterPermission.class) + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class ModuleStatusAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + ModuleLoader loader = ModuleLoader.getInstance(); + VBox vbox = new VBox(); + ModuleStatusBean bean = new ModuleStatusBean(); + + if (loader.isNewInstall()) + bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); + else if (form.getReturnUrl() != null) + { + try + { + bean.nextURL = form.getReturnActionURL(); + } + catch (URLException x) + { + // might not be an ActionURL e.g. /labkey/_webdav/home + } + } + if (null == bean.nextURL) + bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + + if (loader.isNewInstall()) + bean.verb = "Install"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verb = "Upgrade"; + else + bean.verb = "Start"; + + if (loader.isNewInstall()) + bean.verbing = "Installing"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verbing = "Upgrading"; + else + bean.verbing = "Starting"; + + JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + + getPageConfig().setTemplate(Template.Wizard); + getPageConfig().setTitle(bean.verb + " Modules"); + setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); + + return vbox; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class NewInstallSiteSettingsForm extends FileSettingsForm + { + private String _notificationEmail; + private String _siteName; + + public String getNotificationEmail() + { + return _notificationEmail; + } + + public void setNotificationEmail(String notificationEmail) + { + _notificationEmail = notificationEmail; + } + + public String getSiteName() + { + return _siteName; + } + + public void setSiteName(String siteName) + { + _siteName = siteName; + } + } + + @RequiresSiteAdmin + public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction + { + public NewInstallSiteSettingsAction() + { + super(NewInstallSiteSettingsForm.class); + } + + @Override + public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) + { + super.validateCommand(form, errors); + + if (isBlank(form.getNotificationEmail())) + { + errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); + } + try + { + ValidEmail email = new ValidEmail(form.getNotificationEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + + @Override + public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception + { + boolean success = super.handlePost(form, errors); + if (success) + { + WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); + try + { + lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + lafProps.setSystemShortName(form.getSiteName()); + lafProps.save(); + + // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. + UsageReportingLevel.reportNow(); + + return true; + } + return false; + } + + @Override + public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) + { + if (!reshow) + { + File root = _svc.getSiteDefaultRoot(); + + if (root.exists()) + form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); + form.setSiteName(props.getShortName()); + form.setNotificationEmail(props.getSystemEmailAddress()); + } + + JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Set Defaults"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) + { + return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresSiteAdmin + public static class InstallCompleteAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Complete"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static List getInstallUpgradeWizardSteps() + { + List navTrail = new ArrayList<>(); + if (ModuleLoader.getInstance().isNewInstall()) + { + navTrail.add(new NavTree("Account Setup")); + navTrail.add(new NavTree("Install Modules")); + navTrail.add(new NavTree("Set Defaults")); + } + else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) + { + navTrail.add(new NavTree("Upgrade Modules")); + } + else + { + navTrail.add(new NavTree("Start Modules")); + } + navTrail.add(new NavTree("Complete")); + return navTrail; + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DbCheckerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Check Tools", this.getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DoCheckAction extends SimpleViewAction + { + @Override + public ModelAndView getView(DataCheckForm form, BindException errors) + { + try (var ignore=SpringActionController.ignoreSqlUpdates()) + { + ActionURL currentUrl = getViewContext().cloneActionURL(); + String fixRequested = currentUrl.getParameter("_fix"); + HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
    ")); + + if (null != fixRequested) + { + HtmlString sqlCheck = HtmlString.EMPTY_STRING; + if (fixRequested.equalsIgnoreCase("container")) + sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); + else if (fixRequested.equalsIgnoreCase("descriptor")) + sqlCheck = OntologyManager.doProjectColumnCheck(true); + contentBuilder.append(sqlCheck); + } + else + { + LOG.info("Starting database check"); // Debugging test timeout + LOG.info("Checking container column references"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Container Column References..."); + HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "container"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); + strTemp = OntologyManager.doProjectColumnCheck(false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "descriptor"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Schema consistency with tableXML.") + .unsafeAppend("

    "); + Set schemas = DbSchema.getAllSchemasToTest(); + + for (DbSchema schema : schemas) + { + SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); + List results = schemaResult.getResults(null); + if (results.isEmpty()) + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .append(": OK") + .unsafeAppend("
    "); + } + else + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .unsafeAppend(""); + for (var r : results) + { + HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); + contentBuilder.unsafeAppend("
  • ") + .append(item) + .unsafeAppend("
    • \n"); + } + contentBuilder.unsafeAppend(""); + } + } + + LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Consistency of Provisioned Storage...\n"); + StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); + contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); + for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) + { + for (String error : dr.getErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + } + for (String error : pr.getGlobalErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + + LOG.info("Database check complete"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Database Consistency checker complete"); + } + + contentBuilder.unsafeAppend("
    "); + + return new HtmlView(contentBuilder); + } + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Tools", this.getClass()); + } + } + + public static class DataCheckForm + { + private String _dbSchema = ""; + private boolean _full = false; + + public List modules = ModuleLoader.getInstance().getModules(); + public DataCheckForm(){} + + public List getModules() { return modules; } + public String getDbSchema() { return _dbSchema; } + @SuppressWarnings("unused") + public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } + public boolean getFull() { return _full; } + public void setFull(boolean full) { _full = full; } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class GetSchemaXmlDocAction extends ExportAction + { + @Override + public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception + { + String fullyQualifiedSchemaName = form.getDbSchema(); + if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) + { + throw new NotFoundException("Must specify dbSchema parameter"); + } + + boolean bFull = form.getFull(); + + Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); + TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); + StringWriter sw = new StringWriter(); + + XmlOptions xOpt = new XmlOptions(); + xOpt.setSavePrettyPrint(); + xOpt.setUseDefaultNamespace(); + + tdoc.save(sw, xOpt); + + sw.flush(); + PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderInformationAction extends FolderManagementViewAction + { + @Override + protected HtmlView getTabView() + { + Container c = getContainer(); + User currentUser = getUser(); + + User createdBy = UserManager.getUser(c.getCreatedBy()); + Map propValueMap = new LinkedHashMap<>(); + propValueMap.put("Path", c.getPath()); + propValueMap.put("Name", c.getName()); + propValueMap.put("Displayed Title", c.getTitle()); + propValueMap.put("EntityId", c.getId()); + propValueMap.put("RowId", c.getRowId()); + propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); + propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); + propValueMap.put("Folder Type", c.getFolderType().getName()); + propValueMap.put("Description", c.getDescription()); + + return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); + } + } + + public static class MissingValuesForm + { + private boolean _inheritMvIndicators; + private String[] _mvIndicators; + private String[] _mvLabels; + + public boolean isInheritMvIndicators() + { + return _inheritMvIndicators; + } + + public void setInheritMvIndicators(boolean inheritMvIndicators) + { + _inheritMvIndicators = inheritMvIndicators; + } + + public String[] getMvIndicators() + { + return _mvIndicators; + } + + public void setMvIndicators(String[] mvIndicators) + { + _mvIndicators = mvIndicators; + } + + public String[] getMvLabels() + { + return _mvLabels; + } + + public void setMvLabels(String[] mvLabels) + { + _mvLabels = mvLabels; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MissingValuesAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); + } + + @Override + public void validateCommand(MissingValuesForm form, Errors errors) + { + } + + @Override + public boolean handlePost(MissingValuesForm form, BindException errors) + { + if (form.isInheritMvIndicators()) + { + MvUtil.inheritMvIndicators(getContainer()); + return true; + } + else + { + // Javascript should have enforced any constraints + MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); + return true; + } + } + } + + @SuppressWarnings("unused") + public static class RConfigForm + { + private Integer _reportEngine; + private Integer _pipelineEngine; + private boolean _overrideDefault; + + public Integer getReportEngine() + { + return _reportEngine; + } + + public void setReportEngine(Integer reportEngine) + { + _reportEngine = reportEngine; + } + + public Integer getPipelineEngine() + { + return _pipelineEngine; + } + + public void setPipelineEngine(Integer pipelineEngine) + { + _pipelineEngine = pipelineEngine; + } + + public boolean getOverrideDefault() + { + return _overrideDefault; + } + + public void setOverrideDefault(String overrideDefault) + { + _overrideDefault = "override".equals(overrideDefault); + } + } + + @RequiresPermission(AdminPermission.class) + public static class RConfigurationAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); + } + + @Override + public void validateCommand(RConfigForm form, Errors errors) + { + if (form.getOverrideDefault()) + { + if (form.getReportEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); + if (form.getPipelineEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); + } + } + + @Override + public URLHelper getSuccessURL(RConfigForm rConfigForm) + { + return getContainer().getStartURL(getUser()); + } + + @Override + public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception + { + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + if (null != mgr) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + if (rConfigForm.getOverrideDefault()) + { + ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); + ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); + + if (reportEngine != null) + mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + else + { + // need to clear the current scope (if any) + ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); + ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); + + if (reportEngine != null) + mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + transaction.commit(); + } + return true; + } + return false; + } + } + + @SuppressWarnings("unused") + public static class ExportFolderForm + { + private String[] _types; + private int _location; + private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. + private String _exportType; + private boolean _includeSubfolders; + private PHI _exportPhiLevel; // Input: max level when viewing form + private boolean _shiftDates; + private boolean _alternateIds; + private boolean _maskClinic; + + public String[] getTypes() + { + return _types; + } + + public void setTypes(String[] types) + { + _types = types; + } + + public int getLocation() + { + return _location; + } + + public void setLocation(int location) + { + _location = location; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + + public ExportType getExportType() + { + if ("study".equals(_exportType)) + return ExportType.STUDY; + else + return ExportType.ALL; + } + + public void setExportType(String exportType) + { + _exportType = exportType; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public PHI getExportPhiLevel() + { + return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + public boolean isShiftDates() + { + return _shiftDates; + } + + public void setShiftDates(boolean shiftDates) + { + _shiftDates = shiftDates; + } + + public boolean isAlternateIds() + { + return _alternateIds; + } + + public void setAlternateIds(boolean alternateIds) + { + _alternateIds = alternateIds; + } + + public boolean isMaskClinic() + { + return _maskClinic; + } + + public void setMaskClinic(boolean maskClinic) + { + _maskClinic = maskClinic; + } + } + + public enum ExportOption + { + PipelineRootAsFiles("file root as multiple files") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + else if (root.isCloudRoot()) + { + errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); + } + else + { + File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); + try + { + writer.write(container, ctx, new FileSystemFile(exportDir)); + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + return null; + } + }, + + PipelineRootAsZip("file root as a single zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); + FileUtil.createDirectories(exportDir); + exportFolderToFile(exportDir, container, writer, ctx, errors); + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + }, + DownloadAsZip("browser download as a zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + try + { + // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 + // Same pattern as ExportListArchiveAction + Path tempDir = FileUtil.getTempDirectory().toPath(); + Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); + + // No exceptions, so stream the resulting zip file to the browser and delete it + try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) + { + Files.copy(tempZipFile, os); + } + finally + { + Files.delete(tempZipFile); + } + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return null; + } + }; + + private final String _description; + + ExportOption(String description) + { + _description = description; + } + + public String getDescription() + { + return _description; + } + + public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; + + Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception + { + String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); + + try (ZipFile zip = new ZipFile(exportDir, filename)) + { + writer.write(container, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return exportDir.resolve(filename); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ExportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception + { + // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so + // rendering a view would throw. + return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); + } + + @Override + protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) + { + form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); + + ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); + PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); + PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); + form.setExportPhiLevel(maxAllowedPhiForExport); + + return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ExportFolderForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ExportOption exportOption = null; + if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) + { + exportOption = ExportOption.values()[form.getLocation()]; + } + if (exportOption == null) + { + throw new NotFoundException("Invalid export location: " + form.getLocation()); + } + ContainerManager.checkContainerValidity(container); + + FolderWriterImpl writer = new FolderWriterImpl(); + FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), + form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), + form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); + + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); + AuditLogService.get().addEvent(getUser(), event); + + _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) + { + return _successURL; + } + } + + public static class ImportFolderForm + { + private boolean _createSharedDatasets; + private boolean _validateQueries; + private boolean _failForUndefinedVisits; + private String _sourceTemplateFolder; + private String _sourceTemplateFolderId; + private String _origin; + + public boolean isCreateSharedDatasets() + { + return _createSharedDatasets; + } + + public void setCreateSharedDatasets(boolean createSharedDatasets) + { + _createSharedDatasets = createSharedDatasets; + } + + public boolean isValidateQueries() + { + return _validateQueries; + } + + public boolean isFailForUndefinedVisits() + { + return _failForUndefinedVisits; + } + + public void setFailForUndefinedVisits(boolean failForUndefinedVisits) + { + _failForUndefinedVisits = failForUndefinedVisits; + } + + public void setValidateQueries(boolean validateQueries) + { + _validateQueries = validateQueries; + } + + public String getSourceTemplateFolder() + { + return _sourceTemplateFolder; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolder(String sourceTemplateFolder) + { + _sourceTemplateFolder = sourceTemplateFolder; + } + + public String getSourceTemplateFolderId() + { + return _sourceTemplateFolderId; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolderId(String sourceTemplateFolderId) + { + _sourceTemplateFolderId = sourceTemplateFolderId; + } + + public String getOrigin() + { + return _origin; + } + + public void setOrigin(String origin) + { + _origin = origin; + } + + public Container getSourceTemplateFolderContainer() + { + if (null == getSourceTemplateFolderId()) + return null; + return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ImportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL; + + @Override + protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) + { + // default the createSharedDatasets and validateQueries to true if this is not a form error reshow + if (!errors.hasErrors()) + { + form.setCreateSharedDatasets(true); + form.setValidateQueries(true); + } + + return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ImportFolderForm form, Errors errors) + { + // don't allow import into the root container + if (getContainer().isRoot()) + { + throw new NotFoundException(); + } + } + + @Override + public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception + { + ViewContext context = getViewContext(); + ActionURL url = context.getActionURL(); + User user = getUser(); + Container container = getContainer(); + PipeRoot pipelineRoot; + FileLike pipelineUnzipDir; // Should be local & writable + PipelineUrls pipelineUrlProvider; + + if (form.getOrigin() == null) + { + form.setOrigin("Folder"); + } + + // make sure we have a pipeline url provider to use for the success URL redirect + pipelineUrlProvider = urlProvider(PipelineUrls.class); + if (pipelineUrlProvider == null) + { + errors.reject("folderImport", "Pipeline url provider does not exist."); + return false; + } + + // make sure that the pipeline root is valid for this container + pipelineRoot = PipelineService.get().findPipelineRoot(container); + if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) + { + errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); + return false; + } + + // make sure we are able to delete any existing unzip dir in the pipeline root + try + { + pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); + } + catch (DirectoryNotDeletedException e) + { + errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); + return false; + } + + FolderImportConfig fiConfig; + if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) + { + fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); + } + else + { + fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); + if (fiConfig == null || errors.hasErrors()) + { + return false; + } + } + + // get the folder.xml file from the unzipped import archive + FileLike archiveXml = pipelineUnzipDir.resolveChild("folder.xml"); + if (!archiveXml.exists()) + { + errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); + return false; + } + + ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); + options.setSkipQueryValidation(!form.isValidateQueries()); + options.setCreateSharedDatasets(form.isCreateSharedDatasets()); + options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); + options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); + + // finally, create the study or folder import pipeline job + _successURL = pipelineUrlProvider.urlBegin(container); + PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); + + return !errors.hasErrors(); + } + + private @Nullable FolderImportConfig getFolderFromZipArchive(FileLike pipelineUnzipDir, BindException errors) + { + // user chose to import from a zip file + Map map = getFileMap(); + + // make sure we have a single file selected for import + if (map.size() != 1) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // make sure the file is not empty and that it has a .zip extension + MultipartFile zipFile = map.values().iterator().next(); + String originalFilename = zipFile.getOriginalFilename(); + if (0 == zipFile.getSize() || isBlank(originalFilename) || !originalFilename.toLowerCase().endsWith(".zip")) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // copy and unzip the uploaded import archive zip file to the pipeline unzip dir + try + { + FileLike pipelineUnzipFile = pipelineUnzipDir.resolveFile(org.labkey.api.util.Path.parse(originalFilename)); + // Check that the resolved file is under the pipelineUnzipDir + if (!pipelineUnzipFile.toNioPathForRead().normalize().startsWith(pipelineUnzipDir.toNioPathForRead().normalize())) + { + errors.reject("folderImport", "Invalid file path - must be within the unzip directory"); + return null; + } + + FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) + FileUtil.createNewFile(pipelineUnzipFile, true); + try (OutputStream os = pipelineUnzipFile.openOutputStream()) + { + FileUtil.copyData(zipFile.getInputStream(), os); + } + ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); + + return new FolderImportConfig( + false, + originalFilename, + pipelineUnzipFile, + pipelineUnzipFile + ); + } + catch (FileNotFoundException e) + { + LOG.debug("Failed to import '" + originalFilename + "'.", e); + errors.reject("folderImport", "File not found."); + return null; + } + catch (IOException e) + { + LOG.debug("Failed to import '" + originalFilename + "'.", e); + errors.reject("folderImport", "Unable to unzip folder archive."); + return null; + } + } + + private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final FileLike pipelineUnzipDir, final BindException errors) throws Exception + { + // user choose to import from a template source folder + Container sourceContainer = form.getSourceTemplateFolderContainer(); + + // In order to support the Advanced import options to import into multiple target folders we need to zip + // the source template folder so that the zip file can be passed to the pipeline processes. + FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, + getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, + PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + FolderWriterImpl writer = new FolderWriterImpl(); + String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); + FileLike implicitZipFile = pipelineUnzipDir.resolveChild(zipFileName); + if (!pipelineUnzipDir.isDirectory()) + pipelineUnzipDir.mkdirs(); + implicitZipFile.createFile(); + try (OutputStream out = implicitZipFile.openOutputStream(); + ZipFile zip = new ZipFile(out, false)) + { + writer.write(sourceContainer, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container + ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); + + return new FolderImportConfig( + StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), + implicitZipFile.getName(), + implicitZipFile, + null + ); + } + + private static class FolderImportConfig { + FileLike pipelineUnzipFile; + String originalFileName; + FileLike archiveFile; + boolean fromTemplateSourceFolder; + + public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, FileLike archiveFile, @Nullable FileLike pipelineUnzipFile) + { + this.originalFileName = originalFileName; + this.archiveFile = archiveFile; + this.fromTemplateSourceFolder = fromTemplateSourceFolder; + this.pipelineUnzipFile = pipelineUnzipFile; + } + } + + @Override + public URLHelper getSuccessURL(ImportFolderForm importFolderForm) + { + return _successURL; + } + } + + private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) + { + // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of + // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers + // to apply during the implicit export. + Set registeredFolderWriters = new HashSet<>(); + FolderSerializationRegistry registry = FolderSerializationRegistry.get(); + if (null == registry) + { + throw new RuntimeException(); + } + Collection registeredWriters = registry.getRegisteredFolderWriters(); + for (FolderWriter writer : registeredWriters) + { + String dataType = writer.getDataType(); + boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); + boolean excludeForTemplate = !writer.includeWithTemplate(); + + if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) + { + registeredFolderWriters.add(dataType); + + // for each Writer also determine if there are related children Writers, if so include them also + Collection> childWriters = writer.getChildren(true, true); + if (!childWriters.isEmpty()) + { + for (org.labkey.api.writer.Writer child : childWriters) + { + dataType = child.getDataType(); + if (dataType != null) + registeredFolderWriters.add(dataType); + } + } + } + } + return registeredFolderWriters; + } + + public static class FolderSettingsForm + { + private String _defaultDateFormat; + private boolean _defaultDateFormatInherited; + private String _defaultDateTimeFormat; + private boolean _defaultDateTimeFormatInherited; + private String _defaultTimeFormat; + private boolean _defaultTimeFormatInherited; + private String _defaultNumberFormat; + private boolean _defaultNumberFormatInherited; + private boolean _restrictedColumnsEnabled; + private boolean _restrictedColumnsEnabledInherited; + + public String getDefaultDateFormat() + { + return _defaultDateFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormat(String defaultDateFormat) + { + _defaultDateFormat = defaultDateFormat; + } + + public boolean isDefaultDateFormatInherited() + { + return _defaultDateFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) + { + _defaultDateFormatInherited = defaultDateFormatInherited; + } + + public String getDefaultDateTimeFormat() + { + return _defaultDateTimeFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormat(String defaultDateTimeFormat) + { + _defaultDateTimeFormat = defaultDateTimeFormat; + } + + public boolean isDefaultDateTimeFormatInherited() + { + return _defaultDateTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) + { + _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; + } + + public String getDefaultTimeFormat() + { + return _defaultTimeFormat; + } + + @SuppressWarnings("UnusedDeclaration") + public void setDefaultTimeFormat(String defaultTimeFormat) + { + _defaultTimeFormat = defaultTimeFormat; + } + + public boolean isDefaultTimeFormatInherited() + { + return _defaultTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) + { + _defaultTimeFormatInherited = defaultTimeFormatInherited; + } + + public String getDefaultNumberFormat() + { + return _defaultNumberFormat; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormat(String defaultNumberFormat) + { + _defaultNumberFormat = defaultNumberFormat; + } + + public boolean isDefaultNumberFormatInherited() + { + return _defaultNumberFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) + { + _defaultNumberFormatInherited = defaultNumberFormatInherited; + } + + public boolean areRestrictedColumnsEnabled() + { + return _restrictedColumnsEnabled; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) + { + _restrictedColumnsEnabled = restrictedColumnsEnabled; + } + + public boolean isRestrictedColumnsEnabledInherited() + { + return _restrictedColumnsEnabledInherited; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) + { + _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderSettingsAction extends FolderManagementViewPostAction + { + @Override + protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(FolderSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(FolderSettingsForm form, BindException errors) + { + return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + } + } + + // Validate and populate the folder settings; save & log all changes + private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) + { + validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); + validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); + validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); + validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); + + setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); + + if (!errors.hasErrors()) + { + props.save(); + + //write an audit log event + props.writeAuditLogEvent(c, user); + } + + return !errors.hasErrors(); + } + + private interface FormatSaver + { + void save(String format) throws IllegalArgumentException; + } + + private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) + { + String defaultFormat = StringUtils.trimToNull(format); + if (inherited) + { + clearer.run(); + } + else + { + try + { + saver.save(defaultFormat); + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class ModulePropertiesAction extends FolderManagementViewAction + { + @Override + protected JspView getTabView() + { + return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); + } + } + + @SuppressWarnings("unused") + public static class FolderTypeForm + { + private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; + private String _defaultModule; + private String _folderType; + private boolean _wizard; + + public String[] getActiveModules() + { + return _activeModules; + } + + public void setActiveModules(String[] activeModules) + { + _activeModules = activeModules; + } + + public String getDefaultModule() + { + return _defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + _defaultModule = defaultModule; + } + + public String getFolderType() + { + return _folderType; + } + + public void setFolderType(String folderType) + { + _folderType = folderType; + } + + public boolean isWizard() + { + return _wizard; + } + + public void setWizard(boolean wizard) + { + _wizard = wizard; + } + } + + @RequiresPermission(AdminPermission.class) + @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments + public static class FolderTypeAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); + } + + @Override + public void validateCommand(FolderTypeForm form, Errors errors) + { + boolean fEmpty = true; + for (String module : form._activeModules) + { + if (module != null) + { + fEmpty = false; + break; + } + } + if (fEmpty && "None".equals(form.getFolderType())) + { + errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); + } + } + + @Override + public boolean handlePost(FolderTypeForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + String[] modules = form.getActiveModules(); + + if (modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) + { + container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + container.setDefaultModule(defaultModule); + } + else + { + FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); + if (container.isContainerTab() && folderType.hasContainerTabs()) + errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); + else + container.setFolderType(folderType, getUser(), errors, activeModules); + } + if (errors.hasErrors()) + return false; + + if (form.isWizard()) + { + _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + } + else + _successURL = container.getFolderType().getStartURL(container, getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) + { + return _successURL; + } + } + + @SuppressWarnings("unused") + public static class FileRootsForm extends SetupForm implements FileManagementForm + { + private String _folderRootPath; + private String _fileRootOption; + private String _cloudRootName; + private boolean _isFolderSetup; + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _migrateFilesOption; + + // cloud settings + private String[] _enabledCloudStore; + //file management + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public boolean isFolderSetup() + { + return _isFolderSetup; + } + + public void setFolderSetup(boolean folderSetup) + { + _isFolderSetup = folderSetup; + } + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsStandAloneAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + view.setFrame(WebPartView.FrameType.NONE); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); + getPageConfig().setTemplate(PageConfig.Template.Wizard); + getPageConfig().setTitle("Change File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) + .addParameter("folderSetup", true) + .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + /** + * This standalone file root management action can be used on folder types that do not support + * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL + * navigation when a workbook needs it. + */ + @RequiresPermission(AdminPermission.class) + public class ManageFileRootAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + getPageConfig().setTitle("Manage File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = getContainer().getStartURL(getUser()); + + if (getViewContext().getActionURL().getReturnUrl() != null) + { + url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + } + + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) + { + return getFileRootsView(form, errors, getReshow()); + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + } + + private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) + { + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + view.setFrame(WebPartView.FrameType.DIV); + try + { + if (!reshow) + setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return view; + } + + private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception + { + if (form.isPipelineRootForm()) + { + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + } + else + { + setFileRootFromForm(getViewContext(), form, errors); + setEnabledCloudStores(getViewContext(), form, errors); + return !errors.hasErrors(); + } + } + + public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) + { + FileContentService service = FileContentService.get(); + if (null != service) + { + boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); + String cloudRootName = form.getCloudRootName(); + if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) + { + Path defaultRootPath = service.getDefaultRootPath(container, false); + cloudRootName = service.getDefaultRootInfo(container).getCloudName(); + isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); + } + + if (isOrDefaultsToCloudRoot && null != cloudRootName) + { + if (null != form.getEnabledCloudStore()) + { + for (String storeName : form.getEnabledCloudStore()) + { + if (Strings.CI.equals(cloudRootName, storeName)) + return; + } + } + // Didn't find cloud root in enabled list + errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); + } + } + } + + public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) + { + boolean changed = false; + boolean shouldCopyMove = false; + FileContentService service = FileContentService.get(); + if (null != service) + { + // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. + // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. + MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? + MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : + MigrateFilesOption.leave; + List> sourceInfos = + ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? + Collections.emptyList() : + getCopySourceInfo(service, ctx.getContainer()); + + if (form.isDisableFileSharing()) + { + if (!service.isFileRootDisabled(ctx.getContainer())) + { + service.disableFileRoot(ctx.getContainer()); + changed = true; + } + } + else if (form.hasSiteDefaultRoot()) + { + if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), true); + changed = true; + shouldCopyMove = true; + } + } + else if (form.isCloudFileRoot()) + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String cloudRootName = form.getCloudRootName(); + if (null != cloudRootName && + (!service.isCloudRoot(ctx.getContainer()) || + !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setCloudRoot(ctx.getContainer(), cloudRootName); + try + { + PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); + if (form.isFolderSetup() && !sourceInfos.isEmpty()) + { + // File root was set to cloud storage, remove folder created + Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded + if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) + { + try + { + Files.deleteIfExists(fromPath.getParent()); + } + catch (IOException e) + { + LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); + } + } + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + changed = true; + shouldCopyMove = true; + } + } + else + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + URI uri = FileUtil.createUri(root, false); // root is unencoded + Path path = FileUtil.getPath(ctx.getContainer(), uri); + if (null == path || !Files.exists(path)) + { + errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); + } + else + { + Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setFileRootPath(ctx.getContainer(), root); + changed = true; + shouldCopyMove = true; + } + } + } + else + { + service.setFileRootPath(ctx.getContainer(), null); + changed = true; + } + } + + if (!errors.hasErrors()) + { + if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) + { + // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); + if (null != pipeRoot) + { + try + { + initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); + } + catch (PipelineValidationException e) + { + throw new RuntimeValidationException(e); + } + } + else + { + LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); + } + } + + form.setFileRootChanged(changed); + if (changed && null != ctx.getUser()) + { + setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); + String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); + AuditLogService.get().addEvent(ctx.getUser(), event); + } + } + } + } + + private static List> getCopySourceInfo(FileContentService service, Container container) + { + + List> sourceInfo = new ArrayList<>(); + addCopySourceInfo(service, container, sourceInfo, true); + return sourceInfo; + } + + private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) + { + if (isRoot || service.isUseDefaultRoot(container)) + { + Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); + if (null != sourceFileRootDir) + { + String pathStr = FileUtil.pathToString(sourceFileRootDir); + if (null != pathStr) + sourceInfo.add(new Pair<>(container, pathStr)); + else + throw new RuntimeValidationException("Unexpected error converting path to string"); + } + } + for (Container childContainer : container.getChildren()) + addCopySourceInfo(service, childContainer, sourceInfo, false); + } + + private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, + MigrateFilesOption migrateFilesOption) throws PipelineValidationException + { + CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); + PipelineService.get().queueJob(job); + } + + private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) + { + // test permissions. only site admins are able to turn on a custom file root for a folder + // this is only relevant if the folder is either being switched to a custom file root, + // or if the file root is changed. + if (!service.isUseDefaultRoot(ctx.getContainer())) + { + Path fileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null != fileRootPath) + { + String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); + if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) + { + if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) + throw new UnauthorizedException("Only site admins can change file roots"); + } + } + } + } + + public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) + { + String[] enabledCloudStores = form.getEnabledCloudStore(); + CloudStoreService cloud = CloudStoreService.get(); + if (cloud != null) + { + Set enabled = Collections.emptySet(); + if (enabledCloudStores != null) + enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); + + try + { + // Check if anything changed + boolean changed = false; + Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); + if (enabled.size() != storeNames.size()) + changed = true; + else + if (!enabled.containsAll(storeNames)) + changed = true; + if (changed) + cloud.setEnabledCloudStores(ctx.getContainer(), enabled); + form.setEnabledCloudStoresChanged(changed); + } + catch (UncheckedExecutionException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException + // is what BlobStore hands us if bucket (S3 container) does not exist + if (null != e.getCause()) + errors.reject(ERROR_MSG, e.getCause().getMessage()); + else + throw e; + } + catch (RuntimeException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + errors.reject(ERROR_MSG, e.getMessage()); + } + } + } + + + public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException + { + String rootSetParam = ctx.getActionURL().getParameter("rootSet"); + boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); + String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); + boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); + setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); + } + + public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, + String migrateFilesOption) throws IllegalArgumentException + { + FileContentService service = FileContentService.get(); + String confirmMessage = null; + + String migrateFilesMessage = ""; + if (fileRootChanged && !form.isFolderSetup()) + { + if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) + migrateFilesMessage = ". Existing files not copied or moved."; + else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files copied."; + form.setMigrateFilesOption(migrateFilesOption); + } + else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files moved."; + form.setMigrateFilesOption(migrateFilesOption); + } + } + + if (service != null) + { + if (service.isFileRootDisabled(container)) + { + form.setFileRootOption(FileRootProp.disable.name()); + if (fileRootChanged) + confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); + } + else if (service.isUseDefaultRoot(container)) + { + form.setFileRootOption(FileRootProp.siteDefault.name()); + Path root = service.getFileRootPath(container); + if (root != null && Files.exists(root) && fileRootChanged) + confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; + } + else if (!service.isCloudRoot(container)) + { + Path root = service.getFileRootPath(container); + + form.setFileRootOption(FileRootProp.folderOverride.name()); + if (root != null) + { + String absolutePath = FileUtil.getAbsolutePath(container, root); + form.setFolderRootPath(absolutePath); + if (Files.exists(root)) + { + if (fileRootChanged) + confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; + } + } + } + else + { + form.setFileRootOption(FileRootProp.cloudRoot.name()); + form.setCloudRootName(service.getCloudRootName(container)); + Path root = service.getFileRootPath(container); + if (root != null && fileRootChanged) + { + confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; + } + } + } + + if (fileRootChanged && confirmMessage != null) + form.setConfirmMessage(confirmMessage); + else if (enabledCloudChanged) + form.setConfirmMessage("The enabled cloud stores changed."); + } + + @RequiresPermission(AdminPermission.class) + public static class ManageFoldersAction extends FolderManagementViewAction + { + @Override + protected HttpView getTabView() + { + return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); + } + } + + public static class NotificationsForm + { + private String _provider; + + public String getProvider() + { + return _provider; + } + + public void setProvider(String provider) + { + _provider = provider; + } + } + + private static final String DATA_REGION_NAME = "Users"; + + @RequiresPermission(AdminPermission.class) + public static class NotificationsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) + { + final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); + DataRegionSelection.clearAll(getViewContext(), key); + + QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); + settings.setAllowChooseView(true); + settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); + + UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); + QueryView queryView = new QueryView(schema, settings, errors) + { + @Override + public List getDisplayColumns() + { + List columns = new ArrayList<>(); + SecurityPolicy policy = getContainer().getPolicy(); + Set assignmentSet = new HashSet<>(); + + for (RoleAssignment assignment : policy.getAssignments()) + { + Group g = SecurityManager.getGroup(assignment.getUserId()); + if (g != null) + assignmentSet.add(g.getName()); + } + + for (DisplayColumn col : super.getDisplayColumns()) + { + if (col.getName().equalsIgnoreCase("Groups")) + columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); + else + columns.add(col); + } + return columns; + } + + @Override + protected void populateButtonBar(DataView dataView, ButtonBar bar) + { + try + { + // add the provider configuration menu items to the admin panel button + MenuButton adminButton = new MenuButton("Update user settings"); + adminButton.setRequiresSelection(true); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); + + bar.add(adminButton); + super.populateButtonBar(dataView, bar); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }; + queryView.setShadeAlternatingRows(true); + queryView.setShowBorders(true); + queryView.setShowDetailsColumn(false); + queryView.setShowRecordSelectors(true); + queryView.setFrame(WebPartView.FrameType.NONE); + queryView.disableContainerFilterSelection(); + queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + + VBox defaultsView = new VBox( + HtmlView.unsafe( + "
    Default settings
    " + + "You can change this folder's default settings for email notifications here.") + ); + + PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + { + defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); + } + + return new VBox( + new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), + defaultsView, + new VBox( + HtmlView.unsafe( + "
    User settings
    " + + "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + + "notification setting is visible in the appropriately named column.

    " + + "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), + queryView + ) + ); + } + + @Override + public void validateCommand(NotificationsForm form, Errors errors) + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + provider.validateCommand(getViewContext(), errors); + } + + @Override + public boolean handlePost(NotificationsForm form, BindException errors) throws Exception + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + { + return provider.handlePost(getViewContext(), errors); + } + errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); + return false; + } + } + + public static class NotifyOptionsForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public ConfigTypeProvider getProvider() + { + return MessageConfigService.get().getConfigType(getType()); + } + } + + /** + * Action to populate an Ext store with email notification options for admin settings + */ + @RequiresPermission(AdminPermission.class) + public static class GetEmailOptionsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(NotifyOptionsForm form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + + ConfigTypeProvider provider = form.getProvider(); + if (provider != null) + { + List options = new ArrayList<>(); + + // if the list of options is not for the folder default, add an option to use the folder default + if (getViewContext().get("isDefault") == null) + options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); + + for (NotificationOption option : provider.getOptions()) + { + options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); + } + resp.put("success", true); + if (!options.isEmpty()) + resp.put("options", options); + } + else + resp.put("success", false); + + return resp; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetBulkEmailOptionsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(EmailConfigFormImpl form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + ConfigTypeProvider provider = form.getProvider(); + String srcIdentifier = getContainer().getId(); + + Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); + + if (!selections.isEmpty() && provider != null) + { + int newOption = form.getIndividualEmailOption(); + + for (String user : selections) + { + User projectUser = UserManager.getUser(Integer.parseInt(user)); + UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); + + int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; + + //has this projectUser's option changed? if so, update + //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default + if (currentEmailOption != newOption) + { + provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); + } + } + resp.put("success", true); + } + else + { + resp.put("success", false); + resp.put("message", "There were no users selected"); + } + return resp; + } + } + + /** Renders only the groups that are assigned roles in this container */ + private static class FolderGroupColumn extends DataColumn + { + private final Set _assignmentSet; + + public FolderGroupColumn(Set assignmentSet, ColumnInfo col) + { + super(col); + _assignmentSet = assignmentSet; + } + + @Override + public void renderGridCellContents(RenderContext ctx, HtmlWriter out) + { + String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); + + if (value != null) + { + out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) + .filter(_assignmentSet::contains) + .map(HtmlString::of) + .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); + } + } + } + + private static class PanelConfig implements MessageConfigService.PanelInfo + { + private final ActionURL _returnUrl; + private final String _dataRegionSelectionKey; + + public PanelConfig(ActionURL returnUrl, String selectionKey) + { + _returnUrl = returnUrl; + _dataRegionSelectionKey = selectionKey; + } + + @Override + public ActionURL getReturnUrl() + { + return _returnUrl; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + } + + public static class ConceptsForm + { + private String _conceptURI; + private String _containerId; + private String _schemaName; + private String _queryName; + + public String getConceptURI() + { + return _conceptURI; + } + + public void setConceptURI(String conceptURI) + { + _conceptURI = conceptURI; + } + + public String getContainerId() + { + return _containerId; + } + + public void setContainerId(String containerId) + { + _containerId = containerId; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConceptsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); + } + + @Override + public void validateCommand(ConceptsForm form, Errors errors) + { + // validate that the required input fields are provided + String missingRequired = "", sep = ""; + if (form.getConceptURI() == null) + { + missingRequired += "conceptURI"; + sep = ", "; + } + if (form.getSchemaName() == null) + { + missingRequired += sep + "schemaName"; + sep = ", "; + } + if (form.getQueryName() == null) + missingRequired += sep + "queryName"; + if (!missingRequired.isEmpty()) + errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); + + // validate that, if provided, the containerId matches an existing container + Container postContainer = null; + if (form.getContainerId() != null) + { + postContainer = ContainerManager.getForId(form.getContainerId()); + if (postContainer == null) + errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); + } + + // validate that the schema and query names provided exist + if (form.getSchemaName() != null && form.getQueryName() != null) + { + Container c = postContainer != null ? postContainer : getContainer(); + UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); + if (schema == null) + errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); + else if (schema.getTable(form.getQueryName()) == null) + errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); + } + } + + @Override + public boolean handlePost(ConceptsForm form, BindException errors) + { + Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); + ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); + + return true; + } + } + + @RequiresPermission(AdminPermission.class) + public class FolderAliasesAction extends FormViewAction + { + @Override + public void validateCommand(FolderAliasesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/folderAliases.jsp"); + } + + @Override + public boolean handlePost(FolderAliasesForm form, BindException errors) + { + List aliases = new ArrayList<>(); + if (form.getAliases() != null) + { + StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); + while (st.hasMoreTokens()) + { + String alias = st.nextToken().trim(); + if (!alias.startsWith("/")) + { + alias = "/" + alias; + } + while (alias.endsWith("/")) + { + alias = alias.substring(0, alias.lastIndexOf('/')); + } + aliases.add(alias); + } + } + ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); + + return true; + } + + @Override + public ActionURL getSuccessURL(FolderAliasesForm form) + { + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); + } + } + + public static class FolderAliasesForm + { + private String _aliases; + + public String getAliases() + { + return _aliases; + } + + @SuppressWarnings("unused") + public void setAliases(String aliases) + { + _aliases = aliases; + } + } + + @RequiresPermission(AdminPermission.class) + public class CustomizeEmailAction extends FormViewAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) + { + JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); + result.setTitle("Email Template"); + return result; + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + + template.setSubject(form.getEmailSubject()); + template.setSenderName(form.getEmailSender()); + template.setReplyToEmail(form.getEmailReplyTo()); + template.setBody(form.getEmailMessage()); + + String[] errorStrings = new String[1]; + if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. + EmailTemplateService.get().saveEmailTemplate(template, getContainer()); + else + errors.reject(ERROR_MSG, errorStrings[0]); + } + + return !errors.hasErrors(); + } + + @Override + public ActionURL getSuccessURL(CustomEmailForm form) + { + ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); + result.replaceParameter("templateClass", form.getTemplateClass()); + if (form.getReturnActionURL() != null) + { + result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + } + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("customEmail"); + addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class DeleteCustomEmailAction extends FormHandlerAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + template.setSubject(form.getEmailSubject()); + template.setBody(form.getEmailMessage()); + + EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(CustomEmailForm form) + { + return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); + } + } + + @SuppressWarnings("unused") + public static class CustomEmailForm extends ReturnUrlForm + { + private String _templateClass; + private String _emailSubject; + private String _emailSender; + private String _emailReplyTo; + private String _emailMessage; + private String _templateDescription; + + public void setTemplateClass(String name){_templateClass = name;} + public String getTemplateClass(){return _templateClass;} + public void setEmailSubject(String subject){_emailSubject = subject;} + public String getEmailSubject(){return _emailSubject;} + public void setEmailSender(String sender){_emailSender = sender;} + public String getEmailSender(){return _emailSender;} + public void setEmailMessage(String body){_emailMessage = body;} + public String getEmailMessage(){return _emailMessage;} + public String getEmailReplyTo(){return _emailReplyTo;} + public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} + + public String getTemplateDescription() + { + return _templateDescription; + } + + public void setTemplateDescription(String templateDescription) + { + _templateDescription = templateDescription; + } + } + + private ActionURL getManageFoldersURL() + { + return new AdminUrlsImpl().getManageFoldersURL(getContainer()); + } + + public static class ManageFoldersForm extends ReturnUrlForm + { + private String name; + private String title; + private boolean titleSameAsName; + private String folder; + private String target; + private String folderType; + private String defaultModule; + private String[] activeModules; + private boolean hasLoaded = false; + private boolean showAll; + private boolean confirmed = false; + private boolean addAlias = false; + private String templateSourceId; + private String[] templateWriterTypes; + private boolean templateIncludeSubfolders = false; + private String[] targets; + private PHI _exportPhiLevel = PHI.NotPHI; + + public boolean getHasLoaded() + { + return hasLoaded; + } + + public void setHasLoaded(boolean hasLoaded) + { + this.hasLoaded = hasLoaded; + } + + public String[] getActiveModules() + { + return activeModules; + } + + public void setActiveModules(String[] activeModules) + { + this.activeModules = activeModules; + } + + public String getDefaultModule() + { + return defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + this.defaultModule = defaultModule; + } + + public boolean isShowAll() + { + return showAll; + } + + public void setShowAll(boolean showAll) + { + this.showAll = showAll; + } + + public String getFolder() + { + return folder; + } + + public void setFolder(String folder) + { + this.folder = folder; + } + + public String getName() + { + return name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isTitleSameAsName() + { + return titleSameAsName; + } + + public void setTitleSameAsName(boolean updateTitle) + { + this.titleSameAsName = updateTitle; + } + public void setName(String name) + { + this.name = name; + } + + public boolean isConfirmed() + { + return confirmed; + } + + public void setConfirmed(boolean confirmed) + { + this.confirmed = confirmed; + } + + public String getFolderType() + { + return folderType; + } + + public void setFolderType(String folderType) + { + this.folderType = folderType; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + + public String getTarget() + { + return target; + } + + public void setTarget(String target) + { + this.target = target; + } + + public void setTemplateSourceId(String templateSourceId) + { + this.templateSourceId = templateSourceId; + } + + public String getTemplateSourceId() + { + return templateSourceId; + } + + public Container getTemplateSourceContainer() + { + if (null == getTemplateSourceId()) + return null; + return ContainerManager.getForId(getTemplateSourceId()); + } + + public String[] getTemplateWriterTypes() + { + return templateWriterTypes; + } + + public void setTemplateWriterTypes(String[] templateWriterTypes) + { + this.templateWriterTypes = templateWriterTypes; + } + + public boolean getTemplateIncludeSubfolders() + { + return templateIncludeSubfolders; + } + + public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) + { + this.templateIncludeSubfolders = templateIncludeSubfolders; + } + + public String[] getTargets() + { + return targets; + } + + public void setTargets(String[] targets) + { + this.targets = targets; + } + + public PHI getExportPhiLevel() + { + return _exportPhiLevel; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + /** + * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, + * but it will work for non-workbook children as well. + */ + public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException + { + if (getTargets() != null) + { + final List targets = new ArrayList<>(); + final List directChildren = ContainerManager.getChildren(currentContainer); + + Arrays.stream(getTargets()).forEach(x -> { + Container c = ContainerManager.getForId(x); + if (c == null) + { + try + { + Integer rowId = ConvertHelper.convert(x, Integer.class); + if (rowId > 0) + c = ContainerManager.getForRowId(rowId); + } + catch (ConversionException e) + { + //ignore + } + } + + if (c != null) + { + if (!c.equals(currentContainer)) + { + if (!directChildren.contains(c)) + { + throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); + } + + if (c.getContainerType().canHaveChildren()) + { + throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); + } + } + + targets.add(c); + } + else + { + throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); + } + }); + + return targets; + } + else + { + return Collections.singletonList(currentContainer); + } + } + } + + public static class RenameContainerForm + { + private String name; + private String title; + private boolean addAlias = true; + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + } + + // Note that validation checks occur in ContainerManager.rename() + @RequiresPermission(AdminPermission.class) + public static class RenameContainerAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RenameContainerForm form, BindException errors) + { + Container container = getContainer(); + String name = StringUtils.trimToNull(form.getName()); + String title = StringUtils.trimToNull(form.getTitle()); + + String nameValue = name; + String titleValue = title; + if (name == null && title == null) + { + errors.reject(ERROR_MSG, "Please specify a name or a title."); + return new ApiSimpleResponse("success", false); + } + else if (name != null && title == null) + { + titleValue = name; + } + else if (name == null) + { + nameValue = container.getName(); + } + + boolean addAlias = form.isAddAlias(); + + try + { + Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); + return new ApiSimpleResponse(c.toJSON(getUser())); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + return new ApiSimpleResponse("success", false); + } + } + } + + @RequiresPermission(AdminPermission.class) + public class RenameFolderAction extends FormViewAction + { + private ActionURL _returnUrl; + + @Override + public void validateCommand(ManageFoldersForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + try + { + String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); + Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); + _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); + return true; + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + } + + return false; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + return _returnUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + String containerType = getContainer().isProject() ? "Project" : "Folder"; + addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); + } + } + + public static class MoveFolderTreeView extends JspView + { + private MoveFolderTreeView(ManageFoldersForm form, BindException errors) + { + super("/org/labkey/core/admin/moveFolder.jsp", form, errors); + } + } + + @RequiresPermission(AdminPermission.class) + @ActionNames("ShowMoveFolderTree,MoveFolder") + public class MoveFolderAction extends FormViewAction + { + boolean showConfirmPage = false; + boolean moveFailed = false; + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + Container c = getContainer(); + + if (c.isRoot()) + throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root + + if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); + + Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); + if (null == newParent) + { + errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); + } + else if (!newParent.hasPermission(getUser(), AdminPermission.class)) + { + throw new UnauthorizedException(); + } + else if (newParent.hasChild(c.getName())) + { + errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception + { + if (showConfirmPage) + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + if (moveFailed) + return new SimpleErrorView(errors); + else + return new MoveFolderTreeView(form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception + { + Container c = getContainer(); + Container newParent = ContainerManager.getForPath(form.getTarget()); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) + { + showConfirmPage = true; + return false; // reshow + } + + try + { + ContainerManager.move(c, newParent, getUser()); + } + catch (ValidationException e) + { + moveFailed = true; + getPageConfig().setTemplate(Template.Dialog); + for (ValidationError validationError : e.getErrors()) + { + errors.addError(new LabKeyError(validationError.getMessage())); + } + if (!errors.hasErrors()) + errors.addError(new LabKeyError("Move failed")); + return false; + } + + if (form.isAddAlias()) + { + List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); + newAliases.add(c.getPath()); + ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) + { + Container c = getContainer(); + c = ContainerManager.getForId(c.getId()); // Reload container to populate new location + return new AdminUrlsImpl().getManageFoldersURL(c); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Folder Management", getManageFoldersURL()); + root.addChild("Move Folder"); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConfirmProjectMoveAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ManageFoldersForm form, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm Project Move"); + } + } + + private static abstract class AbstractCreateFolderAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FORM target, Errors errors) + { + } + + @Override + public ModelAndView getView(FORM form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + if (!reshow) + { + FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); + if (null != folderType) + { + // If a default folder type has been configured by a site admin set that as the default folder type choice + form.setFolderType(folderType.getName()); + } + form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); + } + JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); + vbox.addView(statusView); + + Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder + + setHelpTopic("createProject"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); + getPageConfig().setTemplate(Template.Wizard); + + if (c.isRoot()) + getPageConfig().setTitle("Create Project"); + else + { + String title = "Create Folder"; + + title += " in /"; + if (c == ContainerManager.getHomeContainer()) + title += "Home"; + else + title += c.getName(); + + getPageConfig().setTitle(title); + } + + return vbox; + } + + @Override + public boolean handlePost(FORM form, BindException errors) throws Exception + { + Container parent = getViewContext().getContainerNoTab(); + String folderName = StringUtils.trimToNull(form.getName()); + String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); + StringBuilder error = new StringBuilder(); + Consumer afterCreateHandler = getAfterCreateHandler(form); + + Container container; + + if (Container.isLegalName(folderName, parent.isRoot(), error)) + { + if (parent.hasChild(folderName)) + { + if (parent.isRoot()) + { + error.append("The server already has a project with this name."); + } + else + { + error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); + } + } + else + { + String folderType = form.getFolderType(); + + if (null == folderType) + { + errors.reject(null, "Folder type must be specified"); + return false; + } + + if ("Template".equals(folderType)) // Create folder from selected template + { + Container sourceContainer = form.getTemplateSourceContainer(); + if (null == sourceContainer) + { + errors.reject(null, "Source template folder not selected"); + return false; + } + else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) + { + errors.reject(null, "User does not have administrator permissions to the source container"); + return false; + } + else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) + { + errors.reject(null, "The source folder has a restricted module for which you do not have permission."); + return false; + } + + FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", + form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, + new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + + container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); + } + else + { + FolderType type = FolderTypeManager.get().getFolderType(folderType); + + if (type == null) + { + errors.reject(null, "Folder type not recognized"); + return false; + } + + String[] modules = form.getActiveModules(); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + if (null == modules || modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + } + + // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. + Consumer configureContainer = (newContainer) -> + { + afterCreateHandler.accept(newContainer); + newContainer.setFolderType(type, getUser(), errors); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + newContainer.setDefaultModule(defaultModule); + } + }; + container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); + } + + _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + } + + errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); + return false; + } + + /** + * Return a Consumer that provides post-creation handling on the new Container + */ + abstract public Consumer getAfterCreateHandler(FORM form); + + @Override + protected String getCommandClassMethodName() + { + return "getAfterCreateHandler"; + } + + @Override + public ActionURL getSuccessURL(FORM form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public static class CreateFolderAction extends AbstractCreateFolderAction + { + @Override + public Consumer getAfterCreateHandler(ManageFoldersForm form) + { + // No special handling + return container -> {}; + } + } + + public static class CreateProjectForm extends ManageFoldersForm + { + private boolean _assignProjectAdmin = false; + + public boolean isAssignProjectAdmin() + { + return _assignProjectAdmin; + } + + @SuppressWarnings("unused") + public void setAssignProjectAdmin(boolean assignProjectAdmin) + { + _assignProjectAdmin = assignProjectAdmin; + } + } + + @RequiresPermission(CreateProjectPermission.class) + public static class CreateProjectAction extends AbstractCreateFolderAction + { + @Override + public void validateCommand(CreateProjectForm target, Errors errors) + { + super.validateCommand(target, errors); + if (!getContainer().isRoot()) + errors.reject(ERROR_MSG, "Must be invoked from the root"); + } + + @Override + public Consumer getAfterCreateHandler(CreateProjectForm form) + { + if (form.isAssignProjectAdmin()) + { + return c -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); + policy.addRoleAssignment(getUser(), ProjectAdminRole.class); + User savePolicyUser = getUser(); + if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + SecurityPolicyManager.savePolicy(policy, savePolicyUser); + }; + } + else + { + return c -> {}; + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetFolderPermissionsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(SetFolderPermissionsForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); + vbox.addView(statusView); + + Container c = getContainer(); + getPageConfig().setTitle("Users / Permissions"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + setHelpTopic("createProject"); + + return vbox; + } + + @Override + public boolean handlePost(SetFolderPermissionsForm form, BindException errors) + { + Container c = getContainer(); + String permissionType = form.getPermissionType(); + + if(c.isProject()){ + _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); + } + else + { + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + if (form.isAdvanced()) + { + _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); + } + else + { + _successURL = getContainer().getStartURL(getUser()); + } + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + } + + if(permissionType == null){ + errors.reject(ERROR_MSG, "You must select one of the options for permissions."); + return false; + } + + switch (permissionType) + { + case "CurrentUser" -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); + policy.addRoleAssignment(getUser(), role); + SecurityPolicyManager.savePolicy(policy, getUser()); + } + case "Inherit" -> SecurityManager.setInheritPermissions(c); + case "CopyExistingProject" -> { + String targetProject = form.getTargetProject(); + if (targetProject == null) + { + errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); + return false; + } + Container source = ContainerManager.getForId(targetProject); + if (source == null) + { + source = ContainerManager.getForPath(targetProject); + } + if (source == null) + { + throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); + } + Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); + + //copy role assignments + SecurityPolicy op = SecurityPolicyManager.getPolicy(source); + MutableSecurityPolicy np = new MutableSecurityPolicy(c); + for (RoleAssignment assignment : op.getAssignments()) + { + int userId = assignment.getUserId(); + UserPrincipal p = SecurityManager.getPrincipal(userId); + Role r = assignment.getRole(); + + if (p instanceof Group g) + { + if (!g.isProjectGroup()) + { + np.addRoleAssignment(p, r, false); + } + else + { + np.addRoleAssignment(groupMap.get(p), r, false); + } + } + else + { + np.addRoleAssignment(p, r, false); + } + } + SecurityPolicyManager.savePolicy(np, getUser()); + } + default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); + } + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + + @Override + public ActionURL getSuccessURL(SetFolderPermissionsForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + } + } + + public static class SetFolderPermissionsForm + { + private String targetProject; + private String permissionType; + private boolean advanced; + + public String getPermissionType() + { + return permissionType; + } + + @SuppressWarnings("unused") + public void setPermissionType(String permissionType) + { + this.permissionType = permissionType; + } + + public String getTargetProject() + { + return targetProject; + } + + @SuppressWarnings("unused") + public void setTargetProject(String targetProject) + { + this.targetProject = targetProject; + } + + public boolean isAdvanced() + { + return advanced; + } + + @SuppressWarnings("unused") + public void setAdvanced(boolean advanced) + { + this.advanced = advanced; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetInitialFolderSettingsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FilesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + Container c = getContainer(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + + String noun = c.isProject() ? "Project": "Folder"; + getPageConfig().setTitle(noun + " Settings"); + + return vbox; + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) + { + Container c = getContainer(); + String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); + String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; + + if(folderRootPath == null && !fileRootOption.equals("default")) + { + errors.reject(ERROR_MSG, "Error: Must supply a default file location."); + return false; + } + + FileContentService service = FileContentService.get(); + if(fileRootOption.equals("default")) + { + service.setIsUseDefaultRoot(c, true); + } + // Requires AdminOperationsPermission to set file root + else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) + { + if (!service.isValidProjectRoot(folderRootPath)) + { + errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + return false; + } + + service.setIsUseDefaultRoot(c.getProject(), false); + service.setFileRootPath(c.getProject(), folderRootPath); + } + + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + _successURL = getContainer().getStartURL(getUser()); + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + + return true; + } + + @Override + public ActionURL getSuccessURL(FilesForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + setHelpTopic("createProject"); + } + } + + @RequiresPermission(DeletePermission.class) + public static class DeleteWorkbooksAction extends SimpleRedirectAction + { + public void validateCommand(ReturnUrlForm target, Errors errors) + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + if (ids.isEmpty()) + { + errors.reject(ERROR_MSG, "No IDs provided"); + } + } + + @Override + public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + + ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); + ids.forEach(id -> ret.addParameter("targets", id)); + + ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + + return ret; + } + } + + //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent + @RequiresPermission(DeletePermission.class) + public static class DeleteFolderAction extends FormViewAction + { + private final List _deleted = new ArrayList<>(); + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + try + { + List targets = form.getTargetContainers(getContainer()); + for (Container target : targets) + { + if (!ContainerManager.isDeletable(target)) + errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); + + if (target.isProject() && !getUser().hasRootAdminPermission()) + { + throw new UnauthorizedException(); + } + + Class permClass = target.getPermissionNeededToDelete(); + if (!target.hasPermission(getUser(), permClass)) + { + Permission perm = RoleManager.getPermission(permClass); + throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); + } + + if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) + { + throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); + } + + if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); + } + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + List targets = form.getTargetContainers(getContainer()); + + // Must be site/app admin to delete a project + for (Container c : targets) + { + ContainerManager.deleteAll(c, getUser()); + } + + _deleted.addAll(targets); + + return true; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: + // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders + if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) + { + Container c = getContainer(); + if (c.isProject()) + return AppProps.getInstance().getHomePageActionURL(); + else + return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); + } + else + { + if (form.getReturnUrl() != null) + { + return form.getReturnActionURL(); + } + else + { + return getContainer().getStartURL(getUser()); + } + } + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersAction extends FormViewAction + { + @Override + public void validateCommand(FolderReorderForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); + } + + @Override + public boolean handlePost(FolderReorderForm form, BindException errors) + { + return ReorderFolders(form, errors); + } + + @Override + public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) + { + if (getContainer().isRoot()) + return getShowAdminURL(); + else + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); + addAdminNavTrail(root, title, this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersApiAction extends MutatingApiAction + { + @Override + public ApiResponse execute(FolderReorderForm form, BindException errors) + { + return new ApiSimpleResponse("success", ReorderFolders(form, errors)); + } + } + + private boolean ReorderFolders(FolderReorderForm form, BindException errors) + { + Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); + if (form.isResetToAlphabetical()) + ContainerManager.setChildOrderToAlphabetical(parent); + else if (form.getOrder() != null) + { + List children = parent.getChildren(); + String[] order = form.getOrder().split(";"); + Map nameToContainer = new HashMap<>(); + for (Container child : children) + nameToContainer.put(child.getName(), child); + List sorted = new ArrayList<>(children.size()); + for (String childName : order) + { + Container child = nameToContainer.get(childName); + sorted.add(child); + } + + try + { + ContainerManager.setChildOrder(parent, sorted); + } + catch (ContainerException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + } + + return true; + } + + public static class FolderReorderForm + { + private String _order; + private boolean _resetToAlphabetical; + + public String getOrder() + { + return _order; + } + + @SuppressWarnings("unused") + public void setOrder(String order) + { + _order = order; + } + + public boolean isResetToAlphabetical() + { + return _resetToAlphabetical; + } + + @SuppressWarnings("unused") + public void setResetToAlphabetical(boolean resetToAlphabetical) + { + _resetToAlphabetical = resetToAlphabetical; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RevertFolderAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RevertFolderForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + + boolean success = false; + Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); + if (null != revertContainer) + { + if (revertContainer.isContainerTab()) + { + FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + revertContainer.setFolderType(origFolderType, getUser(), errors); + if (!errors.hasErrors()) + success = true; + } + } + } + else if (revertContainer.getFolderType().hasContainerTabs()) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + List children = revertContainer.getChildren(); + for (Container container : children) + { + if (container.isContainerTab()) + { + FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + container.setFolderType(origFolderType, getUser(), errors); + } + } + } + } + if (!errors.hasErrors()) + { + transaction.commit(); + success = true; + } + } + } + } + return new ApiSimpleResponse("success", success); + } + } + + public static class RevertFolderForm + { + private String _containerPath; + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + } + + public static class EmailTestForm + { + private String _to; + private String _body; + private ConfigurationException _exception; + + public String getTo() + { + return _to; + } + + public void setTo(String to) + { + _to = to; + } + + public String getBody() + { + return _body; + } + + public void setBody(String body) + { + _body = body; + } + + public ConfigurationException getException() + { + return _exception; + } + + public void setException(ConfigurationException exception) + { + _exception = exception; + } + + public String getFrom(Container c) + { + LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); + return props.getSystemEmailAddress(); + } + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class EmailTestAction extends FormViewAction + { + @Override + public void validateCommand(EmailTestForm form, Errors errors) + { + if(null == form.getTo() || form.getTo().isEmpty()) + { + errors.reject(ERROR_MSG, "To field cannot be blank."); + form.setException(new ConfigurationException("To field cannot be blank")); + return; + } + + try + { + ValidEmail email = new ValidEmail(form.getTo()); + } + catch(ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + form.setException(new ConfigurationException(e.getMessage())); + } + } + + @Override + public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) + { + JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); + testView.setTitle("Send a Test Email"); + + if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) + { + JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); + emailPropsView.setTitle("Current Email Settings"); + + return new VBox(emailPropsView, testView); + } + else + return testView; + } + + @Override + public boolean handlePost(EmailTestForm form, BindException errors) throws Exception + { + if (errors.hasErrors()) + { + return false; + } + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); + try + { + MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); + msg.setSubject("Test email message sent from " + props.getShortName()); + msg.setText(PageFlowUtil.filter(form.getBody())); + + try + { + MailHelper.send(msg, getUser(), getContainer()); + } + catch (ConfigurationException e) + { + form.setException(e); + return false; + } + catch (Exception e) + { + form.setException(new ConfigurationException(e.getMessage())); + return false; + } + } + catch (MessagingException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + return true; + } + + @Override + public URLHelper getSuccessURL(EmailTestForm emailTestForm) + { + return new ActionURL(EmailTestAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Test Email Configuration", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class RecreateViewsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + getPageConfig().setShowHeader(false); + getPageConfig().setTitle("Recreate Views?"); + return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + ModuleLoader.getInstance().recreateViews(); + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull ActionURL getSuccessURL(Object o) + { + return AppProps.getInstance().getHomePageActionURL(); + } + } + + static public class LoggingForm + { + public boolean isLogging() + { + return logging; + } + + public void setLogging(boolean logging) + { + this.logging = logging; + } + + public boolean logging = false; + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class GetSessionLogEventsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + Integer eventId = null; + try + { + String s = getViewContext().getRequest().getParameter("eventId"); + if (null != s) + eventId = Integer.parseInt(s); + } + catch (NumberFormatException ignored) {} + ApiSimpleResponse res = new ApiSimpleResponse(); + res.put("success", true); + res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); + return res; + } + } + + @RequiresLogin + @AllowedBeforeInitialUserIsSet + @AllowedDuringUpgrade + @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ + public static class GetTrackedAllocationsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + long requestId = 0; + try + { + String s = getViewContext().getRequest().getParameter("requestId"); + if (null != s) + requestId = Long.parseLong(s); + } + catch (NumberFormatException ignored) {} + List requests = MemTracker.getInstance().getNewRequests(requestId); + List> jsonRequests = new ArrayList<>(requests.size()); + for (RequestInfo requestInfo : requests) + { + Map m = new HashMap<>(); + m.put("requestId", requestInfo.getId()); + m.put("url", requestInfo.getUrl()); + m.put("date", requestInfo.getDate()); + + + List> sortedObjects = sortByCounts(requestInfo); + + List> jsonObjects = new ArrayList<>(sortedObjects.size()); + for (Map.Entry entry : sortedObjects) + { + Map jsonObject = new HashMap<>(); + jsonObject.put("name", entry.getKey()); + jsonObject.put("count", entry.getValue()); + jsonObjects.add(jsonObject); + } + m.put("objects", jsonObjects); + jsonRequests.add(m); + } + return new ApiSimpleResponse("requests", jsonRequests); + } + + private List> sortByCounts(RequestInfo requestInfo) + { + List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); + objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); + return objects; + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class TrackedAllocationsViewerAction extends SimpleViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + getPageConfig().setTemplate(Template.Print); + return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class SessionLoggingAction extends FormViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) + throw new UnauthorizedException(); + } + + @Override + public boolean handlePost(LoggingForm form, BindException errors) + { + boolean on = SessionAppender.isLogging(getViewContext().getRequest()); + if (form.logging != on) + { + if (!form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging OFF"); + SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); + if (form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging ON"); + } + return true; + } + + @Override + public void validateCommand(LoggingForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) + { + SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); + getPageConfig().setTemplate(Template.Print); + return new LoggingView(); + } + + @Override + public ActionURL getSuccessURL(LoggingForm o) + { + return new ActionURL(SessionLoggingAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); + root.addChild("View Event Log"); + } + } + + static class LoggingView extends JspView + { + LoggingView() + { + super("/org/labkey/core/admin/logging.jsp", null); + } + } + + public static class LogForm + { + private String _message; + private String _level; + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = level; + } + } + + + // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when + // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. + @RequiresLogin + public static class LogAction extends MutatingApiAction + { + @Override + public ApiResponse execute(LogForm logForm, BindException errors) + { + // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate + // \n to newlines to allow them at the beginning or end of the message as well. + StringBuilder message = new StringBuilder(); + message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); + + Level level = Level.toLevel(logForm.getLevel(), Level.INFO); + CLIENT_LOG.log(level, message); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class ValidateDomainsAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file + for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) + { + PipeRoot root = PipelineService.get().findPipelineRoot(project); + if (root != null && root.isValid()) + { + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipelineJob job = new ValidateDomainsPipelineJob(info, root); + PipelineService.get().queueJob(job); + return true; + } + } + return false; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); + } + } + + public static class ModulesForm + { + private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) + private boolean _managedOnly = false; + private boolean _unmanagedOnly = false; + + public double[] getIgnore() + { + return _ignore; + } + + public void setIgnore(double[] ignore) + { + _ignore = ignore; + } + + private Set getIgnoreSet() + { + return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); + } + + public boolean isManagedOnly() + { + return _managedOnly; + } + + @SuppressWarnings("unused") + public void setManagedOnly(boolean managedOnly) + { + _managedOnly = managedOnly; + } + + public boolean isUnmanagedOnly() + { + return _unmanagedOnly; + } + + @SuppressWarnings("unused") + public void setUnmanagedOnly(boolean unmanagedOnly) + { + _unmanagedOnly = unmanagedOnly; + } + } + + public enum ManageFilter + { + ManagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && module.shouldManageVersion(); + } + }, + UnmanagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && !module.shouldManageVersion(); + } + }, + All + { + @Override + public boolean accept(Module module) + { + return true; + } + }; + + public abstract boolean accept(Module module); + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ModulesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModulesForm form, BindException errors) + { + ModuleLoader ml = ModuleLoader.getInstance(); + boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); + + Collection unknownModules = ml.getUnknownModuleContexts().values(); + Collection knownModules = ml.getAllModuleContexts(); + knownModules.removeAll(unknownModules); + + Set ignoreSet = form.getIgnoreSet(); + HtmlString managedLink = HtmlString.EMPTY_STRING; + HtmlString unmanagedLink = HtmlString.EMPTY_STRING; + + // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release + // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. + if (AppProps.getInstance().isDevMode()) + { + if (ignoreSet.isEmpty() && !form.isManagedOnly()) + { + String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("ignore", "0.00," + lowestSchemaVersion); + url.addParameter("managedOnly", true); + managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); + } + else + { + List ignore = ignoreSet + .stream() + .map(ModuleContext::formatVersion) + .collect(Collectors.toCollection(LinkedList::new)); + + String ignoreString = ignore.isEmpty() ? null : ignore.toString(); + String unmanaged = form.isManagedOnly() ? "unmanaged" : null; + + managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); + } + + if (!form.isUnmanagedOnly()) + { + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("unmanagedOnly", true); + unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); + } + else + { + unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); + } + } + + ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); + + HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); + if (hasAdminOpsPerm) + { + deleteInstructions.unsafeAppend("

    ").append( + "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + + "Module files are typically deployed in /modules and /externalModules.") + .unsafeAppend("

    ").append( + LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); + } + + HtmlStringBuilder docLink = HtmlStringBuilder.of(); + docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); + + HtmlStringBuilder knownDescription = HtmlStringBuilder.of() + .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); + HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); + + HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() + .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + + "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + + "is stored is not accessible, or the module file is corrupted.") + .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); + HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); + HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); + + return new VBox(known, unknown); + } + + private class ModulesView extends WebPartView + { + private final Collection _contexts; + private final HtmlString _descriptionHtml; + private final HtmlString _noModulesDescriptionHtml; + private final Set _ignoreVersions; + private final ManageFilter _manageFilter; + + private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) + { + super(FrameType.PORTAL); + List sorted = new ArrayList<>(contexts); + sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); + + _contexts = sorted; + _descriptionHtml = descriptionHtml; + _noModulesDescriptionHtml = noModulesDescriptionHtml; + _ignoreVersions = ignoreVersions; + _manageFilter = manageFilter; + setTitle(type + " Modules"); + } + + @Override + protected void renderView(Object model, HtmlWriter out) + { + boolean isDevMode = AppProps.getInstance().isDevMode(); + boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); + boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); + final AtomicInteger rowCount = new AtomicInteger(); + ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); + final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); + final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); + + if (_contexts.isEmpty()) + { + out.write(_noModulesDescriptionHtml); + } + else + { + DIV( + DIV(_descriptionHtml), + TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), + TR( + TD(cl("labkey-column-header"),"Name"), + TD(cl("labkey-column-header"),"Release Version"), + TD(cl("labkey-column-header"),"Schema Version"), + TD(cl("labkey-column-header"),"Class"), + TD(cl("labkey-column-header"),"Location"), + TD(cl("labkey-column-header"),"Schemas"), + !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions + null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions + !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions + ), + _contexts.stream() + .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) + .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) + .filter(pair -> _manageFilter.accept(pair.getValue())) + .map(pair -> + { + ModuleContext moduleContext = pair.getKey(); + Module module = pair.getValue(); + List schemas = moduleContext.getSchemaList(); + Double schemaVersion = moduleContext.getSchemaVersion(); + boolean replaceableModule = false; + if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) + { + File zip = module.getZippedPath(); + if (null != zip && zip.getParentFile().equals(externalModulesDir)) + replaceableModule = true; + } + boolean deleteableModule = replaceableModule || null == module; + String className = StringUtils.trimToEmpty(moduleContext.getClassName()); + String fullPathToModule = ""; + String shortPathToModule = ""; + if (null != module) + { + Path p = module.getExplodedPath().toPath(); + if (null != module.getZippedPath()) + p = module.getZippedPath().toPath(); + if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) + if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) + p = Paths.get(module.getSourcePath()); + fullPathToModule = p.toString(); + shortPathToModule = fullPathToModule; + Path rel = relativeRoot.relativize(p); + if (!rel.startsWith("..")) + shortPathToModule = rel.toString(); + } + ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); + + return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), + TD(moduleContext.getName()), + TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), + TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), + TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), + TD(SPAN(at(title,fullPathToModule),shortPathToModule)), + TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), + !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), + null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), + !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) + ); + }) + ) + ).appendTo(out); + } + } + } + + private ActionURL getDeleteURL(String name) + { + ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); + if (null != url) + return url; + url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getUpdateURL(String name) + { + ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); + if (null != url) + return url; + url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getModuleEditorURL(String name) + { + return ModuleEditorService.get().getModuleEditorURL(name); + } + + private ActionURL getCreateURL() + { + ActionURL url = ModuleEditorService.get().getCreateModuleURL(); + if (null != url) + return url; + url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("defaultModules"); + addAdminNavTrail(root, "Modules", getClass()); + } + } + + public static class SchemaVersionTestCase extends Assert + { + @Test + public void verifyMinimumSchemaVersion() + { + List modulesTooLow = ModuleLoader.getInstance().getModules().stream() + .filter(ManageFilter.ManagedOnly::accept) + .filter(m -> null != m.getSchemaVersion()) + .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) + .toList(); + + if (!modulesTooLow.isEmpty()) + fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); + } + + @Test + public void modulesWithSchemaVersionButNoScripts() + { + // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. + List moduleNames = ModuleLoader.getInstance().getModules().stream() + .filter(m -> m.getSchemaVersion() != null) + .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) + .map(m -> m.getName() + ": " + m.getSchemaVersion()) + .toList(); + + if (!moduleNames.isEmpty()) + fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); + } + } + + public static class ModuleForm + { + private String _name; + + public String getName() + { + return _name; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setName(String name) + { + _name = name; + } + + @NotNull + private ModuleContext getModuleContext() + { + ModuleLoader ml = ModuleLoader.getInstance(); + ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); + + if (null == ctx) + throw new NotFoundException("Module not found"); + + return ctx; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DeleteModuleAction extends ConfirmAction + { + @Override + public void validateCommand(ModuleForm form, Errors errors) + { + } + + @Override + public ModelAndView getConfirmView(ModuleForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Module"); + + ModuleContext ctx = form.getModuleContext(); + Module module = ModuleLoader.getInstance().getModule(ctx.getName()); + boolean hasSchemas = !ctx.getSchemaList().isEmpty(); + boolean hasFiles = false; + if (null != module) + hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); + + HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); + HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); + if (hasSchemas) + { + SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); + List deleteList = schemaActions.deleteList(); + List skipList = schemaActions.skipList(); + + // List all the schemas that will be deleted + if (!deleteList.isEmpty()) + { + description.append(" and delete all data in "); + description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); + } + + // For unknown modules, also list the schemas that won't be deleted + if (!skipList.isEmpty()) + { + skippedSchemas.append(HtmlString.BR); + skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) + .append("Note: Schema \"") + .append(sam.schema()) + .append("\" will not be deleted because it's in use by module \"") + .append(sam.module()) + .append("\"")); + } + } + + return new HtmlView(DIV( + !hasFiles ? null : DIV(cl("labkey-warning-messages"), + "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", + null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), + null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) + ), + BR(), + "Are you sure you want to remove the ", description, "? ", + "This operation cannot be undone!", + skippedSchemas, + BR(), + !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." + )); + } + + @Override + public boolean handlePost(ModuleForm form, BindException errors) + { + ModuleLoader.getInstance().removeModule(form.getModuleContext()); + + return true; + } + + @Override + public @NotNull URLHelper getSuccessURL(ModuleForm form) + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class UpdateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class CreateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class OptionalFeatureForm + { + private String feature; + private boolean enabled; + + public String getFeature() + { + return feature; + } + + public void setFeature(String feature) + { + this.feature = feature; + } + + public boolean isEnabled() + { + return enabled; + } + + public void setEnabled(boolean enabled) + { + this.enabled = enabled; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @ActionNames("OptionalFeature, ExperimentalFeature") + public static class OptionalFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(OptionalFeatureForm form, BindException errors) + { + String feature = StringUtils.trimToNull(form.getFeature()); + if (feature == null) + throw new ApiUsageException("feature is required"); + + OptionalFeatureService svc = OptionalFeatureService.get(); + if (svc == null) + throw new IllegalStateException(); + + Map ret = new HashMap<>(); + ret.put("feature", feature); + + if (isPost()) + { + ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); + svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); + } + + ret.put("enabled", svc.isFeatureEnabled(feature)); + return new ApiSimpleResponse(ret); + } + } + + public static class OptionalFeaturesForm + { + private String _type; + private boolean _showHidden; + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + public @NotNull FeatureType getTypeEnum() + { + return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); + } + + public boolean isShowHidden() + { + return _showHidden; + } + + @SuppressWarnings("unused") + public void setShowHidden(boolean showHidden) + { + _showHidden = showHidden; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class OptionalFeaturesAction extends SimpleViewAction + { + private FeatureType _type; + + @Override + public ModelAndView getView(OptionalFeaturesForm form, BindException errors) + { + _type = form.getTypeEnum(); + JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("experimental"); + addAdminNavTrail(root, _type.name() + " Features", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ProductFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(ProductConfigForm form, BindException errors) + { + String productKey = StringUtils.trimToNull(form.getProductKey()); + + Map ret = new HashMap<>(); + + if (isPost()) + { + ProductConfiguration.setProductKey(productKey); + } + + ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); + return new ApiSimpleResponse(ret); + } + } + + public static class ProductConfigForm + { + private String productKey; + + public String getProductKey() + { + return productKey; + } + + public void setProductKey(String productKey) + { + this.productKey = productKey; + } + + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class ProductConfigurationAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Product Configuration", getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + } + + + public static class FolderTypesBean + { + private final Collection _allFolderTypes; + private final Collection _enabledFolderTypes; + private final FolderType _defaultFolderType; + + public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) + { + _allFolderTypes = allFolderTypes; + _enabledFolderTypes = enabledFolderTypes; + _defaultFolderType = defaultFolderType; + } + + public Collection getAllFolderTypes() + { + return _allFolderTypes; + } + + public Collection getEnabledFolderTypes() + { + return _enabledFolderTypes; + } + + public FolderType getDefaultFolderType() + { + return _defaultFolderType; + } + } + + @AdminConsoleAction + @RequiresPermission(AdminPermission.class) + public class FolderTypesAction extends FormViewAction + { + @Override + public void validateCommand(Object form, Errors errors) + { + } + + @Override + public ModelAndView getView(Object form, boolean reshow, BindException errors) + { + FolderTypesBean bean; + if (reshow) + { + bean = getOptionsFromRequest(); + } + else + { + FolderTypeManager manager = FolderTypeManager.get(); + var defaultFolderType = manager.getDefaultFolderType(); + // If a default folder type has not yet been configuration use "Collaboration" folder type as the default + defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); + boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); + bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); + } + + return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); + } + + @Override + public boolean handlePost(Object form, BindException errors) + { + FolderTypesBean bean = getOptionsFromRequest(); + var defaultFolderType = bean.getDefaultFolderType(); + if (defaultFolderType == null) + { + errors.reject(ERROR_MSG, "Please select a default folder type."); + return false; + } + var enabledFolderTypes = bean.getEnabledFolderTypes(); + if (!enabledFolderTypes.contains(defaultFolderType)) + { + errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); + return false; + } + + FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); + return true; + } + + private FolderTypesBean getOptionsFromRequest() + { + var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); + List enabledFolderTypes = new ArrayList<>(); + FolderType defaultFolderType = null; + String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); + + for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) + { + boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); + if (enabled) + { + enabledFolderTypes.add(folderType); + } + if (folderType.getName().equals(defaultFolderTypeParam)) + { + defaultFolderType = folderType; + } + } + return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); + } + + @Override + public URLHelper getSuccessURL(Object form) + { + return getShowAdminURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Types", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class CustomizeMenuAction extends MutatingApiAction + { + @Override + public ApiResponse execute(CustomizeMenuForm form, BindException errors) + { + if (null != form.getUrl()) + { + String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); + if (null != errorMessage) + { + errors.reject(ERROR_MSG, errorMessage); + return new ApiSimpleResponse("success", false); + } + } + + setCustomizeMenuForm(form, getContainer(), getUser()); + return new ApiSimpleResponse("success", true); + } + } + + protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; + protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; + protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; + protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; + protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; + protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; + protected static final String CUSTOMMENU_URL = "customMenuUrl"; + protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; + protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; + protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; + protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; + protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; + + public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) + { + CustomizeMenuForm form = new CustomizeMenuForm(); + Map menuProps = webPart.getPropertyMap(); + + String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); + String queryName = menuProps.get(CUSTOMMENU_QUERY); + String columnName = menuProps.get(CUSTOMMENU_COLUMN); + String viewName = menuProps.get(CUSTOMMENU_VIEW); + String folderName = menuProps.get(CUSTOMMENU_FOLDER); + String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; + String urlBottom = menuProps.get(CUSTOMMENU_URL); + String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); + String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); + String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); + boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); + String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); + boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); + String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); + boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); + + form.setSchemaName(schemaName); + form.setQueryName(queryName); + form.setColumnName(columnName); + form.setViewName(viewName); + form.setFolderName(folderName); + form.setTitle(title); + form.setUrl(urlBottom); + form.setRootFolder(rootFolder); + form.setFolderTypes(folderTypes); + form.setChoiceListQuery(choiceListQuery); + form.setIncludeAllDescendants(includeAllDescendants); + form.setCurrentProjectOnly(currentProjectOnly); + + form.setWebPartIndex(webPart.getIndex()); + form.setPageId(webPart.getPageId()); + return form; + } + + private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) + { + Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); + if (null == webPart) + throw new NotFoundException(); + Map menuProps = webPart.getPropertyMap(); + + menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); + menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); + menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); + menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); + menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); + menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); + menuProps.put(CUSTOMMENU_URL, form.getUrl()); + + // If root folder not specified, set as current container + menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); + menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); + menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); + menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); + menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); + + Portal.updatePart(user, webPart); + } + + @RequiresPermission(AdminPermission.class) + public static class AddTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + if(tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived + // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something + // longer than 64 characters. + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + if (name.length() > 50) + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); + + for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) + { + folderTabMap.put(tab.getName(), tab); + } + + if (pages.containsKey(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (page.getCaption() != null && page.getCaption().equals(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + else if (folderTabMap.containsKey(page.getPageId())) + { + if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + validateCommand(form, errors); + + if(errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + String name = form.getTabName(); + String caption = form.getTabName(); + + // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the + // name, and is editable, is allowed to be 64 characters. + if (name.length() > 50) + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); + + Portal.saveParts(container, name); + Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); + + if (!name.equals(caption)) + { + // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(name); + // Get a mutable copy + page = page.copy(); + page.setCaption(caption); + Portal.updatePortalPage(container, page); + } + + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); + tabURL.addParameter("pageId", name); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ShowTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); + + if (form.getTabPageId() == null) + { + errors.reject(ERROR_MSG, "PageId cannot be blank."); + } + + if (!pages.containsKey(form.getTabPageId())) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + validateCommand(form, errors); + if (errors.hasErrors()) + return response; + + Portal.showPage(tabContainer, form.getTabPageId()); + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); + tabURL.addParameter("pageId", form.getTabPageId()); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + + public static class TabActionForm extends ReturnUrlForm + { + // This class is used for tab related actions (add, rename, show, etc.) + String _tabName; + String _tabPageId; + + public String getTabName() + { + return _tabName; + } + + public void setTabName(String name) + { + _tabName = name; + } + + public String getTabPageId() + { + return _tabPageId; + } + + public void setTabPageId(String tabPageId) + { + _tabPageId = tabPageId; + } + } + + @RequiresPermission(AdminPermission.class) + public class MoveTabAction extends MutatingApiAction + { + @Override + public ApiResponse execute(MoveTabForm form, BindException errors) + { + final Map properties = new HashMap<>(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage tab = pages.get(form.getPageId()); + + if (null != tab) + { + int oldIndex = tab.getIndex(); + Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); + + if (null != pageToSwap) + { + properties.put("oldIndex", oldIndex); + properties.put("newIndex", tab.getIndex()); + properties.put("pageId", tab.getPageId()); + properties.put("pageIdToSwap", pageToSwap.getPageId()); + } + else + { + properties.put("error", "Unable to move tab."); + } + } + else + { + properties.put("error", "Requested tab does not exist."); + } + + return new ApiSimpleResponse(properties); + } + } + + public static class MoveTabForm implements HasViewContext + { + private int _direction; + private String _pageId; + private ViewContext _viewContext; + + public int getDirection() + { + // 0 moves left, 1 moves right. + return _direction; + } + + public void setDirection(int direction) + { + _direction = direction; + } + + public String getPageId() + { + return _pageId; + } + + public void setPageId(String pageId) + { + _pageId = pageId; + } + + @Override + public ViewContext getViewContext() + { + return _viewContext; + } + + @Override + public void setViewContext(ViewContext viewContext) + { + _viewContext = viewContext; + } + } + + private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) + { + Map pageMap = new CaseInsensitiveHashMap<>(); + for (Portal.PortalPage pp : Portal.getTabPages(c, true)) + pageMap.put(pp.getPageId(), pp); + + for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) + { + if (pageMap.containsKey(folderTab.getName())) + { + // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to + // be excluded when adjusting the relative indexes. + if (!folderTab.isVisible(c, user)) + pageMap.remove(folderTab.getName()); + } + } + List pagesList = new ArrayList<>(pageMap.values()); + pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); + + int visibleIndex; + for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) + { + if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) + { + break; + } + } + + if (visibleIndex == pagesList.size()) + { + return null; + } + + if (direction == Portal.MOVE_DOWN) + { + if (visibleIndex == pagesList.size() - 1) + { + return page; + } + + Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); + + if (null == nextPage) + return null; + Portal.swapPageIndexes(c, page, nextPage); + return nextPage; + } + else + { + if (visibleIndex < 1) + { + return page; + } + + Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); + + if (null == prevPage) + return null; + Portal.swapPageIndexes(c, page, prevPage); + return prevPage; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RenameTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + if (tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); + if (null == pageToChange) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (!page.equals(pageToChange)) + { + if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + if (page.getPageId().equalsIgnoreCase(name)) + { + if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + else + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + } + } + + List folderTabs = tabContainer.getFolderType().getDefaultTabs(); + for (FolderTab folderTab : folderTabs) + { + String folderTabCaption = folderTab.getCaption(getViewContext()); + if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + return; + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + validateCommand(form, errors); + + if (errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(form.getTabPageId()); + page = page.copy(); + page.setCaption(form.getTabName()); + // Update the page the caption is saved. + Portal.updatePortalPage(container, page); + + response.put("success", true); + return response; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ClearDeletedTabFoldersAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeletedFoldersForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + Container container = ContainerManager.getForPath(form.getContainerPath()); + for (String tabName : form.getResurrectFolders()) + { + ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); + } + return new ApiSimpleResponse("success", true); + } + } + + @SuppressWarnings("unused") + public static class DeletedFoldersForm + { + private String _containerPath; + private String _newFolderType; + private List _resurrectFolders; + + public List getResurrectFolders() + { + return _resurrectFolders; + } + + public void setResurrectFolders(List resurrectFolders) + { + _resurrectFolders = resurrectFolders; + } + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + + public String getNewFolderType() + { + return _newFolderType; + } + + public void setNewFolderType(String newFolderType) + { + _newFolderType = newFolderType; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetFolderTabsAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object form, BindException errors) throws Exception + { + var data = getContainer() + .getFolderType() + .getAppBar(getViewContext(), getPageConfig()) + .getButtons() + .stream() + .map(this::getProperties) + .toList(); + + return success(data); + } + + private Map getProperties(NavTree navTree) + { + Map props = new HashMap<>(); + props.put("id", navTree.getId()); + props.put("text", navTree.getText()); + props.put("href", navTree.getHref()); + props.put("disabled", navTree.isDisabled()); + return props; + } + } + + @SuppressWarnings("unused") + public static class ShortURLForm + { + private String _shortURL; + private String _fullURL; + private boolean _delete; + + private List _savedShortURLs; + + public void setShortURL(String shortURL) + { + _shortURL = shortURL; + } + + public void setFullURL(String fullURL) + { + _fullURL = fullURL; + } + + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getShortURL() + { + return _shortURL; + } + + public String getFullURL() + { + return _fullURL; + } + + public boolean isDelete() + { + return _delete; + } + } + + public abstract static class AbstractShortURLAdminAction extends FormViewAction + { + @Override + public void validateCommand(ShortURLForm target, Errors errors) {} + + @Override + public boolean handlePost(ShortURLForm form, BindException errors) throws Exception + { + String shortURL = StringUtils.trimToEmpty(form.getShortURL()); + if (StringUtils.isEmpty(shortURL)) + { + errors.addError(new LabKeyError("Short URL must not be blank")); + } + if (shortURL.endsWith(".url")) + shortURL = shortURL.substring(0,shortURL.length()-".url".length()); + if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) + { + errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); + } + URLHelper fullURL = null; + if (!form.isDelete()) + { + String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); + if (trimmedFullURL == null) + { + errors.addError(new LabKeyError("Target URL must not be blank")); + } + else + { + try + { + fullURL = new URLHelper(trimmedFullURL); + } + catch (URISyntaxException e) + { + errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); + } + } + } + if (errors.getErrorCount() > 0) + { + return false; + } + + ShortURLService service = ShortURLService.get(); + if (form.isDelete()) + { + ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); + if (shortURLRecord == null) + { + throw new NotFoundException("No such short URL: " + shortURL); + } + try + { + service.deleteShortURL(shortURLRecord, getUser()); + } + catch (ValidationException e) + { + errors.addError(new LabKeyError("Error deleting short URL:")); + for(ValidationError error: e.getErrors()) + { + errors.addError(new LabKeyError(error.getMessage())); + } + } + + if (errors.getErrorCount() > 0) + { + return false; + } + } + else + { + ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); + MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); + // Add a role assignment to let another group manage the URL. This grants permission to the journal + // to change where the URL redirects you to after they copy the data + SecurityPolicyManager.savePolicy(policy, getUser()); + } + return true; + } + } + + @AdminConsoleAction + public class ShortURLAdminAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); + boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); + newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); + newView.setFrame(WebPartView.FrameType.PORTAL); + + QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); + qSettings.setBaseSort(new Sort("-Created")); + QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); + if (!isAppAdmin) + { + existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + } + existingView.setTitle("Existing Short URLs"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Short URL Admin", getClass()); + } + } + + @RequiresPermission(ApplicationAdminPermission.class) + public class UpdateShortURLAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); + if (shortUrlRecord == null) + { + errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); + return new SimpleErrorView(errors); + } + form.setFullURL(shortUrlRecord.getFullURL()); + + JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); + view.setTitle("Update Short URL"); + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Update Short URL", getClass()); + } + } + + // API for reporting client-side exceptions. + // UNDONE: Throttle by IP to avoid DOS from buggy clients. + @Marshal(Marshaller.Jackson) + @SuppressWarnings("UnusedDeclaration") + @RequiresLogin // Issue 52520: Prevent bots from submitting reports + @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root + public static class LogClientExceptionAction extends MutatingApiAction + { + @Override + public Object execute(ExceptionForm form, BindException errors) + { + String errorCode = ExceptionUtil.logClientExceptionToMothership( + form.getStackTrace(), + form.getExceptionMessage(), + form.getBrowser(), + null, + form.getRequestURL(), + form.getReferrerURL(), + form.getUsername() + ); + + Map results = new HashMap<>(); + results.put("errorCode", errorCode); + results.put("loggedToMothership", errorCode != null); + + return success(results); + } + } + + @SuppressWarnings("unused") + public static class ExceptionForm + { + private String _exceptionMessage; + private String _stackTrace; + private String _requestURL; + private String _browser; + private String _username; + private String _referrerURL; + private String _file; + private String _line; + private String _platform; + + public String getExceptionMessage() + { + return _exceptionMessage; + } + + public void setExceptionMessage(String exceptionMessage) + { + _exceptionMessage = exceptionMessage; + } + + public String getUsername() + { + return _username; + } + + public void setUsername(String username) + { + _username = username; + } + + public String getStackTrace() + { + return _stackTrace; + } + + public void setStackTrace(String stackTrace) + { + _stackTrace = stackTrace; + } + + public String getRequestURL() + { + return _requestURL; + } + + public void setRequestURL(String requestURL) + { + _requestURL = requestURL; + } + + public String getBrowser() + { + return _browser; + } + + public void setBrowser(String browser) + { + _browser = browser; + } + + public String getReferrerURL() + { + return _referrerURL; + } + + public void setReferrerURL(String referrerURL) + { + _referrerURL = referrerURL; + } + + public String getFile() + { + return _file; + } + + public void setFile(String file) + { + _file = file; + } + + public String getLine() + { + return _line; + } + + public void setLine(String line) + { + _line = line; + } + + public String getPlatform() + { + return _platform; + } + + public void setPlatform(String platform) + { + _platform = platform; + } + } + + + /** generate URLS to seed web-site scanner */ + @SuppressWarnings("UnusedDeclaration") + @RequiresSiteAdmin + public static class SpiderAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Spider Initialization"); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + List urls = new ArrayList<>(1000); + + if (getContainer().equals(ContainerManager.getRoot())) + { + for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) + { + urls.add(c.getStartURL(getUser()).toString()); + urls.add(new ActionURL(SpiderAction.class, c).toString()); + } + + Container home = ContainerManager.getHomeContainer(); + for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) + { + ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); + urls.add(url.toString()); + } + } + else + { + DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); + def.getSchemaNames().forEach(name -> + { + QuerySchema q = def.getSchema(name); + if (null == q) + return; + var tableNames = q.getTableNames(); + if (null == tableNames) + return; + tableNames.forEach(table -> + { + try + { + var t = q.getTable(table); + if (null != t) + { + ActionURL grid = t.getGridURL(getContainer()); + if (null != grid) + urls.add(grid.toString()); + else + urls.add(new ActionURL("query", "executeQuery.view", getContainer()) + .addParameter("schemaName", q.getSchemaName()) + .addParameter("query.queryName", t.getName()) + .toString()); + } + } + catch (Exception x) + { + // pass + } + }); + }); + + ModuleLoader.getInstance().getModules().forEach(m -> + { + ActionURL url = m.getTabURL(getContainer(), getUser()); + if (null != url) + urls.add(url.toString()); + }); + } + + return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); + } + } + + @SuppressWarnings("UnusedDeclaration") + @RequiresPermission(TroubleshooterPermission.class) + public static class TestMothershipReportAction extends ReadOnlyApiAction + { + @Override + public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception + { + MothershipReport report; + MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; + if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) + { + report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); + } + else + { + report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), + new SQLException("Intentional exception for testing purposes", "400"), + (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), + target, + ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); + } + + final Map params; + if (report == null) + { + params = new LinkedHashMap<>(); + } + else + { + params = report.getJsonFriendlyParams(); + if (form.isSubmit()) + { + report.setForwardedFor(form.getForwardedFor()); + report.run(); + if (null != report.getUpgradeMessage()) + params.put("upgradeMessage", report.getUpgradeMessage()); + } + } + if (form.isDownload()) + { + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); + } + return new ApiSimpleResponse(params); + } + } + + + static class MothershipReportSelectionForm + { + private String _type = MothershipReport.Type.CheckForUpdates.toString(); + private String _level = UsageReportingLevel.ON.toString(); + private boolean _submit = false; + private boolean _download = false; + private String _forwardedFor = null; + // indicates action is being invoked for dev/test + private boolean _testMode = false; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = StringUtils.upperCase(level); + } + + public boolean isSubmit() + { + return _submit; + } + + public void setSubmit(boolean submit) + { + _submit = submit; + } + + public String getForwardedFor() + { + return _forwardedFor; + } + + public void setForwardedFor(String forwardedFor) + { + _forwardedFor = forwardedFor; + } + + public boolean isTestMode() + { + return _testMode; + } + + public void setTestMode(boolean testMode) + { + _testMode = testMode; + } + + public boolean isDownload() + { + return _download; + } + + public void setDownload(boolean download) + { + _download = download; + } + } + + + @RequiresPermission(TroubleshooterPermission.class) + public class SuspiciousAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + Collection list = BlockListFilter.reportSuspicious(); + HtmlStringBuilder html = HtmlStringBuilder.of(); + if (list.isEmpty()) + { + html.append("No suspicious activity.\n"); + } + else + { + html.unsafeAppend("") + .unsafeAppend("\n"); + for (BlockListFilter.Suspicious s : list) + { + html.unsafeAppend("\n"); + } + html.unsafeAppend("
    host (user)user-agentcount
    ") + .append(s.host); + if (!isBlank(s.user)) + html.append(HtmlString.NBSP).append("(" + s.user + ")"); + html.unsafeAppend("") + .append(s.userAgent) + .unsafeAppend("") + .append(s.count) + .unsafeAppend("
    "); + } + return new HtmlView(html); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); + } + } + + /** This is a very crude API right now, mostly using default serialization of pre-existing objects + * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedBeforeInitialUserIsSet + public static class ConfigurationSummaryAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) + { + if (!getContainer().isRoot()) + throw new NotFoundException("Must be invoked in the root"); + + // requires site-admin, unless there are no users + if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) + throw new UnauthorizedException(); + + return getConfigurationJson(); + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + ObjectMapper result = JsonUtil.createDefaultMapper(); + result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); + return result; + } + + /* returns a jackson serializable object that reports superset of information returned in admin console */ + private JSONObject getConfigurationJson() + { + JSONObject res = new JSONObject(); + + res.put("server", AdminBean.getPropertyMap()); + + final Map> sets = new TreeMap<>(); + new SqlSelector(CoreSchema.getInstance().getScope(), + new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + + "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> + { + String category = (String)m.get("category"); + String name = (String)m.get("name"); + Object value = m.get("value"); + if (!sets.containsKey(category)) + sets.put(category, new TreeMap<>()); + sets.get(category).put(name,value); + } + ); + res.put("siteSettings", sets); + + HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); + res.put("health", result); + + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + res.put("scriptEngines", mgr.getEngineDefinitions()); + + return res; + } + } + + @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) + private static class IgnorePasswordMixIn + { + } + + @AdminConsoleAction() + public class AllowListAction extends FormViewAction + { + private AllowListType _type; + + @Override + public void validateCommand(AllowListForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) + { + _type = form.getTypeEnum(); + + form.setExistingValuesList(form.getTypeEnum().getValues()); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); + newView.setTitle("Register New " + form.getTypeEnum().getTitle()); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); + existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + AllowListType allowListType = form.getTypeEnum(); + //handle delete of existing value + if (form.isDelete()) + { + String urlToDelete = form.getExistingValue(); + List values = new ArrayList<>(allowListType.getValues()); + for (String value : values) + { + if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) + { + values.remove(value); + allowListType.setValues(values, getUser()); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated urls + else if (form.isSaveAll()) + { + Set validatedValues = form.validateValues(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(validatedValues.stream().toList(), getUser()); + } + //save new external value + else if (form.isSaveNew()) + { + Set valueSet = form.validateNewValue(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(valueSet, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic(_type.getHelpTopic()); + addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); + } + } + + public static class AllowListForm + { + private String _newValue; + private String _existingValue; + private boolean _delete; + private String _existingValues; + private boolean _saveAll; + private boolean _saveNew; + private String _type; + + private List _existingValuesList; + + public String getNewValue() + { + return _newValue; + } + + @SuppressWarnings("unused") + public void setNewValue(String newValue) + { + _newValue = newValue; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getExistingValues() + { + return _existingValues; + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public List getExistingValuesList() + { + //for updated urls that comes in as String values from the jsp/html form + if (null != getExistingValues()) + { + // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. + return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); + } + return _existingValuesList; + } + + public void setExistingValuesList(List valuesList) + { + _existingValuesList = valuesList; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + @NotNull + public AllowListType getTypeEnum() + { + return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); + } + + @JsonIgnore + public Set validateNewValue(BindException errors) + { + String value = StringUtils.trimToEmpty(getNewValue()); + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + return null; + + Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); + checkDuplicatesByAddition(value, valueSet, errors); + return valueSet; + } + + @JsonIgnore + public Set validateValues(BindException errors) + { + List values = getExistingValuesList(); //get values from the form, this includes updated values + Set valueSet = new CaseInsensitiveHashSet(); + + if (null != values && !values.isEmpty()) + { + for (String value : values) + { + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + continue; + + checkDuplicatesByAddition(value, valueSet, errors); + } + } + + return valueSet; + } + + /** + * Adds value to value set unless it is a duplicate, in which case it adds an error + * @param value to check + * @param valueSet of existing values + * @param errors collections of errors observed + */ + @JsonIgnore + private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) + { + String trimValue = StringUtils.trimToEmpty(value); + if (!valueSet.add(trimValue)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); + } + } + + @AdminConsoleAction + public static class DeleteAllValuesAction extends FormHandlerAction + { + @Override + public void validateCommand(AllowListForm form, Errors errors) + { + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + form.getTypeEnum().setValues(Collections.emptyList(), getUser()); + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + } + + public static class ExternalSourcesForm + { + private boolean _delete; + private boolean _saveNew; + private boolean _saveAll; + + private String _newDirective; + private String _newHost; + private String _existingValue; + private String _existingValues; + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public String getNewDirective() + { + return _newDirective; + } + + @SuppressWarnings("unused") + public void setNewDirective(String newDirective) + { + _newDirective = newDirective; + } + + public String getNewHost() + { + return _newHost; + } + + @SuppressWarnings("unused") + public void setNewHost(String newHost) + { + _newHost = newHost; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public List getExistingValues() + { + return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + private AllowedHost getExistingAllowedHost(BindException errors) + { + return getAllowedHost(getExistingValue(), errors); + } + + private AllowedHost getAllowedHost(String value, BindException errors) + { + String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts + if (parts.length != 2) + { + errors.addError(new LabKeyError("Can't parse allowed host.")); + return null; + } + return validateHost(parts[0], parts[1], errors); + } + + private List getExistingAllowedHosts(BindException errors) + { + List existing = getExistingValues().stream() + .map(value-> getAllowedHost(value, errors)) + .toList(); + + if (errors.hasErrors()) + return null; + + return checkDuplicates(existing, errors); + } + + private List validateNewAllowedHost(BindException errors) throws JsonProcessingException + { + AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); + + if (errors.hasErrors()) + return null; + + List hosts = getSavedAllowedHosts(); + hosts.add(newAllowedHost); + + return checkDuplicates(hosts, errors); + } + + // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons + public static AllowedHost validateHost(String directiveString, String host, BindException errors) + { + AllowedHost ret = null; + + if (StringUtils.isEmpty(directiveString)) + { + errors.addError(new LabKeyError("Directive must not be blank")); + } + else if (StringUtils.isEmpty(host)) + { + errors.addError(new LabKeyError("Host must not be blank")); + } + else if (host.contains(";")) + { + errors.addError(new LabKeyError("Semicolons are not allowed in host names")); + } + else + { + Directive directive = EnumUtils.getEnum(Directive.class, directiveString); + + if (null == directive) + { + errors.addError(new LabKeyError("Unknown directive: " + directiveString)); + } + else + { + ret = new AllowedHost(directive, host.trim()); + } + } + + return ret; + } + + /** + * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons + + * @param hosts a list of AllowedHost objects to check for duplicates + * @param errors errors to populate + * @return hosts if there are no duplicates, otherwise {@code null} + */ + public static @Nullable List checkDuplicates(List hosts, BindException errors) + { + // Not a simple Set check since we want host check to be case-insensitive + MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); + + hosts.forEach(allowedHost -> { + String host = allowedHost.host().trim(); + if (!map.put(allowedHost.directive(), host)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); + }); + + return errors.hasErrors() ? null : hosts; + } + + // Returns a mutable list + public List getSavedAllowedHosts() throws JsonProcessingException + { + return AllowedExternalResourceHosts.readAllowedHosts(); + } + } + + @AdminConsoleAction() + public class ExternalSourcesAction extends FormViewAction + { + @Override + public void validateCommand(ExternalSourcesForm form, Errors errors) + { + } + + @Override + public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) + { + boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); + newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); + existingView.setTitle("Existing External Resource Hosts"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + private static final Object HOST_LOCK = new Object(); + + @Override + public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception + { + List allowedHosts = null; + + // Multiple requests could access this in parallel, so synchronize access, Issue 53457 + synchronized (HOST_LOCK) + { + //handle delete of an existing value + if (form.isDelete()) + { + AllowedHost subToDelete = form.getExistingAllowedHost(errors); + if (errors.hasErrors()) + return false; + allowedHosts = form.getSavedAllowedHosts(); + var iter = allowedHosts.listIterator(); + while (iter.hasNext()) + { + AllowedHost sub = iter.next(); + if (sub.equals(subToDelete)) + { + iter.remove(); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated hosts + else if (form.isSaveAll()) + { + allowedHosts = form.getExistingAllowedHosts(errors); + if (errors.hasErrors()) + return false; + } + //save new external value + else if (form.isSaveNew()) + { + allowedHosts = form.validateNewAllowedHost(errors); + } + + if (errors.hasErrors()) + return false; + + AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(ExternalSourcesForm form) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("externalHosts"); + addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ProjectSettingsAction extends ProjectSettingsViewPostAction + { + @Override + protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(ProjectSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception + { + return saveProjectSettings(getContainer(), getUser(), form, errors); + } + } + + private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) + { + WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); + boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); + + // Site-only properties + + if (c.isRoot()) + { + DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); + props.setDateParsingMode(dateParsingMode); + + if (hasAdminOpsPerm) + { + String customWelcome = form.getCustomWelcome(); + String welcomeUrl = StringUtils.trimToNull(customWelcome); + if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); + } + else + { + props.setCustomWelcome(welcomeUrl); + } + } + } + + // Site & project properties + + boolean shouldInherit = form.getShouldInherit(); + if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) + { + SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); + } + + setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); + setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); + setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); + setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); + setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); + setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); + + // a few properties on this page should be restricted to operational permissions (i.e. site admin) + if (hasAdminOpsPerm) + { + setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { + String systemEmailAddress = form.getSystemEmailAddress(); + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(systemEmailAddress); + props.setSystemEmailAddress(email); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + }); + + setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { + String customLogin = form.getCustomLogin(); + if (props.isValidUrl(customLogin)) + { + props.setCustomLogin(customLogin); + } + else + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); + } + }); + } + + setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); + setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); + setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); + setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { + String supportEmail = form.getSupportEmail(); + + if (!isBlank(supportEmail)) + { + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(supportEmail); + props.setSupportEmail(email.toString()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + } + else + { + // This stores a blank value, not null (which would mean inherit) + props.setSupportEmail(null); + } + }); + + boolean noErrors = !saveFolderSettings(c, user, props, form, errors); + + if (noErrors) + { + // Bump the look & feel revision so browsers retrieve the new theme stylesheet + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + } + + return noErrors; + } + + private static void setProperty(boolean inherited, Runnable clear, Runnable set) + { + if (inherited) + clear.run(); + else + set.run(); + } + + // Same as ProjectSettingsAction, but provides special admin console permissions handling + @AdminConsoleAction(ApplicationAdminPermission.class) + public static class LookAndFeelSettingsAction extends ProjectSettingsAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class UpdateContainerSettingsAction extends MutatingApiAction + { + @Override + public Object execute(FolderSettingsForm form, BindException errors) + { + boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", saved && !errors.hasErrors()); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResourcesAction extends ProjectSettingsViewPostAction + { + @Override + protected JspView getTabView(Object o, boolean reshow, BindException errors) + { + LookAndFeelBean bean = new LookAndFeelBean(); + return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); + } + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + Container c = getContainer(); + Map fileMap = getFileMap(); + + for (ResourceType type : ResourceType.values()) + { + MultipartFile file = fileMap.get(type.name()); + + if (file != null && !file.isEmpty()) + { + try + { + type.save(file, c, getUser()); + } + catch (Exception e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + } + } + + // Note that audit logging happens via the attachment code, so we don't log separately here + + // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + return true; + } + } + + // Same as ResourcesAction, but provides special admin console permissions handling + @AdminConsoleAction + public static class AdminConsoleResourcesAction extends ResourcesAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MenuBarAction extends ProjectSettingsViewAction + { + @Override + protected HttpView getTabView() + { + if (getContainer().isRoot()) + return HtmlView.err("Menu bar must be configured for each project separately."); + + WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); + v.setView("menubar", new VBox()); + Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); + + return v; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FilesAction extends ProjectSettingsViewPostAction + { + @Override + protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) + { + Container c = getContainer(); + + if (c.isRoot()) + return HtmlView.err("Files must be configured for each project separately."); + + if (!reshow || form.isPipelineRootForm()) + { + try + { + AdminController.setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + VBox box = new VBox(); + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + box.addView(view); + + // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root + if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) + { + SetupForm setupForm = SetupForm.init(c); + setupForm.setShowAdditionalOptionsLink(true); + setupForm.setErrors(errors); + PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); + + if (pipeRoot != null) + { + for (String errorMessage : pipeRoot.validate()) + errors.addError(new LabKeyError(errorMessage)); + } + JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); + pipelineView.setTitle("Configure Data Processing Pipeline"); + box.addView(pipelineView); + } + + return box; + } + + @Override + public void validateCommand(FilesForm form, Errors errors) + { + if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) + { + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + File f = new File(root); + if (!f.exists() || !f.isDirectory()) + { + errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + } + } + else + errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); + } + else if (form.isCloudFileRoot()) + { + AdminController.validateCloudFileRoot(form, getContainer(), errors); + } + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) throws Exception + { + FileContentService service = FileContentService.get(); + if (service != null) + { + if (form.isPipelineRootForm()) + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + else + { + AdminController.setFileRootFromForm(getViewContext(), form, errors); + } + } + + // Cloud settings + AdminController.setEnabledCloudStores(getViewContext(), form, errors); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(FilesForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); + if (form.isPipelineRootForm()) + { + url.addParameter("piperootSet", true); + } + else + { + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + } + return url; + } + } + + public static class LookAndFeelView extends JspView + { + LookAndFeelView(BindException errors) + { + super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); + } + } + + + public static class LookAndFeelBean + { + public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); + } + + @RequiresPermission(AdminPermission.class) + public static class AdjustSystemTimestampsAction extends FormViewAction + { + @Override + public void addNavTrail(NavTree root) + { + } + + @Override + public void validateCommand(AdjustTimestampsForm form, Errors errors) + { + if (form.getHourDelta() == null || form.getHourDelta() == 0) + errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); + } + + @Override + public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); + } + + private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) + { + SQLFragment sql = new SQLFragment(); + DbSchema schema = tInfo.getSchema(); + String comma = ""; + List updating = new ArrayList<>(); + + for (String fieldName: fieldNames) + { + ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); + if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) + { + updating.add(fieldName); + if (sql.isEmpty()) + sql.append("UPDATE ").append(tInfo, "").append(" SET "); + sql.append(comma) + .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); + comma = ", "; + } + } + + if (!sql.isEmpty()) + { + logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); + logger.debug(sql.toDebugString()); + int numRows = new SqlExecutor(schema).execute(sql); + logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); + } + } + + @Override + public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception + { + List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); + logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + DbScope scope = DbScope.getLabKeyScope(); + try (DbScope.Transaction t = scope.ensureTransaction()) + { + ModuleLoader.getInstance().getModules().forEach(module -> { + logger.info("==> Beginning update of timestamps for module: " + module.getName()); + module.getSchemaNames().stream().sorted().forEach(schemaName -> { + DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); + scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from + schema.getTableNames().forEach(tableName -> { + TableInfo tInfo = schema.getTable(tableName); + if (tInfo.getTableType() == DatabaseTableType.TABLE) + { + updateFields(tInfo, toUpdate, form.getHourDelta()); + } + }); + }); + logger.info("<== DONE updating timestamps for module: " + module.getName()); + }); + t.commit(); + } + logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + return true; + } + + @Override + public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) + { + return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); + } + } + + public static class AdjustTimestampsForm + { + private Integer hourDelta; + + public Integer getHourDelta() + { + return hourDelta; + } + + public void setHourDelta(Integer hourDelta) + { + this.hourDelta = hourDelta; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ViewUsageStatistics extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Usage Statistics", this.getClass()); + } + } + + private static final URI LABKEY_ORG_REPORT_ACTION; + + static + { + LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); + } + + @RequiresNoPermission + @CSRF(CSRF.Method.NONE) + public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction + { + private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); + + // recent reports, to help avoid log spam + private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); + + @Override + public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + var ret = new JSONObject().put("success", true); + + // fail fast + if (!_log.isWarnEnabled()) + return ret; + + var request = getViewContext().getRequest(); + assert null != request; + + var userAgent = request.getHeader("User-Agent"); + if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) + return ret; + + // NOTE User may be "guest", and will always be guest if being relayed to labkey.org + var jsonObj = form.getJsonObject(); + if (null != jsonObj) + { + JSONObject cspReport = jsonObj.optJSONObject("csp-report"); + if (cspReport != null) + { + String blockedUri = cspReport.optString("blocked-uri", null); + + // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org + if (blockedUri != null && + blockedUri.startsWith("https://labkey.org%2C") && + blockedUri.endsWith("undefined") && + !_log.isDebugEnabled()) + { + return ret; + } + + String urlString = cspReport.optString("document-uri", null); + if (urlString != null) + { + String path = new URLHelper(urlString).deleteParameters().getURIString(); + if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) + { + // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. + boolean forwarded = jsonObj.optBoolean("forwarded", false); + if (!forwarded) + { + User user = getUser(); + String email = null; + // If the user is not logged in, we may still be able to snag the email address from our cookie + if (user.isGuest()) + email = LoginController.getEmailFromCookie(getViewContext().getRequest()); + if (null == email) + email = user.getEmail(); + jsonObj.put("user", email); + String ipAddress = request.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null) + ipAddress = request.getRemoteAddr(); + jsonObj.put("ip", ipAddress); + if (isNotBlank(userAgent)) + jsonObj.put("user-agent", userAgent); + String labkeyVersion = request.getParameter("labkeyVersion"); + if (null != labkeyVersion) + jsonObj.put("labkeyVersion", labkeyVersion); + String cspVersion = request.getParameter("cspVersion"); + if (null != cspVersion) + jsonObj.put("cspVersion", cspVersion); + } + + var jsonStr = jsonObj.toString(2); + _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); + + if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) + { + jsonObj.put("forwarded", true); + + // Create an HttpClient + HttpClient client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + // Create the POST request + HttpRequest remoteRequest = HttpRequest.newBuilder() + .uri(LABKEY_ORG_REPORT_ACTION) + .header("Content-Type", request.getContentType()) // Use whatever the browser set + .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) + .build(); + + // Send the request and get the response + HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); + } + else + { + JSONObject jsonResponse = new JSONObject(response.body()); + boolean success = jsonResponse.optBoolean("success", false); + if (!success) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); + } + } + } + } + } + } + } + return ret; + } + } + + + public static class TestCase extends AbstractActionPermissionTest + { + @Override + @Test + public void testActionPermissions() + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + AdminController controller = new AdminController(); + + // @RequiresPermission(ReadPermission.class) + assertForReadPermission(user, false, + new GetModulesAction(), + new GetFolderTabsAction(), + new ClearDeletedTabFoldersAction() + ); + + // @RequiresPermission(DeletePermission.class) + assertForUpdateOrDeletePermission(user, + new DeleteFolderAction() + ); + + // @RequiresPermission(AdminPermission.class) + assertForAdminPermission(user, + controller.new CustomizeEmailAction(), + controller.new FolderAliasesAction(), + controller.new MoveFolderAction(), + controller.new MoveTabAction(), + controller.new RenameFolderAction(), + controller.new ReorderFoldersAction(), + controller.new ReorderFoldersApiAction(), + controller.new SiteValidationAction(), + new AddTabAction(), + new ConfirmProjectMoveAction(), + new CreateFolderAction(), + new CustomizeMenuAction(), + new DeleteCustomEmailAction(), + new FilesAction(), + new MenuBarAction(), + new ProjectSettingsAction(), + new RenameContainerAction(), + new RenameTabAction(), + new ResetPropertiesAction(), + new ResetQueryStatisticsAction(), + new ResetResourceAction(), + new ResourcesAction(), + new RevertFolderAction(), + new SetFolderPermissionsAction(), + new SetInitialFolderSettingsAction(), + new ShowTabAction() + ); + + //TODO @RequiresPermission(AdminReadPermission.class) + //controller.new TestMothershipReportAction() + + // @RequiresPermission(AdminOperationsPermission.class) + assertForAdminOperationsPermission(ContainerManager.getRoot(), user, + controller.new DbCheckerAction(), + controller.new DeleteModuleAction(), + controller.new DoCheckAction(), + controller.new EmailTestAction(), + controller.new ShowNetworkDriveTestAction(), + controller.new ValidateDomainsAction(), + new OptionalFeatureAction(), + new GetSchemaXmlDocAction(), + new RecreateViewsAction() + ); + + // @AdminConsoleAction + assertForAdminPermission(ContainerManager.getRoot(), user, + controller.new ActionsAction(), + controller.new CachesAction(), + controller.new ConfigureSystemMaintenanceAction(), + controller.new CustomizeSiteAction(), + controller.new DumpHeapAction(), + controller.new ExecutionPlanAction(), + controller.new FolderTypesAction(), + controller.new MemTrackerAction(), + controller.new ModulesAction(), + controller.new QueriesAction(), + controller.new QueryStackTracesAction(), + controller.new ResetErrorMarkAction(), + controller.new ShortURLAdminAction(), + controller.new ShowAllErrorsAction(), + controller.new ShowErrorsSinceMarkAction(), + controller.new ShowPrimaryLogAction(), + controller.new ShowCspReportLogAction(), + controller.new ShowThreadsAction(), + new ExportActionsAction(), + new ExportQueriesAction(), + new MemoryChartAction(), + new ShowAdminAction() + ); + + // @RequiresSiteAdmin + assertForRequiresSiteAdmin(user, + controller.new EnvironmentVariablesAction(), + controller.new SystemMaintenanceAction(), + controller.new SystemPropertiesAction(), + new GetPendingRequestCountAction(), + new InstallCompleteAction(), + new NewInstallSiteSettingsAction() + ); + + assertForTroubleshooterPermission(ContainerManager.getRoot(), user, + controller.new OptionalFeaturesAction(), + controller.new ShowModuleErrorsAction(), + new ModuleStatusAction() + ); + } + } + + public static class SerializationTest extends PipelineJob.TestSerialization + { + static class TestJob extends PipelineJob + { + ImpersonationContext _impersonationContext; + ImpersonationContext _impersonationContext1; + ImpersonationContext _impersonationContext2; + + @Override + public URLHelper getStatusHref() + { + return null; + } + + @Override + public String getDescription() + { + return "Test Job"; + } + } + + @Test + public void testSerialization() + { + TestJob job = new TestJob(); + TestContext ctx = TestContext.get(); + ViewContext viewContext = new ViewContext(); + viewContext.setContainer(ContainerManager.getSharedContainer()); + viewContext.setUser(ctx.getUser()); + RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( + viewContext.getContainer(), viewContext.getUser(), + Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); + job._impersonationContext = factory.getImpersonationContext(); + + try + { + UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + UserManager.getGuestUser(), null); + job._impersonationContext1 = factory1.getImpersonationContext(); + } + catch (Exception e) + { + LOG.error("Invalid user email for impersonating."); + } + + GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); + job._impersonationContext2 = factory2.getImpersonationContext(); + testSerialize(job, LOG); + } + } + + public static class WorkbookDeleteTestCase extends Assert + { + private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; + private static final String TEST_EMAIL = "testDelete@myDomain.com"; + + @Test + public void testWorkbookDelete() throws Exception + { + doCleanup(); + + Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); + Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); + + ValidEmail email = new ValidEmail(TEST_EMAIL); + SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); + User nonAdminUser = newUserStatus.getUser(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, ReaderRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + // User lacks any permission, throw unauthorized for parent and workbook: + HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Grant permission, should be able to delete the workbook but not parent: + policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, EditorRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Hitting delete action results in a redirect: + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); + + doCleanup(); + } + + protected static void doCleanup() throws Exception + { + Container project = ContainerManager.getForPath(FOLDER_NAME); + if (project != null) + { + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) + { + User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); + UserManager.deleteUser(u.getUserId()); + } + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunInput.java b/experiment/src/org/labkey/experiment/api/AbstractRunInput.java index 11a1cda4b3c..43349a47470 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunInput.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunInput.java @@ -1,124 +1,125 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment.api; - -import org.jetbrains.annotations.Nullable; -import org.labkey.api.exp.IdentifiableBase; -import org.labkey.api.exp.Lsid; - -import java.util.Objects; - -/** - * Base class for the beans that wire up material and data objects to be inputs to a protocol application. - * User: jeckels - * Date: Oct 31, 2008 - */ -public abstract class AbstractRunInput extends IdentifiableBase -{ - private final String _defaultRole; - - private long _targetApplicationId; - protected String _role; - protected Long _protocolInputId; - - protected AbstractRunInput(String defaultRole) - { - _defaultRole = defaultRole; - } - - public long getTargetApplicationId() - { - return _targetApplicationId; - } - - public void setTargetApplicationId(long targetApplicationId) - { - _targetApplicationId = targetApplicationId; - } - - public String getRole() - { - return _role; - } - - public void setRole(@Nullable String role) - { - if (role == null) - { - role = _defaultRole; - } - // Issue 17590. For a while, we've had a bug with exporting material role names in XARs. We exported the - // material name instead of the role name itself. The material names can be longer than the role names, - // so truncate here if needed to prevent a SQLException later - if (role.length() > 50) - { - role = role.substring(0, 49); - } - _role = role; - } - - public Long getProtocolInputId() - { - return _protocolInputId; - } - - public void setProtocolInputId(Long protocolInputId) - { - _protocolInputId = protocolInputId; - } - - protected abstract long getInputKey(); - - - protected static String lsid(String namespace, long inputKey, long targetApplicationId) - { - if (targetApplicationId == 0 || inputKey == 0) - throw new IllegalStateException("LSID requires targetApplicationId and input id"); - Lsid lsid = new Lsid(namespace, inputKey + "." + targetApplicationId); - return lsid.toString(); - } - - - @Override - public void setLSID(String lsid) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AbstractRunInput input = (AbstractRunInput) o; - - return getInputKey() == input.getInputKey() && - Objects.equals(_role, input._role) && - _targetApplicationId == input._targetApplicationId && - Objects.equals(_protocolInputId, input._protocolInputId); - } - - @Override - public int hashCode() - { - int result = (int)getInputKey(); - result = 31 * result + (int)_targetApplicationId; - result = 31 * result + (_role == null ? 0 : _role.hashCode()); - result = 31 * result + (_protocolInputId == null ? 0 : _protocolInputId.hashCode()); - return result; - } -} +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment.api; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.exp.IdentifiableBase; +import org.labkey.api.exp.Lsid; +import org.labkey.api.util.StringUtilsLabKey; + +import java.util.Objects; + +/** + * Base class for the beans that wire up material and data objects to be inputs to a protocol application. + * User: jeckels + * Date: Oct 31, 2008 + */ +public abstract class AbstractRunInput extends IdentifiableBase +{ + private final String _defaultRole; + + private long _targetApplicationId; + protected String _role; + protected Long _protocolInputId; + + protected AbstractRunInput(String defaultRole) + { + _defaultRole = defaultRole; + } + + public long getTargetApplicationId() + { + return _targetApplicationId; + } + + public void setTargetApplicationId(long targetApplicationId) + { + _targetApplicationId = targetApplicationId; + } + + public String getRole() + { + return _role; + } + + public void setRole(@Nullable String role) + { + if (role == null) + { + role = _defaultRole; + } + // Issue 17590. For a while, we've had a bug with exporting material role names in XARs. We exported the + // material name instead of the role name itself. The material names can be longer than the role names, + // so truncate here if needed to prevent a SQLException later + if (role.length() > 50) + { + role = StringUtilsLabKey.leftSurrogatePairFriendly(role, 49); + } + _role = role; + } + + public Long getProtocolInputId() + { + return _protocolInputId; + } + + public void setProtocolInputId(Long protocolInputId) + { + _protocolInputId = protocolInputId; + } + + protected abstract long getInputKey(); + + + protected static String lsid(String namespace, long inputKey, long targetApplicationId) + { + if (targetApplicationId == 0 || inputKey == 0) + throw new IllegalStateException("LSID requires targetApplicationId and input id"); + Lsid lsid = new Lsid(namespace, inputKey + "." + targetApplicationId); + return lsid.toString(); + } + + + @Override + public void setLSID(String lsid) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AbstractRunInput input = (AbstractRunInput) o; + + return getInputKey() == input.getInputKey() && + Objects.equals(_role, input._role) && + _targetApplicationId == input._targetApplicationId && + Objects.equals(_protocolInputId, input._protocolInputId); + } + + @Override + public int hashCode() + { + int result = (int)getInputKey(); + result = 31 * result + (int)_targetApplicationId; + result = 31 * result + (_role == null ? 0 : _role.hashCode()); + result = 31 * result + (_protocolInputId == null ? 0 : _protocolInputId.hashCode()); + return result; + } +} diff --git a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java index 79de048c200..a59e474df7c 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java @@ -1,977 +1,978 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.ExperimentDataHandler; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Handler; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.XarFormatException; -import org.labkey.api.exp.XarSource; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.query.ExpDataClassDataTable; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryRowReference; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchResultTemplate; -import org.labkey.api.search.SearchScope; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DataClassReadPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.MediaReadPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.MimeMap; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.InputBuilder; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.ViewContext; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.labkey.api.exp.query.ExpSchema.SCHEMA_EXP_DATA; - -public class ExpDataImpl extends AbstractRunItemImpl implements ExpData -{ - public enum DataOperations - { - Edit("editing", UpdatePermission.class), - EditLineage("editing lineage", UpdatePermission.class), - Delete("deleting", DeletePermission.class), - Move("moving", MoveEntitiesPermission.class); - - private final String _description; // used as a suffix in messaging users about what is not allowed - private final Class _permissionClass; - - DataOperations(String description, Class permissionClass) - { - _description = description; - _permissionClass = permissionClass; - } - - public String getDescription() - { - return _description; - } - - public Class getPermissionClass() - { - return _permissionClass; - } - } - - public static final SearchService.SearchCategory expDataCategory = new SearchService.SearchCategory("data", "ExpData", false) { - @Override - public Set getPermittedContainerIds(User user, Map containers) - { - return getPermittedContainerIds(user, containers, DataClassReadPermission.class); - } - }; - public static final SearchService.SearchCategory expMediaDataCategory = new SearchService.SearchCategory("mediaData", "ExpData for media objects", false) { - @Override - public Set getPermittedContainerIds(User user, Map containers) - { - return getPermittedContainerIds(user, containers, MediaReadPermission.class); - } - }; - - /** Cache this because it can be expensive to recompute */ - private Boolean _finalRunOutput; - - /** - * Temporary mapping until experiment.xml contains the mime type - */ - private static final MimeMap MIME_MAP = new MimeMap(); - - static public List fromDatas(List datas) - { - List ret = new ArrayList<>(datas.size()); - for (Data data : datas) - { - ret.add(new ExpDataImpl(data)); - } - return ret; - } - - // For serialization - protected ExpDataImpl() {} - - public ExpDataImpl(Data data) - { - super(data); - } - - @Override - public void setComment(User user, String comment) throws ValidationException - { - setComment(user, comment, true); - } - - @Override - public void setComment(User user, String comment, boolean index) throws ValidationException - { - super.setComment(user, comment); - - if (index) - index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); - } - - @Override - @Nullable - public ActionURL detailsURL() - { - DataType dataType = getDataType(); - if (dataType != null) - { - ActionURL url = dataType.getDetailsURL(this); - if (url != null) - return url; - } - - return _object.detailsURL(); - } - - @Override - public @Nullable QueryRowReference getQueryRowReference() - { - return getQueryRowReference(null); - } - - @Override - public @Nullable QueryRowReference getQueryRowReference(@Nullable User user) - { - ExpDataClassImpl dc = getDataClass(user); - if (dc != null) - return new QueryRowReference(getContainer(), SCHEMA_EXP_DATA, dc.getName(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); - - // Issue 40123: see MedImmuneDataHandler MEDIMMUNE_DATA_TYPE, this claims the "Data" namespace - DataType type = getDataType(); - if (type != null) - { - QueryRowReference queryRowReference = type.getQueryRowReference(this); - if (queryRowReference != null) - return queryRowReference; - } - - return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Data.name(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); - } - - @Override - public List getTargetApplications() - { - return getTargetApplications(new SimpleFilter(FieldKey.fromParts("DataId"), getRowId()), ExperimentServiceImpl.get().getTinfoDataInput()); - } - - @Override - public List getTargetRuns() - { - return getTargetRuns(ExperimentServiceImpl.get().getTinfoDataInput(), "DataId"); - } - - @Override - public DataType getDataType() - { - return ExperimentService.get().getDataType(getLSIDNamespacePrefix()); - } - - @Override - public void setDataFileURI(URI uri) - { - ensureUnlocked(); - _object.setDataFileUrl(ExpData.normalizeDataFileURI(uri)); - } - - @Override - public void save(User user) - { - // Replace the default "Data" cpastype if the Data belongs to a DataClass - ExpDataClassImpl dataClass = getDataClass(); - if (dataClass != null && ExpData.DEFAULT_CPAS_TYPE.equals(getCpasType())) - setCpasType(dataClass.getLSID()); - - boolean isNew = getRowId() == 0; - save(user, ExperimentServiceImpl.get().getTinfoData(), true); - - if (isNew) - { - if (dataClass != null) - { - Map map = new HashMap<>(); - map.put("lsid", getLSID()); - Table.insert(user, dataClass.getTinfo(), map); - } - } - index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); - } - - @Override - protected void save(User user, TableInfo table, boolean ensureObject) - { - assert ensureObject; - super.save(user, table, true); - } - - @Override - public URI getDataFileURI() - { - String url = _object.getDataFileUrl(); - if (url == null) - return null; - try - { - return new URI(_object.getDataFileUrl()); - } - catch (URISyntaxException use) - { - return null; - } - } - - @Override - public ExperimentDataHandler findDataHandler() - { - return Handler.Priority.findBestHandler(ExperimentServiceImpl.get().getExperimentDataHandlers(), this); - } - - @Override - public String getDataFileUrl() - { - return _object.getDataFileUrl(); - } - - @Override - public boolean hasFileScheme() - { - return !FileUtil.hasCloudScheme(getDataFileUrl()); - } - - @Override - @Nullable - public File getFile() - { - return _object.getFile(); - } - - @Override - public @Nullable FileLike getFileLike() - { - return _object.getFileLike(); - } - - @Override - @Nullable - public java.nio.file.Path getFilePath() - { - return _object.getFilePath(); - } - - @Override - public boolean isInlineImage() - { - return null != getFile() && MIME_MAP.isInlineImageFor(getFile()); - } - - @Override - public void delete(User user) - { - delete(user, true); - } - - @Override - public void delete(User user, boolean deleteRunsUsingData) - { - ExperimentServiceImpl.get().deleteDataByRowIds(user, getContainer(), Collections.singleton(getRowId()), deleteRunsUsingData); - } - - public String getMimeType() - { - if (null != getDataFileUrl()) - return MIME_MAP.getContentTypeFor(getDataFileUrl()); - else - return null; - } - - @Override - public boolean isFileOnDisk() - { - java.nio.file.Path f = getFilePath(); - if (f != null) - if (!FileUtil.hasCloudScheme(f)) - return NetworkDrive.exists(f.toFile()) && !Files.isDirectory(f); - else - return Files.exists(f); - else - return false; - } - - public boolean isPathAccessible() - { - java.nio.file.Path path = getFilePath(); - return (null != path && Files.exists(path)); - } - - @Override - public String getCpasType() - { - String result = _object.getCpasType(); - if (result != null) - return result; - - ExpDataClass dataClass = getDataClass(); - if (dataClass != null) - return dataClass.getLSID(); - - return ExpData.DEFAULT_CPAS_TYPE; - } - - public void setGenerated(boolean generated) - { - ensureUnlocked(); - _object.setGenerated(generated); - } - - @Override - public boolean isGenerated() - { - return _object.isGenerated(); - } - - @Override - public boolean isFinalRunOutput() - { - if (_finalRunOutput == null) - { - ExpRun run = getRun(); - _finalRunOutput = run != null && run.isFinalOutput(this); - } - return _finalRunOutput.booleanValue(); - } - - @Override - @Nullable - public ExpDataClassImpl getDataClass() - { - return getDataClass(null); - } - - @Override - @Nullable - public ExpDataClassImpl getDataClass(@Nullable User user) - { - if (_object.getClassId() != null && getContainer() != null) - { - if (user == null) - return ExperimentServiceImpl.get().getDataClass(getContainer(), _object.getClassId()); - else - return ExperimentServiceImpl.get().getDataClass(getContainer(), user, _object.getClassId()); - } - - return null; - } - - @Override - public void importDataFile(PipelineJob job, XarSource xarSource) throws ExperimentException - { - String dataFileURL = getDataFileUrl(); - if (dataFileURL == null) - return; - - if (xarSource.shouldIgnoreDataFiles()) - { - job.debug("Skipping load of data file " + dataFileURL + " based on the XAR source"); - return; - } - - job.debug("Trying to load data file " + dataFileURL + " into the system"); - - java.nio.file.Path path = FileUtil.stringToPath(getContainer(), dataFileURL); - - if (!Files.exists(path)) - { - job.debug("Unable to find the data file " + FileUtil.getAbsolutePath(getContainer(), path) + " on disk."); - return; - } - - // Check that the file is under the pipeline root to prevent users from referencing a file that they - // don't have permission to import - PipeRoot pr = PipelineService.get().findPipelineRoot(job.getContainer()); - if (!xarSource.allowImport(pr, job.getContainer(), path)) - { - if (pr == null) - { - job.warn("No pipeline root was set, skipping load of file " + FileUtil.getAbsolutePath(getContainer(), path)); - return; - } - job.debug("The data file " + FileUtil.getAbsolutePath(getContainer(), path) + " is not under the folder's pipeline root: " + pr + ". It will not be loaded directly, but may be loaded if referenced from other files that are under the pipeline root."); - return; - } - - ExperimentDataHandler handler = findDataHandler(); - try - { - handler.importFile(this, FileSystemLike.wrapFile(path), job.getInfo(), job.getLogger(), xarSource.getXarContext()); - } - catch (ExperimentException e) - { - throw new XarFormatException(e); - } - - job.debug("Finished trying to load data file " + dataFileURL + " into the system"); - } - - // Get all text and int strings from the data class for indexing - private void getIndexValues( - Map props, - @NotNull ExpDataClassDataTableImpl table, - Set identifiersHi, - Set identifiersMed, - Set identifiersLo, - Set keywordHi, - Set keywordMed, - Set keywordsLo, - JSONObject jsonData - ) - { - CaseInsensitiveHashSet skipColumns = new CaseInsensitiveHashSet(); - for (ExpDataClassDataTable.Column column : ExpDataClassDataTable.Column.values()) - skipColumns.add(column.name()); - skipColumns.add("Ancestors"); - skipColumns.add("Container"); - - processIndexValues(props, table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); - } - - @Override - @NotNull - public Collection getAliases() - { - TableInfo mapTi = ExperimentService.get().getTinfoDataAliasMap(); - TableInfo ti = ExperimentService.get().getTinfoAlias(); - SQLFragment sql = new SQLFragment() - .append("SELECT a.name FROM ").append(mapTi, "m") - .append(" JOIN ").append(ti, "a") - .append(" ON m.alias = a.RowId WHERE m.lsid = ? "); - sql.add(getLSID()); - ArrayList aliases = new SqlSelector(mapTi.getSchema(), sql).getArrayList(String.class); - return Collections.unmodifiableList(aliases); - } - - @Override - public String getDocumentId() - { - String dataClassName = "-"; - ExpDataClass dc = getDataClass(); - if (dc != null) - dataClassName = dc.getName(); - // why not just data:rowId? - return "data:" + new Path(getContainer().getId(), dataClassName, Long.toString(getRowId())).encode(); - } - - @Override - public Map getObjectProperties() - { - return getObjectProperties(getDataClass()); - } - - @Override - public Map getObjectProperties(@Nullable User user) - { - return getObjectProperties(getDataClass(user)); - } - - private Map getObjectProperties(ExpDataClassImpl dataClass) - { - HashMap ret = new HashMap<>(super.getObjectProperties()); - var ti = null == dataClass ? null : dataClass.getTinfo(); - if (null != ti) - { - ret.putAll(getObjectProperties(ti)); - } - return ret; - } - - private static Pair getRowIdClassNameContainerFromDocumentId(String resourceIdentifier, Map dcCache) - { - if (resourceIdentifier.startsWith("data:")) - resourceIdentifier = resourceIdentifier.substring("data:".length()); - - Path path = Path.parse(resourceIdentifier); - if (path.size() != 3) - return null; - String containerId = path.get(0); - String dataClassName = path.get(1); - String rowIdString = path.get(2); - - long rowId; - try - { - rowId = Long.parseLong(rowIdString); - if (rowId == 0) - return null; - } - catch (NumberFormatException ex) - { - return null; - } - - Container c = ContainerManager.getForId(containerId); - if (c == null) - return null; - - ExpDataClass dc = null; - if (!StringUtils.isEmpty(dataClassName) && !dataClassName.equals("-")) - { - String dcKey = containerId + '-' + dataClassName; - dc = dcCache.computeIfAbsent(dcKey, (x) -> ExperimentServiceImpl.get().getDataClass(c, dataClassName)); - } - - return new Pair<>(rowId, dc); - } - - @Nullable - public static ExpDataImpl fromDocumentId(String resourceIdentifier) - { - Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, new HashMap<>()); - if (rowIdDataClass == null) - return null; - - Long rowId = rowIdDataClass.first; - ExpDataClass dc = rowIdDataClass.second; - - if (dc != null) - return ExperimentServiceImpl.get().getExpData(dc, rowId); - else - return ExperimentServiceImpl.get().getExpData(rowId); - } - - @Nullable - public static Map fromDocumentIds(Collection resourceIdentifiers) - { - Map rowIdIdentifierMap = new LongHashMap<>(); - Map dcCache = new HashMap<>(); - Map dcMap = new LongHashMap<>(); - Map> dcRowIdMap = new LongHashMap<>(); // data rowIds with dataClass - List rowIds = new ArrayList<>(); // data rowIds without dataClass - for (String resourceIdentifier : resourceIdentifiers) - { - Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, dcCache); - if (rowIdDataClass == null) - continue; - - Long rowId = rowIdDataClass.first; - ExpDataClass dc = rowIdDataClass.second; - - rowIdIdentifierMap.put(rowId, resourceIdentifier); - - if (dc != null) - { - dcMap.put(dc.getRowId(), dc); - dcRowIdMap - .computeIfAbsent(dc.getRowId(), (k) -> new ArrayList<>()) - .add(rowId); - } - else - rowIds.add(rowId); - } - - List expDatas = new ArrayList<>(); - if (!rowIds.isEmpty()) - expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(rowIds)); - - if (!dcRowIdMap.isEmpty()) - { - for (Long dataClassId : dcRowIdMap.keySet()) - { - ExpDataClass dc = dcMap.get(dataClassId); - if (dc != null) - expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(dc, dcRowIdMap.get(dataClassId))); - } - } - - Map identifierDatas = new HashMap<>(); - for (ExpData data : expDatas) - { - identifierDatas.put(rowIdIdentifierMap.get(data.getRowId()), data); - } - - return identifierDatas; - } - - @Override - public @Nullable URI getWebDavURL(@NotNull FileContentService.PathType type) - { - java.nio.file.Path path = getFilePath(); - if (path == null) - { - return null; - } - - Container c = getContainer(); - if (c == null) - { - return null; - } - - return FileContentService.get().getWebDavUrl(path, c, type); - } - - @Override - public @Nullable WebdavResource createIndexDocument(@Nullable TableInfo tableInfo) - { - Container container = getContainer(); - if (container == null) - return null; - - Map props = new HashMap<>(); - JSONObject jsonData = new JSONObject(); - Set keywordsHi = new HashSet<>(); - Set keywordsMed = new HashSet<>(); - Set keywordsLo = new HashSet<>(); - - Set identifiersHi = new HashSet<>(); - Set identifiersMed = new HashSet<>(); - Set identifiersLo = new HashSet<>(); - - StringBuilder body = new StringBuilder(); - - // Name is an identifier with the highest weight - identifiersHi.add(getName()); - keywordsMed.add(getName()); // also add to keywords since those are stemmed - - // Description is added as a keywordsLo -- in Biologics it is common for the description to - // contain names of other DataClasses, e.g., "Mature desK of PS-10", which would be tokenized as - // [mature, desk, ps, 10] if added it as a keyword so we lower its priority to avoid useless results. - // CONSIDER: tokenize the description and extract identifiers - if (null != getDescription()) - keywordsLo.add(getDescription()); - - String comment = getComment(); - if (comment != null) - keywordsMed.add(comment); - - // Add aliases in parentheses in the title - StringBuilder title = new StringBuilder(getName()); - Collection aliases = getAliases(); - if (!aliases.isEmpty()) - { - title.append(" (").append(StringUtils.join(aliases, ", ")).append(")"); - identifiersHi.addAll(aliases); - } - - ExpDataClassImpl dc = getDataClass(User.getSearchUser()); - if (dc != null) - { - ActionURL show = new ActionURL(ExperimentController.ShowDataClassAction.class, container).addParameter("rowId", dc.getRowId()); - NavTree t = new NavTree(dc.getName(), show); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - props.put(SearchService.PROPERTY.navtrail.toString(), nav); - - props.put(DataSearchResultTemplate.PROPERTY, dc.getName()); - body.append(dc.getName()); - - if (tableInfo == null) - tableInfo = QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_EXP_DATA).getTable(dc.getName()); - - if (!(tableInfo instanceof ExpDataClassDataTableImpl expDataClassDataTable)) - throw new IllegalArgumentException(String.format("Unable to index data class item in %s. Table must be an instance of %s", dc.getName(), ExpDataClassDataTableImpl.class.getName())); - - if (!expDataClassDataTable.getDataClass().equals(dc)) - throw new IllegalArgumentException(String.format("Data class table mismatch for %s", dc.getName())); - - // Collect other text columns and lookup display columns - getIndexValues(props, expDataClassDataTable, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); - } - - // === Stored, not indexed - if (dc != null && dc.isMedia()) - props.put(SearchService.PROPERTY.categories.toString(), expMediaDataCategory.toString()); - else - props.put(SearchService.PROPERTY.categories.toString(), expDataCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), title.toString()); - props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); - - ActionURL view = ExperimentController.ExperimentUrlsImpl.get().getDataDetailsURL(this); - view.setExtraPath(container.getId()); - String docId = getDocumentId(); - - // Generate a summary explicitly instead of relying on a summary to be extracted - // from the document body. Placing lookup values and the description in the body - // would tokenize using the English analyzer and index "PS-12" as ["ps", "12"] which leads to poor results. - StringBuilder summary = new StringBuilder(); - if (StringUtils.isNotEmpty(getDescription())) - summary.append(getDescription()).append("\n"); - - appendTokens(summary, keywordsMed); - appendTokens(summary, identifiersMed); - appendTokens(summary, identifiersLo); - - props.put(SearchService.PROPERTY.summary.toString(), summary); - - return new ExpDataResource( - getRowId(), - new Path(docId), - docId, - container.getEntityId(), - "text/plain", - body.toString(), - view, - props, - getCreatedBy(), - getCreated(), - getModifiedBy(), - getModified() - ); - } - - private static void appendTokens(StringBuilder sb, Collection toks) - { - if (toks.isEmpty()) - return; - - sb.append(toks.stream().map(s -> s.length() > 30 ? s.substring(0, 30) + "\u2026" : s).collect(Collectors.joining(", "))).append("\n"); - } - - private static class ExpDataResource extends SimpleDocumentResource - { - final long _rowId; - - public ExpDataResource(long rowId, Path path, String documentId, GUID containerId, String contentType, String body, URLHelper executeUrl, Map properties, User createdBy, Date created, User modifiedBy, Date modified) - { - super(path, documentId, containerId, contentType, body, executeUrl, createdBy, created, modifiedBy, modified, properties); - _rowId = rowId; - } - - @Override - public void setLastIndexed(long ms, long modified) - { - ExperimentServiceImpl.get().setDataLastIndexed(_rowId, ms); - } - } - - public static class DataSearchResultTemplate implements SearchResultTemplate - { - public static final String NAME = "data"; - public static final String PROPERTY = "dataclass"; - - @Nullable - @Override - public String getName() - { - return NAME; - } - - private ExpDataClass getDataClass() - { - if (HttpView.hasCurrentView()) - { - ViewContext ctx = HttpView.currentContext(); - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - return ExperimentService.get().getDataClass(ctx.getContainer(), ctx.getUser(), dataclass); - } - return null; - } - - @Nullable - @Override - public String getCategories() - { - ExpDataClass dataClass = getDataClass(); - - if (dataClass != null && dataClass.isMedia()) - return expMediaDataCategory.getName(); - - return expDataCategory.getName(); - } - - @Nullable - @Override - public SearchScope getSearchScope() - { - return SearchScope.FolderAndSubfolders; - } - - @NotNull - @Override - public String getResultNameSingular() - { - ExpDataClass dc = getDataClass(); - if (dc != null) - return dc.getName(); - return "data"; - } - - @NotNull - @Override - public String getResultNamePlural() - { - return getResultNameSingular(); - } - - @Override - public boolean includeNavigationLinks() - { - return true; - } - - @Override - public boolean includeAdvanceUI() - { - return false; - } - - @Nullable - @Override - public HtmlString getExtraHtml(ViewContext ctx) - { - String q = ctx.getActionURL().getParameter("q"); - - if (StringUtils.isNotBlank(q)) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - ActionURL url = ctx.cloneActionURL().deleteParameter(PROPERTY); - url.replaceParameter(ActionURL.Param._dc, (int)Math.round(1000 * Math.random())); - - StringBuilder html = new StringBuilder(); - html.append("
    "); - - appendParam(html, null, dataclass, "All", false, url); - for (ExpDataClass dc : ExperimentService.get().getDataClasses(ctx.getContainer(), ctx.getUser(), true)) - { - appendParam(html, dc.getName(), dataclass, dc.getName(), true, url); - } - - html.append("
    "); - return HtmlString.unsafe(html.toString()); - } - else - { - return null; - } - } - - private void appendParam(StringBuilder sb, @Nullable String dataclass, @Nullable String current, @NotNull String label, boolean addParam, ActionURL url) - { - sb.append(""); - - if (!Objects.equals(dataclass, current)) - { - if (addParam) - url = url.clone().addParameter(PROPERTY, dataclass); - - sb.append(LinkBuilder.simpleLink(label, url)); - } - else - { - sb.append(label); - } - - sb.append(" "); - } - - @Override - public HtmlString getHiddenInputsHtml(ViewContext ctx) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - { - return InputBuilder.hidden().id("search-type").name(PROPERTY).value(dataclass).getHtmlString(); - } - - return null; - } - - - @Override - public String reviseQuery(ViewContext ctx, String q) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - - if (null != dataclass) - return "+(" + q + ") +" + PROPERTY + ":" + dataclass; - else - return q; - } - - @Override - public void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) - { - SearchResultTemplate.super.addNavTrail(root, ctx, scope, category); - - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - { - String text = root.getText(); - root.setText(text + " - " + dataclass); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.ExperimentDataHandler; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Handler; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.XarFormatException; +import org.labkey.api.exp.XarSource; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataClassDataTable; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryRowReference; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchResultTemplate; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DataClassReadPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MediaReadPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.MimeMap; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.InputBuilder; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.ViewContext; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.labkey.api.exp.query.ExpSchema.SCHEMA_EXP_DATA; + +public class ExpDataImpl extends AbstractRunItemImpl implements ExpData +{ + public enum DataOperations + { + Edit("editing", UpdatePermission.class), + EditLineage("editing lineage", UpdatePermission.class), + Delete("deleting", DeletePermission.class), + Move("moving", MoveEntitiesPermission.class); + + private final String _description; // used as a suffix in messaging users about what is not allowed + private final Class _permissionClass; + + DataOperations(String description, Class permissionClass) + { + _description = description; + _permissionClass = permissionClass; + } + + public String getDescription() + { + return _description; + } + + public Class getPermissionClass() + { + return _permissionClass; + } + } + + public static final SearchService.SearchCategory expDataCategory = new SearchService.SearchCategory("data", "ExpData", false) { + @Override + public Set getPermittedContainerIds(User user, Map containers) + { + return getPermittedContainerIds(user, containers, DataClassReadPermission.class); + } + }; + public static final SearchService.SearchCategory expMediaDataCategory = new SearchService.SearchCategory("mediaData", "ExpData for media objects", false) { + @Override + public Set getPermittedContainerIds(User user, Map containers) + { + return getPermittedContainerIds(user, containers, MediaReadPermission.class); + } + }; + + /** Cache this because it can be expensive to recompute */ + private Boolean _finalRunOutput; + + /** + * Temporary mapping until experiment.xml contains the mime type + */ + private static final MimeMap MIME_MAP = new MimeMap(); + + static public List fromDatas(List datas) + { + List ret = new ArrayList<>(datas.size()); + for (Data data : datas) + { + ret.add(new ExpDataImpl(data)); + } + return ret; + } + + // For serialization + protected ExpDataImpl() {} + + public ExpDataImpl(Data data) + { + super(data); + } + + @Override + public void setComment(User user, String comment) throws ValidationException + { + setComment(user, comment, true); + } + + @Override + public void setComment(User user, String comment, boolean index) throws ValidationException + { + super.setComment(user, comment); + + if (index) + index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); + } + + @Override + @Nullable + public ActionURL detailsURL() + { + DataType dataType = getDataType(); + if (dataType != null) + { + ActionURL url = dataType.getDetailsURL(this); + if (url != null) + return url; + } + + return _object.detailsURL(); + } + + @Override + public @Nullable QueryRowReference getQueryRowReference() + { + return getQueryRowReference(null); + } + + @Override + public @Nullable QueryRowReference getQueryRowReference(@Nullable User user) + { + ExpDataClassImpl dc = getDataClass(user); + if (dc != null) + return new QueryRowReference(getContainer(), SCHEMA_EXP_DATA, dc.getName(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + + // Issue 40123: see MedImmuneDataHandler MEDIMMUNE_DATA_TYPE, this claims the "Data" namespace + DataType type = getDataType(); + if (type != null) + { + QueryRowReference queryRowReference = type.getQueryRowReference(this); + if (queryRowReference != null) + return queryRowReference; + } + + return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Data.name(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + } + + @Override + public List getTargetApplications() + { + return getTargetApplications(new SimpleFilter(FieldKey.fromParts("DataId"), getRowId()), ExperimentServiceImpl.get().getTinfoDataInput()); + } + + @Override + public List getTargetRuns() + { + return getTargetRuns(ExperimentServiceImpl.get().getTinfoDataInput(), "DataId"); + } + + @Override + public DataType getDataType() + { + return ExperimentService.get().getDataType(getLSIDNamespacePrefix()); + } + + @Override + public void setDataFileURI(URI uri) + { + ensureUnlocked(); + _object.setDataFileUrl(ExpData.normalizeDataFileURI(uri)); + } + + @Override + public void save(User user) + { + // Replace the default "Data" cpastype if the Data belongs to a DataClass + ExpDataClassImpl dataClass = getDataClass(); + if (dataClass != null && ExpData.DEFAULT_CPAS_TYPE.equals(getCpasType())) + setCpasType(dataClass.getLSID()); + + boolean isNew = getRowId() == 0; + save(user, ExperimentServiceImpl.get().getTinfoData(), true); + + if (isNew) + { + if (dataClass != null) + { + Map map = new HashMap<>(); + map.put("lsid", getLSID()); + Table.insert(user, dataClass.getTinfo(), map); + } + } + index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); + } + + @Override + protected void save(User user, TableInfo table, boolean ensureObject) + { + assert ensureObject; + super.save(user, table, true); + } + + @Override + public URI getDataFileURI() + { + String url = _object.getDataFileUrl(); + if (url == null) + return null; + try + { + return new URI(_object.getDataFileUrl()); + } + catch (URISyntaxException use) + { + return null; + } + } + + @Override + public ExperimentDataHandler findDataHandler() + { + return Handler.Priority.findBestHandler(ExperimentServiceImpl.get().getExperimentDataHandlers(), this); + } + + @Override + public String getDataFileUrl() + { + return _object.getDataFileUrl(); + } + + @Override + public boolean hasFileScheme() + { + return !FileUtil.hasCloudScheme(getDataFileUrl()); + } + + @Override + @Nullable + public File getFile() + { + return _object.getFile(); + } + + @Override + public @Nullable FileLike getFileLike() + { + return _object.getFileLike(); + } + + @Override + @Nullable + public java.nio.file.Path getFilePath() + { + return _object.getFilePath(); + } + + @Override + public boolean isInlineImage() + { + return null != getFile() && MIME_MAP.isInlineImageFor(getFile()); + } + + @Override + public void delete(User user) + { + delete(user, true); + } + + @Override + public void delete(User user, boolean deleteRunsUsingData) + { + ExperimentServiceImpl.get().deleteDataByRowIds(user, getContainer(), Collections.singleton(getRowId()), deleteRunsUsingData); + } + + public String getMimeType() + { + if (null != getDataFileUrl()) + return MIME_MAP.getContentTypeFor(getDataFileUrl()); + else + return null; + } + + @Override + public boolean isFileOnDisk() + { + java.nio.file.Path f = getFilePath(); + if (f != null) + if (!FileUtil.hasCloudScheme(f)) + return NetworkDrive.exists(f.toFile()) && !Files.isDirectory(f); + else + return Files.exists(f); + else + return false; + } + + public boolean isPathAccessible() + { + java.nio.file.Path path = getFilePath(); + return (null != path && Files.exists(path)); + } + + @Override + public String getCpasType() + { + String result = _object.getCpasType(); + if (result != null) + return result; + + ExpDataClass dataClass = getDataClass(); + if (dataClass != null) + return dataClass.getLSID(); + + return ExpData.DEFAULT_CPAS_TYPE; + } + + public void setGenerated(boolean generated) + { + ensureUnlocked(); + _object.setGenerated(generated); + } + + @Override + public boolean isGenerated() + { + return _object.isGenerated(); + } + + @Override + public boolean isFinalRunOutput() + { + if (_finalRunOutput == null) + { + ExpRun run = getRun(); + _finalRunOutput = run != null && run.isFinalOutput(this); + } + return _finalRunOutput.booleanValue(); + } + + @Override + @Nullable + public ExpDataClassImpl getDataClass() + { + return getDataClass(null); + } + + @Override + @Nullable + public ExpDataClassImpl getDataClass(@Nullable User user) + { + if (_object.getClassId() != null && getContainer() != null) + { + if (user == null) + return ExperimentServiceImpl.get().getDataClass(getContainer(), _object.getClassId()); + else + return ExperimentServiceImpl.get().getDataClass(getContainer(), user, _object.getClassId()); + } + + return null; + } + + @Override + public void importDataFile(PipelineJob job, XarSource xarSource) throws ExperimentException + { + String dataFileURL = getDataFileUrl(); + if (dataFileURL == null) + return; + + if (xarSource.shouldIgnoreDataFiles()) + { + job.debug("Skipping load of data file " + dataFileURL + " based on the XAR source"); + return; + } + + job.debug("Trying to load data file " + dataFileURL + " into the system"); + + java.nio.file.Path path = FileUtil.stringToPath(getContainer(), dataFileURL); + + if (!Files.exists(path)) + { + job.debug("Unable to find the data file " + FileUtil.getAbsolutePath(getContainer(), path) + " on disk."); + return; + } + + // Check that the file is under the pipeline root to prevent users from referencing a file that they + // don't have permission to import + PipeRoot pr = PipelineService.get().findPipelineRoot(job.getContainer()); + if (!xarSource.allowImport(pr, job.getContainer(), path)) + { + if (pr == null) + { + job.warn("No pipeline root was set, skipping load of file " + FileUtil.getAbsolutePath(getContainer(), path)); + return; + } + job.debug("The data file " + FileUtil.getAbsolutePath(getContainer(), path) + " is not under the folder's pipeline root: " + pr + ". It will not be loaded directly, but may be loaded if referenced from other files that are under the pipeline root."); + return; + } + + ExperimentDataHandler handler = findDataHandler(); + try + { + handler.importFile(this, FileSystemLike.wrapFile(path), job.getInfo(), job.getLogger(), xarSource.getXarContext()); + } + catch (ExperimentException e) + { + throw new XarFormatException(e); + } + + job.debug("Finished trying to load data file " + dataFileURL + " into the system"); + } + + // Get all text and int strings from the data class for indexing + private void getIndexValues( + Map props, + @NotNull ExpDataClassDataTableImpl table, + Set identifiersHi, + Set identifiersMed, + Set identifiersLo, + Set keywordHi, + Set keywordMed, + Set keywordsLo, + JSONObject jsonData + ) + { + CaseInsensitiveHashSet skipColumns = new CaseInsensitiveHashSet(); + for (ExpDataClassDataTable.Column column : ExpDataClassDataTable.Column.values()) + skipColumns.add(column.name()); + skipColumns.add("Ancestors"); + skipColumns.add("Container"); + + processIndexValues(props, table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); + } + + @Override + @NotNull + public Collection getAliases() + { + TableInfo mapTi = ExperimentService.get().getTinfoDataAliasMap(); + TableInfo ti = ExperimentService.get().getTinfoAlias(); + SQLFragment sql = new SQLFragment() + .append("SELECT a.name FROM ").append(mapTi, "m") + .append(" JOIN ").append(ti, "a") + .append(" ON m.alias = a.RowId WHERE m.lsid = ? "); + sql.add(getLSID()); + ArrayList aliases = new SqlSelector(mapTi.getSchema(), sql).getArrayList(String.class); + return Collections.unmodifiableList(aliases); + } + + @Override + public String getDocumentId() + { + String dataClassName = "-"; + ExpDataClass dc = getDataClass(); + if (dc != null) + dataClassName = dc.getName(); + // why not just data:rowId? + return "data:" + new Path(getContainer().getId(), dataClassName, Long.toString(getRowId())).encode(); + } + + @Override + public Map getObjectProperties() + { + return getObjectProperties(getDataClass()); + } + + @Override + public Map getObjectProperties(@Nullable User user) + { + return getObjectProperties(getDataClass(user)); + } + + private Map getObjectProperties(ExpDataClassImpl dataClass) + { + HashMap ret = new HashMap<>(super.getObjectProperties()); + var ti = null == dataClass ? null : dataClass.getTinfo(); + if (null != ti) + { + ret.putAll(getObjectProperties(ti)); + } + return ret; + } + + private static Pair getRowIdClassNameContainerFromDocumentId(String resourceIdentifier, Map dcCache) + { + if (resourceIdentifier.startsWith("data:")) + resourceIdentifier = resourceIdentifier.substring("data:".length()); + + Path path = Path.parse(resourceIdentifier); + if (path.size() != 3) + return null; + String containerId = path.get(0); + String dataClassName = path.get(1); + String rowIdString = path.get(2); + + long rowId; + try + { + rowId = Long.parseLong(rowIdString); + if (rowId == 0) + return null; + } + catch (NumberFormatException ex) + { + return null; + } + + Container c = ContainerManager.getForId(containerId); + if (c == null) + return null; + + ExpDataClass dc = null; + if (!StringUtils.isEmpty(dataClassName) && !dataClassName.equals("-")) + { + String dcKey = containerId + '-' + dataClassName; + dc = dcCache.computeIfAbsent(dcKey, (x) -> ExperimentServiceImpl.get().getDataClass(c, dataClassName)); + } + + return new Pair<>(rowId, dc); + } + + @Nullable + public static ExpDataImpl fromDocumentId(String resourceIdentifier) + { + Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, new HashMap<>()); + if (rowIdDataClass == null) + return null; + + Long rowId = rowIdDataClass.first; + ExpDataClass dc = rowIdDataClass.second; + + if (dc != null) + return ExperimentServiceImpl.get().getExpData(dc, rowId); + else + return ExperimentServiceImpl.get().getExpData(rowId); + } + + @Nullable + public static Map fromDocumentIds(Collection resourceIdentifiers) + { + Map rowIdIdentifierMap = new LongHashMap<>(); + Map dcCache = new HashMap<>(); + Map dcMap = new LongHashMap<>(); + Map> dcRowIdMap = new LongHashMap<>(); // data rowIds with dataClass + List rowIds = new ArrayList<>(); // data rowIds without dataClass + for (String resourceIdentifier : resourceIdentifiers) + { + Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, dcCache); + if (rowIdDataClass == null) + continue; + + Long rowId = rowIdDataClass.first; + ExpDataClass dc = rowIdDataClass.second; + + rowIdIdentifierMap.put(rowId, resourceIdentifier); + + if (dc != null) + { + dcMap.put(dc.getRowId(), dc); + dcRowIdMap + .computeIfAbsent(dc.getRowId(), (k) -> new ArrayList<>()) + .add(rowId); + } + else + rowIds.add(rowId); + } + + List expDatas = new ArrayList<>(); + if (!rowIds.isEmpty()) + expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(rowIds)); + + if (!dcRowIdMap.isEmpty()) + { + for (Long dataClassId : dcRowIdMap.keySet()) + { + ExpDataClass dc = dcMap.get(dataClassId); + if (dc != null) + expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(dc, dcRowIdMap.get(dataClassId))); + } + } + + Map identifierDatas = new HashMap<>(); + for (ExpData data : expDatas) + { + identifierDatas.put(rowIdIdentifierMap.get(data.getRowId()), data); + } + + return identifierDatas; + } + + @Override + public @Nullable URI getWebDavURL(@NotNull FileContentService.PathType type) + { + java.nio.file.Path path = getFilePath(); + if (path == null) + { + return null; + } + + Container c = getContainer(); + if (c == null) + { + return null; + } + + return FileContentService.get().getWebDavUrl(path, c, type); + } + + @Override + public @Nullable WebdavResource createIndexDocument(@Nullable TableInfo tableInfo) + { + Container container = getContainer(); + if (container == null) + return null; + + Map props = new HashMap<>(); + JSONObject jsonData = new JSONObject(); + Set keywordsHi = new HashSet<>(); + Set keywordsMed = new HashSet<>(); + Set keywordsLo = new HashSet<>(); + + Set identifiersHi = new HashSet<>(); + Set identifiersMed = new HashSet<>(); + Set identifiersLo = new HashSet<>(); + + StringBuilder body = new StringBuilder(); + + // Name is an identifier with the highest weight + identifiersHi.add(getName()); + keywordsMed.add(getName()); // also add to keywords since those are stemmed + + // Description is added as a keywordsLo -- in Biologics it is common for the description to + // contain names of other DataClasses, e.g., "Mature desK of PS-10", which would be tokenized as + // [mature, desk, ps, 10] if added it as a keyword so we lower its priority to avoid useless results. + // CONSIDER: tokenize the description and extract identifiers + if (null != getDescription()) + keywordsLo.add(getDescription()); + + String comment = getComment(); + if (comment != null) + keywordsMed.add(comment); + + // Add aliases in parentheses in the title + StringBuilder title = new StringBuilder(getName()); + Collection aliases = getAliases(); + if (!aliases.isEmpty()) + { + title.append(" (").append(StringUtils.join(aliases, ", ")).append(")"); + identifiersHi.addAll(aliases); + } + + ExpDataClassImpl dc = getDataClass(User.getSearchUser()); + if (dc != null) + { + ActionURL show = new ActionURL(ExperimentController.ShowDataClassAction.class, container).addParameter("rowId", dc.getRowId()); + NavTree t = new NavTree(dc.getName(), show); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + props.put(SearchService.PROPERTY.navtrail.toString(), nav); + + props.put(DataSearchResultTemplate.PROPERTY, dc.getName()); + body.append(dc.getName()); + + if (tableInfo == null) + tableInfo = QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_EXP_DATA).getTable(dc.getName()); + + if (!(tableInfo instanceof ExpDataClassDataTableImpl expDataClassDataTable)) + throw new IllegalArgumentException(String.format("Unable to index data class item in %s. Table must be an instance of %s", dc.getName(), ExpDataClassDataTableImpl.class.getName())); + + if (!expDataClassDataTable.getDataClass().equals(dc)) + throw new IllegalArgumentException(String.format("Data class table mismatch for %s", dc.getName())); + + // Collect other text columns and lookup display columns + getIndexValues(props, expDataClassDataTable, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); + } + + // === Stored, not indexed + if (dc != null && dc.isMedia()) + props.put(SearchService.PROPERTY.categories.toString(), expMediaDataCategory.toString()); + else + props.put(SearchService.PROPERTY.categories.toString(), expDataCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), title.toString()); + props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); + + ActionURL view = ExperimentController.ExperimentUrlsImpl.get().getDataDetailsURL(this); + view.setExtraPath(container.getId()); + String docId = getDocumentId(); + + // Generate a summary explicitly instead of relying on a summary to be extracted + // from the document body. Placing lookup values and the description in the body + // would tokenize using the English analyzer and index "PS-12" as ["ps", "12"] which leads to poor results. + StringBuilder summary = new StringBuilder(); + if (StringUtils.isNotEmpty(getDescription())) + summary.append(getDescription()).append("\n"); + + appendTokens(summary, keywordsMed); + appendTokens(summary, identifiersMed); + appendTokens(summary, identifiersLo); + + props.put(SearchService.PROPERTY.summary.toString(), summary); + + return new ExpDataResource( + getRowId(), + new Path(docId), + docId, + container.getEntityId(), + "text/plain", + body.toString(), + view, + props, + getCreatedBy(), + getCreated(), + getModifiedBy(), + getModified() + ); + } + + private static void appendTokens(StringBuilder sb, Collection toks) + { + if (toks.isEmpty()) + return; + + sb.append(toks.stream().map(s -> s.length() > 30 ? StringUtilsLabKey.leftSurrogatePairFriendly(s, 30) + "\u2026" : s).collect(Collectors.joining(", "))).append("\n"); + } + + private static class ExpDataResource extends SimpleDocumentResource + { + final long _rowId; + + public ExpDataResource(long rowId, Path path, String documentId, GUID containerId, String contentType, String body, URLHelper executeUrl, Map properties, User createdBy, Date created, User modifiedBy, Date modified) + { + super(path, documentId, containerId, contentType, body, executeUrl, createdBy, created, modifiedBy, modified, properties); + _rowId = rowId; + } + + @Override + public void setLastIndexed(long ms, long modified) + { + ExperimentServiceImpl.get().setDataLastIndexed(_rowId, ms); + } + } + + public static class DataSearchResultTemplate implements SearchResultTemplate + { + public static final String NAME = "data"; + public static final String PROPERTY = "dataclass"; + + @Nullable + @Override + public String getName() + { + return NAME; + } + + private ExpDataClass getDataClass() + { + if (HttpView.hasCurrentView()) + { + ViewContext ctx = HttpView.currentContext(); + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + return ExperimentService.get().getDataClass(ctx.getContainer(), ctx.getUser(), dataclass); + } + return null; + } + + @Nullable + @Override + public String getCategories() + { + ExpDataClass dataClass = getDataClass(); + + if (dataClass != null && dataClass.isMedia()) + return expMediaDataCategory.getName(); + + return expDataCategory.getName(); + } + + @Nullable + @Override + public SearchScope getSearchScope() + { + return SearchScope.FolderAndSubfolders; + } + + @NotNull + @Override + public String getResultNameSingular() + { + ExpDataClass dc = getDataClass(); + if (dc != null) + return dc.getName(); + return "data"; + } + + @NotNull + @Override + public String getResultNamePlural() + { + return getResultNameSingular(); + } + + @Override + public boolean includeNavigationLinks() + { + return true; + } + + @Override + public boolean includeAdvanceUI() + { + return false; + } + + @Nullable + @Override + public HtmlString getExtraHtml(ViewContext ctx) + { + String q = ctx.getActionURL().getParameter("q"); + + if (StringUtils.isNotBlank(q)) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + ActionURL url = ctx.cloneActionURL().deleteParameter(PROPERTY); + url.replaceParameter(ActionURL.Param._dc, (int)Math.round(1000 * Math.random())); + + StringBuilder html = new StringBuilder(); + html.append("
    "); + + appendParam(html, null, dataclass, "All", false, url); + for (ExpDataClass dc : ExperimentService.get().getDataClasses(ctx.getContainer(), ctx.getUser(), true)) + { + appendParam(html, dc.getName(), dataclass, dc.getName(), true, url); + } + + html.append("
    "); + return HtmlString.unsafe(html.toString()); + } + else + { + return null; + } + } + + private void appendParam(StringBuilder sb, @Nullable String dataclass, @Nullable String current, @NotNull String label, boolean addParam, ActionURL url) + { + sb.append(""); + + if (!Objects.equals(dataclass, current)) + { + if (addParam) + url = url.clone().addParameter(PROPERTY, dataclass); + + sb.append(LinkBuilder.simpleLink(label, url)); + } + else + { + sb.append(label); + } + + sb.append(" "); + } + + @Override + public HtmlString getHiddenInputsHtml(ViewContext ctx) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + { + return InputBuilder.hidden().id("search-type").name(PROPERTY).value(dataclass).getHtmlString(); + } + + return null; + } + + + @Override + public String reviseQuery(ViewContext ctx, String q) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + + if (null != dataclass) + return "+(" + q + ") +" + PROPERTY + ":" + dataclass; + else + return q; + } + + @Override + public void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) + { + SearchResultTemplate.super.addNavTrail(root, ctx, scope, category); + + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + { + String text = root.getText(); + root.setText(text + " - " + dataclass); + } + } + } +} diff --git a/mothership/src/org/labkey/mothership/MothershipManager.java b/mothership/src/org/labkey/mothership/MothershipManager.java index f5a9c22f104..7d60c0a1979 100644 --- a/mothership/src/org/labkey/mothership/MothershipManager.java +++ b/mothership/src/org/labkey/mothership/MothershipManager.java @@ -1,626 +1,627 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.mothership; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.query.FieldKey; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.ReentrantLockWithName; -import org.labkey.api.util.logging.LogHelper; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; - -import static org.labkey.api.security.UserManager.USER_DISPLAY_NAME_COMPARATOR; - -public class MothershipManager -{ - private static final MothershipManager INSTANCE = new MothershipManager(); - private static final String SCHEMA_NAME = "mothership"; - private static final String UPGRADE_MESSAGE_PROPERTY_CATEGORY = "upgradeMessage"; - private static final String MOTHERSHIP_SECURE_CATEGORY = "mothershipSecure"; - private static final String CURRENT_BUILD_DATE_PROP = "currentBuildDate"; - private static final String UPGRADE_MESSAGE_PROP = "upgradeMessage"; - private static final String CREATE_ISSUE_URL_PROP = "createIssueURL"; - private static final String ISSUES_CONTAINER_PROP = "issuesContainer"; - private static final String MARKETING_MESSAGE_PROP = "marketingMessage"; - private static final String UPTIME_CONTAINER_PROP = "uptimeContainer"; - private static final String STATUS_CAKE_API_KEY_PROP = "statusCakeApiKey"; - private static final ReentrantLock INSERT_EXCEPTION_LOCK = new ReentrantLockWithName(MothershipManager.class, "INSERT_EXCEPTION_LOCK"); - - private static final Logger log = LogHelper.getLogger(MothershipManager.class, "Persists mothership records like sessions and installs"); - - public static MothershipManager get() - { - return INSTANCE; - } - - private MothershipManager() {} - - String getSchemaName() - { - return SCHEMA_NAME; - } - - /* package */ - public DbSchema getSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); - } - - public void insertException(ExceptionStackTrace stackTrace, ExceptionReport report) - { - // Synchronize to prevent two different threads from creating duplicate rows in the ExceptionStackTrace table - try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction(INSERT_EXCEPTION_LOCK)) - { - boolean isNew = false; - ExceptionStackTrace existingStackTrace = getExceptionStackTrace(stackTrace.getStackTraceHash(), stackTrace.getContainer()); - if (existingStackTrace != null) - { - stackTrace = existingStackTrace; - } - else - { - stackTrace = Table.insert(null, getTableInfoExceptionStackTrace(), stackTrace); - isNew = true; - } - - report.setExceptionStackTraceId(stackTrace.getExceptionStackTraceId()); - - String url = report.getUrl(); - if (null != url && url.length() > 512) - report.setURL(url.substring(0, 506) + "..."); - - String referrerURL = report.getReferrerURL(); - if (null != referrerURL && referrerURL.length() > 512) - report.setReferrerURL(referrerURL.substring(0, 506) + "..."); - - String browser = report.getBrowser(); - if (null != browser && browser.length() > 100) - report.setBrowser(browser.substring(0,90) + "..."); - - String exceptionMessage = report.getExceptionMessage(); - if (null != exceptionMessage && exceptionMessage.length() > 1000) - report.setExceptionMessage(exceptionMessage.substring(0,990) + "..."); - - String actionName = report.getPageflowAction(); - if (null != actionName && actionName.length() > 40) - { - report.setPageflowAction(actionName.substring(0, 39)); - } - - String controllerName = report.getPageflowName(); - if (null != controllerName && controllerName.length() > 30) - { - report.setPageflowName(controllerName.substring(0, 29)); - } - - String errorCode = report.getErrorCode(); - if (null != errorCode && errorCode.length() > MothershipReport.ERROR_CODE_LENGTH) - { - report.setErrorCode(errorCode.substring(0, MothershipReport.ERROR_CODE_LENGTH - 1)); - } - - report = Table.insert(null, getTableInfoExceptionReport(), report); - stackTrace.setInstances(stackTrace.getInstances() + 1); - stackTrace.setLastReport(report.getCreated()); - if (isNew) - { - stackTrace.setFirstReport(report.getCreated()); - } - Table.update(null, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); - - transaction.commit(); - } - } - - private static final Object ENSURE_SOFTWARE_RELEASE_LOCK = new Object(); - - private void addFilter(SimpleFilter filter, String fieldKey, Object value) - { - if (value == null) - { - filter.addCondition(FieldKey.fromString(fieldKey), null, CompareType.ISBLANK); - } - else - { - filter.addCondition(FieldKey.fromString(fieldKey), value); - } - - } - - public SoftwareRelease ensureSoftwareRelease(Container container, String revision, String url, String branch, String tag, Date buildTime, String buildNumber) - { - synchronized (ENSURE_SOFTWARE_RELEASE_LOCK) - { - // Issue 48270 - transform empty strings to nulls before querying for existing row - revision = StringUtils.trimToNull(revision); - url = StringUtils.trimToNull(url); - branch = StringUtils.trimToNull(branch); - tag = StringUtils.trimToNull(tag); - revision = StringUtils.trimToNull(revision); - buildNumber = StringUtils.trimToNull(buildNumber); - - // Filter on the columns that are part of the unique constraint - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - addFilter(filter, "VcsRevision", revision); - addFilter(filter, "VcsUrl", url); - addFilter(filter, "VcsBranch", branch); - addFilter(filter, "VcsTag", tag); - addFilter(filter, "BuildTime", buildTime); - - SoftwareRelease result = new TableSelector(getTableInfoSoftwareRelease(), filter, null).getObject(SoftwareRelease.class); - if (result == null) - { - if (buildNumber == null) - { - buildNumber = "Unknown VCS"; - } - - result = new SoftwareRelease(); - result.setVcsUrl(url); - result.setVcsRevision(revision); - result.setVcsBranch(branch); - result.setVcsTag(tag); - result.setBuildTime(buildTime); - result.setBuildNumber(buildNumber); - result.setContainer(container.getId()); - result = Table.insert(null, getTableInfoSoftwareRelease(), result); - } - return result; - } - } - - public ServerInstallation getServerInstallation(@NotNull String serverGUID, @NotNull String serverHostName, @NotNull Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerInstallationGUID"), serverGUID); - filter.addCondition(FieldKey.fromString("ServerHostName"), serverHostName); - return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); - } - - public ServerSession getServerSession(String serverSessionGUID, Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerSessionGUID"), serverSessionGUID); - return new TableSelector(getTableInfoServerSession(), filter, null).getObject(ServerSession.class); - } - - public ExceptionStackTrace getExceptionStackTrace(String stackTraceHash, String containerId) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), containerId); - filter.addCondition(FieldKey.fromString("StackTraceHash"), stackTraceHash); - return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); - } - - public ExceptionStackTrace getExceptionStackTrace(int exceptionStackTraceId, Container container) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromString("ExceptionStackTraceId"), exceptionStackTraceId); - return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); - } - - public void deleteForContainer(Container c) - { - SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); - sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionReport() + " WHERE ExceptionStackTraceId IN (SELECT ExceptionStackTraceId FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?)", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoServerSession() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoServerInstallation() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoSoftwareRelease() + " WHERE Container = ?", c); - } - - public void deleteForUser(User u) - { - SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); - sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET AssignedTo = NULL WHERE AssignedTo = ?", u.getUserId()); - sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET ModifiedBy = NULL WHERE ModifiedBy = ?", u.getUserId()); - } - - public synchronized ServerSession updateServerSession(MothershipController.ServerInfoForm form, String serverIP, ServerSession session, ServerInstallation installation, Container container) - { - try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction()) - { - String hostName = form.getBestServerHostName(serverIP); - ServerInstallation existingInstallation = getServerInstallation(installation.getServerInstallationGUID(), hostName, container); - - if (existingInstallation == null) - { - installation.setContainer(container.getId()); - installation.setServerHostName(hostName); - installation = Table.insert(null, getTableInfoServerInstallation(), installation); - } - else - { - existingInstallation.setServerHostName(hostName); - installation = Table.update(null, getTableInfoServerInstallation(), existingInstallation, existingInstallation.getServerInstallationId()); - } - - Date now = new Date(); - ServerSession existingSession = getServerSession(session.getServerSessionGUID(), container); - if (existingSession != null) - { - // Issue 50876: Reparent mothership server session when the base URL changes mid-session - existingSession.setServerInstallationId(installation.getServerInstallationId()); - - Calendar existingCal = Calendar.getInstance(); - existingCal.setTime(existingSession.getLastKnownTime()); - Calendar nowCal = Calendar.getInstance(); - - // Check if this session is straddling months. If so, break it into two so that we - // retain metrics with month level granularity - if (existingCal.get(Calendar.MONTH) != nowCal.get(Calendar.MONTH)) - { - // Chain the two sessions together - session.setOriginalServerSessionId(existingSession.getServerSessionId()); - - // Update the GUID for the old one as we're capturing in a new record going forward - existingSession.setServerSessionGUID(GUID.makeGUID()); - Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); - existingSession = null; - } - } - - if (existingSession == null) - { - session.setEarliestKnownTime(now); - session.setServerInstallationId(installation.getServerInstallationId()); - - configureSession(session, now, serverIP, form); - session = Table.insert(null, getTableInfoServerSession(), session); - } - else - { - configureSession(existingSession, now, serverIP, form); - session = Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); - } - - transaction.commit(); - return session; - } - } - - private void configureSession(@NotNull ServerSession session, Date now, String serverIP, MothershipController.ServerInfoForm form) - { - session.setLastKnownTime(now); - session.setServerIP(serverIP); - session.setServerHostName(form.getBestServerHostName(serverIP)); - - session.setLogoLink(getBestString(session.getLogoLink(), form.getLogoLink())); - session.setOrganizationName(getBestString(session.getOrganizationName(), form.getOrganizationName())); - session.setSystemDescription(getBestString(session.getSystemDescription(), form.getSystemDescription())); - session.setSystemShortName(getBestString(session.getSystemShortName(), form.getSystemShortName())); - - session.setContainerCount(getBestInteger(session.getContainerCount(), form.getContainerCount())); - session.setProjectCount(getBestInteger(session.getProjectCount(), form.getProjectCount())); - session.setRecentUserCount(getBestInteger(session.getRecentUserCount(), form.getRecentUserCount())); - session.setUserCount(getBestInteger(session.getUserCount(), form.getUserCount())); - session.setAdministratorEmail(getBestString(session.getAdministratorEmail(), form.getAdministratorEmail())); - session.setDistribution(getBestString(session.getDistribution(), form.getDistribution())); - session.setUsageReportingLevel(getBestString(session.getUsageReportingLevel(), form.getUsageReportingLevel())); - session.setExceptionReportingLevel(getBestString(session.getExceptionReportingLevel(), form.getExceptionReportingLevel())); - session.setJsonMetrics(getBestJson(session.getJsonMetrics(), form.getJsonMetrics(), session.getServerSessionGUID())); - - } - - private String getBestString(String currentValue, String newValue) - { - if (StringUtils.isEmpty(newValue)) - { - return currentValue; - } - return newValue; - } - - private Integer getBestInteger(Integer currentValue, Integer newValue) - { - if (newValue == null) - { - return currentValue; - } - return newValue; - } - - private Boolean getBestBoolean(Boolean currentValue, Boolean newValue) - { - if (newValue == null) - { - return currentValue; - } - return newValue; - } - - private String getBestJson(String currentValue, String newValue, String serverSessionGUID) - { - if (StringUtils.isEmpty(newValue)) - { - return currentValue; - } - if (StringUtils.isEmpty(currentValue)) - { - // Verify the newValue as valid json; if it is, return it. Otherwise, return null. - try - { - JsonUtil.DEFAULT_MAPPER.readTree(newValue); - return newValue; - } - catch (IOException e) - { - logJsonError(newValue, serverSessionGUID, e); - return null; - } - } - - // Rather than overwrite the current json map, merge the new with the current. - ObjectMapper mapper = JsonUtil.createDefaultMapper(); - try - { - log.debug("Merging JSON. Old is " + currentValue.length() + " characters, new is " + newValue.length()); - Map currentMap = mapper.readValue(currentValue, Map.class); - Map newMap = mapper.readValue(newValue, Map.class); - merge(currentMap, newMap); - return mapper.writeValueAsString(currentMap); - } - catch (IOException e) - { - logJsonError(newValue, serverSessionGUID, e); - return currentValue; - } - } - - /** Merges the values from newMap into currentMap, recursing through child maps. See issue 50665 */ - private void merge(Map currentMap, Map newMap) - { - for (Map.Entry entry : newMap.entrySet()) - { - String key = entry.getKey(); - Object currentChild = currentMap.get(key); - if (currentChild instanceof Map currentChildMap && entry.getValue() instanceof Map newChildMap) - { - merge(currentChildMap, newChildMap); - } - else - { - currentMap.put(entry.getKey(), entry.getValue()); - } - } - } - - private void logJsonError(String newValue, String serverSessionGUID, Exception e) - { - log.error("Malformed json in mothership report from server session '"+serverSessionGUID + "': " + newValue, e); - } - - public TableInfo getTableInfoExceptionStackTrace() - { - return getSchema().getTable("ExceptionStackTrace"); - } - - public TableInfo getTableInfoExceptionReport() - { - return getSchema().getTable("ExceptionReport"); - } - - public TableInfo getTableInfoSoftwareRelease() - { - return getSchema().getTable("SoftwareRelease"); - } - - public TableInfo getTableInfoServerSession() - { - return getSchema().getTable("ServerSession"); - } - - public TableInfo getTableInfoServerInstallation() - { - return getSchema().getTable("ServerInstallation"); - } - - public SqlDialect getDialect() - { - return getSchema().getSqlDialect(); - } - - private WritablePropertyMap getWritableProperties(Container c, boolean secure) - { - if (secure) - { - return PropertyManager.getEncryptedStore().getWritableProperties(c, MOTHERSHIP_SECURE_CATEGORY, true); - } - else - { - return PropertyManager.getWritableProperties(c, UPGRADE_MESSAGE_PROPERTY_CATEGORY, true); - } - } - - private @NotNull Map getProperties(boolean secure) - { - if (secure) - { - return PropertyManager.getEncryptedStore().getProperties(getContainer(), MOTHERSHIP_SECURE_CATEGORY); - } - else - { - return PropertyManager.getProperties(getContainer(), UPGRADE_MESSAGE_PROPERTY_CATEGORY); - } - } - - private static Container getContainer() - { - return ContainerManager.getForPath(MothershipReport.CONTAINER_PATH); - } - - public Date getCurrentBuildDate() - { - Map props = getProperties(false); - String buildDate = props.get(CURRENT_BUILD_DATE_PROP); - return null == buildDate ? null : new Date(DateUtil.parseISODateTime(buildDate)); - } - - private String getStringProperty(String name) - { - return getStringProperty(name, false); - } - - private String getStringProperty(String name, boolean secure) - { - Map props = getProperties(secure); - String message = props.get(name); - if (message == null) - { - return ""; - } - return message; - } - - public String getUpgradeMessage() - { - return getStringProperty(UPGRADE_MESSAGE_PROP); - } - - public String getMarketingMessage() - { - return getStringProperty(MARKETING_MESSAGE_PROP); - } - - private void saveProperty(String name, String value) - { - saveProperty(name, value, false); - } - - private void saveProperty(String name, String value, boolean secure) - { - WritablePropertyMap props = getWritableProperties(getContainer(), secure); - props.put(name, value); - props.save(); - } - - public void setCurrentBuildDate(Date buildDate) - { - saveProperty(CURRENT_BUILD_DATE_PROP, DateUtil.formatIsoDateShortTime(buildDate)); - } - - public void setUpgradeMessage(String message) - { - saveProperty(UPGRADE_MESSAGE_PROP, message); - } - - public void setMarketingMessage(String message) - { - saveProperty(MARKETING_MESSAGE_PROP, message); - } - - public String getCreateIssueURL() - { - return getStringProperty(CREATE_ISSUE_URL_PROP); - } - - public void setCreateIssueURL(String url) - { - saveProperty(CREATE_ISSUE_URL_PROP, url); - } - - public void updateExceptionStackTrace(ExceptionStackTrace stackTrace, User user) - { - Table.update(user, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); - } - - public String getIssuesContainer() - { - return getStringProperty(ISSUES_CONTAINER_PROP); - } - - public void setIssuesContainer(String container) - { - saveProperty(ISSUES_CONTAINER_PROP, container); - } - - public String getUptimeContainer() - { - return getStringProperty(UPTIME_CONTAINER_PROP); - } - - public void setUptimeContainer(String uptimeContainer) - { - saveProperty(UPTIME_CONTAINER_PROP, uptimeContainer); - } - - public String getStatusCakeApiKey() - { - return getStringProperty(STATUS_CAKE_API_KEY_PROP, true); - } - - public void setStatusCakeApiKey(String statusCakeApiKey) - { - saveProperty(STATUS_CAKE_API_KEY_PROP, statusCakeApiKey, true); - } - - public void updateSoftwareRelease(Container container, User user, SoftwareRelease bean) - { - bean.setContainer(container.getId()); - Table.update(user, getTableInfoSoftwareRelease(), bean, bean.getSoftwareReleaseId()); - } - - public ServerInstallation getServerInstallation(int id, Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerInstallationId"), id); - return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); - } - - public List getAssignedToList(Container container) - { - List projectUsers = org.labkey.api.security.SecurityManager.getProjectUsers(container.getProject()); - List list = new ArrayList<>(); - // Filter list to only show active users - for (User user : projectUsers) - { - if (user.isActive()) - { - list.add(user); - } - } - list.sort(USER_DISPLAY_NAME_COMPARATOR); - return list; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mothership; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.ReentrantLockWithName; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import static org.labkey.api.security.UserManager.USER_DISPLAY_NAME_COMPARATOR; + +public class MothershipManager +{ + private static final MothershipManager INSTANCE = new MothershipManager(); + private static final String SCHEMA_NAME = "mothership"; + private static final String UPGRADE_MESSAGE_PROPERTY_CATEGORY = "upgradeMessage"; + private static final String MOTHERSHIP_SECURE_CATEGORY = "mothershipSecure"; + private static final String CURRENT_BUILD_DATE_PROP = "currentBuildDate"; + private static final String UPGRADE_MESSAGE_PROP = "upgradeMessage"; + private static final String CREATE_ISSUE_URL_PROP = "createIssueURL"; + private static final String ISSUES_CONTAINER_PROP = "issuesContainer"; + private static final String MARKETING_MESSAGE_PROP = "marketingMessage"; + private static final String UPTIME_CONTAINER_PROP = "uptimeContainer"; + private static final String STATUS_CAKE_API_KEY_PROP = "statusCakeApiKey"; + private static final ReentrantLock INSERT_EXCEPTION_LOCK = new ReentrantLockWithName(MothershipManager.class, "INSERT_EXCEPTION_LOCK"); + + private static final Logger log = LogHelper.getLogger(MothershipManager.class, "Persists mothership records like sessions and installs"); + + public static MothershipManager get() + { + return INSTANCE; + } + + private MothershipManager() {} + + String getSchemaName() + { + return SCHEMA_NAME; + } + + /* package */ + public DbSchema getSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); + } + + public void insertException(ExceptionStackTrace stackTrace, ExceptionReport report) + { + // Synchronize to prevent two different threads from creating duplicate rows in the ExceptionStackTrace table + try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction(INSERT_EXCEPTION_LOCK)) + { + boolean isNew = false; + ExceptionStackTrace existingStackTrace = getExceptionStackTrace(stackTrace.getStackTraceHash(), stackTrace.getContainer()); + if (existingStackTrace != null) + { + stackTrace = existingStackTrace; + } + else + { + stackTrace = Table.insert(null, getTableInfoExceptionStackTrace(), stackTrace); + isNew = true; + } + + report.setExceptionStackTraceId(stackTrace.getExceptionStackTraceId()); + + String url = report.getUrl(); + if (null != url && url.length() > 512) + report.setURL(StringUtilsLabKey.leftSurrogatePairFriendly(url, 506) + "..."); + + String referrerURL = report.getReferrerURL(); + if (null != referrerURL && referrerURL.length() > 512) + report.setReferrerURL(StringUtilsLabKey.leftSurrogatePairFriendly(referrerURL, 506) + "..."); + + String browser = report.getBrowser(); + if (null != browser && browser.length() > 100) + report.setBrowser(browser.substring(0,90) + "..."); + + String exceptionMessage = report.getExceptionMessage(); + if (null != exceptionMessage && exceptionMessage.length() > 1000) + report.setExceptionMessage(StringUtilsLabKey.leftSurrogatePairFriendly(exceptionMessage, 990) + "..."); + + String actionName = report.getPageflowAction(); + if (null != actionName && actionName.length() > 40) + { + report.setPageflowAction(actionName.substring(0, 39)); + } + + String controllerName = report.getPageflowName(); + if (null != controllerName && controllerName.length() > 30) + { + report.setPageflowName(controllerName.substring(0, 29)); + } + + String errorCode = report.getErrorCode(); + if (null != errorCode && errorCode.length() > MothershipReport.ERROR_CODE_LENGTH) + { + report.setErrorCode(errorCode.substring(0, MothershipReport.ERROR_CODE_LENGTH - 1)); + } + + report = Table.insert(null, getTableInfoExceptionReport(), report); + stackTrace.setInstances(stackTrace.getInstances() + 1); + stackTrace.setLastReport(report.getCreated()); + if (isNew) + { + stackTrace.setFirstReport(report.getCreated()); + } + Table.update(null, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); + + transaction.commit(); + } + } + + private static final Object ENSURE_SOFTWARE_RELEASE_LOCK = new Object(); + + private void addFilter(SimpleFilter filter, String fieldKey, Object value) + { + if (value == null) + { + filter.addCondition(FieldKey.fromString(fieldKey), null, CompareType.ISBLANK); + } + else + { + filter.addCondition(FieldKey.fromString(fieldKey), value); + } + + } + + public SoftwareRelease ensureSoftwareRelease(Container container, String revision, String url, String branch, String tag, Date buildTime, String buildNumber) + { + synchronized (ENSURE_SOFTWARE_RELEASE_LOCK) + { + // Issue 48270 - transform empty strings to nulls before querying for existing row + revision = StringUtils.trimToNull(revision); + url = StringUtils.trimToNull(url); + branch = StringUtils.trimToNull(branch); + tag = StringUtils.trimToNull(tag); + revision = StringUtils.trimToNull(revision); + buildNumber = StringUtils.trimToNull(buildNumber); + + // Filter on the columns that are part of the unique constraint + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + addFilter(filter, "VcsRevision", revision); + addFilter(filter, "VcsUrl", url); + addFilter(filter, "VcsBranch", branch); + addFilter(filter, "VcsTag", tag); + addFilter(filter, "BuildTime", buildTime); + + SoftwareRelease result = new TableSelector(getTableInfoSoftwareRelease(), filter, null).getObject(SoftwareRelease.class); + if (result == null) + { + if (buildNumber == null) + { + buildNumber = "Unknown VCS"; + } + + result = new SoftwareRelease(); + result.setVcsUrl(url); + result.setVcsRevision(revision); + result.setVcsBranch(branch); + result.setVcsTag(tag); + result.setBuildTime(buildTime); + result.setBuildNumber(buildNumber); + result.setContainer(container.getId()); + result = Table.insert(null, getTableInfoSoftwareRelease(), result); + } + return result; + } + } + + public ServerInstallation getServerInstallation(@NotNull String serverGUID, @NotNull String serverHostName, @NotNull Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerInstallationGUID"), serverGUID); + filter.addCondition(FieldKey.fromString("ServerHostName"), serverHostName); + return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); + } + + public ServerSession getServerSession(String serverSessionGUID, Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerSessionGUID"), serverSessionGUID); + return new TableSelector(getTableInfoServerSession(), filter, null).getObject(ServerSession.class); + } + + public ExceptionStackTrace getExceptionStackTrace(String stackTraceHash, String containerId) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), containerId); + filter.addCondition(FieldKey.fromString("StackTraceHash"), stackTraceHash); + return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); + } + + public ExceptionStackTrace getExceptionStackTrace(int exceptionStackTraceId, Container container) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromString("ExceptionStackTraceId"), exceptionStackTraceId); + return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); + } + + public void deleteForContainer(Container c) + { + SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); + sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionReport() + " WHERE ExceptionStackTraceId IN (SELECT ExceptionStackTraceId FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?)", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoServerSession() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoServerInstallation() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoSoftwareRelease() + " WHERE Container = ?", c); + } + + public void deleteForUser(User u) + { + SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); + sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET AssignedTo = NULL WHERE AssignedTo = ?", u.getUserId()); + sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET ModifiedBy = NULL WHERE ModifiedBy = ?", u.getUserId()); + } + + public synchronized ServerSession updateServerSession(MothershipController.ServerInfoForm form, String serverIP, ServerSession session, ServerInstallation installation, Container container) + { + try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction()) + { + String hostName = form.getBestServerHostName(serverIP); + ServerInstallation existingInstallation = getServerInstallation(installation.getServerInstallationGUID(), hostName, container); + + if (existingInstallation == null) + { + installation.setContainer(container.getId()); + installation.setServerHostName(hostName); + installation = Table.insert(null, getTableInfoServerInstallation(), installation); + } + else + { + existingInstallation.setServerHostName(hostName); + installation = Table.update(null, getTableInfoServerInstallation(), existingInstallation, existingInstallation.getServerInstallationId()); + } + + Date now = new Date(); + ServerSession existingSession = getServerSession(session.getServerSessionGUID(), container); + if (existingSession != null) + { + // Issue 50876: Reparent mothership server session when the base URL changes mid-session + existingSession.setServerInstallationId(installation.getServerInstallationId()); + + Calendar existingCal = Calendar.getInstance(); + existingCal.setTime(existingSession.getLastKnownTime()); + Calendar nowCal = Calendar.getInstance(); + + // Check if this session is straddling months. If so, break it into two so that we + // retain metrics with month level granularity + if (existingCal.get(Calendar.MONTH) != nowCal.get(Calendar.MONTH)) + { + // Chain the two sessions together + session.setOriginalServerSessionId(existingSession.getServerSessionId()); + + // Update the GUID for the old one as we're capturing in a new record going forward + existingSession.setServerSessionGUID(GUID.makeGUID()); + Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); + existingSession = null; + } + } + + if (existingSession == null) + { + session.setEarliestKnownTime(now); + session.setServerInstallationId(installation.getServerInstallationId()); + + configureSession(session, now, serverIP, form); + session = Table.insert(null, getTableInfoServerSession(), session); + } + else + { + configureSession(existingSession, now, serverIP, form); + session = Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); + } + + transaction.commit(); + return session; + } + } + + private void configureSession(@NotNull ServerSession session, Date now, String serverIP, MothershipController.ServerInfoForm form) + { + session.setLastKnownTime(now); + session.setServerIP(serverIP); + session.setServerHostName(form.getBestServerHostName(serverIP)); + + session.setLogoLink(getBestString(session.getLogoLink(), form.getLogoLink())); + session.setOrganizationName(getBestString(session.getOrganizationName(), form.getOrganizationName())); + session.setSystemDescription(getBestString(session.getSystemDescription(), form.getSystemDescription())); + session.setSystemShortName(getBestString(session.getSystemShortName(), form.getSystemShortName())); + + session.setContainerCount(getBestInteger(session.getContainerCount(), form.getContainerCount())); + session.setProjectCount(getBestInteger(session.getProjectCount(), form.getProjectCount())); + session.setRecentUserCount(getBestInteger(session.getRecentUserCount(), form.getRecentUserCount())); + session.setUserCount(getBestInteger(session.getUserCount(), form.getUserCount())); + session.setAdministratorEmail(getBestString(session.getAdministratorEmail(), form.getAdministratorEmail())); + session.setDistribution(getBestString(session.getDistribution(), form.getDistribution())); + session.setUsageReportingLevel(getBestString(session.getUsageReportingLevel(), form.getUsageReportingLevel())); + session.setExceptionReportingLevel(getBestString(session.getExceptionReportingLevel(), form.getExceptionReportingLevel())); + session.setJsonMetrics(getBestJson(session.getJsonMetrics(), form.getJsonMetrics(), session.getServerSessionGUID())); + + } + + private String getBestString(String currentValue, String newValue) + { + if (StringUtils.isEmpty(newValue)) + { + return currentValue; + } + return newValue; + } + + private Integer getBestInteger(Integer currentValue, Integer newValue) + { + if (newValue == null) + { + return currentValue; + } + return newValue; + } + + private Boolean getBestBoolean(Boolean currentValue, Boolean newValue) + { + if (newValue == null) + { + return currentValue; + } + return newValue; + } + + private String getBestJson(String currentValue, String newValue, String serverSessionGUID) + { + if (StringUtils.isEmpty(newValue)) + { + return currentValue; + } + if (StringUtils.isEmpty(currentValue)) + { + // Verify the newValue as valid json; if it is, return it. Otherwise, return null. + try + { + JsonUtil.DEFAULT_MAPPER.readTree(newValue); + return newValue; + } + catch (IOException e) + { + logJsonError(newValue, serverSessionGUID, e); + return null; + } + } + + // Rather than overwrite the current json map, merge the new with the current. + ObjectMapper mapper = JsonUtil.createDefaultMapper(); + try + { + log.debug("Merging JSON. Old is " + currentValue.length() + " characters, new is " + newValue.length()); + Map currentMap = mapper.readValue(currentValue, Map.class); + Map newMap = mapper.readValue(newValue, Map.class); + merge(currentMap, newMap); + return mapper.writeValueAsString(currentMap); + } + catch (IOException e) + { + logJsonError(newValue, serverSessionGUID, e); + return currentValue; + } + } + + /** Merges the values from newMap into currentMap, recursing through child maps. See issue 50665 */ + private void merge(Map currentMap, Map newMap) + { + for (Map.Entry entry : newMap.entrySet()) + { + String key = entry.getKey(); + Object currentChild = currentMap.get(key); + if (currentChild instanceof Map currentChildMap && entry.getValue() instanceof Map newChildMap) + { + merge(currentChildMap, newChildMap); + } + else + { + currentMap.put(entry.getKey(), entry.getValue()); + } + } + } + + private void logJsonError(String newValue, String serverSessionGUID, Exception e) + { + log.error("Malformed json in mothership report from server session '"+serverSessionGUID + "': " + newValue, e); + } + + public TableInfo getTableInfoExceptionStackTrace() + { + return getSchema().getTable("ExceptionStackTrace"); + } + + public TableInfo getTableInfoExceptionReport() + { + return getSchema().getTable("ExceptionReport"); + } + + public TableInfo getTableInfoSoftwareRelease() + { + return getSchema().getTable("SoftwareRelease"); + } + + public TableInfo getTableInfoServerSession() + { + return getSchema().getTable("ServerSession"); + } + + public TableInfo getTableInfoServerInstallation() + { + return getSchema().getTable("ServerInstallation"); + } + + public SqlDialect getDialect() + { + return getSchema().getSqlDialect(); + } + + private WritablePropertyMap getWritableProperties(Container c, boolean secure) + { + if (secure) + { + return PropertyManager.getEncryptedStore().getWritableProperties(c, MOTHERSHIP_SECURE_CATEGORY, true); + } + else + { + return PropertyManager.getWritableProperties(c, UPGRADE_MESSAGE_PROPERTY_CATEGORY, true); + } + } + + private @NotNull Map getProperties(boolean secure) + { + if (secure) + { + return PropertyManager.getEncryptedStore().getProperties(getContainer(), MOTHERSHIP_SECURE_CATEGORY); + } + else + { + return PropertyManager.getProperties(getContainer(), UPGRADE_MESSAGE_PROPERTY_CATEGORY); + } + } + + private static Container getContainer() + { + return ContainerManager.getForPath(MothershipReport.CONTAINER_PATH); + } + + public Date getCurrentBuildDate() + { + Map props = getProperties(false); + String buildDate = props.get(CURRENT_BUILD_DATE_PROP); + return null == buildDate ? null : new Date(DateUtil.parseISODateTime(buildDate)); + } + + private String getStringProperty(String name) + { + return getStringProperty(name, false); + } + + private String getStringProperty(String name, boolean secure) + { + Map props = getProperties(secure); + String message = props.get(name); + if (message == null) + { + return ""; + } + return message; + } + + public String getUpgradeMessage() + { + return getStringProperty(UPGRADE_MESSAGE_PROP); + } + + public String getMarketingMessage() + { + return getStringProperty(MARKETING_MESSAGE_PROP); + } + + private void saveProperty(String name, String value) + { + saveProperty(name, value, false); + } + + private void saveProperty(String name, String value, boolean secure) + { + WritablePropertyMap props = getWritableProperties(getContainer(), secure); + props.put(name, value); + props.save(); + } + + public void setCurrentBuildDate(Date buildDate) + { + saveProperty(CURRENT_BUILD_DATE_PROP, DateUtil.formatIsoDateShortTime(buildDate)); + } + + public void setUpgradeMessage(String message) + { + saveProperty(UPGRADE_MESSAGE_PROP, message); + } + + public void setMarketingMessage(String message) + { + saveProperty(MARKETING_MESSAGE_PROP, message); + } + + public String getCreateIssueURL() + { + return getStringProperty(CREATE_ISSUE_URL_PROP); + } + + public void setCreateIssueURL(String url) + { + saveProperty(CREATE_ISSUE_URL_PROP, url); + } + + public void updateExceptionStackTrace(ExceptionStackTrace stackTrace, User user) + { + Table.update(user, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); + } + + public String getIssuesContainer() + { + return getStringProperty(ISSUES_CONTAINER_PROP); + } + + public void setIssuesContainer(String container) + { + saveProperty(ISSUES_CONTAINER_PROP, container); + } + + public String getUptimeContainer() + { + return getStringProperty(UPTIME_CONTAINER_PROP); + } + + public void setUptimeContainer(String uptimeContainer) + { + saveProperty(UPTIME_CONTAINER_PROP, uptimeContainer); + } + + public String getStatusCakeApiKey() + { + return getStringProperty(STATUS_CAKE_API_KEY_PROP, true); + } + + public void setStatusCakeApiKey(String statusCakeApiKey) + { + saveProperty(STATUS_CAKE_API_KEY_PROP, statusCakeApiKey, true); + } + + public void updateSoftwareRelease(Container container, User user, SoftwareRelease bean) + { + bean.setContainer(container.getId()); + Table.update(user, getTableInfoSoftwareRelease(), bean, bean.getSoftwareReleaseId()); + } + + public ServerInstallation getServerInstallation(int id, Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerInstallationId"), id); + return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); + } + + public List getAssignedToList(Container container) + { + List projectUsers = org.labkey.api.security.SecurityManager.getProjectUsers(container.getProject()); + List list = new ArrayList<>(); + // Filter list to only show active users + for (User user : projectUsers) + { + if (user.isActive()) + { + list.add(user); + } + } + list.sort(USER_DISPLAY_NAME_COMPARATOR); + return list; + } +} diff --git a/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java b/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java index a8b49ecaa20..0b16a0b4eb8 100644 --- a/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java +++ b/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java @@ -1,594 +1,594 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.pipeline.api; - -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.pipeline.WorkDirFactory; -import org.labkey.api.pipeline.WorkDirectory; -import org.labkey.api.pipeline.file.FileAnalysisJobSupport; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.URIUtil; -import org.springframework.beans.factory.InitializingBean; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Used to copy files from (and back to) a remote file system so that they can be used directly on the local file system, - * improving performance on high-latency and/or low-bandwidth network file systems - * - * @author jeckels - */ -public class WorkDirectoryRemote extends AbstractWorkDirectory -{ - private static final Logger _systemLog = LogManager.getLogger(WorkDirectoryRemote.class); - - private static final int FILE_LOCKS_DEFAULT = 5; - - private final File _lockDirectory; - private final File _folderToClean; - - private static final Map _locks = new HashMap<>(); - - @Override - public File inputFile(File fileInput, boolean forceCopy) throws IOException - { - return inputFile(fileInput, newFile(fileInput.getName()), forceCopy); - } - - @Override - public File inputFile(File fileInput, File fileWork, boolean forceCopy) throws IOException - { - //can be used to prevent duplicate copy attempts - if (fileWork.exists() && !forceCopy) - { - _copiedInputs.put(fileInput, fileWork); - } - - return copyInputFile(fileInput, fileWork); - } - - public static class Factory extends AbstractFactory implements InitializingBean - { - private String _lockDirectory; - private String _tempDirectory; - private boolean _sharedTempDirectory; - private boolean _allowReuseExistingTempDirectory; - private boolean _deterministicWorkingDirName; - private boolean _cleanupOnStartup; - private String _transferToDirOnFailure = null; - - @Override - public void afterPropertiesSet() - { - if (_tempDirectory == null) - { - throw new IllegalStateException("tempDirectory not set - set it directly using the tempDirectory property or use the tempDirectoryEnv property to point to an environment variable"); - } - if (_cleanupOnStartup) - { - FileUtil.deleteDirectoryContents(new File(_tempDirectory)); - } - } - - @Override - public WorkDirectory createWorkDirectory(String jobId, FileAnalysisJobSupport support, boolean useDeterministicFolderPath, Logger log) throws IOException - { - if (useDeterministicFolderPath) - { - _sharedTempDirectory = true; - _allowReuseExistingTempDirectory = true; - _deterministicWorkingDirName = true; - } - - File tempDir; - File tempDirBase = null; - int attempt = 0; - do - { - // We've seen very intermittent problems failing to create temp files in the past during the DRTs, - // so try a few times before failing - File dirParent = (_tempDirectory == null ? null : new File(_tempDirectory)); - - // If the temp directory is shared, then create a jobId directory to be sure the - // work directory path is unique. - try - { - if (_sharedTempDirectory) - { - if (_deterministicWorkingDirName) - { - dirParent = new File(dirParent, jobId); - tempDirBase = dirParent; - } - else - { - dirParent = FileUtil.createTempFile(jobId, "", dirParent); - tempDirBase = dirParent; - } - - if (_allowReuseExistingTempDirectory && dirParent.exists()) - { - log.info("parent directory exists, reusing: " + dirParent.getPath()); - } - else - { - dirParent.delete(); - FileUtil.mkdirs(dirParent); - } - } - - String name = support.getBaseName(); - if (name.length() > 10) - { - // Don't let the total path get too long - Windows doesn't like paths longer than 255 characters - // so if there's a ridiculously long file name, we don't want to duplicate its name in the - // directory too - name = name.substring(0, 9); - } - else if (name.length() < 3) - { - //File.createTempFile() does not allow prefixes <3 chars - name = "wd_" + name; - } - - if (_deterministicWorkingDirName) - { - tempDir = new File(dirParent, name + WORK_DIR_SUFFIX); - } - else - { - tempDir = FileUtil.createTempFile(name, WORK_DIR_SUFFIX, dirParent); - } - - if (_allowReuseExistingTempDirectory && tempDir.exists()) - { - log.info("working directory exists, reusing: " + dirParent.getPath()); - } - else - { - tempDir.delete(); - FileUtil.mkdirs(tempDir); - } - } - catch (IOException e) - { - IOException ioException = new IOException("Failed to create local working directory in the tempDirectory " - + dirParent + ", specified in the tempDirectory property in the pipeline configuration"); - ioException.initCause(e); - _systemLog.error(ioException.getMessage(), e); - throw ioException; - } - attempt++; - } - while (attempt < 5 && !tempDir.isDirectory()); - if (!tempDir.isDirectory()) - { - throw new IOException("Failed to create local working directory " + tempDir); - } - - File lockDir = (_lockDirectory == null ? null : new File(_lockDirectory)); - File transferToDirOnFailure = (_transferToDirOnFailure == null ? null : new File(_transferToDirOnFailure)); - return new WorkDirectoryRemote(support, this, log, lockDir, tempDir, transferToDirOnFailure, _allowReuseExistingTempDirectory, tempDirBase); - } - - public String getLockDirectory() - { - if (_lockDirectory!= null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File lockDir = new File(_lockDirectory); - if (!NetworkDrive.exists(lockDir) || !lockDir.isDirectory()) - throw new IllegalArgumentException("The lock directory " + _lockDirectory + " does not exist."); - } - return _lockDirectory; - } - - public void setLockDirectory(String directoryString) - { - _lockDirectory = directoryString; - } - - public String getTempDirectory() - { - if (_tempDirectory != null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File tempDir = new File(_tempDirectory); - if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) - throw new IllegalArgumentException("The temporary directory " + _tempDirectory + " does not exist."); - } - return _tempDirectory; - } - - /** @param directoryString path of the directory to be used as scratch space */ - public void setTempDirectory(String directoryString) - { - _tempDirectory = directoryString; - } - - public void setCleanupOnStartup(boolean cleanupOnStartup) - { - _cleanupOnStartup = cleanupOnStartup; - } - - /** - * Set to an environment variable set to the path to use for the temporary directory. - * (e.g. some cluster schedulers initialize TMPDIR to a job specific temporary directory - * which will be removed, if the job is cancelled) - * - * @param tempDirectoryVar environment variable name - */ - public void setTempDirectoryEnv(String tempDirectoryVar) - { - String tempDirectory = System.getenv(tempDirectoryVar); - if (tempDirectory == null || tempDirectory.isEmpty()) - throw new IllegalArgumentException("The environment variable " + tempDirectoryVar + " does not exist:\n" + System.getenv()); - setTempDirectory(tempDirectory); - } - - /** - * @return true if the root temporary directory will be shared by multiple tasks - */ - public boolean isSharedTempDirectory() - { - return _sharedTempDirectory; - } - - /** - * Set to true, if the root temporary directory will be shared by multiple tasks. - * This is usually not necessary on a scheduled computational cluster, where each - * task is given a separate working environment. - * - * @param sharedTempDirectory true if the root temporary directory will be shared by multiple tasks - */ - public void setSharedTempDirectory(boolean sharedTempDirectory) - { - _sharedTempDirectory = sharedTempDirectory; - } - - public String getTransferToDirOnFailure() - { - if (_transferToDirOnFailure != null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File tempDir = new File(_transferToDirOnFailure); - if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) - throw new IllegalArgumentException("The directory " + _transferToDirOnFailure + " does not exist."); - } - - return _transferToDirOnFailure; - } - - /** - * If a directory is provided, when a remote job fails, the working directory will - * be moved from the working location to a directory under this folder - */ - public void setTransferToDirOnFailure(String transferToDirOnFailure) - { - _transferToDirOnFailure = transferToDirOnFailure; - } - - public boolean isAllowReuseExistingTempDirectory() - { - return _allowReuseExistingTempDirectory; - } - - /** - * If true, instead of deleting an existing working directory on job startup, an existing directory will be reused. - * This is mostly used to allow job resume, and should only be used - */ - public void setAllowReuseExistingTempDirectory(boolean allowReuseExistingTempDirectory) - { - _allowReuseExistingTempDirectory = allowReuseExistingTempDirectory; - } - - public boolean isDeterministicWorkingDirName() - { - return _deterministicWorkingDirName; - } - - /** - * If true, the working directory for each job will be named using the job' name alone (as opposed to a random temp file based on jobName) - * This is intended to support job resume, and should be used with sharedTempDirectory=true to avoid conflicts. - */ - public void setDeterministicWorkingDirName(boolean deterministicWorkingDirName) - { - _deterministicWorkingDirName = deterministicWorkingDirName; - } - } - - public WorkDirectoryRemote(FileAnalysisJobSupport support, WorkDirFactory factory, Logger log, File lockDir, File tempDir, File transferToDirOnFailure, boolean reuseExistingDirectory, File folderToClean) throws IOException - { - super(support, factory, tempDir, reuseExistingDirectory, log); - - _lockDirectory = lockDir; - _transferToDirOnFailure = transferToDirOnFailure; - _folderToClean = folderToClean; - } - - /** - * @return a pair, where the first value is the total number of locks, and the second value is the lock index that - * should be used next - */ - private MasterLockInfo parseMasterLock(RandomAccessFile masterIn, File masterLockFile) throws IOException - { - ByteArrayOutputStream bOut = new ByteArrayOutputStream(); - byte[] b = new byte[128]; - int i; - while ((i = masterIn.read(b)) != -1) - { - bOut.write(b, 0, i); - } - String line = new String(bOut.toByteArray(), StringUtilsLabKey.DEFAULT_CHARSET).trim(); - int totalLocks = FILE_LOCKS_DEFAULT; - int currentIndex = 0; - if (!line.isEmpty()) - { - String[] parts = line.split(" "); - try - { - currentIndex = Integer.parseInt(parts[0]); - } - catch (NumberFormatException e) - { - throw new IOException("Could not parse the current lock index from the master lock file " + masterLockFile + ", the value was: " + parts[0]); - } - - if (parts.length > 1) - { - try - { - totalLocks = Integer.parseInt(parts[1]); - } - catch (NumberFormatException e) - { - throw new IOException("Could not parse the total number of locks from the master lock file " + masterLockFile + ", the value was: " + parts[1]); - } - } - - if (totalLocks < 1) - totalLocks = FILE_LOCKS_DEFAULT; - } - - if (currentIndex >= totalLocks) - { - currentIndex = 0; - } - return new MasterLockInfo(totalLocks, currentIndex); - } - - /** - * File system locks are fine to communicate locking between two different processes, but they don't work for - * multiple threads inside the same VM. We need to do Java-level locking as well. - */ - private static synchronized Lock getInMemoryLockObject(File f) - { - Lock result = _locks.get(f); - if (result == null) - { - result = new ReentrantLock(); - _locks.put(f, result); - } - return result; - } - - @Override - public void remove(boolean success) throws IOException - { - super.remove(success); - - // Issue 25166: this was a pre-existing potential bug. If _sharedTempDirectory is true, we create a second level - // of temp directory above the primary working dir. this is added to make sure we clean this up. - _jobLog.debug("inspecting remote work dir: " + (_folderToClean == null ? _dir.getPath() : _folderToClean.getPath())); - if (success && _folderToClean != null && !_dir.equals(_folderToClean)) - { - _jobLog.debug("removing entire work dir through: " + _folderToClean.getPath()); - _jobLog.debug("starting with: " + _dir.getPath()); - File toCheck = _dir; - - //debugging only: - if (!URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) - { - _jobLog.warn("not a descendant!"); - } - - while (toCheck != null && URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) - { - if (!toCheck.exists()) - { - _jobLog.debug("directory does not exist: " + toCheck.getPath()); - toCheck = toCheck.getParentFile(); - continue; - } - - String[] children = toCheck.list(); - if (children != null && children.length == 0) - { - _jobLog.debug("removing directory: " + toCheck.getPath()); - FileUtils.deleteDirectory(toCheck); - toCheck = toCheck.getParentFile(); - } - else if (children == null) - { - _jobLog.debug("unable to list children, will not delete: " + toCheck.getPath()); - continue; - } - else - { - _jobLog.debug("work directory has children, will not delete: " + toCheck.getPath()); - _jobLog.debug("files:"); - for (String fn : children) - { - _jobLog.debug(fn); - } - break; - } - } - } - } - - @Override - protected CopyingResource createCopyingLock() throws IOException - { - if (_lockDirectory == null) - { - return new SimpleCopyingResource(); - } - - _jobLog.debug("Starting to acquire lock for copying files"); - - MasterLockInfo lockInfo; - - // Synchronize to prevent multiple threads from trying to lock the master file from within the same VM - synchronized (WorkDirectoryRemote.class) - { - RandomAccessFile randomAccessFile = null; - FileLock masterLock = null; - - try - { - File masterLockFile = new File(_lockDirectory, "counter"); - randomAccessFile = new RandomAccessFile(masterLockFile, "rw"); - FileChannel masterChannel = randomAccessFile.getChannel(); - masterLock = masterChannel.lock(); - - lockInfo = parseMasterLock(randomAccessFile, masterLockFile); - int nextIndex = (lockInfo.getCurrentLock() + 1) % lockInfo.getTotalLocks(); - rewriteMasterLock(randomAccessFile, new MasterLockInfo(lockInfo.getTotalLocks(), nextIndex)); - } - finally - { - if (randomAccessFile != null) { try { randomAccessFile.close(); } catch (IOException e) {} } - if (masterLock != null) { try { masterLock.release(); } catch (IOException e) {} } - } - } - - _jobLog.debug("Acquiring lock #" + lockInfo.getCurrentLock()); - File f = new File(_lockDirectory, "lock" + lockInfo.getCurrentLock()); - FileChannel lockChannel = new FileOutputStream(f, true).getChannel(); - FileLockCopyingResource result = new FileLockCopyingResource(lockChannel, lockInfo.getCurrentLock(), f); - _jobLog.debug("Lock #" + lockInfo.getCurrentLock() + " acquired"); - - return result; - } - - private void rewriteMasterLock(RandomAccessFile masterFile, MasterLockInfo lockInfo) - throws IOException - { - masterFile.seek(0); - - String output = Integer.toString(lockInfo.getCurrentLock()); - if (lockInfo.getTotalLocks() != FILE_LOCKS_DEFAULT) - output += " " + Integer.toString(lockInfo.getTotalLocks()); - byte[] outputBytes = output.getBytes(StringUtilsLabKey.DEFAULT_CHARSET); - masterFile.write(outputBytes); - masterFile.setLength(outputBytes.length); - } - - private static class MasterLockInfo - { - private final int _totalLocks; - private final int _currentLock; - - private MasterLockInfo(int totalLocks, int currentLock) - { - assert totalLocks > 0 : "Total locks must be greater than 0."; - - _totalLocks = totalLocks; - _currentLock = currentLock; - } - - public int getTotalLocks() - { - return _totalLocks; - } - - public int getCurrentLock() - { - return _currentLock; - } - } - - public class FileLockCopyingResource extends SimpleCopyingResource - { - private FileChannel _channel; - private final int _lockNumber; - private FileLock _lock; - private final Throwable _creationStack; - private Lock _memoryLock; - - public FileLockCopyingResource(FileChannel channel, int lockNumber, File f) throws IOException - { - _channel = channel; - _lockNumber = lockNumber; - _creationStack = new Throwable(); - - // Lock the memory part first to eliminate multi-threaded access to the same file - _memoryLock = getInMemoryLockObject(f); - _memoryLock.lock(); - - // Lock the file part second - _lock = _channel.lock(); - } - - @Override - protected void finalize() throws Throwable - { - super.finalize(); - if (_lock != null) - { - _systemLog.error("FileLockCopyingResource was not released before it was garbage collected. Creation stack is: ", _creationStack); - } - close(); - } - - @Override - public void close() - { - if (_lock != null) - { - // Unlock the file part first - try { _lock.release(); } catch (IOException e) {} - try { _channel.close(); } catch (IOException e) {} - _jobLog.debug("Lock #" + _lockNumber + " released"); - _lock = null; - _channel = null; - super.close(); - - // Unlock the memory part last - _memoryLock.unlock(); - _memoryLock = null; - } - } - } +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.pipeline.api; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.pipeline.WorkDirFactory; +import org.labkey.api.pipeline.WorkDirectory; +import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URIUtil; +import org.springframework.beans.factory.InitializingBean; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Used to copy files from (and back to) a remote file system so that they can be used directly on the local file system, + * improving performance on high-latency and/or low-bandwidth network file systems + * + * @author jeckels + */ +public class WorkDirectoryRemote extends AbstractWorkDirectory +{ + private static final Logger _systemLog = LogManager.getLogger(WorkDirectoryRemote.class); + + private static final int FILE_LOCKS_DEFAULT = 5; + + private final File _lockDirectory; + private final File _folderToClean; + + private static final Map _locks = new HashMap<>(); + + @Override + public File inputFile(File fileInput, boolean forceCopy) throws IOException + { + return inputFile(fileInput, newFile(fileInput.getName()), forceCopy); + } + + @Override + public File inputFile(File fileInput, File fileWork, boolean forceCopy) throws IOException + { + //can be used to prevent duplicate copy attempts + if (fileWork.exists() && !forceCopy) + { + _copiedInputs.put(fileInput, fileWork); + } + + return copyInputFile(fileInput, fileWork); + } + + public static class Factory extends AbstractFactory implements InitializingBean + { + private String _lockDirectory; + private String _tempDirectory; + private boolean _sharedTempDirectory; + private boolean _allowReuseExistingTempDirectory; + private boolean _deterministicWorkingDirName; + private boolean _cleanupOnStartup; + private String _transferToDirOnFailure = null; + + @Override + public void afterPropertiesSet() + { + if (_tempDirectory == null) + { + throw new IllegalStateException("tempDirectory not set - set it directly using the tempDirectory property or use the tempDirectoryEnv property to point to an environment variable"); + } + if (_cleanupOnStartup) + { + FileUtil.deleteDirectoryContents(new File(_tempDirectory)); + } + } + + @Override + public WorkDirectory createWorkDirectory(String jobId, FileAnalysisJobSupport support, boolean useDeterministicFolderPath, Logger log) throws IOException + { + if (useDeterministicFolderPath) + { + _sharedTempDirectory = true; + _allowReuseExistingTempDirectory = true; + _deterministicWorkingDirName = true; + } + + File tempDir; + File tempDirBase = null; + int attempt = 0; + do + { + // We've seen very intermittent problems failing to create temp files in the past during the DRTs, + // so try a few times before failing + File dirParent = (_tempDirectory == null ? null : new File(_tempDirectory)); + + // If the temp directory is shared, then create a jobId directory to be sure the + // work directory path is unique. + try + { + if (_sharedTempDirectory) + { + if (_deterministicWorkingDirName) + { + dirParent = new File(dirParent, jobId); + tempDirBase = dirParent; + } + else + { + dirParent = FileUtil.createTempFile(jobId, "", dirParent); + tempDirBase = dirParent; + } + + if (_allowReuseExistingTempDirectory && dirParent.exists()) + { + log.info("parent directory exists, reusing: " + dirParent.getPath()); + } + else + { + dirParent.delete(); + FileUtil.mkdirs(dirParent); + } + } + + String name = support.getBaseName(); + if (name.length() > 10) + { + // Don't let the total path get too long - Windows doesn't like paths longer than 255 characters + // so if there's a ridiculously long file name, we don't want to duplicate its name in the + // directory too + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 9); + } + else if (name.length() < 3) + { + //File.createTempFile() does not allow prefixes <3 chars + name = "wd_" + name; + } + + if (_deterministicWorkingDirName) + { + tempDir = new File(dirParent, name + WORK_DIR_SUFFIX); + } + else + { + tempDir = FileUtil.createTempFile(name, WORK_DIR_SUFFIX, dirParent); + } + + if (_allowReuseExistingTempDirectory && tempDir.exists()) + { + log.info("working directory exists, reusing: " + dirParent.getPath()); + } + else + { + tempDir.delete(); + FileUtil.mkdirs(tempDir); + } + } + catch (IOException e) + { + IOException ioException = new IOException("Failed to create local working directory in the tempDirectory " + + dirParent + ", specified in the tempDirectory property in the pipeline configuration"); + ioException.initCause(e); + _systemLog.error(ioException.getMessage(), e); + throw ioException; + } + attempt++; + } + while (attempt < 5 && !tempDir.isDirectory()); + if (!tempDir.isDirectory()) + { + throw new IOException("Failed to create local working directory " + tempDir); + } + + File lockDir = (_lockDirectory == null ? null : new File(_lockDirectory)); + File transferToDirOnFailure = (_transferToDirOnFailure == null ? null : new File(_transferToDirOnFailure)); + return new WorkDirectoryRemote(support, this, log, lockDir, tempDir, transferToDirOnFailure, _allowReuseExistingTempDirectory, tempDirBase); + } + + public String getLockDirectory() + { + if (_lockDirectory!= null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File lockDir = new File(_lockDirectory); + if (!NetworkDrive.exists(lockDir) || !lockDir.isDirectory()) + throw new IllegalArgumentException("The lock directory " + _lockDirectory + " does not exist."); + } + return _lockDirectory; + } + + public void setLockDirectory(String directoryString) + { + _lockDirectory = directoryString; + } + + public String getTempDirectory() + { + if (_tempDirectory != null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File tempDir = new File(_tempDirectory); + if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) + throw new IllegalArgumentException("The temporary directory " + _tempDirectory + " does not exist."); + } + return _tempDirectory; + } + + /** @param directoryString path of the directory to be used as scratch space */ + public void setTempDirectory(String directoryString) + { + _tempDirectory = directoryString; + } + + public void setCleanupOnStartup(boolean cleanupOnStartup) + { + _cleanupOnStartup = cleanupOnStartup; + } + + /** + * Set to an environment variable set to the path to use for the temporary directory. + * (e.g. some cluster schedulers initialize TMPDIR to a job specific temporary directory + * which will be removed, if the job is cancelled) + * + * @param tempDirectoryVar environment variable name + */ + public void setTempDirectoryEnv(String tempDirectoryVar) + { + String tempDirectory = System.getenv(tempDirectoryVar); + if (tempDirectory == null || tempDirectory.isEmpty()) + throw new IllegalArgumentException("The environment variable " + tempDirectoryVar + " does not exist:\n" + System.getenv()); + setTempDirectory(tempDirectory); + } + + /** + * @return true if the root temporary directory will be shared by multiple tasks + */ + public boolean isSharedTempDirectory() + { + return _sharedTempDirectory; + } + + /** + * Set to true, if the root temporary directory will be shared by multiple tasks. + * This is usually not necessary on a scheduled computational cluster, where each + * task is given a separate working environment. + * + * @param sharedTempDirectory true if the root temporary directory will be shared by multiple tasks + */ + public void setSharedTempDirectory(boolean sharedTempDirectory) + { + _sharedTempDirectory = sharedTempDirectory; + } + + public String getTransferToDirOnFailure() + { + if (_transferToDirOnFailure != null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File tempDir = new File(_transferToDirOnFailure); + if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) + throw new IllegalArgumentException("The directory " + _transferToDirOnFailure + " does not exist."); + } + + return _transferToDirOnFailure; + } + + /** + * If a directory is provided, when a remote job fails, the working directory will + * be moved from the working location to a directory under this folder + */ + public void setTransferToDirOnFailure(String transferToDirOnFailure) + { + _transferToDirOnFailure = transferToDirOnFailure; + } + + public boolean isAllowReuseExistingTempDirectory() + { + return _allowReuseExistingTempDirectory; + } + + /** + * If true, instead of deleting an existing working directory on job startup, an existing directory will be reused. + * This is mostly used to allow job resume, and should only be used + */ + public void setAllowReuseExistingTempDirectory(boolean allowReuseExistingTempDirectory) + { + _allowReuseExistingTempDirectory = allowReuseExistingTempDirectory; + } + + public boolean isDeterministicWorkingDirName() + { + return _deterministicWorkingDirName; + } + + /** + * If true, the working directory for each job will be named using the job' name alone (as opposed to a random temp file based on jobName) + * This is intended to support job resume, and should be used with sharedTempDirectory=true to avoid conflicts. + */ + public void setDeterministicWorkingDirName(boolean deterministicWorkingDirName) + { + _deterministicWorkingDirName = deterministicWorkingDirName; + } + } + + public WorkDirectoryRemote(FileAnalysisJobSupport support, WorkDirFactory factory, Logger log, File lockDir, File tempDir, File transferToDirOnFailure, boolean reuseExistingDirectory, File folderToClean) throws IOException + { + super(support, factory, tempDir, reuseExistingDirectory, log); + + _lockDirectory = lockDir; + _transferToDirOnFailure = transferToDirOnFailure; + _folderToClean = folderToClean; + } + + /** + * @return a pair, where the first value is the total number of locks, and the second value is the lock index that + * should be used next + */ + private MasterLockInfo parseMasterLock(RandomAccessFile masterIn, File masterLockFile) throws IOException + { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + byte[] b = new byte[128]; + int i; + while ((i = masterIn.read(b)) != -1) + { + bOut.write(b, 0, i); + } + String line = new String(bOut.toByteArray(), StringUtilsLabKey.DEFAULT_CHARSET).trim(); + int totalLocks = FILE_LOCKS_DEFAULT; + int currentIndex = 0; + if (!line.isEmpty()) + { + String[] parts = line.split(" "); + try + { + currentIndex = Integer.parseInt(parts[0]); + } + catch (NumberFormatException e) + { + throw new IOException("Could not parse the current lock index from the master lock file " + masterLockFile + ", the value was: " + parts[0]); + } + + if (parts.length > 1) + { + try + { + totalLocks = Integer.parseInt(parts[1]); + } + catch (NumberFormatException e) + { + throw new IOException("Could not parse the total number of locks from the master lock file " + masterLockFile + ", the value was: " + parts[1]); + } + } + + if (totalLocks < 1) + totalLocks = FILE_LOCKS_DEFAULT; + } + + if (currentIndex >= totalLocks) + { + currentIndex = 0; + } + return new MasterLockInfo(totalLocks, currentIndex); + } + + /** + * File system locks are fine to communicate locking between two different processes, but they don't work for + * multiple threads inside the same VM. We need to do Java-level locking as well. + */ + private static synchronized Lock getInMemoryLockObject(File f) + { + Lock result = _locks.get(f); + if (result == null) + { + result = new ReentrantLock(); + _locks.put(f, result); + } + return result; + } + + @Override + public void remove(boolean success) throws IOException + { + super.remove(success); + + // Issue 25166: this was a pre-existing potential bug. If _sharedTempDirectory is true, we create a second level + // of temp directory above the primary working dir. this is added to make sure we clean this up. + _jobLog.debug("inspecting remote work dir: " + (_folderToClean == null ? _dir.getPath() : _folderToClean.getPath())); + if (success && _folderToClean != null && !_dir.equals(_folderToClean)) + { + _jobLog.debug("removing entire work dir through: " + _folderToClean.getPath()); + _jobLog.debug("starting with: " + _dir.getPath()); + File toCheck = _dir; + + //debugging only: + if (!URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) + { + _jobLog.warn("not a descendant!"); + } + + while (toCheck != null && URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) + { + if (!toCheck.exists()) + { + _jobLog.debug("directory does not exist: " + toCheck.getPath()); + toCheck = toCheck.getParentFile(); + continue; + } + + String[] children = toCheck.list(); + if (children != null && children.length == 0) + { + _jobLog.debug("removing directory: " + toCheck.getPath()); + FileUtils.deleteDirectory(toCheck); + toCheck = toCheck.getParentFile(); + } + else if (children == null) + { + _jobLog.debug("unable to list children, will not delete: " + toCheck.getPath()); + continue; + } + else + { + _jobLog.debug("work directory has children, will not delete: " + toCheck.getPath()); + _jobLog.debug("files:"); + for (String fn : children) + { + _jobLog.debug(fn); + } + break; + } + } + } + } + + @Override + protected CopyingResource createCopyingLock() throws IOException + { + if (_lockDirectory == null) + { + return new SimpleCopyingResource(); + } + + _jobLog.debug("Starting to acquire lock for copying files"); + + MasterLockInfo lockInfo; + + // Synchronize to prevent multiple threads from trying to lock the master file from within the same VM + synchronized (WorkDirectoryRemote.class) + { + RandomAccessFile randomAccessFile = null; + FileLock masterLock = null; + + try + { + File masterLockFile = new File(_lockDirectory, "counter"); + randomAccessFile = new RandomAccessFile(masterLockFile, "rw"); + FileChannel masterChannel = randomAccessFile.getChannel(); + masterLock = masterChannel.lock(); + + lockInfo = parseMasterLock(randomAccessFile, masterLockFile); + int nextIndex = (lockInfo.getCurrentLock() + 1) % lockInfo.getTotalLocks(); + rewriteMasterLock(randomAccessFile, new MasterLockInfo(lockInfo.getTotalLocks(), nextIndex)); + } + finally + { + if (randomAccessFile != null) { try { randomAccessFile.close(); } catch (IOException e) {} } + if (masterLock != null) { try { masterLock.release(); } catch (IOException e) {} } + } + } + + _jobLog.debug("Acquiring lock #" + lockInfo.getCurrentLock()); + File f = new File(_lockDirectory, "lock" + lockInfo.getCurrentLock()); + FileChannel lockChannel = new FileOutputStream(f, true).getChannel(); + FileLockCopyingResource result = new FileLockCopyingResource(lockChannel, lockInfo.getCurrentLock(), f); + _jobLog.debug("Lock #" + lockInfo.getCurrentLock() + " acquired"); + + return result; + } + + private void rewriteMasterLock(RandomAccessFile masterFile, MasterLockInfo lockInfo) + throws IOException + { + masterFile.seek(0); + + String output = Integer.toString(lockInfo.getCurrentLock()); + if (lockInfo.getTotalLocks() != FILE_LOCKS_DEFAULT) + output += " " + Integer.toString(lockInfo.getTotalLocks()); + byte[] outputBytes = output.getBytes(StringUtilsLabKey.DEFAULT_CHARSET); + masterFile.write(outputBytes); + masterFile.setLength(outputBytes.length); + } + + private static class MasterLockInfo + { + private final int _totalLocks; + private final int _currentLock; + + private MasterLockInfo(int totalLocks, int currentLock) + { + assert totalLocks > 0 : "Total locks must be greater than 0."; + + _totalLocks = totalLocks; + _currentLock = currentLock; + } + + public int getTotalLocks() + { + return _totalLocks; + } + + public int getCurrentLock() + { + return _currentLock; + } + } + + public class FileLockCopyingResource extends SimpleCopyingResource + { + private FileChannel _channel; + private final int _lockNumber; + private FileLock _lock; + private final Throwable _creationStack; + private Lock _memoryLock; + + public FileLockCopyingResource(FileChannel channel, int lockNumber, File f) throws IOException + { + _channel = channel; + _lockNumber = lockNumber; + _creationStack = new Throwable(); + + // Lock the memory part first to eliminate multi-threaded access to the same file + _memoryLock = getInMemoryLockObject(f); + _memoryLock.lock(); + + // Lock the file part second + _lock = _channel.lock(); + } + + @Override + protected void finalize() throws Throwable + { + super.finalize(); + if (_lock != null) + { + _systemLog.error("FileLockCopyingResource was not released before it was garbage collected. Creation stack is: ", _creationStack); + } + close(); + } + + @Override + public void close() + { + if (_lock != null) + { + // Unlock the file part first + try { _lock.release(); } catch (IOException e) {} + try { _channel.close(); } catch (IOException e) {} + _jobLog.debug("Lock #" + _lockNumber + " released"); + _lock = null; + _channel = null; + super.close(); + + // Unlock the memory part last + _memoryLock.unlock(); + _memoryLock = null; + } + } + } } \ No newline at end of file diff --git a/search/src/org/labkey/search/SearchController.java b/search/src/org/labkey/search/SearchController.java index 20036ac9877..fe1ddb5291a 100644 --- a/search/src/org/labkey/search/SearchController.java +++ b/search/src/org/labkey/search/SearchController.java @@ -1,1197 +1,1198 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.search; - -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasBindParameters; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.search.SearchResultTemplate; -import org.labkey.api.search.SearchScope; -import org.labkey.api.search.SearchService; -import org.labkey.api.search.SearchService.SearchResult; -import org.labkey.api.search.SearchUrls; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.Path; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.webdav.WebdavService; -import org.labkey.search.audit.SearchAuditProvider; -import org.labkey.search.model.AbstractSearchService; -import org.labkey.search.model.CrawlerRunningState; -import org.labkey.search.model.IndexInspector; -import org.labkey.search.model.LuceneDirectoryType; -import org.labkey.search.model.SearchPropertyManager; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.PropertyValues; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; - -import java.io.IOException; -import java.sql.Date; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import static org.labkey.api.action.BaseViewAction.springBindParameters; - -public class SearchController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); - - private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); - - public SearchController() - { - setActionResolver(_actionResolver); - } - - - @SuppressWarnings("unused") - public static class SearchUrlsImpl implements SearchUrls - { - @Override - public ActionURL getSearchURL(Container c, @Nullable String query) - { - return SearchController.getSearchURL(c, query); - } - - @Override - public ActionURL getSearchURL(String query, String category) - { - return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); - } - - @Override - public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) - { - return SearchController.getSearchURL(c, query, null, template); - } - } - - - @RequiresPermission(ReadPermission.class) - public class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getSearchURL(); - } - } - - - public static class AdminForm - { - public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; - private int msg = 0; - private boolean pause; - private boolean start; - private boolean delete; - private String indexPath; - - private boolean limit; - private int fileLimitMB; - - private boolean _path; - - private boolean _directory; - private String _directoryType; - - public String getMessage() - { - return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; - } - - public void setMsg(int m) - { - msg = m; - } - - public boolean isDelete() - { - return delete; - } - - public void setDelete(boolean delete) - { - this.delete = delete; - } - - public boolean isStart() - { - return start; - } - - public void setStart(boolean start) - { - this.start = start; - } - - public boolean isPause() - { - return pause; - } - - public void setPause(boolean pause) - { - this.pause = pause; - } - - public String getIndexPath() - { - return indexPath; - } - - public void setIndexPath(String indexPath) - { - this.indexPath = indexPath; - } - - public boolean isPath() - { - return _path; - } - - public void setPath(boolean path) - { - _path = path; - } - - public boolean isDirectory() - { - return _directory; - } - - public void setDirectory(boolean directory) - { - _directory = directory; - } - - public String getDirectoryType() - { - return _directoryType; - } - - public void setDirectoryType(String directoryType) - { - _directoryType = directoryType; - } - - public boolean isLimit() - { - return limit; - } - - public void setLimit(boolean limit) - { - this.limit = limit; - } - - public int getFileLimitMB() - { - return fileLimitMB; - } - - public int setFileLimitMB(int fileLimitMB) - { - return this.fileLimitMB = fileLimitMB; - } - } - - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class AdminAction extends FormViewAction - { - @SuppressWarnings("UnusedDeclaration") - public AdminAction() - { - } - - public AdminAction(ViewContext ctx, PageConfig pageConfig) - { - setViewContext(ctx); - setPageConfig(pageConfig); - } - - private int _msgid = 0; - - @Override - public void validateCommand(AdminForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) - { - @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) - Throwable t = SearchService.get().getConfigurationError(); - - VBox vbox = new VBox(); - - if (null != t) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); - builder.append(ExceptionUtil.renderException(t)); - WebPartView configErrorView = new HtmlView(builder); - configErrorView.setTitle("Search Configuration Error"); - configErrorView.setFrame(WebPartView.FrameType.PORTAL); - vbox.addView(configErrorView); - } - - // Spring errors get displayed in the "Index Configuration" pane - WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); - indexerView.setTitle("Index Configuration"); - vbox.addView(indexerView); - - // Won't be able to gather statistics if the search index is misconfigured - if (null == t) - { - WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); - indexerStatsView.setTitle("Index Statistics"); - vbox.addView(indexerStatsView); - } - - WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); - searchStatsView.setTitle("Search Statistics"); - vbox.addView(searchStatsView); - - return vbox; - } - - @Override - public boolean handlePost(AdminForm form, BindException errors) - { - SearchService ss = SearchService.get(); - - if (form.isStart()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); - ss.startCrawler(); - } - else if (form.isPause()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); - ss.pauseCrawler(); - } - else if (form.isDelete()) - { - ss.deleteIndex("a site admin requested it"); - ss.start(); - SearchPropertyManager.audit(getUser(), "Index Deleted"); - _msgid = 1; - } - else if (form.isPath()) - { - SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); - ss.updateIndex(); - _msgid = 2; - } - else if (form.isDirectory()) - { - LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); - if (null == type) - { - errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); - return false; - } - SearchPropertyManager.setDirectoryType(getUser(), type); - ss.resetIndex(); - _msgid = 3; - } - else if (form.isLimit()) - { - SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); - ss.resetIndex(); - _msgid = 4; - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AdminForm o) - { - ActionURL success = new ActionURL(AdminAction.class, getContainer()); - if (0 != _msgid) - success.addParameter("msg", String.valueOf(_msgid)); - return success; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("searchAdmin"); - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); - } - } - - - @AdminConsoleAction - public static class IndexContentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/exportContents.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); - root.addChild("Index Contents"); - } - } - - - public static class ExportForm - { - private String _format = "Text"; - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - - @AdminConsoleAction - public static class ExportIndexContentsAction extends ExportAction - { - @Override - public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception - { - new IndexInspector().export(response, form.getFormat()); - } - } - - - /** for selenium testing */ - @RequiresSiteAdmin - public static class WaitForIdleAction extends SimpleRedirectAction - { - @Override - public URLHelper getRedirectURL(Object o) throws Exception - { - SearchService ss = AbstractSearchService.get(); - ss.waitForIdle(); - return new ActionURL(AdminAction.class, getContainer()); - } - } - - // UNDONE: remove; for testing only - @RequiresSiteAdmin - public class CancelAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - SearchService.IndexTask defaultTask = ss.defaultTask(); - for (SearchService.IndexTask task : ss.getTasks()) - { - if (task != defaultTask && !task.isCancelled()) - task.cancel(true); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - - // UNDONE: remove; for testing only - // cause the current directory to be crawled soon - @RequiresSiteAdmin - public class CrawlAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - - ss.addPathToCrawl( - WebdavService.getPath().append(getContainer().getParsedPath()), - new Date(System.currentTimeMillis())); - - return form.getReturnActionURL(getSearchURL()); - } - } - - public static class IndexForm extends ReturnUrlForm - { - boolean _full = false; - boolean _wait = false; - boolean _since = false; - - public boolean isFull() - { - return _full; - } - - @SuppressWarnings("unused") - public void setFull(boolean full) - { - _full = full; - } - - public boolean isWait() - { - return _wait; - } - - @SuppressWarnings("unused") - public void setWait(boolean wait) - { - _wait = wait; - } - - public boolean isSince() - { - return _since; - } - - @SuppressWarnings("unused") - public void setSince(boolean since) - { - _since = since; - } - } - - // for testing only - @RequiresSiteAdmin - public class IndexAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(IndexForm form) throws Exception - { - SearchService ss = SearchService.get(); - - SearchService.IndexTask task = null; - - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - if (form.isFull()) - { - ss.indexFull(true, "a site admin requested it"); - } - else if (form.isSince()) - { - task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); - } - else - { - task = ss.indexContainer(null, getContainer(), null); - } - } - - if (form.isWait() && null != task) - { - task.get(); // wait for completion - if (ss instanceof AbstractSearchService) - ((AbstractSearchService)ss).commit(); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - @RequiresPermission(ReadPermission.class) - public class JsonAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SearchForm form, BindException errors) - { - SearchService ss = SearchService.get(); - - audit(form); - - final Path contextPath = Path.parse(getViewContext().getContextPath()); - - final String query = form.getQ() - .replaceAll("(? hits = result.hits; - totalHits = result.totalHits; - - arr = new Object[hits.size()]; - - int i = 0; - int batchSize = 1000; - Map> docDataMap = new HashMap<>(); - for (int ind = 0; ind < hits.size(); ind++) - { - SearchService.SearchHit hit = hits.get(ind); - JSONObject o = new JSONObject(); - String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; - - o.put("id", id); - o.put("title", hit.title); - o.put("container", hit.container); - o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); - o.put("summary", StringUtils.trimToEmpty(hit.summary)); - o.put("score", hit.score); - o.put("identifiers", hit.identifiers); - o.put("category", StringUtils.trimToEmpty(hit.category)); - - if (form.isExperimentalCustomJson()) - { - o.put("jsonData", hit.jsonData); - - if (ind % batchSize == 0) - { - int batchEnd = Math.min(hits.size(), ind + batchSize); - List docIds = new ArrayList<>(); - for (int j = ind; j < batchEnd; j++) - docIds.add(hits.get(j).docid); - - docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); - } - - Map custom = docDataMap.get(hit.docid); - if (custom != null) - o.put("data", custom); - } - - arr[i++] = o; - } - } - - JSONObject metaData = new JSONObject(); - metaData.put("idProperty","id"); - metaData.put("root", "hits"); - metaData.put("successProperty", "success"); - - response.put("metaData", metaData); - response.put("success",true); - response.put("hits", arr); - response.put("totalHits", totalHits); - response.put("q", query); - - return new ApiSimpleResponse(response); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class TestJson extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - public static ActionURL getSearchURL(Container c) - { - return new ActionURL(SearchAction.class, c); - } - - private ActionURL getSearchURL() - { - return getSearchURL(getContainer()); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString) - { - return getSearchURL(c, queryString, null, null); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) - { - ActionURL url = getSearchURL(c); - - if (null != queryString) - url.addParameter("q", queryString); - - if (null != category) - url.addParameter("category", category); - - if (null != template) - url.addParameter("template", template); - - return url; - } - - // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. - public interface SearchConfiguration - { - ActionURL getPostURL(Container c); // Search does not actually post - String getDescription(Container c); - SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; - boolean includeAdvancedUI(); - boolean includeNavigationLinks(); - } - - - public static class InternalSearchConfiguration implements SearchConfiguration - { - private final SearchService _ss = SearchService.get(); - - private InternalSearchConfiguration() - { - } - - @Override - public ActionURL getPostURL(Container c) - { - return getSearchURL(c); - } - - @Override - public String getDescription(Container c) - { - return LookAndFeelProperties.getInstance(c).getShortName(); - } - - @Override - public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException - { - SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); - options.categories = _ss.getCategories(category); - options.invertResults = invertSort; - options.limit = limit; - options.offset = offset; - options.scope = scope; - options.sortField = sortField; - - return _ss.search(options.build()); - } - - @Override - public boolean includeAdvancedUI() - { - return true; - } - - @Override - public boolean includeNavigationLinks() - { - return true; - } - } - - - @RequiresPermission(ReadPermission.class) - public class SearchAction extends SimpleViewAction - { - private String _category = null; - private SearchScope _scope = null; - private SearchForm _form = null; - - @Override - public ModelAndView getView(SearchForm form, BindException errors) - { - _category = form.getCategory(); - _scope = form.getSearchScope(); - _form = form; - - if (null == _scope || null == _scope.getRoot(getContainer())) - { - throw new NotFoundException(); - } - - form.setPrint(isPrint()); - - audit(form); - - // reenable caching for search results page (fast browser back button) - HttpServletResponse response = getViewContext().getResponse(); - ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); - getPageConfig().setNoIndex(); - setHelpTopic("luceneSearch"); - - return new JspView<>("/org/labkey/search/view/search.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); - } - } - - public static class PriorityForm - { - SearchService.PRIORITY priority = SearchService.PRIORITY.modified; - - public SearchService.PRIORITY getPriority() - { - return priority; - } - - public void setPriority(SearchService.PRIORITY priority) - { - this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); - } - } - - // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue - // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). - // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It - // does not guarantee that all indexed content has been committed... but that may not be required in practice. - - @RequiresPermission(ApplicationAdminPermission.class) - public static class WaitForIndexerAction extends ExportAction - { - @Override - public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception - { - SearchService ss = SearchService.get(); - long startTime = System.currentTimeMillis(); - boolean success = ss.drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); - - LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); - - // Return an error if we time out - if (!success) - response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); - } - } - - @RequiresPermission(ReadPermission.class) - public class CommentAction extends FormHandlerAction - { - @Override - public void validateCommand(SearchForm target, Errors errors) - { - } - - @Override - public boolean handlePost(SearchForm searchForm, BindException errors) - { - audit(searchForm); - return true; - } - - @Override - public URLHelper getSuccessURL(SearchForm searchForm) - { - return getSearchURL(); - } - } - - - @SuppressWarnings("unused") - public static class SearchForm implements HasBindParameters - { - private String _query = ""; - private String _sortField; - private boolean _print = false; - private int _offset = 0; - private int _limit = 1000; - private String _category = null; - private String _comment = null; - private int _textBoxWidth = 50; // default size - private List _fields; - private boolean _includeHelpLink = true; - private boolean _webpart = false; - private boolean _showAdvanced = false; - private boolean _invertSort = false; - private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) - private String _template = null; - private SearchScope _scope = SearchScope.All; - private boolean _normalizeUrls = false; - private boolean _experimentalCustomJson = false; - - public void setConfiguration(SearchConfiguration config) - { - _config = config; - } - - public SearchConfiguration getConfig() - { - return _config; - } - - public String getQ() - { - return _query; - } - - public void setQ(String query) - { - _query = StringUtils.trimToEmpty(query); - } - - public String getSortField() - { - return _sortField; - } - - public void setSortField(String sortField) - { - _sortField = sortField; - } - - public boolean isPrint() - { - return _print; - } - - public void setPrint(boolean print) - { - _print = print; - } - - public int getOffset() - { - return _offset; - } - - public void setOffset(int o) - { - _offset = o; - } - - public int getLimit() - { - return _limit; - } - - public void setLimit(int o) - { - _limit = o; - } - - public String getScope() - { - return _scope.name(); - } - - public void setScope(String scope) - { - try - { - _scope = SearchScope.valueOf(scope); - } - catch (IllegalArgumentException e) - { - _scope = SearchScope.All; - } - } - - public SearchScope getSearchScope() - { - return _scope; - } - - public String getCategory() - { - return _category; - } - - public void setCategory(String category) - { - _category = category; - } - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public int getTextBoxWidth() - { - return _textBoxWidth; - } - - public void setTextBoxWidth(int textBoxWidth) - { - _textBoxWidth = textBoxWidth; - } - - public boolean getIncludeHelpLink() - { - return _includeHelpLink; - } - - public void setIncludeHelpLink(boolean includeHelpLink) - { - _includeHelpLink = includeHelpLink; - } - - public boolean isWebPart() - { - return _webpart; - } - - public void setWebPart(boolean webpart) - { - _webpart = webpart; - } - - public boolean isShowAdvanced() - { - return _showAdvanced; - } - - public void setShowAdvanced(boolean showAdvanced) - { - _showAdvanced = showAdvanced; - } - - public boolean isInvertSort() - { - return _invertSort; - } - - public void setInvertSort(boolean invertSort) - { - _invertSort = invertSort; - } - - public String getTemplate() - { - return _template; - } - - public void setTemplate(String template) - { - _template = template; - } - - public SearchResultTemplate getSearchResultTemplate() - { - SearchService ss = AbstractSearchService.get(); - return ss.getSearchResultTemplate(getTemplate()); - } - - public boolean isNormalizeUrls() - { - return _normalizeUrls; - } - - public void setNormalizeUrls(boolean normalizeUrls) - { - _normalizeUrls = normalizeUrls; - } - - public boolean isExperimentalCustomJson() - { - return _experimentalCustomJson; - } - - public void setExperimentalCustomJson(boolean experimentalCustomJson) - { - _experimentalCustomJson = experimentalCustomJson; - } - - public List getFields() - { - return _fields; - } - - public void setFields(List fields) - { - _fields = fields; - } - - @Override - public @NotNull BindException bindParameters(PropertyValues m) - { - MutablePropertyValues mpvs = new MutablePropertyValues(m); - var q = mpvs.getPropertyValue("q"); - if (null != q && q.getValue() instanceof String[] arr) - { - mpvs.removePropertyValue("q"); - mpvs.addPropertyValue("q", StringUtils.join(arr," ")); - } - return springBindParameters(this, "form", mpvs); - } - } - - - protected void audit(SearchForm form) - { - ViewContext ctx = getViewContext(); - String comment = form.getComment(); - - audit(ctx.getUser(), ctx.getContainer(), form.getQ(), comment); - } - - - public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) - { - if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) - return; - - AuditLogService audit = AuditLogService.get(); - if (null == audit) - return; - - if (null == c) - c = ContainerManager.getRoot(); - - if (query.length() > 200) - query = query.substring(0, 197) + "..."; - - SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); - event.setQuery(query); - - AuditLogService.get().addEvent(user, event); - } - - - public static class SearchSettingsForm - { - private boolean _searchable; - - public boolean isSearchable() - { - return _searchable; - } - - @SuppressWarnings("unused") - public void setSearchable(boolean searchable) - { - _searchable = searchable; - } - } - - - @RequiresPermission(AdminPermission.class) - public static class SearchSettingsAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); - } - - @Override - public void validateCommand(SearchSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SearchSettingsForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SearchSettingsForm searchForm) - { - // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) - return getViewContext().getActionURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - setHelpTopic("searchAdmin"); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.search; + +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasBindParameters; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.search.SearchResultTemplate; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.search.SearchService.SearchResult; +import org.labkey.api.search.SearchUrls; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.Path; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.webdav.WebdavService; +import org.labkey.search.audit.SearchAuditProvider; +import org.labkey.search.model.AbstractSearchService; +import org.labkey.search.model.CrawlerRunningState; +import org.labkey.search.model.IndexInspector; +import org.labkey.search.model.LuceneDirectoryType; +import org.labkey.search.model.SearchPropertyManager; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValues; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +import java.io.IOException; +import java.sql.Date; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.labkey.api.action.BaseViewAction.springBindParameters; + +public class SearchController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); + + private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); + + public SearchController() + { + setActionResolver(_actionResolver); + } + + + @SuppressWarnings("unused") + public static class SearchUrlsImpl implements SearchUrls + { + @Override + public ActionURL getSearchURL(Container c, @Nullable String query) + { + return SearchController.getSearchURL(c, query); + } + + @Override + public ActionURL getSearchURL(String query, String category) + { + return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); + } + + @Override + public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) + { + return SearchController.getSearchURL(c, query, null, template); + } + } + + + @RequiresPermission(ReadPermission.class) + public class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getSearchURL(); + } + } + + + public static class AdminForm + { + public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; + private int msg = 0; + private boolean pause; + private boolean start; + private boolean delete; + private String indexPath; + + private boolean limit; + private int fileLimitMB; + + private boolean _path; + + private boolean _directory; + private String _directoryType; + + public String getMessage() + { + return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; + } + + public void setMsg(int m) + { + msg = m; + } + + public boolean isDelete() + { + return delete; + } + + public void setDelete(boolean delete) + { + this.delete = delete; + } + + public boolean isStart() + { + return start; + } + + public void setStart(boolean start) + { + this.start = start; + } + + public boolean isPause() + { + return pause; + } + + public void setPause(boolean pause) + { + this.pause = pause; + } + + public String getIndexPath() + { + return indexPath; + } + + public void setIndexPath(String indexPath) + { + this.indexPath = indexPath; + } + + public boolean isPath() + { + return _path; + } + + public void setPath(boolean path) + { + _path = path; + } + + public boolean isDirectory() + { + return _directory; + } + + public void setDirectory(boolean directory) + { + _directory = directory; + } + + public String getDirectoryType() + { + return _directoryType; + } + + public void setDirectoryType(String directoryType) + { + _directoryType = directoryType; + } + + public boolean isLimit() + { + return limit; + } + + public void setLimit(boolean limit) + { + this.limit = limit; + } + + public int getFileLimitMB() + { + return fileLimitMB; + } + + public int setFileLimitMB(int fileLimitMB) + { + return this.fileLimitMB = fileLimitMB; + } + } + + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class AdminAction extends FormViewAction + { + @SuppressWarnings("UnusedDeclaration") + public AdminAction() + { + } + + public AdminAction(ViewContext ctx, PageConfig pageConfig) + { + setViewContext(ctx); + setPageConfig(pageConfig); + } + + private int _msgid = 0; + + @Override + public void validateCommand(AdminForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) + { + @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) + Throwable t = SearchService.get().getConfigurationError(); + + VBox vbox = new VBox(); + + if (null != t) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); + builder.append(ExceptionUtil.renderException(t)); + WebPartView configErrorView = new HtmlView(builder); + configErrorView.setTitle("Search Configuration Error"); + configErrorView.setFrame(WebPartView.FrameType.PORTAL); + vbox.addView(configErrorView); + } + + // Spring errors get displayed in the "Index Configuration" pane + WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); + indexerView.setTitle("Index Configuration"); + vbox.addView(indexerView); + + // Won't be able to gather statistics if the search index is misconfigured + if (null == t) + { + WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); + indexerStatsView.setTitle("Index Statistics"); + vbox.addView(indexerStatsView); + } + + WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); + searchStatsView.setTitle("Search Statistics"); + vbox.addView(searchStatsView); + + return vbox; + } + + @Override + public boolean handlePost(AdminForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + if (form.isStart()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); + ss.startCrawler(); + } + else if (form.isPause()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); + ss.pauseCrawler(); + } + else if (form.isDelete()) + { + ss.deleteIndex("a site admin requested it"); + ss.start(); + SearchPropertyManager.audit(getUser(), "Index Deleted"); + _msgid = 1; + } + else if (form.isPath()) + { + SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); + ss.updateIndex(); + _msgid = 2; + } + else if (form.isDirectory()) + { + LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); + if (null == type) + { + errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); + return false; + } + SearchPropertyManager.setDirectoryType(getUser(), type); + ss.resetIndex(); + _msgid = 3; + } + else if (form.isLimit()) + { + SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); + ss.resetIndex(); + _msgid = 4; + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AdminForm o) + { + ActionURL success = new ActionURL(AdminAction.class, getContainer()); + if (0 != _msgid) + success.addParameter("msg", String.valueOf(_msgid)); + return success; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("searchAdmin"); + urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); + } + } + + + @AdminConsoleAction + public static class IndexContentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/exportContents.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); + root.addChild("Index Contents"); + } + } + + + public static class ExportForm + { + private String _format = "Text"; + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + + @AdminConsoleAction + public static class ExportIndexContentsAction extends ExportAction + { + @Override + public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception + { + new IndexInspector().export(response, form.getFormat()); + } + } + + + /** for selenium testing */ + @RequiresSiteAdmin + public static class WaitForIdleAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(Object o) throws Exception + { + SearchService ss = AbstractSearchService.get(); + ss.waitForIdle(); + return new ActionURL(AdminAction.class, getContainer()); + } + } + + // UNDONE: remove; for testing only + @RequiresSiteAdmin + public class CancelAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + SearchService.IndexTask defaultTask = ss.defaultTask(); + for (SearchService.IndexTask task : ss.getTasks()) + { + if (task != defaultTask && !task.isCancelled()) + task.cancel(true); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + + // UNDONE: remove; for testing only + // cause the current directory to be crawled soon + @RequiresSiteAdmin + public class CrawlAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + + ss.addPathToCrawl( + WebdavService.getPath().append(getContainer().getParsedPath()), + new Date(System.currentTimeMillis())); + + return form.getReturnActionURL(getSearchURL()); + } + } + + public static class IndexForm extends ReturnUrlForm + { + boolean _full = false; + boolean _wait = false; + boolean _since = false; + + public boolean isFull() + { + return _full; + } + + @SuppressWarnings("unused") + public void setFull(boolean full) + { + _full = full; + } + + public boolean isWait() + { + return _wait; + } + + @SuppressWarnings("unused") + public void setWait(boolean wait) + { + _wait = wait; + } + + public boolean isSince() + { + return _since; + } + + @SuppressWarnings("unused") + public void setSince(boolean since) + { + _since = since; + } + } + + // for testing only + @RequiresSiteAdmin + public class IndexAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(IndexForm form) throws Exception + { + SearchService ss = SearchService.get(); + + SearchService.IndexTask task = null; + + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + if (form.isFull()) + { + ss.indexFull(true, "a site admin requested it"); + } + else if (form.isSince()) + { + task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); + } + else + { + task = ss.indexContainer(null, getContainer(), null); + } + } + + if (form.isWait() && null != task) + { + task.get(); // wait for completion + if (ss instanceof AbstractSearchService) + ((AbstractSearchService)ss).commit(); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + @RequiresPermission(ReadPermission.class) + public class JsonAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SearchForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + audit(form); + + final Path contextPath = Path.parse(getViewContext().getContextPath()); + + final String query = form.getQ() + .replaceAll("(? hits = result.hits; + totalHits = result.totalHits; + + arr = new Object[hits.size()]; + + int i = 0; + int batchSize = 1000; + Map> docDataMap = new HashMap<>(); + for (int ind = 0; ind < hits.size(); ind++) + { + SearchService.SearchHit hit = hits.get(ind); + JSONObject o = new JSONObject(); + String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; + + o.put("id", id); + o.put("title", hit.title); + o.put("container", hit.container); + o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); + o.put("summary", StringUtils.trimToEmpty(hit.summary)); + o.put("score", hit.score); + o.put("identifiers", hit.identifiers); + o.put("category", StringUtils.trimToEmpty(hit.category)); + + if (form.isExperimentalCustomJson()) + { + o.put("jsonData", hit.jsonData); + + if (ind % batchSize == 0) + { + int batchEnd = Math.min(hits.size(), ind + batchSize); + List docIds = new ArrayList<>(); + for (int j = ind; j < batchEnd; j++) + docIds.add(hits.get(j).docid); + + docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); + } + + Map custom = docDataMap.get(hit.docid); + if (custom != null) + o.put("data", custom); + } + + arr[i++] = o; + } + } + + JSONObject metaData = new JSONObject(); + metaData.put("idProperty","id"); + metaData.put("root", "hits"); + metaData.put("successProperty", "success"); + + response.put("metaData", metaData); + response.put("success",true); + response.put("hits", arr); + response.put("totalHits", totalHits); + response.put("q", query); + + return new ApiSimpleResponse(response); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class TestJson extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + public static ActionURL getSearchURL(Container c) + { + return new ActionURL(SearchAction.class, c); + } + + private ActionURL getSearchURL() + { + return getSearchURL(getContainer()); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString) + { + return getSearchURL(c, queryString, null, null); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) + { + ActionURL url = getSearchURL(c); + + if (null != queryString) + url.addParameter("q", queryString); + + if (null != category) + url.addParameter("category", category); + + if (null != template) + url.addParameter("template", template); + + return url; + } + + // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. + public interface SearchConfiguration + { + ActionURL getPostURL(Container c); // Search does not actually post + String getDescription(Container c); + SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; + boolean includeAdvancedUI(); + boolean includeNavigationLinks(); + } + + + public static class InternalSearchConfiguration implements SearchConfiguration + { + private final SearchService _ss = SearchService.get(); + + private InternalSearchConfiguration() + { + } + + @Override + public ActionURL getPostURL(Container c) + { + return getSearchURL(c); + } + + @Override + public String getDescription(Container c) + { + return LookAndFeelProperties.getInstance(c).getShortName(); + } + + @Override + public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException + { + SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); + options.categories = _ss.getCategories(category); + options.invertResults = invertSort; + options.limit = limit; + options.offset = offset; + options.scope = scope; + options.sortField = sortField; + + return _ss.search(options.build()); + } + + @Override + public boolean includeAdvancedUI() + { + return true; + } + + @Override + public boolean includeNavigationLinks() + { + return true; + } + } + + + @RequiresPermission(ReadPermission.class) + public class SearchAction extends SimpleViewAction + { + private String _category = null; + private SearchScope _scope = null; + private SearchForm _form = null; + + @Override + public ModelAndView getView(SearchForm form, BindException errors) + { + _category = form.getCategory(); + _scope = form.getSearchScope(); + _form = form; + + if (null == _scope || null == _scope.getRoot(getContainer())) + { + throw new NotFoundException(); + } + + form.setPrint(isPrint()); + + audit(form); + + // reenable caching for search results page (fast browser back button) + HttpServletResponse response = getViewContext().getResponse(); + ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); + getPageConfig().setNoIndex(); + setHelpTopic("luceneSearch"); + + return new JspView<>("/org/labkey/search/view/search.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); + } + } + + public static class PriorityForm + { + SearchService.PRIORITY priority = SearchService.PRIORITY.modified; + + public SearchService.PRIORITY getPriority() + { + return priority; + } + + public void setPriority(SearchService.PRIORITY priority) + { + this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); + } + } + + // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue + // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). + // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It + // does not guarantee that all indexed content has been committed... but that may not be required in practice. + + @RequiresPermission(ApplicationAdminPermission.class) + public static class WaitForIndexerAction extends ExportAction + { + @Override + public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception + { + SearchService ss = SearchService.get(); + long startTime = System.currentTimeMillis(); + boolean success = ss.drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); + + LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); + + // Return an error if we time out + if (!success) + response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); + } + } + + @RequiresPermission(ReadPermission.class) + public class CommentAction extends FormHandlerAction + { + @Override + public void validateCommand(SearchForm target, Errors errors) + { + } + + @Override + public boolean handlePost(SearchForm searchForm, BindException errors) + { + audit(searchForm); + return true; + } + + @Override + public URLHelper getSuccessURL(SearchForm searchForm) + { + return getSearchURL(); + } + } + + + @SuppressWarnings("unused") + public static class SearchForm implements HasBindParameters + { + private String _query = ""; + private String _sortField; + private boolean _print = false; + private int _offset = 0; + private int _limit = 1000; + private String _category = null; + private String _comment = null; + private int _textBoxWidth = 50; // default size + private List _fields; + private boolean _includeHelpLink = true; + private boolean _webpart = false; + private boolean _showAdvanced = false; + private boolean _invertSort = false; + private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) + private String _template = null; + private SearchScope _scope = SearchScope.All; + private boolean _normalizeUrls = false; + private boolean _experimentalCustomJson = false; + + public void setConfiguration(SearchConfiguration config) + { + _config = config; + } + + public SearchConfiguration getConfig() + { + return _config; + } + + public String getQ() + { + return _query; + } + + public void setQ(String query) + { + _query = StringUtils.trimToEmpty(query); + } + + public String getSortField() + { + return _sortField; + } + + public void setSortField(String sortField) + { + _sortField = sortField; + } + + public boolean isPrint() + { + return _print; + } + + public void setPrint(boolean print) + { + _print = print; + } + + public int getOffset() + { + return _offset; + } + + public void setOffset(int o) + { + _offset = o; + } + + public int getLimit() + { + return _limit; + } + + public void setLimit(int o) + { + _limit = o; + } + + public String getScope() + { + return _scope.name(); + } + + public void setScope(String scope) + { + try + { + _scope = SearchScope.valueOf(scope); + } + catch (IllegalArgumentException e) + { + _scope = SearchScope.All; + } + } + + public SearchScope getSearchScope() + { + return _scope; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public int getTextBoxWidth() + { + return _textBoxWidth; + } + + public void setTextBoxWidth(int textBoxWidth) + { + _textBoxWidth = textBoxWidth; + } + + public boolean getIncludeHelpLink() + { + return _includeHelpLink; + } + + public void setIncludeHelpLink(boolean includeHelpLink) + { + _includeHelpLink = includeHelpLink; + } + + public boolean isWebPart() + { + return _webpart; + } + + public void setWebPart(boolean webpart) + { + _webpart = webpart; + } + + public boolean isShowAdvanced() + { + return _showAdvanced; + } + + public void setShowAdvanced(boolean showAdvanced) + { + _showAdvanced = showAdvanced; + } + + public boolean isInvertSort() + { + return _invertSort; + } + + public void setInvertSort(boolean invertSort) + { + _invertSort = invertSort; + } + + public String getTemplate() + { + return _template; + } + + public void setTemplate(String template) + { + _template = template; + } + + public SearchResultTemplate getSearchResultTemplate() + { + SearchService ss = AbstractSearchService.get(); + return ss.getSearchResultTemplate(getTemplate()); + } + + public boolean isNormalizeUrls() + { + return _normalizeUrls; + } + + public void setNormalizeUrls(boolean normalizeUrls) + { + _normalizeUrls = normalizeUrls; + } + + public boolean isExperimentalCustomJson() + { + return _experimentalCustomJson; + } + + public void setExperimentalCustomJson(boolean experimentalCustomJson) + { + _experimentalCustomJson = experimentalCustomJson; + } + + public List getFields() + { + return _fields; + } + + public void setFields(List fields) + { + _fields = fields; + } + + @Override + public @NotNull BindException bindParameters(PropertyValues m) + { + MutablePropertyValues mpvs = new MutablePropertyValues(m); + var q = mpvs.getPropertyValue("q"); + if (null != q && q.getValue() instanceof String[] arr) + { + mpvs.removePropertyValue("q"); + mpvs.addPropertyValue("q", StringUtils.join(arr," ")); + } + return springBindParameters(this, "form", mpvs); + } + } + + + protected void audit(SearchForm form) + { + ViewContext ctx = getViewContext(); + String comment = form.getComment(); + + audit(ctx.getUser(), ctx.getContainer(), form.getQ(), comment); + } + + + public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) + { + if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) + return; + + AuditLogService audit = AuditLogService.get(); + if (null == audit) + return; + + if (null == c) + c = ContainerManager.getRoot(); + + if (query.length() > 200) + query = StringUtilsLabKey.leftSurrogatePairFriendly(query, 197) + "..."; + + SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); + event.setQuery(query); + + AuditLogService.get().addEvent(user, event); + } + + + public static class SearchSettingsForm + { + private boolean _searchable; + + public boolean isSearchable() + { + return _searchable; + } + + @SuppressWarnings("unused") + public void setSearchable(boolean searchable) + { + _searchable = searchable; + } + } + + + @RequiresPermission(AdminPermission.class) + public static class SearchSettingsAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); + } + + @Override + public void validateCommand(SearchSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SearchSettingsForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SearchSettingsForm searchForm) + { + // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) + return getViewContext().getActionURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + setHelpTopic("searchAdmin"); + } + } +} From bb0eb4a4c5a7b47680c5545c9ff75bda402e3119 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 13 Nov 2025 15:46:58 -0800 Subject: [PATCH 2/4] CRLF --- api/src/org/labkey/api/data/SQLFragment.java | 2702 +- .../api/data/dialect/StatementWrapper.java | 6030 ++-- .../org/labkey/api/exp/OntologyManager.java | 7830 ++--- api/src/org/labkey/api/util/MemTracker.java | 972 +- .../labkey/core/admin/AdminController.java | 24546 ++++++++-------- .../experiment/api/AbstractRunInput.java | 250 +- .../labkey/experiment/api/ExpDataImpl.java | 1956 +- .../labkey/mothership/MothershipManager.java | 1254 +- .../pipeline/api/WorkDirectoryRemote.java | 1186 +- .../org/labkey/search/SearchController.java | 2396 +- 10 files changed, 24561 insertions(+), 24561 deletions(-) diff --git a/api/src/org/labkey/api/data/SQLFragment.java b/api/src/org/labkey/api/data/SQLFragment.java index 79c8f5ce242..87de41aff1b 100644 --- a/api/src/org/labkey/api/data/SQLFragment.java +++ b/api/src/org/labkey/api/data/SQLFragment.java @@ -1,1351 +1,1351 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.query.AliasManager; -import org.labkey.api.query.FieldKey; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JdbcUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; - -/** - * Holds both the SQL text and JDBC parameter values to use during invocation. - */ -public class SQLFragment implements Appendable, CharSequence -{ - public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; - - private String sql; - private StringBuilder sb = null; - private List params; // TODO: Should be List - - private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd - - // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) - private LinkedHashMap commonTableExpressionsMap = null; - - private static class CTE - { - CTE(@NotNull SqlDialect dialect, @NotNull String name) - { - this.dialect = dialect; - this.preferredName = name; - tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); - } - - CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) - { - this(dialect, name); - this.sqlf = sqlf; - this.recursive = recursive; - } - - CTE(CTE from) - { - this.dialect = from.dialect; - this.preferredName = from.preferredName; - this.tokens.addAll(from.tokens); - this.sqlf = from.sqlf; - this.recursive = from.recursive; - } - - public CTE copy(boolean deep) - { - CTE copy = new CTE(this); - if (deep) - copy.sqlf = new SQLFragment().append(copy.sqlf); - return copy; - } - - private String token() - { - return tokens.iterator().next(); - } - - private final @NotNull SqlDialect dialect; - final String preferredName; - boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) - final Set tokens = new TreeSet<>(); - SQLFragment sqlf = null; - } - - public SQLFragment() - { - sql = ""; - } - - public SQLFragment(CharSequence charseq, @Nullable List params) - { - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - // allow statement separators - this.sql = charseq.toString(); - if (null != params) - this.params = new ArrayList<>(params); - } - - - public SQLFragment(CharSequence sql, Object... params) - { - this(sql, Arrays.asList(params)); - } - - - public SQLFragment(SQLFragment other) - { - this(other,false); - } - - - public SQLFragment(SQLFragment other, boolean deep) - { - sql = other.getSqlCharSequence().toString(); - if (null != other.params) - addAll(other.params); - if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) - { - if (null == this.commonTableExpressionsMap) - this.commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) - { - CTE cte = e.getValue().copy(deep); - this.commonTableExpressionsMap.put(e.getKey(),cte); - } - } - this.tempTokens.addAll(other.tempTokens); - } - - - @Override - public boolean isEmpty() - { - return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); - } - - - /* same as getSQL() but without CTE handling */ - public String getRawSQL() - { - return null != sb ? sb.toString() : null != sql ? sql : ""; - } - - /* - * Directly set the current SQL. - * - * This is useful for wrapping existing SQL, for instance adding a cast - * Obviously parameter number and order must remain unchanged - * - * This can also be used for processing sql scripts (e.g. module .sql update scripts) - */ - public SQLFragment setSqlUnsafe(String unsafe) - { - this.sql = unsafe; - this.sb = null; - return this; - } - - public static SQLFragment unsafe(String unsafe) - { - return new SQLFragment().setSqlUnsafe(unsafe); - } - - - private String replaceCteTokens(String self, String select, List> ctes) - { - for (Pair pair : ctes) - { - String alias = pair.first; - CTE cte = pair.second; - for (String token : cte.tokens) - { - select = Strings.CS.replace(select, token, alias); - } - } - if (null != self) - select = Strings.CS.replace(select, "$SELF$", self); - return select; - } - - - private List collectCommonTableExpressions() - { - List list = new ArrayList<>(); - _collectCommonTableExpressions(list); - return list; - } - - private void _collectCommonTableExpressions(List list) - { - if (null != commonTableExpressionsMap) - { - commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); - list.addAll(commonTableExpressionsMap.values()); - } - } - - - public String getSQL() - { - if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) - return null != sb ? sb.toString() : null != sql ? sql : ""; - - List commonTableExpressions = collectCommonTableExpressions(); - assert !commonTableExpressions.isEmpty(); - - boolean recursive = commonTableExpressions.stream() - .anyMatch(cte -> cte.recursive); - StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); - - // generate final aliases for each CTE */ - SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); - AliasManager am = new AliasManager(dialect); - List> ctes = commonTableExpressions.stream() - .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) - .collect(Collectors.toList()); - - String comma = "\n/*CTE*/\n\t"; - for (Pair p : ctes) - { - String alias = p.first; - CTE cte = p.second; - SQLFragment expr = cte.sqlf; - String sql = expr._getOwnSql(alias, ctes); - ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); - comma = "\n,/*CTE*/\n\t"; - } - ret.append("\n"); - - String select = _getOwnSql( null, ctes ); - ret.append(replaceCteTokens(null, select, ctes)); - return ret.toString(); - } - - - private String _getOwnSql(String alias, List> ctes) - { - String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; - return replaceCteTokens(alias, ownSql, ctes); - } - - - static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); - - /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ - static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) - { - String select1 = sql1.getRawSQL(); - String select2 = sql2.getRawSQL(); - - if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && - (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) - return select1.equals(select2); - - select1 = markerPattern.matcher(select1).replaceAll("CTE"); - select2 = markerPattern.matcher(select2).replaceAll("CTE"); - if (!select1.equals(select2)) - return false; - - Set ctes1 = sql1.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - Set ctes2 = sql2.commonTableExpressionsMap.values().stream() - .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) - .collect(Collectors.toSet()); - return ctes1.equals(ctes2); - } - - - // It is a little confusing that getString() does not return the same charsequence that this object purports to - // represent. However, this is a good "display value" for this object. - // see getSqlCharSequence() - @NotNull - public String toString() - { - return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); - } - - - public String toDebugString() - { - return JdbcUtil.format(this); - } - - - public List getParams() - { - var ctes = collectCommonTableExpressions(); - List ret = new ArrayList<>(); - - for (var cte : ctes) - ret.addAll(cte.sqlf.getParamsNoCTEs()); - ret.addAll(getParamsNoCTEs()); - return Collections.unmodifiableList(ret); - } - - - public List> getParamsWithFragments() - { - var ctes = collectCommonTableExpressions(); - List> ret = new ArrayList<>(); - - for (CTE cte : ctes) - { - if (null != cte.sqlf && null != cte.sqlf.params) - { - for (int i = 0; i < cte.sqlf.params.size(); i++) - { - ret.add(new Pair<>(cte.sqlf, i)); - } - } - } - - if (null != params) - { - for (int i = 0; i < params.size(); i++) - { - ret.add(new Pair<>(this, i)); - } - } - return ret; - } - - private final static Object[] EMPTY_ARRAY = new Object[0]; - - public Object[] getParamsArray() - { - return null == params ? EMPTY_ARRAY : params.toArray(); - } - - public List getParamsNoCTEs() - { - return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); - } - - private List getMutableParams() - { - if (!(params instanceof ArrayList)) - { - List t = new ArrayList<>(); - if (params != null) - t.addAll(params); - params = t; - } - return params; - } - - - private StringBuilder getStringBuilder() - { - if (null == sb) - sb = new StringBuilder(null==sql?"":sql); - return sb; - } - - - @Override - public SQLFragment append(CharSequence charseq) - { - if (null == charseq) - return this; - - if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || - (StringUtils.countMatches(charseq, '\"') % 2) != 0 || - StringUtils.contains(charseq, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().append(charseq); - return this; - } - - public SQLFragment appendIdentifier(DatabaseIdentifier id) - { - return append(id.getSql()); - } - - /** Functionally the same as append(CharSequence). This method just has different asserts */ - public SQLFragment appendIdentifier(CharSequence charseq) - { - if (null == charseq) - return this; - if (charseq instanceof SQLFragment sqlf) - { - if (0 != sqlf.getParamsArray().length) - throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); - charseq = sqlf.getRawSQL(); - } - - String identifier = charseq.toString().strip(); - - if (STR_TABLE_ALIAS.equals(identifier)) - { - getStringBuilder().append(identifier); - return this; - } - - boolean malformed; - if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) - malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; - else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) - malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; - else - malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); - if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); - - getStringBuilder().append(charseq); - return this; - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) - { - return appendIdentifier(table).append(".").appendIdentifier(col); - } - - // just to save some typing - public SQLFragment appendDottedIdentifiers(CharSequence... ids) - { - var dot = ""; - for (var id : ids) - { - append(dot).appendIdentifier(id); - dot = "."; - } - return this; - } - - /** append End Of Statement */ - public SQLFragment appendEOS() - { - getStringBuilder().append(";\n"); - return this; - } - - - @Override - public SQLFragment append(CharSequence csq, int start, int end) - { - append(csq.subSequence(start, end)); - return this; - } - - /** Adds the container's ID as an in-line string constant to the SQL */ - public SQLFragment appendValue(Container c) - { - if (null == c) - return appendNull(); - return appendValue(c, null); - } - - public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) - { - appendValue(c.getEntityId(), dialect); - String name = c.getName(); - if (!StringUtils.containsAny(name,"*/\\'\"?")) - append("/* ").append(name).append(" */"); - return this; - } - - public SQLFragment appendNull() - { - getStringBuilder().append("NULL"); - return this; - } - - public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) - { - if (null == B) - return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); - getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); - return this; - } - - public SQLFragment appendValue(Integer I) - { - if (null == I) - return appendNull(); - getStringBuilder().append(I.intValue()); - return this; - } - - public SQLFragment appendValue(int i) - { - getStringBuilder().append(i); - return this; - } - - - public SQLFragment appendValue(Long L) - { - if (null == L) - return appendNull(); - getStringBuilder().append((long)L); - return this; - } - - public SQLFragment appendValue(long l) - { - getStringBuilder().append(l); - return this; - } - - public SQLFragment appendValue(Float F) - { - if (null == F) - return appendNull(); - return appendValue(F.floatValue()); - } - - public SQLFragment appendValue(float f) - { - if (Float.isFinite(f)) - { - getStringBuilder().append(f); - } - else - { - getStringBuilder().append("?"); - add(f); - } - return this; - } - - public SQLFragment appendValue(Double D) - { - if (null == D) - return appendNull(); - else - return appendValue(D.doubleValue()); - } - - public SQLFragment appendValue(double d) - { - if (Double.isFinite(d)) - { - getStringBuilder().append(d); - } - else - { - getStringBuilder().append("?"); - add(d); - } - return this; - } - - public SQLFragment appendValue(Number N) - { - if (null == N) - return appendNull(); - - if (N instanceof Quantity q) - N = q.value(); - - if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) - { - getStringBuilder().append(N); - } - else if (Double.isFinite(N.doubleValue())) - { - getStringBuilder().append(N); - } - else - { - getStringBuilder().append(" ? "); - add(N); - } - return this; - } - - public final SQLFragment appendNowTimestamp() - { - return appendValue(new NowTimestamp()); - } - - // Issue 27534: Stop using {fn now()} in function declarations - // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps - public final SQLFragment appendValue(NowTimestamp now) - { - if (null == now) - return appendNull(); - getStringBuilder().append("CURRENT_TIMESTAMP"); - return this; - } - - public final SQLFragment appendValue(java.util.Date d) - { - if (null == d) - return appendNull(); - if (d.getClass() == java.util.Date.class) - getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); - else if (d.getClass() == java.sql.Timestamp.class) - getStringBuilder().append("{ts '").append(d).append("'}"); - else if (d.getClass() == java.sql.Date.class) - getStringBuilder().append("{d '").append(d).append("'}"); - else - throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); - return this; - } - - public SQLFragment appendValue(GUID g) - { - return appendValue(g, null); - } - - public SQLFragment appendValue(GUID g, SqlDialect d) - { - if (null == g) - return appendNull(); - // doesn't need StringHandler, just hex and hyphen - String sqlGUID = "'" + g + "'"; - // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway - if (null != d && d.isSqlServer()) - getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); - else - getStringBuilder().append(sqlGUID); - return this; - } - - public SQLFragment appendValue(Enum e) - { - if (null == e) - return appendNull(); - String name = e.name(); - // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. - if (name.contains("'")) - throw new IllegalStateException(); - getStringBuilder().append("'").append(name).append("'"); - return this; - } - - public SQLFragment append(FieldKey fk) - { - if (null == fk) - return appendNull(); - append(String.valueOf(fk)); - return this; - } - - - /** Adds the object as a JDBC parameter value */ - public SQLFragment add(Object p) - { - getMutableParams().add(p); - return this; - } - - public SQLFragment add(Object p, JdbcType type) - { - getMutableParams().add(new Parameter.TypedValue(p, type)); - return this; - } - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Collection l) - { - getMutableParams().addAll(l); - return this; - } - - - /** Adds the objects as JDBC parameter values */ - public SQLFragment addAll(Object... values) - { - if (values == null) - return this; - addAll(Arrays.asList(values)); - return this; - } - - - /** Sets the parameter at the index to the object's value */ - public void set(int i, Object p) - { - getMutableParams().set(i,p); - } - - /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ - public SQLFragment append(SQLFragment f) - { - if (null != f.sb) - getStringBuilder().append(f.sb); - else - getStringBuilder().append(f.sql); - if (null != f.params) - addAll(f.params); - mergeCommonTableExpressions(f); - tempTokens.addAll(f.tempTokens); - return this; - } - - public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) - { - String s = ""; - for (SQLFragment fragment : fragments) - { - append(s); - s = separator; - append(fragment); - } - return this; - } - - // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out - // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the - // need to parse them) isn't particularly easy... so punt for now. - public boolean appendComment(String comment, SqlDialect dialect) - { - if (dialect.supportsComments()) - { - StringBuilder sb = getStringBuilder(); - int len = sb.length(); - if (len > 0 && sb.charAt(len-1) != '\n') - sb.append('\n'); - sb.append("\n-- "); - boolean truncated = comment.length() > 1000; - if (truncated) - comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); - sb.append(comment); - if (StringUtils.countMatches(comment, "'")%2==1) - sb.append("'"); - if (truncated) - sb.append("..."); - sb.append('\n'); - } - return true; - } - - - /** see also append(TableInfo, String alias) */ - public SQLFragment append(TableInfo table) - { - SQLFragment s = table.getSQLName(); - if (s != null) - return append(s); - - String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); - return append(table.getFromSQL(alias)); - } - - /** Add a table/query to the SQL with an alias, as used in a FROM clause */ - public SQLFragment append(TableInfo table, String alias) - { - return append(table.getFromSQL(alias)); - } - - /** Add to the SQL */ - @Override - public SQLFragment append(char ch) - { - getStringBuilder().append(ch); - return this; - } - - /** This is like appendValue(CharSequence s), but force use of literal syntax - * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions - * array_agg('string') fails array_agg('string'::VARCHAR) works - * json_object('{}) works json_object('string'::VARCHAR) fails - * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR - */ - public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) - { - if (null==s) - return appendNull(); - getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); - return this; - } - - /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ - public SQLFragment appendValue(CharSequence s) - { - return appendValue(s, null); - } - - public SQLFragment appendValue(CharSequence s, SqlDialect d) - { - if (null==s) - return appendNull(); - if (null==d || s.length() > 200) - return append("?").add(s.toString()); - appendStringLiteral(s, d); - return this; - } - - public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) - { - dialect.appendInClauseSql(this, params); - return this; - } - - public CharSequence getSqlCharSequence() - { - if (null != sb) - { - return sb; - } - return sql; - } - - public void insert(int index, SQLFragment sql) - { - if (!sql.getParams().isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); - } - if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); - } - if (!tempTokens.isEmpty()) - { - throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); - } - getStringBuilder().insert(index, sql.getRawSQL()); - } - - /** Insert into the SQL */ - public void insert(int index, String str) - { - if ((StringUtils.countMatches(str, '\'') % 2) != 0 || - (StringUtils.countMatches(str, '\"') % 2) != 0 || - StringUtils.contains(str, ';')) - { - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); - } - - getStringBuilder().insert(index, str); - } - - /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ - public void prepend(SQLFragment sql) - { - getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); - if (null != sql.params) - getMutableParams().addAll(0, sql.params); - mergeCommonTableExpressions(sql); - } - - - public int indexOf(String str) - { - return getStringBuilder().indexOf(str); - } - - - // Display query in "English" (display SQL with params substituted) - // with a little more work could probably be made to be SQL legal - public String getFilterText() - { - String sql = getSQL().replaceFirst("WHERE ", ""); - List params = getParams(); - for (Object param1 : params) - { - String param = param1.toString(); - param = param.replaceAll("\\\\", "\\\\\\\\"); - param = param.replaceAll("\\$", "\\\\\\$"); - sql = sql.replaceFirst("\\?", param); - } - return sql.replaceAll("\"", ""); - } - - - @Override - public char charAt(int index) - { - return getSqlCharSequence().charAt(index); - } - - @Override - public int length() - { - return getSqlCharSequence().length(); - } - - @Override - public @NotNull CharSequence subSequence(int start, int end) - { - return getSqlCharSequence().subSequence(start, end); - } - - /** - * KEY is used as a faster way to look for equivalent CTE expressions. - * returning a name here allows us to potentially merge CTE at add time - * - * if you don't have a key you can just use sqlf.toString() - */ - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) - { - return addCommonTableExpression(dialect, key, proposedName, sqlf, false); - } - - public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - return prev.token(); - CTE cte = new CTE(dialect, proposedName, sqlf, recursive); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - CTE prev = commonTableExpressionsMap.get(key); - if (null != prev) - throw new IllegalStateException("Cannot create CTE token from already used key."); - CTE cte = new CTE(dialect ,proposedName); - commonTableExpressionsMap.put(key, cte); - return cte.token(); - } - - public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) - { - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - - if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) - { - // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first - SQLFragment newSql = new SQLFragment(sqlf); - LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); - for (Map.Entry e : commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = toMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - toMap.put(e.getKey(), from.copy(false)); - } - - commonTableExpressionsMap = toMap; - newSql.commonTableExpressionsMap = null; - sqlf = newSql; - } - - CTE cte = commonTableExpressionsMap.get(key); - if (null == cte) - throw new IllegalStateException("CTE not found."); - cte.sqlf = sqlf; - cte.recursive = recursive; - } - - - private void mergeCommonTableExpressions(SQLFragment sqlFrom) - { - if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) - return; - if (null == commonTableExpressionsMap) - commonTableExpressionsMap = new LinkedHashMap<>(); - for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) - { - CTE from = e.getValue(); - CTE to = commonTableExpressionsMap.get(e.getKey()); - if (null != to) - to.tokens.addAll(from.tokens); - else - commonTableExpressionsMap.put(e.getKey(), from.copy(false)); - } - } - - - public void addTempToken(Object tempToken) - { - tempTokens.add(tempToken); - } - - public void addTempTokens(SQLFragment other) - { - tempTokens.add(other.tempTokens); - } - - public static SQLFragment prettyPrint(SQLFragment from) - { - SQLFragment sqlf = new SQLFragment(from); - - String s = from.getSqlCharSequence().toString(); - StringBuilder sb = new StringBuilder(s.length() + 200); - String[] lines = StringUtils.split(s, '\n'); - int indent = 0; - - for (String line : lines) - { - String t = line.trim(); - - if (t.isEmpty()) - continue; - - if (t.startsWith("-- params = b.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - SQLFragment c = new SQLFragment(b); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=?) - SELECT * FROM CTE WHERE y=?""", - c.getSQL()); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT a FROM b WHERE x=5) - SELECT * FROM CTE WHERE y='xxyzzy'""", - filterDebugString(c.toDebugString())); - params = c.getParams(); - assertEquals(2,params.size()); - assertEquals(5, params.get(0)); - assertEquals("xxyzzy", params.get(1)); - - - // combining - - SQLFragment sqlf = new SQLFragment(); - String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); - sqlf.append("SELECT * FROM ").append(token).append(" _1"); - - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - SELECT * FROM cte1 _1""", - sqlf.getSQL()); - - SQLFragment sqlf2 = new SQLFragment(); - String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); - sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); - assertEquals(""" - WITH - /*CTE*/ - \tcte2 AS (SELECT * FROM a) - SELECT * FROM cte2 _2""", - sqlf2.getSQL()); - - SQLFragment sqlf3 = new SQLFragment(); - String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); - sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); - assertEquals(""" - WITH - /*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte3 _3""", - sqlf3.getSQL()); - - SQLFragment union = new SQLFragment(); - union.append(sqlf); - union.append("\nUNION\n"); - union.append(sqlf2); - union.append("\nUNION\n"); - union.append(sqlf3); - assertEquals(""" - WITH - /*CTE*/ - \tcte1 AS (SELECT * FROM a) - ,/*CTE*/ - \tcte3 AS (SELECT * FROM b) - SELECT * FROM cte1 _1 - UNION - SELECT * FROM cte1 _2 - UNION - SELECT * FROM cte3 _3""", - union.getSQL()); - } - - @Test - public void nested_cte() - { - // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); - b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); - assertEquals(""" - WITH - /*CTE*/ - \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - SELECT * FROM CTE WHERE p='parameterTWO'""", - filterDebugString(b.toDebugString())); - assertEquals("parameterONE", b.getParams().get(0)); - } - - // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); - assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') - SELECT * FROM B_ WHERE i=3""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(3, params.size()); - assertEquals("parameterONE", params.get(0)); - assertEquals("parameterTWO", params.get(1)); - assertEquals(3, params.get(2)); - } - - // Same as previous but top-level query has both a nested and non-nested CTE - { - SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAtwo", params.get(2)); - assertEquals(4, params.get(3)); - } - - // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) - // TODO, we only collapse CTEs that are siblings - { - SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); - SQLFragment b = new SQLFragment(); - String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); - b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); - SQLFragment c = new SQLFragment(); - String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); - String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); - c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); - assertEquals(""" - WITH - /*CTE*/ - \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - ,/*CTE*/ - \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') - ,/*CTE*/ - \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) - SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", - filterDebugString(c.toDebugString())); - List params = c.getParams(); - assertEquals(4, params.size()); - assertEquals("parameterAone", params.get(0)); - assertEquals("parameterB", params.get(1)); - assertEquals("parameterAone", params.get(2)); - assertEquals(4, params.get(3)); - } - } - - - private void shouldFail(Runnable r) - { - try - { - r.run(); - if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Expected IllegalArgumentException"); - } - catch (IllegalArgumentException e) - { - if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) - fail("Did not expect IllegalArgumentException"); - } - } - - - @Test - public void testIllegalArgument() - { - shouldFail(() -> new SQLFragment(";")); - shouldFail(() -> new SQLFragment().append(";")); - shouldFail(() -> new SQLFragment("AND name='")); - shouldFail(() -> new SQLFragment().append("AND name = '")); - shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); - - shouldFail(() -> new SQLFragment().appendIdentifier("column name")); - shouldFail(() -> new SQLFragment().appendIdentifier("?")); - shouldFail(() -> new SQLFragment().appendIdentifier(";")); - shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); - } - - - String mysqlQuoteIdentifier(String id) - { - return "`" + id.replaceAll("`", "``") + "`"; - } - - @Test - public void testMysql() - { - // OK - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); - new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); - - // not OK - shouldFail(() -> new SQLFragment().appendIdentifier("`")); - shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); - } - } - - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof SQLFragment other)) - { - return false; - } - return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); - } - - /** - * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string - * concatenation using the provided separator. The parameters are combined to form the new parameter list. - * - * @param fragments SQLFragments to join together - * @param separator Separator to use on the SQL portion - * @return A new SQLFragment that joins all the SQLFragments - */ - public static SQLFragment join(Iterable fragments, String separator) - { - if (separator.contains("?")) - throw new IllegalStateException("separator must not include a parameter marker"); - - // Join all the SQL statements - String sql = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getSQL) - .collect(Collectors.joining(separator)); - - // Collect all the parameters to a single list - List params = StreamSupport.stream(fragments.spliterator(), false) - .map(SQLFragment::getParams) - .flatMap(Collection::stream) - .collect(Collectors.toList()); - - return new SQLFragment(sql, params); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.query.AliasManager; +import org.labkey.api.query.FieldKey; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JdbcUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import static org.labkey.api.query.ExprColumn.STR_TABLE_ALIAS; + +/** + * Holds both the SQL text and JDBC parameter values to use during invocation. + */ +public class SQLFragment implements Appendable, CharSequence +{ + public static final String FEATUREFLAG_DISABLE_STRICT_CHECKS = "sqlfragment-disable-strict-checks"; + + private String sql; + private StringBuilder sb = null; + private List params; // TODO: Should be List + + private final List tempTokens = new ArrayList<>(); // Hold refs to ensure they're not GC'd + + // use ordered map to make sql generation more deterministic (see collectCommonTableExpressions()) + private LinkedHashMap commonTableExpressionsMap = null; + + private static class CTE + { + CTE(@NotNull SqlDialect dialect, @NotNull String name) + { + this.dialect = dialect; + this.preferredName = name; + tokens.add("/*$*/" + GUID.makeGUID() + ":" + name + "/*$*/"); + } + + CTE(@NotNull SqlDialect dialect, @NotNull String name, SQLFragment sqlf, boolean recursive) + { + this(dialect, name); + this.sqlf = sqlf; + this.recursive = recursive; + } + + CTE(CTE from) + { + this.dialect = from.dialect; + this.preferredName = from.preferredName; + this.tokens.addAll(from.tokens); + this.sqlf = from.sqlf; + this.recursive = from.recursive; + } + + public CTE copy(boolean deep) + { + CTE copy = new CTE(this); + if (deep) + copy.sqlf = new SQLFragment().append(copy.sqlf); + return copy; + } + + private String token() + { + return tokens.iterator().next(); + } + + private final @NotNull SqlDialect dialect; + final String preferredName; + boolean recursive = false; // NOTE this is dialect dependant (getSql() does not take a dialect) + final Set tokens = new TreeSet<>(); + SQLFragment sqlf = null; + } + + public SQLFragment() + { + sql = ""; + } + + public SQLFragment(CharSequence charseq, @Nullable List params) + { + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + // allow statement separators + this.sql = charseq.toString(); + if (null != params) + this.params = new ArrayList<>(params); + } + + + public SQLFragment(CharSequence sql, Object... params) + { + this(sql, Arrays.asList(params)); + } + + + public SQLFragment(SQLFragment other) + { + this(other,false); + } + + + public SQLFragment(SQLFragment other, boolean deep) + { + sql = other.getSqlCharSequence().toString(); + if (null != other.params) + addAll(other.params); + if (null != other.commonTableExpressionsMap && !other.commonTableExpressionsMap.isEmpty()) + { + if (null == this.commonTableExpressionsMap) + this.commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : other.commonTableExpressionsMap.entrySet()) + { + CTE cte = e.getValue().copy(deep); + this.commonTableExpressionsMap.put(e.getKey(),cte); + } + } + this.tempTokens.addAll(other.tempTokens); + } + + + @Override + public boolean isEmpty() + { + return (null == sb || sb.isEmpty()) && (sql == null || sql.isEmpty()); + } + + + /* same as getSQL() but without CTE handling */ + public String getRawSQL() + { + return null != sb ? sb.toString() : null != sql ? sql : ""; + } + + /* + * Directly set the current SQL. + * + * This is useful for wrapping existing SQL, for instance adding a cast + * Obviously parameter number and order must remain unchanged + * + * This can also be used for processing sql scripts (e.g. module .sql update scripts) + */ + public SQLFragment setSqlUnsafe(String unsafe) + { + this.sql = unsafe; + this.sb = null; + return this; + } + + public static SQLFragment unsafe(String unsafe) + { + return new SQLFragment().setSqlUnsafe(unsafe); + } + + + private String replaceCteTokens(String self, String select, List> ctes) + { + for (Pair pair : ctes) + { + String alias = pair.first; + CTE cte = pair.second; + for (String token : cte.tokens) + { + select = Strings.CS.replace(select, token, alias); + } + } + if (null != self) + select = Strings.CS.replace(select, "$SELF$", self); + return select; + } + + + private List collectCommonTableExpressions() + { + List list = new ArrayList<>(); + _collectCommonTableExpressions(list); + return list; + } + + private void _collectCommonTableExpressions(List list) + { + if (null != commonTableExpressionsMap) + { + commonTableExpressionsMap.values().forEach(cte -> cte.sqlf._collectCommonTableExpressions(list)); + list.addAll(commonTableExpressionsMap.values()); + } + } + + + public String getSQL() + { + if (null == commonTableExpressionsMap || commonTableExpressionsMap.isEmpty()) + return null != sb ? sb.toString() : null != sql ? sql : ""; + + List commonTableExpressions = collectCommonTableExpressions(); + assert !commonTableExpressions.isEmpty(); + + boolean recursive = commonTableExpressions.stream() + .anyMatch(cte -> cte.recursive); + StringBuilder ret = new StringBuilder("WITH" + (recursive ? " RECURSIVE" : "")); + + // generate final aliases for each CTE */ + SqlDialect dialect = Objects.requireNonNull(commonTableExpressions.get(0).dialect); + AliasManager am = new AliasManager(dialect); + List> ctes = commonTableExpressions.stream() + .map(cte -> new Pair<>(am.decideAlias(cte.preferredName),cte)) + .collect(Collectors.toList()); + + String comma = "\n/*CTE*/\n\t"; + for (Pair p : ctes) + { + String alias = p.first; + CTE cte = p.second; + SQLFragment expr = cte.sqlf; + String sql = expr._getOwnSql(alias, ctes); + ret.append(comma).append(alias).append(" AS (").append(sql).append(")"); + comma = "\n,/*CTE*/\n\t"; + } + ret.append("\n"); + + String select = _getOwnSql( null, ctes ); + ret.append(replaceCteTokens(null, select, ctes)); + return ret.toString(); + } + + + private String _getOwnSql(String alias, List> ctes) + { + String ownSql = null != sb ? sb.toString() : null != this.sql ? this.sql : ""; + return replaceCteTokens(alias, ownSql, ctes); + } + + + static Pattern markerPattern = Pattern.compile("/\\*\\$\\*/.*/\\*\\$\\*/"); + + /* This is not an exhaustive .equals() test, but it give pretty good confidence that these statements are the same */ + static boolean debugCompareSQL(SQLFragment sql1, SQLFragment sql2) + { + String select1 = sql1.getRawSQL(); + String select2 = sql2.getRawSQL(); + + if ((null == sql1.commonTableExpressionsMap || sql1.commonTableExpressionsMap.isEmpty()) && + (null == sql2.commonTableExpressionsMap || sql2.commonTableExpressionsMap.isEmpty())) + return select1.equals(select2); + + select1 = markerPattern.matcher(select1).replaceAll("CTE"); + select2 = markerPattern.matcher(select2).replaceAll("CTE"); + if (!select1.equals(select2)) + return false; + + Set ctes1 = sql1.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + Set ctes2 = sql2.commonTableExpressionsMap.values().stream() + .map(cte -> markerPattern.matcher(cte.sqlf.getRawSQL()).replaceAll("CTE")) + .collect(Collectors.toSet()); + return ctes1.equals(ctes2); + } + + + // It is a little confusing that getString() does not return the same charsequence that this object purports to + // represent. However, this is a good "display value" for this object. + // see getSqlCharSequence() + @NotNull + public String toString() + { + return "SQLFragment@" + System.identityHashCode(this) + "\n" + JdbcUtil.format(this); + } + + + public String toDebugString() + { + return JdbcUtil.format(this); + } + + + public List getParams() + { + var ctes = collectCommonTableExpressions(); + List ret = new ArrayList<>(); + + for (var cte : ctes) + ret.addAll(cte.sqlf.getParamsNoCTEs()); + ret.addAll(getParamsNoCTEs()); + return Collections.unmodifiableList(ret); + } + + + public List> getParamsWithFragments() + { + var ctes = collectCommonTableExpressions(); + List> ret = new ArrayList<>(); + + for (CTE cte : ctes) + { + if (null != cte.sqlf && null != cte.sqlf.params) + { + for (int i = 0; i < cte.sqlf.params.size(); i++) + { + ret.add(new Pair<>(cte.sqlf, i)); + } + } + } + + if (null != params) + { + for (int i = 0; i < params.size(); i++) + { + ret.add(new Pair<>(this, i)); + } + } + return ret; + } + + private final static Object[] EMPTY_ARRAY = new Object[0]; + + public Object[] getParamsArray() + { + return null == params ? EMPTY_ARRAY : params.toArray(); + } + + public List getParamsNoCTEs() + { + return params == null ? Collections.emptyList() : Collections.unmodifiableList(params); + } + + private List getMutableParams() + { + if (!(params instanceof ArrayList)) + { + List t = new ArrayList<>(); + if (params != null) + t.addAll(params); + params = t; + } + return params; + } + + + private StringBuilder getStringBuilder() + { + if (null == sb) + sb = new StringBuilder(null==sql?"":sql); + return sb; + } + + + @Override + public SQLFragment append(CharSequence charseq) + { + if (null == charseq) + return this; + + if ((StringUtils.countMatches(charseq, '\'') % 2) != 0 || + (StringUtils.countMatches(charseq, '\"') % 2) != 0 || + StringUtils.contains(charseq, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.append(String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().append(charseq); + return this; + } + + public SQLFragment appendIdentifier(DatabaseIdentifier id) + { + return append(id.getSql()); + } + + /** Functionally the same as append(CharSequence). This method just has different asserts */ + public SQLFragment appendIdentifier(CharSequence charseq) + { + if (null == charseq) + return this; + if (charseq instanceof SQLFragment sqlf) + { + if (0 != sqlf.getParamsArray().length) + throw new IllegalStateException("Unexpected SQL in appendIdentifier()"); + charseq = sqlf.getRawSQL(); + } + + String identifier = charseq.toString().strip(); + + if (STR_TABLE_ALIAS.equals(identifier)) + { + getStringBuilder().append(identifier); + return this; + } + + boolean malformed; + if (identifier.length() >= 2 && identifier.startsWith("\"") && identifier.endsWith("\"")) + malformed = (StringUtils.countMatches(identifier, '\"') % 2) != 0; + else if (identifier.length() >= 2 && identifier.startsWith("`") && identifier.endsWith("`")) + malformed = (StringUtils.countMatches(identifier, '`') % 2) != 0; + else + malformed = StringUtils.containsAny(identifier, "*/\\'\"`?;- \t\n"); + if (malformed && !AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.appendIdentifier(String) value appears to be incorrectly formatted: " + identifier); + + getStringBuilder().append(charseq); + return this; + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence table, DatabaseIdentifier col) + { + return appendIdentifier(table).append(".").appendIdentifier(col); + } + + // just to save some typing + public SQLFragment appendDottedIdentifiers(CharSequence... ids) + { + var dot = ""; + for (var id : ids) + { + append(dot).appendIdentifier(id); + dot = "."; + } + return this; + } + + /** append End Of Statement */ + public SQLFragment appendEOS() + { + getStringBuilder().append(";\n"); + return this; + } + + + @Override + public SQLFragment append(CharSequence csq, int start, int end) + { + append(csq.subSequence(start, end)); + return this; + } + + /** Adds the container's ID as an in-line string constant to the SQL */ + public SQLFragment appendValue(Container c) + { + if (null == c) + return appendNull(); + return appendValue(c, null); + } + + public SQLFragment appendValue(@NotNull Container c, SqlDialect dialect) + { + appendValue(c.getEntityId(), dialect); + String name = c.getName(); + if (!StringUtils.containsAny(name,"*/\\'\"?")) + append("/* ").append(name).append(" */"); + return this; + } + + public SQLFragment appendNull() + { + getStringBuilder().append("NULL"); + return this; + } + + public SQLFragment appendValue(Boolean B, @NotNull SqlDialect dialect) + { + if (null == B) + return append("CAST(NULL AS ").append(dialect.getBooleanDataType()).append(")"); + getStringBuilder().append(B ? dialect.getBooleanTRUE() : dialect.getBooleanFALSE()); + return this; + } + + public SQLFragment appendValue(Integer I) + { + if (null == I) + return appendNull(); + getStringBuilder().append(I.intValue()); + return this; + } + + public SQLFragment appendValue(int i) + { + getStringBuilder().append(i); + return this; + } + + + public SQLFragment appendValue(Long L) + { + if (null == L) + return appendNull(); + getStringBuilder().append((long)L); + return this; + } + + public SQLFragment appendValue(long l) + { + getStringBuilder().append(l); + return this; + } + + public SQLFragment appendValue(Float F) + { + if (null == F) + return appendNull(); + return appendValue(F.floatValue()); + } + + public SQLFragment appendValue(float f) + { + if (Float.isFinite(f)) + { + getStringBuilder().append(f); + } + else + { + getStringBuilder().append("?"); + add(f); + } + return this; + } + + public SQLFragment appendValue(Double D) + { + if (null == D) + return appendNull(); + else + return appendValue(D.doubleValue()); + } + + public SQLFragment appendValue(double d) + { + if (Double.isFinite(d)) + { + getStringBuilder().append(d); + } + else + { + getStringBuilder().append("?"); + add(d); + } + return this; + } + + public SQLFragment appendValue(Number N) + { + if (null == N) + return appendNull(); + + if (N instanceof Quantity q) + N = q.value(); + + if (N instanceof BigDecimal || N instanceof BigInteger || N instanceof Long) + { + getStringBuilder().append(N); + } + else if (Double.isFinite(N.doubleValue())) + { + getStringBuilder().append(N); + } + else + { + getStringBuilder().append(" ? "); + add(N); + } + return this; + } + + public final SQLFragment appendNowTimestamp() + { + return appendValue(new NowTimestamp()); + } + + // Issue 27534: Stop using {fn now()} in function declarations + // Issue 48864: Query Table's use of web server time can cause discrepancies in created/modified timestamps + public final SQLFragment appendValue(NowTimestamp now) + { + if (null == now) + return appendNull(); + getStringBuilder().append("CURRENT_TIMESTAMP"); + return this; + } + + public final SQLFragment appendValue(java.util.Date d) + { + if (null == d) + return appendNull(); + if (d.getClass() == java.util.Date.class) + getStringBuilder().append("{ts '").append(new Timestamp(d.getTime())).append("'}"); + else if (d.getClass() == java.sql.Timestamp.class) + getStringBuilder().append("{ts '").append(d).append("'}"); + else if (d.getClass() == java.sql.Date.class) + getStringBuilder().append("{d '").append(d).append("'}"); + else + throw new IllegalStateException("Unexpected date type: " + d.getClass().getName()); + return this; + } + + public SQLFragment appendValue(GUID g) + { + return appendValue(g, null); + } + + public SQLFragment appendValue(GUID g, SqlDialect d) + { + if (null == g) + return appendNull(); + // doesn't need StringHandler, just hex and hyphen + String sqlGUID = "'" + g + "'"; + // I'm testing dialect type, because some dialects do not support getGuidType(), and postgers uses VARCHAR anyway + if (null != d && d.isSqlServer()) + getStringBuilder().append("CAST(").append(sqlGUID).append(" AS UNIQUEIDENTIFIER)"); + else + getStringBuilder().append(sqlGUID); + return this; + } + + public SQLFragment appendValue(Enum e) + { + if (null == e) + return appendNull(); + String name = e.name(); + // Enum.name() usually returns a simple string (a legal java identifier), this is a paranoia check. + if (name.contains("'")) + throw new IllegalStateException(); + getStringBuilder().append("'").append(name).append("'"); + return this; + } + + public SQLFragment append(FieldKey fk) + { + if (null == fk) + return appendNull(); + append(String.valueOf(fk)); + return this; + } + + + /** Adds the object as a JDBC parameter value */ + public SQLFragment add(Object p) + { + getMutableParams().add(p); + return this; + } + + public SQLFragment add(Object p, JdbcType type) + { + getMutableParams().add(new Parameter.TypedValue(p, type)); + return this; + } + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Collection l) + { + getMutableParams().addAll(l); + return this; + } + + + /** Adds the objects as JDBC parameter values */ + public SQLFragment addAll(Object... values) + { + if (values == null) + return this; + addAll(Arrays.asList(values)); + return this; + } + + + /** Sets the parameter at the index to the object's value */ + public void set(int i, Object p) + { + getMutableParams().set(i,p); + } + + /** Append both the SQL and the parameters from the other SQLFragment to this SQLFragment */ + public SQLFragment append(SQLFragment f) + { + if (null != f.sb) + getStringBuilder().append(f.sb); + else + getStringBuilder().append(f.sql); + if (null != f.params) + addAll(f.params); + mergeCommonTableExpressions(f); + tempTokens.addAll(f.tempTokens); + return this; + } + + public SQLFragment append(@NotNull Iterable fragments, @NotNull String separator) + { + String s = ""; + for (SQLFragment fragment : fragments) + { + append(s); + s = separator; + append(fragment); + } + return this; + } + + // return boolean so this can be used in an assert. passing in a dialect is not ideal, but parsing comments out + // before submitting the fragment is not reliable and holding statements & comments separately (to eliminate the + // need to parse them) isn't particularly easy... so punt for now. + public boolean appendComment(String comment, SqlDialect dialect) + { + if (dialect.supportsComments()) + { + StringBuilder sb = getStringBuilder(); + int len = sb.length(); + if (len > 0 && sb.charAt(len-1) != '\n') + sb.append('\n'); + sb.append("\n-- "); + boolean truncated = comment.length() > 1000; + if (truncated) + comment = StringUtilsLabKey.leftSurrogatePairFriendly(comment, 1000); + sb.append(comment); + if (StringUtils.countMatches(comment, "'")%2==1) + sb.append("'"); + if (truncated) + sb.append("..."); + sb.append('\n'); + } + return true; + } + + + /** see also append(TableInfo, String alias) */ + public SQLFragment append(TableInfo table) + { + SQLFragment s = table.getSQLName(); + if (s != null) + return append(s); + + String alias = table.getSqlDialect().makeLegalIdentifier(table.getName()); + return append(table.getFromSQL(alias)); + } + + /** Add a table/query to the SQL with an alias, as used in a FROM clause */ + public SQLFragment append(TableInfo table, String alias) + { + return append(table.getFromSQL(alias)); + } + + /** Add to the SQL */ + @Override + public SQLFragment append(char ch) + { + getStringBuilder().append(ch); + return this; + } + + /** This is like appendValue(CharSequence s), but force use of literal syntax + * CAUTIONARY NOTE: String literals in PostgresSQL are tricky because of overloaded functions + * array_agg('string') fails array_agg('string'::VARCHAR) works + * json_object('{}) works json_object('string'::VARCHAR) fails + * In the case of json_object() it expects TEXT. Postgres will promote 'json' to TEXT, but not 'json'::VARCHAR + */ + public SQLFragment appendStringLiteral(CharSequence s, @NotNull SqlDialect d) + { + if (null==s) + return appendNull(); + getStringBuilder().append(d.getStringHandler().quoteStringLiteral(s.toString())); + return this; + } + + /** Add to the SQL as either an in-line string literal or as a JDBC parameter depending on whether it would need escaping */ + public SQLFragment appendValue(CharSequence s) + { + return appendValue(s, null); + } + + public SQLFragment appendValue(CharSequence s, SqlDialect d) + { + if (null==s) + return appendNull(); + if (null==d || s.length() > 200) + return append("?").add(s.toString()); + appendStringLiteral(s, d); + return this; + } + + public SQLFragment appendInClause(@NotNull Collection params, SqlDialect dialect) + { + dialect.appendInClauseSql(this, params); + return this; + } + + public CharSequence getSqlCharSequence() + { + if (null != sb) + { + return sb; + } + return sql; + } + + public void insert(int index, SQLFragment sql) + { + if (!sql.getParams().isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with parameters - they must be inserted/merged separately"); + } + if (sql.commonTableExpressionsMap != null && !sql.commonTableExpressionsMap.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with CTEs - they must be inserted/merged separately"); + } + if (!tempTokens.isEmpty()) + { + throw new IllegalArgumentException("Not supported for SQLFragments with temp tokens - they must be inserted/merged separately"); + } + getStringBuilder().insert(index, sql.getRawSQL()); + } + + /** Insert into the SQL */ + public void insert(int index, String str) + { + if ((StringUtils.countMatches(str, '\'') % 2) != 0 || + (StringUtils.countMatches(str, '\"') % 2) != 0 || + StringUtils.contains(str, ';')) + { + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + throw new IllegalArgumentException("SQLFragment.insert(int,String) does not allow semicolons or unmatched quotes"); + } + + getStringBuilder().insert(index, str); + } + + /** Insert this SQLFragment's SQL and parameters at the start of the existing SQL and parameters */ + public void prepend(SQLFragment sql) + { + getStringBuilder().insert(0, sql.getSqlCharSequence().toString()); + if (null != sql.params) + getMutableParams().addAll(0, sql.params); + mergeCommonTableExpressions(sql); + } + + + public int indexOf(String str) + { + return getStringBuilder().indexOf(str); + } + + + // Display query in "English" (display SQL with params substituted) + // with a little more work could probably be made to be SQL legal + public String getFilterText() + { + String sql = getSQL().replaceFirst("WHERE ", ""); + List params = getParams(); + for (Object param1 : params) + { + String param = param1.toString(); + param = param.replaceAll("\\\\", "\\\\\\\\"); + param = param.replaceAll("\\$", "\\\\\\$"); + sql = sql.replaceFirst("\\?", param); + } + return sql.replaceAll("\"", ""); + } + + + @Override + public char charAt(int index) + { + return getSqlCharSequence().charAt(index); + } + + @Override + public int length() + { + return getSqlCharSequence().length(); + } + + @Override + public @NotNull CharSequence subSequence(int start, int end) + { + return getSqlCharSequence().subSequence(start, end); + } + + /** + * KEY is used as a faster way to look for equivalent CTE expressions. + * returning a name here allows us to potentially merge CTE at add time + * + * if you don't have a key you can just use sqlf.toString() + */ + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf) + { + return addCommonTableExpression(dialect, key, proposedName, sqlf, false); + } + + public String addCommonTableExpression(SqlDialect dialect, Object key, String proposedName, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + return prev.token(); + CTE cte = new CTE(dialect, proposedName, sqlf, recursive); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public String createCommonTableExpressionToken(SqlDialect dialect, Object key, String proposedName) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + CTE prev = commonTableExpressionsMap.get(key); + if (null != prev) + throw new IllegalStateException("Cannot create CTE token from already used key."); + CTE cte = new CTE(dialect ,proposedName); + commonTableExpressionsMap.put(key, cte); + return cte.token(); + } + + public void setCommonTableExpressionSql(Object key, SQLFragment sqlf, boolean recursive) + { + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + + if (null != sqlf.commonTableExpressionsMap && !sqlf.commonTableExpressionsMap.isEmpty()) + { + // Need to merge CTEs up; this.cte depends on newSql.ctes, so they need to come first + SQLFragment newSql = new SQLFragment(sqlf); + LinkedHashMap toMap = new LinkedHashMap<>(newSql.commonTableExpressionsMap); + for (Map.Entry e : commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = toMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + toMap.put(e.getKey(), from.copy(false)); + } + + commonTableExpressionsMap = toMap; + newSql.commonTableExpressionsMap = null; + sqlf = newSql; + } + + CTE cte = commonTableExpressionsMap.get(key); + if (null == cte) + throw new IllegalStateException("CTE not found."); + cte.sqlf = sqlf; + cte.recursive = recursive; + } + + + private void mergeCommonTableExpressions(SQLFragment sqlFrom) + { + if (null == sqlFrom.commonTableExpressionsMap || sqlFrom.commonTableExpressionsMap.isEmpty()) + return; + if (null == commonTableExpressionsMap) + commonTableExpressionsMap = new LinkedHashMap<>(); + for (Map.Entry e : sqlFrom.commonTableExpressionsMap.entrySet()) + { + CTE from = e.getValue(); + CTE to = commonTableExpressionsMap.get(e.getKey()); + if (null != to) + to.tokens.addAll(from.tokens); + else + commonTableExpressionsMap.put(e.getKey(), from.copy(false)); + } + } + + + public void addTempToken(Object tempToken) + { + tempTokens.add(tempToken); + } + + public void addTempTokens(SQLFragment other) + { + tempTokens.add(other.tempTokens); + } + + public static SQLFragment prettyPrint(SQLFragment from) + { + SQLFragment sqlf = new SQLFragment(from); + + String s = from.getSqlCharSequence().toString(); + StringBuilder sb = new StringBuilder(s.length() + 200); + String[] lines = StringUtils.split(s, '\n'); + int indent = 0; + + for (String line : lines) + { + String t = line.trim(); + + if (t.isEmpty()) + continue; + + if (t.startsWith("-- params = b.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + SQLFragment c = new SQLFragment(b); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=?) + SELECT * FROM CTE WHERE y=?""", + c.getSQL()); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT a FROM b WHERE x=5) + SELECT * FROM CTE WHERE y='xxyzzy'""", + filterDebugString(c.toDebugString())); + params = c.getParams(); + assertEquals(2,params.size()); + assertEquals(5, params.get(0)); + assertEquals("xxyzzy", params.get(1)); + + + // combining + + SQLFragment sqlf = new SQLFragment(); + String token = sqlf.addCommonTableExpression(dialect, "KEY_A", "cte1", new SQLFragment("SELECT * FROM a")); + sqlf.append("SELECT * FROM ").append(token).append(" _1"); + + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + SELECT * FROM cte1 _1""", + sqlf.getSQL()); + + SQLFragment sqlf2 = new SQLFragment(); + String token2 = sqlf2.addCommonTableExpression(dialect, "KEY_A", "cte2", new SQLFragment("SELECT * FROM a")); + sqlf2.append("SELECT * FROM ").append(token2).append(" _2"); + assertEquals(""" + WITH + /*CTE*/ + \tcte2 AS (SELECT * FROM a) + SELECT * FROM cte2 _2""", + sqlf2.getSQL()); + + SQLFragment sqlf3 = new SQLFragment(); + String token3 = sqlf3.addCommonTableExpression(dialect, "KEY_B", "cte3", new SQLFragment("SELECT * FROM b")); + sqlf3.append("SELECT * FROM ").append(token3).append(" _3"); + assertEquals(""" + WITH + /*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte3 _3""", + sqlf3.getSQL()); + + SQLFragment union = new SQLFragment(); + union.append(sqlf); + union.append("\nUNION\n"); + union.append(sqlf2); + union.append("\nUNION\n"); + union.append(sqlf3); + assertEquals(""" + WITH + /*CTE*/ + \tcte1 AS (SELECT * FROM a) + ,/*CTE*/ + \tcte3 AS (SELECT * FROM b) + SELECT * FROM cte1 _1 + UNION + SELECT * FROM cte1 _2 + UNION + SELECT * FROM cte3 _3""", + union.getSQL()); + } + + @Test + public void nested_cte() + { + // one-level cte using cteToken (CTE fragment 'a' does not contain a CTE) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteToken = b.addCommonTableExpression(dialect, new Object(), "CTE", a); + b.append("SELECT * FROM ").append(cteToken).append(" WHERE p=?").add("parameterTWO"); + assertEquals(""" + WITH + /*CTE*/ + \tCTE AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + SELECT * FROM CTE WHERE p='parameterTWO'""", + filterDebugString(b.toDebugString())); + assertEquals("parameterONE", b.getParams().get(0)); + } + + // two-level cte using cteTokens (CTE fragment 'b' contains a CTE of fragment a) + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'one' as s, CAST(? AS VARCHAR) as p", "parameterONE"); + assertEquals("SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p", filterDebugString(a.toDebugString())); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterTWO"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + c.append("SELECT * FROM ").append(cteTokenB).append(" WHERE i=?").add(3); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'one' as s, CAST('parameterONE' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterTWO') + SELECT * FROM B_ WHERE i=3""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(3, params.size()); + assertEquals("parameterONE", params.get(0)); + assertEquals("parameterTWO", params.get(1)); + assertEquals(3, params.get(2)); + } + + // Same as previous but top-level query has both a nested and non-nested CTE + { + SQLFragment a = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment a2 = new SQLFragment("SELECT 2 as i, 'Atwo' as s, CAST(? AS VARCHAR) as p", "parameterAtwo"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, new Object(), "A_", a); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, new Object(), "A2_", a2); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 2 as i, 'Atwo' as s, CAST('parameterAtwo' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAtwo", params.get(2)); + assertEquals(4, params.get(3)); + } + + // Same as previous but two of the CTEs are the same and should be collapsed (e.g. imagine a container filter implemented with a CTE) + // TODO, we only collapse CTEs that are siblings + { + SQLFragment cf = new SQLFragment("SELECT 1 as i, 'Aone' as s, CAST(? AS VARCHAR) as p", "parameterAone"); + SQLFragment b = new SQLFragment(); + String cteTokenA = b.addCommonTableExpression(dialect, "CTE_KEY_CF", "A_", cf); + b.append("SELECT * FROM ").append(cteTokenA).append(" WHERE p=?").add("parameterB"); + SQLFragment c = new SQLFragment(); + String cteTokenB = c.addCommonTableExpression(dialect, new Object(), "B_", b); + String cteTokenA2 = c.addCommonTableExpression(dialect, "CTE_KEY_CF", "A2_", cf); + c.append("SELECT *, ? as xyz FROM ").add(4).append(cteTokenB).append(" B, ").append(cteTokenA2).append(" A WHERE B.i=A.i"); + assertEquals(""" + WITH + /*CTE*/ + \tA_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + ,/*CTE*/ + \tB_ AS (SELECT * FROM A_ WHERE p='parameterB') + ,/*CTE*/ + \tA2_ AS (SELECT 1 as i, 'Aone' as s, CAST('parameterAone' AS VARCHAR) as p) + SELECT *, 4 as xyz FROM B_ B, A2_ A WHERE B.i=A.i""", + filterDebugString(c.toDebugString())); + List params = c.getParams(); + assertEquals(4, params.size()); + assertEquals("parameterAone", params.get(0)); + assertEquals("parameterB", params.get(1)); + assertEquals("parameterAone", params.get(2)); + assertEquals(4, params.get(3)); + } + } + + + private void shouldFail(Runnable r) + { + try + { + r.run(); + if (!AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Expected IllegalArgumentException"); + } + catch (IllegalArgumentException e) + { + if (AppProps.getInstance().isOptionalFeatureEnabled(FEATUREFLAG_DISABLE_STRICT_CHECKS)) + fail("Did not expect IllegalArgumentException"); + } + } + + + @Test + public void testIllegalArgument() + { + shouldFail(() -> new SQLFragment(";")); + shouldFail(() -> new SQLFragment().append(";")); + shouldFail(() -> new SQLFragment("AND name='")); + shouldFail(() -> new SQLFragment().append("AND name = '")); + shouldFail(() -> new SQLFragment().append("AND name = 'Robert'); DROP TABLE Students; --")); + + shouldFail(() -> new SQLFragment().appendIdentifier("column name")); + shouldFail(() -> new SQLFragment().appendIdentifier("?")); + shouldFail(() -> new SQLFragment().appendIdentifier(";")); + shouldFail(() -> new SQLFragment().appendIdentifier("\"column\"name\"")); + } + + + String mysqlQuoteIdentifier(String id) + { + return "`" + id.replaceAll("`", "``") + "`"; + } + + @Test + public void testMysql() + { + // OK + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("mysql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my`sql")); + new SQLFragment().appendIdentifier(mysqlQuoteIdentifier("my\"sql")); + + // not OK + shouldFail(() -> new SQLFragment().appendIdentifier("`")); + shouldFail(() -> new SQLFragment().appendIdentifier("`a`a`")); + } + } + + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof SQLFragment other)) + { + return false; + } + return getSQL().equals(other.getSQL()) && getParams().equals(other.getParams()); + } + + /** + * Joins the SQLFragments in the provided {@code Iterable} into a single SQLFragment. The SQL is joined by string + * concatenation using the provided separator. The parameters are combined to form the new parameter list. + * + * @param fragments SQLFragments to join together + * @param separator Separator to use on the SQL portion + * @return A new SQLFragment that joins all the SQLFragments + */ + public static SQLFragment join(Iterable fragments, String separator) + { + if (separator.contains("?")) + throw new IllegalStateException("separator must not include a parameter marker"); + + // Join all the SQL statements + String sql = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getSQL) + .collect(Collectors.joining(separator)); + + // Collect all the parameters to a single list + List params = StreamSupport.stream(fragments.spliterator(), false) + .map(SQLFragment::getParams) + .flatMap(Collection::stream) + .collect(Collectors.toList()); + + return new SQLFragment(sql, params); + } +} diff --git a/api/src/org/labkey/api/data/dialect/StatementWrapper.java b/api/src/org/labkey/api/data/dialect/StatementWrapper.java index 94171f65d4f..25c8f6ca9d1 100644 --- a/api/src/org/labkey/api/data/dialect/StatementWrapper.java +++ b/api/src/org/labkey/api/data/dialect/StatementWrapper.java @@ -1,3015 +1,3015 @@ -/* - * Copyright (c) 2010-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.data.dialect; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.collections.OneBasedList; -import org.labkey.api.data.ConnectionWrapper; -import org.labkey.api.data.Container; -import org.labkey.api.data.QueryLogging; -import org.labkey.api.data.ResultSetWrapper; -import org.labkey.api.data.queryprofiler.Query; -import org.labkey.api.data.queryprofiler.QueryProfiler; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.DebugInfoDumper; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.view.ViewServlet; - -import java.io.InputStream; -import java.io.Reader; -import java.math.BigDecimal; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.Date; -import java.sql.NClob; -import java.sql.ParameterMetaData; -import java.sql.PreparedStatement; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; - -public class StatementWrapper implements Statement, PreparedStatement, CallableStatement -{ - private final ConnectionWrapper _conn; - private final Statement _stmt; - private final Logger _log; - private String _debugSql = ""; - private long _msStart = 0; - private boolean userCancelled = false; - // NOTE: CallableStatement supports getObject(), but PreparedStatement doesn't - private OneBasedList _parameters = null; - private @Nullable StackTraceElement[] _stackTrace = null; - /** Track the place that closed this statement for troubleshooting purposes */ - private @Nullable Throwable _closingStackTrace = null; - private @Nullable Boolean _requestThread = null; - private QueryLogging _queryLogging = QueryLogging.emptyQueryLogging(); - - String _sqlStateTestException = null; - - - public StatementWrapper(ConnectionWrapper conn, Statement stmt) - { - _conn = conn; - _log = conn.getLogger(); - _stmt = stmt; - assert MemTracker.getInstance().put(this); - } - - public StatementWrapper(ConnectionWrapper conn, Statement stmt, String sql) - { - this(conn, stmt); - _debugSql = sql; - } - - public void setStackTrace(@Nullable StackTraceElement[] stackTrace) - { - _stackTrace = stackTrace; - } - - public @NotNull Boolean isRequestThread() - { - return null != _requestThread ? _requestThread : ViewServlet.isRequestThread(); - } - - public @Nullable Throwable getClosingStackTrace() - { - return _closingStackTrace; - } - - public void setRequestThread(@Nullable Boolean requestThread) - { - _requestThread = requestThread; - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - public QueryLogging getQueryLogging() - { - return _queryLogging; - } - - public void setQueryLogging(QueryLogging queryLogging) - { - _queryLogging = queryLogging; - } - - @Override - public boolean wasNull() - throws SQLException - { - try - { - return ((CallableStatement)_stmt).wasNull(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String getString(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getString(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getBoolean(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBoolean(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte getByte(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getByte(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public short getShort(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getShort(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getInt(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getInt(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public long getLong(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getLong(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public float getFloat(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getFloat(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public double getDouble(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDouble(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(int parameterIndex, int scale) - throws SQLException - { - try - { - //noinspection deprecation - return ((CallableStatement)_stmt).getBigDecimal(parameterIndex, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte[] getBytes(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBytes(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBigDecimal(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(int i, Map> map) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(i, map); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Ref getRef(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getRef(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Blob getBlob(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBlob(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Clob getClob(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getClob(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Array getArray(int i) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getArray(i); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(int parameterIndex, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterIndex, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(int parameterIndex, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void registerOutParameter(String parameterName, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public URL getURL(int parameterIndex) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getURL(parameterIndex); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setURL(String parameterName, URL val) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setURL(parameterName, val); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(String parameterName, int sqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setNull(parameterName, sqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBoolean(String parameterName, boolean x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBoolean(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setByte(String parameterName, byte x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setByte(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setShort(String parameterName, short x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setShort(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setInt(String parameterName, int x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setInt(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setLong(String parameterName, long x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setLong(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFloat(String parameterName, float x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setFloat(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDouble(String parameterName, double x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDouble(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBigDecimal(String parameterName, BigDecimal x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBigDecimal(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setString(String parameterName, String x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setString(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBytes(String parameterName, byte[] x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBytes(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(String parameterName, Date x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDate(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(String parameterName, Time x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTime(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(String parameterName, Timestamp x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTimestamp(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setAsciiStream(String parameterName, InputStream x, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setAsciiStream(parameterName, x, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBinaryStream(String parameterName, InputStream x, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setBinaryStream(parameterName, x, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x, int targetSqlType, int scale) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType, scale); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x, int targetSqlType) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(String parameterName, Object x) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setObject(parameterName, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setCharacterStream(String parameterName, Reader reader, int length) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setCharacterStream(parameterName, reader, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(String parameterName, Date x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setDate(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(String parameterName, Time x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTime(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(String parameterName, Timestamp x, Calendar cal) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setTimestamp(parameterName, x, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(String parameterName, int sqlType, String typeName) - throws SQLException - { - try - { - ((CallableStatement)_stmt).setNull(parameterName, sqlType, typeName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String getString(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getString(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getBoolean(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBoolean(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte getByte(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getByte(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public short getShort(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getShort(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getInt(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getInt(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public long getLong(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getLong(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public float getFloat(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getFloat(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public double getDouble(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDouble(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public byte[] getBytes(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBytes(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public BigDecimal getBigDecimal(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBigDecimal(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Object getObject(String parameterName, Map> map) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getObject(parameterName, map); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Ref getRef(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getRef(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Blob getBlob(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getBlob(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Clob getClob(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getClob(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Array getArray(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getArray(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Date getDate(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getDate(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Time getTime(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTime(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public Timestamp getTimestamp(String parameterName, Calendar cal) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getTimestamp(parameterName, cal); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public URL getURL(String parameterName) - throws SQLException - { - try - { - return ((CallableStatement)_stmt).getURL(parameterName); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet executeQuery() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - if (null != _sqlStateTestException) - throw new SQLException("Test sql exception", _sqlStateTestException); - - ResultSet rs = ((PreparedStatement)_stmt).executeQuery(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int executeUpdate() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - int rows = -1; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = ((PreparedStatement)_stmt).executeUpdate(); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - private void _set(int i, @Nullable Object o) - { - if (null == _parameters) - _parameters = new OneBasedList<>(10); - while (_parameters.size() < i) - _parameters.add(null); - _parameters.set(i, o); - } - - @Override - public void setNull(int parameterIndex, int sqlType) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType); - _set(parameterIndex, null); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBoolean(int parameterIndex, boolean x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBoolean(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setByte(int parameterIndex, byte x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setByte(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setShort(int parameterIndex, short x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setShort(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setInt(int parameterIndex, int x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setInt(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setLong(int parameterIndex, long x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setLong(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFloat(int parameterIndex, float x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setFloat(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDouble(int parameterIndex, double x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDouble(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBigDecimal(int parameterIndex, BigDecimal x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBigDecimal(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setString(int parameterIndex, String x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setString(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBytes(int parameterIndex, byte[] x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBytes(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(int parameterIndex, Date x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDate(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(int parameterIndex, Time x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTime(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(int parameterIndex, Timestamp x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setAsciiStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setUnicodeStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - //noinspection deprecation - ((PreparedStatement)_stmt).setUnicodeStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBinaryStream(parameterIndex, x, length); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearParameters() - throws SQLException - { - try - { - ((PreparedStatement)_stmt).clearParameters(); - if (null != _parameters) _parameters.clear(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x, int targetSqlType, int scale) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType, scale); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x, int targetSqlType) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setObject(int parameterIndex, Object x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setObject(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean execute() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - Boolean ret=null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - ret = ((PreparedStatement)_stmt).execute(); - return ret; - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - int rows = (ret==Boolean.FALSE) ? _stmt.getUpdateCount() : -1; - afterExecute(preQuery, ex, rows); - } - } - - @Override - public void addBatch() - throws SQLException - { - try - { - ((PreparedStatement)_stmt).addBatch(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - // NOTE: We intentionally do not store potentially large parameters (reader, blob, etc.) - - @Override - public void setCharacterStream(int parameterIndex, Reader reader, int length) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setCharacterStream(parameterIndex, reader, length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setRef(int i, Ref x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setRef(i, x); - _set(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBlob(int i, Blob x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setBlob(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setClob(int i, Clob x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setClob(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setArray(int i, Array x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setArray(i, x); - _set(i, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSetMetaData getMetaData() - throws SQLException - { - try - { - ResultSetMetaData rs = ((PreparedStatement)_stmt).getMetaData(); - assert MemTracker.getInstance().put(rs); - return rs; - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setDate(int parameterIndex, Date x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setDate(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTime(int parameterIndex, Time x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTime(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x, cal); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setNull(int parameterIndex, int sqlType, String typeName) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType, typeName); - _set(parameterIndex, null); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setURL(int parameterIndex, URL x) - throws SQLException - { - try - { - ((PreparedStatement)_stmt).setURL(parameterIndex, x); - _set(parameterIndex, x); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ParameterMetaData getParameterMetaData() - throws SQLException - { - try - { - return ((PreparedStatement)_stmt).getParameterMetaData(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet executeQuery(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - ResultSet rs = _stmt.executeQuery(sql); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int executeUpdate(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public void close() - throws SQLException - { - try - { - _stmt.close(); - if (AppProps.getInstance().isDevMode() && _closingStackTrace == null) - { - _closingStackTrace = new Throwable("Remembering stack for closing Statement on thread " + Thread.currentThread().getName()); - } - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getMaxFieldSize() - throws SQLException - { - try - { - return _stmt.getMaxFieldSize(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setMaxFieldSize(int max) - throws SQLException - { - try - { - _stmt.setMaxFieldSize(max); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getMaxRows() - throws SQLException - { - try - { - return _stmt.getMaxRows(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setMaxRows(int max) - throws SQLException - { - try - { - _stmt.setMaxRows(max); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setEscapeProcessing(boolean enable) - throws SQLException - { - try - { - _stmt.setEscapeProcessing(enable); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getQueryTimeout() - throws SQLException - { - try - { - return _stmt.getQueryTimeout(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setQueryTimeout(int seconds) - throws SQLException - { - try - { - _stmt.setQueryTimeout(seconds); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void cancel() - throws SQLException - { - try - { - userCancelled = true; - _stmt.cancel(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public SQLWarning getWarnings() - throws SQLException - { - try - { - return _stmt.getWarnings(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearWarnings() - throws SQLException - { - try - { - _stmt.clearWarnings(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setCursorName(String name) - throws SQLException - { - try - { - _stmt.setCursorName(name); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean execute(String sql) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public ResultSet getResultSet() - throws SQLException - { - try - { - ResultSet rs = _stmt.getResultSet(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getUpdateCount() - throws SQLException - { - try - { - int updateCount; - updateCount = _stmt.getUpdateCount(); - return updateCount; - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public boolean getMoreResults() - throws SQLException - { - try - { - return _stmt.getMoreResults(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFetchDirection(int direction) - throws SQLException - { - try - { - _stmt.setFetchDirection(direction); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getFetchDirection() - throws SQLException - { - try - { - return _stmt.getFetchDirection(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setFetchSize(int rows) - throws SQLException - { - try - { - _stmt.setFetchSize(rows); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getFetchSize() - throws SQLException - { - try - { - return _stmt.getFetchSize(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getResultSetConcurrency() - throws SQLException - { - try - { - return _stmt.getResultSetConcurrency(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int getResultSetType() - throws SQLException - { - try - { - return _stmt.getResultSetType(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void addBatch(String sql) - throws SQLException - { - try - { - _stmt.addBatch(sql); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void clearBatch() - throws SQLException - { - try - { - _stmt.clearBatch(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int[] executeBatch() - throws SQLException - { - Query preQuery = beforeExecute(_debugSql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return _stmt.executeBatch(); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public Connection getConnection() - { - return _conn; - } - - @Override - public boolean getMoreResults(int current) - throws SQLException - { - try - { - return _stmt.getMoreResults(current); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public ResultSet getGeneratedKeys() - throws SQLException - { - try - { - ResultSet rs = _stmt.getGeneratedKeys(); - assert MemTracker.getInstance().put(rs); - return wrap(rs); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public int executeUpdate(String sql, int autoGeneratedKeys) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, autoGeneratedKeys); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public int executeUpdate(String sql, int[] columnIndexes) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, columnIndexes); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public int executeUpdate(String sql, String[] columnNames) - throws SQLException - { - Query preQuery = beforeExecute(sql); - int rows = -1; - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) - { - return rows = _stmt.executeUpdate(sql, columnNames); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, rows); - } - } - - @Override - public boolean execute(String sql, int autoGeneratedKeys) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, autoGeneratedKeys); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public boolean execute(String sql, int[] columnIndexes) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, columnIndexes); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public boolean execute(String sql, String[] columnNames) - throws SQLException - { - Query preQuery = beforeExecute(sql); - SQLException ex = null; - try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) - { - return _stmt.execute(sql, columnNames); - } - catch (SQLException sqlx) - { - ex = sqlx; - throw sqlx; - } - finally - { - afterExecute(preQuery, ex, -1); - } - } - - @Override - public int getResultSetHoldability() - throws SQLException - { - try - { - return _stmt.getResultSetHoldability(); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public String toString() - { - return _stmt.toString(); - } - - // TODO: These methods should be properly implemented via delegation. - - @Override - public boolean isWrapperFor(Class iface) - { - throw new UnsupportedOperationException(); - } - - @Override - public T unwrap(Class iface) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isClosed() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isPoolable() - { - throw new UnsupportedOperationException(); - } - - @Override - public void setPoolable(boolean poolable) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(int parameterIndex, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException - { - try - { - if (length > Integer.MAX_VALUE) - throw new IllegalArgumentException("File length exceeds " + Integer.MAX_VALUE); - setBinaryStream(parameterIndex, x, (int)length); - } - catch (SQLException e) - { - throw _conn.logAndCheckException(e); - } - } - - @Override - public void setBlob(int parameterIndex, InputStream inputStream) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(int parameterIndex, InputStream inputStream, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(int parameterIndex, Reader value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(int parameterIndex, Reader value, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(int parameterIndex, NClob value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNString(int parameterIndex, String value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setRowId(int parameterIndex, RowId x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setSQLXML(int parameterIndex, SQLXML xmlObject) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getCharacterStream(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getCharacterStream(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getNCharacterStream(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public Reader getNCharacterStream(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public NClob getNClob(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public NClob getNClob(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public String getNString(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public String getNString(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public RowId getRowId(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public RowId getRowId(String parameterName) - { - throw new UnsupportedOperationException(); - } - - @Override - public SQLXML getSQLXML(int parameterIndex) - { - throw new UnsupportedOperationException(); - } - - @Override - public SQLXML getSQLXML(String parameterName) - { - throw new UnsupportedOperationException(); - }//-- - - @Override - public void setAsciiStream(String parameterName, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAsciiStream(String parameterName, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(String parameterName, InputStream x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBinaryStream(String parameterName, InputStream x, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, InputStream inputStream) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, InputStream inputStream, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setBlob(String parameterName, Blob x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setCharacterStream(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setClob(String parameterName, Clob x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(String parameterName, Reader value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNCharacterStream(String parameterName, Reader value, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, Reader reader) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, Reader reader, long length) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNClob(String parameterName, NClob value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setNString(String parameterName, String value) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setRowId(String parameterName, RowId x) - { - throw new UnsupportedOperationException(); - } - - @Override - public void setSQLXML(String parameterName, SQLXML xmlObject) - { - throw new UnsupportedOperationException(); - } - - @Override - public T getObject(int parameterIndex, Class type) - { - throw new UnsupportedOperationException(); - } - - // JDBC 4.1 methods below must be here so we compile on JDK 7; implement once we require JRE 7. - - @Override - public T getObject(String parameterName, Class type) - { - throw new UnsupportedOperationException(); - } - - @Override - public void closeOnCompletion() - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isCloseOnCompletion() - { - throw new UnsupportedOperationException(); - } - - private Query beforeExecute(String sql) - { - _debugSql = sql; - // Crawler.java and BaseWebDriverTest.java use "8(" as attempted injection string - if (_debugSql.contains("\"8(\"") && !_debugSql.contains("\"\"8(\"\"")) // 18196 - throw new IllegalArgumentException("SQL injection test failed: " + _debugSql); - _msStart = System.currentTimeMillis(); - - List zeroBasedList = translateParametersForQueryTracking(); - return QueryProfiler.getInstance().preTrack(_conn.getScope(), sql, zeroBasedList, _stackTrace, isRequestThread()); - } - - - private void afterExecute(Query query, @Nullable SQLException x, int rowsAffected) - { - if (null != x) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.DialectSQL, query.getOriginalSql(), true); - if (SqlDialect.isConfigurationException(x)) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - } - } - _logStatement(query.getOriginalSql(), x, rowsAffected, getQueryLogging()); - } - - - private static final Package JAVA_LANG = java.lang.String.class.getPackage(); - - private void _logStatement(String sql, @Nullable SQLException x, int rowsAffected, QueryLogging queryLogging) - { - long elapsed = System.currentTimeMillis() - _msStart; - boolean isAssertEnabled = false; - assert isAssertEnabled = true; - - if (isAssertEnabled && AppProps.getInstance().isDevMode() && isMutatingSql(sql)) - SpringActionController.executingMutatingSql(sql); - - // Hold on to this stack trace so that we can reuse it later (if collection has been enabled) - Query query = QueryProfiler.getInstance().track(_conn.getScope(), sql, translateParametersForQueryTracking(), elapsed, _stackTrace, isRequestThread(), queryLogging); - - if (x != null) - { - _conn.logAndCheckException(x); - } - - //noinspection ConstantConditions - if (!_log.isEnabled(Level.DEBUG) && !isAssertEnabled) - return; - - StringBuilder logEntry = new StringBuilder(sql.length() * 2); - logEntry.append("SQL "); - - Integer sid = _conn.getSPID(); - if (sid != null) - logEntry.append(" [").append(sid).append("]"); - if (_msStart != 0) - logEntry.append(" time ").append(DateUtil.formatDuration(elapsed)); - - if (-1 != rowsAffected) - logEntry.append("\n ").append(rowsAffected).append(" rows affected"); - - logEntry.append("\n "); - logEntry.append(sql.trim().replace("\n", "\n ")); - - if (null != _parameters) - { - for (int i = 1; i <= _parameters.size(); i++) - { - try - { - Object o = _parameters.get(i); - String value; - if (o == null) - value = "NULL"; - else if (o instanceof Container) - value = "'" + ((Container)o).getId() + "' " + ((Container)o).getPath(); - else if (o instanceof String) - value = "'" + escapeSql((String) o) + "'"; - else - value = String.valueOf(o); - if (value.length() > 100) - value = StringUtilsLabKey.leftSurrogatePairFriendly(value, 100) + ". . ."; - logEntry.append("\n --[").append(i).append("] "); - logEntry.append(value); - Class c = null==o ? null : o.getClass(); - if (null != c && c != String.class && c != Integer.class) - logEntry.append(" :").append(c.getPackage() == JAVA_LANG ? c.getSimpleName() : c.getName()); - } - catch (Exception ex) - { - /* */ - } - } - } - _parameters = null; - - if (userCancelled) - logEntry.append("\n cancelled by user"); - if (null != x) - logEntry.append("\n ").append(x); - _appendTableStackTrace(logEntry, 5, query.getStackTraceElements()); - - final String logString = logEntry.toString(); - _log.log(Level.DEBUG, logString); - - // check for deadlock or transaction related error - if (SqlDialect.isTransactionException(x)) - { - DebugInfoDumper.dumpThreads(_log); - } - } - - private @Nullable List translateParametersForQueryTracking() - { - // Make a copy of the parameters list (it gets modified by callers) and switch to zero-based list (_parameters is a one-based list) - - List zeroBasedList; - - if (null != _parameters) - { - zeroBasedList = new ArrayList<>(_parameters.size()); - - // Translate parameters that can't be cached (for now, just JDBC arrays). I'd rather stash the original parameters and send - // those to the query profiler, but this would require one or more non-standard methods on StatementWrapper. See #24314. - for (Object o : _parameters) - { - if (o instanceof Array a) - { - try - { - o = a.getArray(); - } - catch (Exception e) - { - _log.error("Could not retrieve array", e); - o = null; - } - } - - zeroBasedList.add(o); - } - } - else - { - zeroBasedList = null; - } - return zeroBasedList; - } - - private boolean isMutatingSql(String sql) - { - return new MutatingSqlDetector(sql).isMutating(); - } - - - // Copied from Commons Lang 2.5 StringEscapeUtils. The method has been removed from Commons Lang 3.1 because it's - // simplistic and misleading. But we're only using it for logging. - private static String escapeSql(String str) - { - if (str == null) - { - return null; - } - return StringUtils.replace(str, "'", "''"); - } - - - private void _appendTableStackTrace(StringBuilder sb, int count, @Nullable StackTraceElement[] ste) - { - if (ste != null) - { - int i = 1; // Always skip getStackTrace() call - for (; i < ste.length; i++) - { - String line = ste[i].toString(); - if (!(line.startsWith("org.labkey.api.data.") || line.startsWith("java.lang.Thread"))) - break; - } - int last = Math.min(ste.length, i + count); - for (; i < last; i++) - { - String line = ste[i].toString(); - if (line.startsWith("javax.servlet.http.HttpServlet.service(")) - break; - sb.append("\n ").append(line); - } - } - } - - - public String getDebugSql() - { - return _debugSql; - } - - - ResultSet wrap(ResultSet rs) - { - return new ResultSetWrapper(rs) - { - @Override - public boolean next() throws SQLException - { - try - { - return super.next(); - } - catch (SQLException x) - { - if (SqlDialect.isTransactionException(x)) - _logStatement(_debugSql, x, -1, getQueryLogging()); - throw x; - } - } - }; - } -} +/* + * Copyright (c) 2010-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.data.dialect; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.collections.OneBasedList; +import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.Container; +import org.labkey.api.data.QueryLogging; +import org.labkey.api.data.ResultSetWrapper; +import org.labkey.api.data.queryprofiler.Query; +import org.labkey.api.data.queryprofiler.QueryProfiler; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.DebugInfoDumper; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.MemTracker; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.view.ViewServlet; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.Date; +import java.sql.NClob; +import java.sql.ParameterMetaData; +import java.sql.PreparedStatement; +import java.sql.Ref; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.RowId; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.Map; + +public class StatementWrapper implements Statement, PreparedStatement, CallableStatement +{ + private final ConnectionWrapper _conn; + private final Statement _stmt; + private final Logger _log; + private String _debugSql = ""; + private long _msStart = 0; + private boolean userCancelled = false; + // NOTE: CallableStatement supports getObject(), but PreparedStatement doesn't + private OneBasedList _parameters = null; + private @Nullable StackTraceElement[] _stackTrace = null; + /** Track the place that closed this statement for troubleshooting purposes */ + private @Nullable Throwable _closingStackTrace = null; + private @Nullable Boolean _requestThread = null; + private QueryLogging _queryLogging = QueryLogging.emptyQueryLogging(); + + String _sqlStateTestException = null; + + + public StatementWrapper(ConnectionWrapper conn, Statement stmt) + { + _conn = conn; + _log = conn.getLogger(); + _stmt = stmt; + assert MemTracker.getInstance().put(this); + } + + public StatementWrapper(ConnectionWrapper conn, Statement stmt, String sql) + { + this(conn, stmt); + _debugSql = sql; + } + + public void setStackTrace(@Nullable StackTraceElement[] stackTrace) + { + _stackTrace = stackTrace; + } + + public @NotNull Boolean isRequestThread() + { + return null != _requestThread ? _requestThread : ViewServlet.isRequestThread(); + } + + public @Nullable Throwable getClosingStackTrace() + { + return _closingStackTrace; + } + + public void setRequestThread(@Nullable Boolean requestThread) + { + _requestThread = requestThread; + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + public QueryLogging getQueryLogging() + { + return _queryLogging; + } + + public void setQueryLogging(QueryLogging queryLogging) + { + _queryLogging = queryLogging; + } + + @Override + public boolean wasNull() + throws SQLException + { + try + { + return ((CallableStatement)_stmt).wasNull(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String getString(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getString(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getBoolean(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBoolean(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte getByte(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getByte(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public short getShort(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getShort(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getInt(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getInt(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public long getLong(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getLong(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public float getFloat(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getFloat(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public double getDouble(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDouble(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(int parameterIndex, int scale) + throws SQLException + { + try + { + //noinspection deprecation + return ((CallableStatement)_stmt).getBigDecimal(parameterIndex, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte[] getBytes(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBytes(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBigDecimal(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(int i, Map> map) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(i, map); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Ref getRef(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getRef(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Blob getBlob(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBlob(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Clob getClob(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getClob(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Array getArray(int i) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getArray(i); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(int parameterIndex, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterIndex, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(int parameterIndex, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterIndex, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void registerOutParameter(String parameterName, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).registerOutParameter(parameterName, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public URL getURL(int parameterIndex) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getURL(parameterIndex); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setURL(String parameterName, URL val) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setURL(parameterName, val); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(String parameterName, int sqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setNull(parameterName, sqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBoolean(String parameterName, boolean x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBoolean(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setByte(String parameterName, byte x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setByte(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setShort(String parameterName, short x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setShort(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setInt(String parameterName, int x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setInt(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setLong(String parameterName, long x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setLong(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFloat(String parameterName, float x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setFloat(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDouble(String parameterName, double x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDouble(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBigDecimal(String parameterName, BigDecimal x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBigDecimal(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setString(String parameterName, String x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setString(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBytes(String parameterName, byte[] x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBytes(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(String parameterName, Date x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDate(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(String parameterName, Time x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTime(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(String parameterName, Timestamp x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTimestamp(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setAsciiStream(String parameterName, InputStream x, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setAsciiStream(parameterName, x, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBinaryStream(String parameterName, InputStream x, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setBinaryStream(parameterName, x, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x, int targetSqlType, int scale) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType, scale); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x, int targetSqlType) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x, targetSqlType); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(String parameterName, Object x) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setObject(parameterName, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setCharacterStream(String parameterName, Reader reader, int length) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setCharacterStream(parameterName, reader, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(String parameterName, Date x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setDate(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(String parameterName, Time x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTime(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(String parameterName, Timestamp x, Calendar cal) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setTimestamp(parameterName, x, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(String parameterName, int sqlType, String typeName) + throws SQLException + { + try + { + ((CallableStatement)_stmt).setNull(parameterName, sqlType, typeName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String getString(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getString(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getBoolean(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBoolean(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte getByte(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getByte(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public short getShort(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getShort(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getInt(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getInt(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public long getLong(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getLong(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public float getFloat(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getFloat(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public double getDouble(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDouble(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public byte[] getBytes(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBytes(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public BigDecimal getBigDecimal(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBigDecimal(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Object getObject(String parameterName, Map> map) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getObject(parameterName, map); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Ref getRef(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getRef(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Blob getBlob(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getBlob(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Clob getClob(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getClob(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Array getArray(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getArray(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Date getDate(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getDate(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Time getTime(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTime(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public Timestamp getTimestamp(String parameterName, Calendar cal) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getTimestamp(parameterName, cal); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public URL getURL(String parameterName) + throws SQLException + { + try + { + return ((CallableStatement)_stmt).getURL(parameterName); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet executeQuery() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + if (null != _sqlStateTestException) + throw new SQLException("Test sql exception", _sqlStateTestException); + + ResultSet rs = ((PreparedStatement)_stmt).executeQuery(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int executeUpdate() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + int rows = -1; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = ((PreparedStatement)_stmt).executeUpdate(); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + private void _set(int i, @Nullable Object o) + { + if (null == _parameters) + _parameters = new OneBasedList<>(10); + while (_parameters.size() < i) + _parameters.add(null); + _parameters.set(i, o); + } + + @Override + public void setNull(int parameterIndex, int sqlType) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType); + _set(parameterIndex, null); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBoolean(int parameterIndex, boolean x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBoolean(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setByte(int parameterIndex, byte x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setByte(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setShort(int parameterIndex, short x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setShort(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setInt(int parameterIndex, int x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setInt(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setLong(int parameterIndex, long x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setLong(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFloat(int parameterIndex, float x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setFloat(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDouble(int parameterIndex, double x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDouble(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBigDecimal(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setString(int parameterIndex, String x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setString(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBytes(int parameterIndex, byte[] x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBytes(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(int parameterIndex, Date x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDate(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(int parameterIndex, Time x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTime(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setAsciiStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + //noinspection deprecation + ((PreparedStatement)_stmt).setUnicodeStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBinaryStream(parameterIndex, x, length); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearParameters() + throws SQLException + { + try + { + ((PreparedStatement)_stmt).clearParameters(); + if (null != _parameters) _parameters.clear(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scale) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType, scale); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x, targetSqlType); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setObject(int parameterIndex, Object x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setObject(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean execute() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + Boolean ret=null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + ret = ((PreparedStatement)_stmt).execute(); + return ret; + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + int rows = (ret==Boolean.FALSE) ? _stmt.getUpdateCount() : -1; + afterExecute(preQuery, ex, rows); + } + } + + @Override + public void addBatch() + throws SQLException + { + try + { + ((PreparedStatement)_stmt).addBatch(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + // NOTE: We intentionally do not store potentially large parameters (reader, blob, etc.) + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setCharacterStream(parameterIndex, reader, length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setRef(int i, Ref x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setRef(i, x); + _set(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBlob(int i, Blob x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setBlob(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setClob(int i, Clob x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setClob(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setArray(int i, Array x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setArray(i, x); + _set(i, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSetMetaData getMetaData() + throws SQLException + { + try + { + ResultSetMetaData rs = ((PreparedStatement)_stmt).getMetaData(); + assert MemTracker.getInstance().put(rs); + return rs; + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setDate(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTime(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setTimestamp(parameterIndex, x, cal); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setNull(parameterIndex, sqlType, typeName); + _set(parameterIndex, null); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setURL(int parameterIndex, URL x) + throws SQLException + { + try + { + ((PreparedStatement)_stmt).setURL(parameterIndex, x); + _set(parameterIndex, x); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ParameterMetaData getParameterMetaData() + throws SQLException + { + try + { + return ((PreparedStatement)_stmt).getParameterMetaData(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet executeQuery(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + ResultSet rs = _stmt.executeQuery(sql); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int executeUpdate(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public void close() + throws SQLException + { + try + { + _stmt.close(); + if (AppProps.getInstance().isDevMode() && _closingStackTrace == null) + { + _closingStackTrace = new Throwable("Remembering stack for closing Statement on thread " + Thread.currentThread().getName()); + } + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getMaxFieldSize() + throws SQLException + { + try + { + return _stmt.getMaxFieldSize(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setMaxFieldSize(int max) + throws SQLException + { + try + { + _stmt.setMaxFieldSize(max); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getMaxRows() + throws SQLException + { + try + { + return _stmt.getMaxRows(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setMaxRows(int max) + throws SQLException + { + try + { + _stmt.setMaxRows(max); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setEscapeProcessing(boolean enable) + throws SQLException + { + try + { + _stmt.setEscapeProcessing(enable); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getQueryTimeout() + throws SQLException + { + try + { + return _stmt.getQueryTimeout(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setQueryTimeout(int seconds) + throws SQLException + { + try + { + _stmt.setQueryTimeout(seconds); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void cancel() + throws SQLException + { + try + { + userCancelled = true; + _stmt.cancel(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public SQLWarning getWarnings() + throws SQLException + { + try + { + return _stmt.getWarnings(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearWarnings() + throws SQLException + { + try + { + _stmt.clearWarnings(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setCursorName(String name) + throws SQLException + { + try + { + _stmt.setCursorName(name); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean execute(String sql) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public ResultSet getResultSet() + throws SQLException + { + try + { + ResultSet rs = _stmt.getResultSet(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getUpdateCount() + throws SQLException + { + try + { + int updateCount; + updateCount = _stmt.getUpdateCount(); + return updateCount; + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public boolean getMoreResults() + throws SQLException + { + try + { + return _stmt.getMoreResults(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFetchDirection(int direction) + throws SQLException + { + try + { + _stmt.setFetchDirection(direction); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getFetchDirection() + throws SQLException + { + try + { + return _stmt.getFetchDirection(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setFetchSize(int rows) + throws SQLException + { + try + { + _stmt.setFetchSize(rows); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getFetchSize() + throws SQLException + { + try + { + return _stmt.getFetchSize(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getResultSetConcurrency() + throws SQLException + { + try + { + return _stmt.getResultSetConcurrency(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int getResultSetType() + throws SQLException + { + try + { + return _stmt.getResultSetType(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void addBatch(String sql) + throws SQLException + { + try + { + _stmt.addBatch(sql); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void clearBatch() + throws SQLException + { + try + { + _stmt.clearBatch(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int[] executeBatch() + throws SQLException + { + Query preQuery = beforeExecute(_debugSql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return _stmt.executeBatch(); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public Connection getConnection() + { + return _conn; + } + + @Override + public boolean getMoreResults(int current) + throws SQLException + { + try + { + return _stmt.getMoreResults(current); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public ResultSet getGeneratedKeys() + throws SQLException + { + try + { + ResultSet rs = _stmt.getGeneratedKeys(); + assert MemTracker.getInstance().put(rs); + return wrap(rs); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, autoGeneratedKeys); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, columnIndexes); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public int executeUpdate(String sql, String[] columnNames) + throws SQLException + { + Query preQuery = beforeExecute(sql); + int rows = -1; + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(_debugSql)) + { + return rows = _stmt.executeUpdate(sql, columnNames); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, rows); + } + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, autoGeneratedKeys); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public boolean execute(String sql, int[] columnIndexes) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, columnIndexes); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public boolean execute(String sql, String[] columnNames) + throws SQLException + { + Query preQuery = beforeExecute(sql); + SQLException ex = null; + try (var ignore = DebugInfoDumper.pushThreadDumpContext(sql)) + { + return _stmt.execute(sql, columnNames); + } + catch (SQLException sqlx) + { + ex = sqlx; + throw sqlx; + } + finally + { + afterExecute(preQuery, ex, -1); + } + } + + @Override + public int getResultSetHoldability() + throws SQLException + { + try + { + return _stmt.getResultSetHoldability(); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public String toString() + { + return _stmt.toString(); + } + + // TODO: These methods should be properly implemented via delegation. + + @Override + public boolean isWrapperFor(Class iface) + { + throw new UnsupportedOperationException(); + } + + @Override + public T unwrap(Class iface) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isClosed() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isPoolable() + { + throw new UnsupportedOperationException(); + } + + @Override + public void setPoolable(boolean poolable) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException + { + try + { + if (length > Integer.MAX_VALUE) + throw new IllegalArgumentException("File length exceeds " + Integer.MAX_VALUE); + setBinaryStream(parameterIndex, x, (int)length); + } + catch (SQLException e) + { + throw _conn.logAndCheckException(e); + } + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(int parameterIndex, NClob value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNString(int parameterIndex, String value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setRowId(int parameterIndex, RowId x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getCharacterStream(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getCharacterStream(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getNCharacterStream(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public Reader getNCharacterStream(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public NClob getNClob(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public NClob getNClob(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public String getNString(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public String getNString(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public RowId getRowId(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public RowId getRowId(String parameterName) + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLXML getSQLXML(int parameterIndex) + { + throw new UnsupportedOperationException(); + } + + @Override + public SQLXML getSQLXML(String parameterName) + { + throw new UnsupportedOperationException(); + }//-- + + @Override + public void setAsciiStream(String parameterName, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAsciiStream(String parameterName, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(String parameterName, InputStream x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBinaryStream(String parameterName, InputStream x, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, InputStream inputStream) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, InputStream inputStream, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setBlob(String parameterName, Blob x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setCharacterStream(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setClob(String parameterName, Clob x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(String parameterName, Reader value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNCharacterStream(String parameterName, Reader value, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, Reader reader) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, Reader reader, long length) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNClob(String parameterName, NClob value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setNString(String parameterName, String value) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setRowId(String parameterName, RowId x) + { + throw new UnsupportedOperationException(); + } + + @Override + public void setSQLXML(String parameterName, SQLXML xmlObject) + { + throw new UnsupportedOperationException(); + } + + @Override + public T getObject(int parameterIndex, Class type) + { + throw new UnsupportedOperationException(); + } + + // JDBC 4.1 methods below must be here so we compile on JDK 7; implement once we require JRE 7. + + @Override + public T getObject(String parameterName, Class type) + { + throw new UnsupportedOperationException(); + } + + @Override + public void closeOnCompletion() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isCloseOnCompletion() + { + throw new UnsupportedOperationException(); + } + + private Query beforeExecute(String sql) + { + _debugSql = sql; + // Crawler.java and BaseWebDriverTest.java use "8(" as attempted injection string + if (_debugSql.contains("\"8(\"") && !_debugSql.contains("\"\"8(\"\"")) // 18196 + throw new IllegalArgumentException("SQL injection test failed: " + _debugSql); + _msStart = System.currentTimeMillis(); + + List zeroBasedList = translateParametersForQueryTracking(); + return QueryProfiler.getInstance().preTrack(_conn.getScope(), sql, zeroBasedList, _stackTrace, isRequestThread()); + } + + + private void afterExecute(Query query, @Nullable SQLException x, int rowsAffected) + { + if (null != x) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.DialectSQL, query.getOriginalSql(), true); + if (SqlDialect.isConfigurationException(x)) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + } + } + _logStatement(query.getOriginalSql(), x, rowsAffected, getQueryLogging()); + } + + + private static final Package JAVA_LANG = java.lang.String.class.getPackage(); + + private void _logStatement(String sql, @Nullable SQLException x, int rowsAffected, QueryLogging queryLogging) + { + long elapsed = System.currentTimeMillis() - _msStart; + boolean isAssertEnabled = false; + assert isAssertEnabled = true; + + if (isAssertEnabled && AppProps.getInstance().isDevMode() && isMutatingSql(sql)) + SpringActionController.executingMutatingSql(sql); + + // Hold on to this stack trace so that we can reuse it later (if collection has been enabled) + Query query = QueryProfiler.getInstance().track(_conn.getScope(), sql, translateParametersForQueryTracking(), elapsed, _stackTrace, isRequestThread(), queryLogging); + + if (x != null) + { + _conn.logAndCheckException(x); + } + + //noinspection ConstantConditions + if (!_log.isEnabled(Level.DEBUG) && !isAssertEnabled) + return; + + StringBuilder logEntry = new StringBuilder(sql.length() * 2); + logEntry.append("SQL "); + + Integer sid = _conn.getSPID(); + if (sid != null) + logEntry.append(" [").append(sid).append("]"); + if (_msStart != 0) + logEntry.append(" time ").append(DateUtil.formatDuration(elapsed)); + + if (-1 != rowsAffected) + logEntry.append("\n ").append(rowsAffected).append(" rows affected"); + + logEntry.append("\n "); + logEntry.append(sql.trim().replace("\n", "\n ")); + + if (null != _parameters) + { + for (int i = 1; i <= _parameters.size(); i++) + { + try + { + Object o = _parameters.get(i); + String value; + if (o == null) + value = "NULL"; + else if (o instanceof Container) + value = "'" + ((Container)o).getId() + "' " + ((Container)o).getPath(); + else if (o instanceof String) + value = "'" + escapeSql((String) o) + "'"; + else + value = String.valueOf(o); + if (value.length() > 100) + value = StringUtilsLabKey.leftSurrogatePairFriendly(value, 100) + ". . ."; + logEntry.append("\n --[").append(i).append("] "); + logEntry.append(value); + Class c = null==o ? null : o.getClass(); + if (null != c && c != String.class && c != Integer.class) + logEntry.append(" :").append(c.getPackage() == JAVA_LANG ? c.getSimpleName() : c.getName()); + } + catch (Exception ex) + { + /* */ + } + } + } + _parameters = null; + + if (userCancelled) + logEntry.append("\n cancelled by user"); + if (null != x) + logEntry.append("\n ").append(x); + _appendTableStackTrace(logEntry, 5, query.getStackTraceElements()); + + final String logString = logEntry.toString(); + _log.log(Level.DEBUG, logString); + + // check for deadlock or transaction related error + if (SqlDialect.isTransactionException(x)) + { + DebugInfoDumper.dumpThreads(_log); + } + } + + private @Nullable List translateParametersForQueryTracking() + { + // Make a copy of the parameters list (it gets modified by callers) and switch to zero-based list (_parameters is a one-based list) + + List zeroBasedList; + + if (null != _parameters) + { + zeroBasedList = new ArrayList<>(_parameters.size()); + + // Translate parameters that can't be cached (for now, just JDBC arrays). I'd rather stash the original parameters and send + // those to the query profiler, but this would require one or more non-standard methods on StatementWrapper. See #24314. + for (Object o : _parameters) + { + if (o instanceof Array a) + { + try + { + o = a.getArray(); + } + catch (Exception e) + { + _log.error("Could not retrieve array", e); + o = null; + } + } + + zeroBasedList.add(o); + } + } + else + { + zeroBasedList = null; + } + return zeroBasedList; + } + + private boolean isMutatingSql(String sql) + { + return new MutatingSqlDetector(sql).isMutating(); + } + + + // Copied from Commons Lang 2.5 StringEscapeUtils. The method has been removed from Commons Lang 3.1 because it's + // simplistic and misleading. But we're only using it for logging. + private static String escapeSql(String str) + { + if (str == null) + { + return null; + } + return StringUtils.replace(str, "'", "''"); + } + + + private void _appendTableStackTrace(StringBuilder sb, int count, @Nullable StackTraceElement[] ste) + { + if (ste != null) + { + int i = 1; // Always skip getStackTrace() call + for (; i < ste.length; i++) + { + String line = ste[i].toString(); + if (!(line.startsWith("org.labkey.api.data.") || line.startsWith("java.lang.Thread"))) + break; + } + int last = Math.min(ste.length, i + count); + for (; i < last; i++) + { + String line = ste[i].toString(); + if (line.startsWith("javax.servlet.http.HttpServlet.service(")) + break; + sb.append("\n ").append(line); + } + } + } + + + public String getDebugSql() + { + return _debugSql; + } + + + ResultSet wrap(ResultSet rs) + { + return new ResultSetWrapper(rs) + { + @Override + public boolean next() throws SQLException + { + try + { + return super.next(); + } + catch (SQLException x) + { + if (SqlDialect.isTransactionException(x)) + _logStatement(_debugSql, x, -1, getQueryLogging()); + throw x; + } + } + }; + } +} diff --git a/api/src/org/labkey/api/exp/OntologyManager.java b/api/src/org/labkey/api/exp/OntologyManager.java index bd2a07600eb..b30147543c8 100644 --- a/api/src/org/labkey/api/exp/OntologyManager.java +++ b/api/src/org/labkey/api/exp/OntologyManager.java @@ -1,3915 +1,3915 @@ -/* - * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.exp; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.cache.BlockingCache; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.data.*; -import org.labkey.api.data.DbScope.Transaction; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.gwt.client.ui.domain.CancellationException; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.test.TestTimeout; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.ResultSetUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.view.HttpView; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Collections.emptySet; -import static java.util.Collections.unmodifiableCollection; -import static java.util.Collections.unmodifiableList; -import static java.util.Collections.unmodifiableMap; -import static java.util.stream.Collectors.joining; -import static org.labkey.api.util.IntegerUtils.asLong; - -/** - * Lots of static methods for dealing with domains and property descriptors. Tends to operate primarily on the bean-style - * classes like {@link PropertyDescriptor} and {@link DomainDescriptor}. When possible, it's usually preferable to use - * {@link PropertyService}, {@link Domain}, and {@link DomainProperty} instead as they tend to provide higher-level - * abstractions. - */ -public class OntologyManager -{ - private static final Logger _log = LogManager.getLogger(OntologyManager.class); - private static final Cache, Map> PROPERTY_MAP_CACHE = DatabaseCache.get(getExpSchema().getScope(), 100000, "Property maps", new PropertyMapCacheLoader()); - private static final BlockingCache OBJECT_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "ObjectIds", new ObjectIdCacheLoader()); - private static final Cache, PropertyDescriptor> PROP_DESCRIPTOR_CACHE = DatabaseCache.get(getExpSchema().getScope(), 40000, "Property descriptors", new CacheLoader<>() - { - @Override - public PropertyDescriptor load(@NotNull Pair key, @Nullable Object argument) - { - PropertyDescriptor ret = null; - String propertyURI = key.first; - Container c = ContainerManager.getForId(key.second); - if (null != c) - { - Container proj = c.getProject(); - if (null == proj) - proj = c; - _log.debug("Loading a property descriptor for key " + key + " using project " + proj); - String sql = " SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyURI = ? AND Project IN (?,?)"; - List pdArray = new SqlSelector(getExpSchema(), sql, propertyURI, proj, _sharedContainer.getId()).getArrayList(PropertyDescriptor.class); - if (!pdArray.isEmpty()) - { - PropertyDescriptor pd = pdArray.get(0); - - // if someone has explicitly inserted a descriptor with the same URI as an existing one, - // and one of the two is in the shared project, use the project-level descriptor. - if (pdArray.size() > 1) - { - _log.debug("Multiple PropertyDescriptors found for " + propertyURI); - if (pd.getProject().equals(_sharedContainer)) - pd = pdArray.get(1); - } - _log.debug("Loaded property descriptor " + pd); - ret = pd; - } - } - return ret; - } - }); - - /** DomainURI, ContainerEntityId -> DomainDescriptor */ - private static final Cache, DomainDescriptor> DOMAIN_DESCRIPTORS_BY_URI_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, CacheManager.UNLIMITED, "Domain descriptors by URI", (key, argument) -> { - String domainURI = key.first; - Container c = ContainerManager.getForId(key.second); - - if (c == null) - { - return null; - } - - return fetchDomainDescriptorFromDB(domainURI, c); - }); - - @Nullable - private static DomainDescriptor fetchDomainDescriptorFromDB(String domainURI, Container c) - { - return fetchDomainDescriptorFromDB(domainURI, c, false); - } - - /** Goes against the DB, bypassing the cache */ - @Nullable - public static DomainDescriptor fetchDomainDescriptorFromDB(String uriOrName, Container c, boolean isName) - { - Container proj = c.getProject(); - if (null == proj) - proj = c; - - String sql = " SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE " + (isName ? "Name" : "DomainURI") + " = ? AND Project IN (?,?) "; - List ddArray = new SqlSelector(getExpSchema(), sql, uriOrName, - proj, - ContainerManager.getSharedContainer().getId()).getArrayList(DomainDescriptor.class); - DomainDescriptor dd = null; - if (!ddArray.isEmpty()) - { - dd = ddArray.get(0); - - // if someone has explicitly inserted a descriptor with the same URI as an existing one , - // and one of the two is in the shared project, use the project-level descriptor. - if (ddArray.size() > 1) - { - _log.debug("Multiple DomainDescriptors found for " + uriOrName); - if (dd.getProject().equals(ContainerManager.getSharedContainer())) - dd = ddArray.get(0); - } - } - return dd; - } - - private static final BlockingCache DOMAIN_DESC_BY_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(),2000, CacheManager.UNLIMITED,"Domain descriptors by ID", new DomainDescriptorLoader()); - private static final BlockingCache, List>> DOMAIN_PROPERTIES_CACHE = DatabaseCache.get(getExpSchema().getScope(), 5000, CacheManager.UNLIMITED, "Domain properties", new CacheLoader<>() - { - @Override - public List> load(@NotNull Pair key, @Nullable Object argument) - { - String typeURI = key.first; - Container c = ContainerManager.getForId(key.second); - if (null == c) - return Collections.emptyList(); - SQLFragment sql = new SQLFragment("SELECT PropertyURI, Required " + - "FROM " + getTinfoPropertyDescriptor() + " PD\n" + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (PD.PropertyId = PDM.PropertyId)\n" + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)\n" + - "WHERE DD.DomainURI = ? AND DD.Project IN (?, ?) ORDER BY PDM.SortOrder, PD.PropertyId"); - - sql.addAll( - typeURI, - // protect against null project, just double-up shared project - c.isRoot() ? c.getId() : (c.getProject() == null ? _sharedContainer.getProject().getId() : c.getProject().getId()), - _sharedContainer.getProject().getId() - ); - - return new SqlSelector(getExpSchema(), sql).mapStream() - .map(map -> Pair.of((String)map.get("PropertyURI"), (Boolean)map.get("Required"))) - .toList(); - } - }); - private static final Cache> DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "Domain descriptors by container", (c, argument) -> { - String sql = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - - Map dds = new LinkedHashMap<>(); - for (DomainDescriptor dd : new SqlSelector(getExpSchema(), sql, c).getArrayList(DomainDescriptor.class)) - { - dds.putIfAbsent(dd.getDomainURI(), dd); - } - - return unmodifiableMap(dds); - }); - - private static final Container _sharedContainer = ContainerManager.getSharedContainer(); - - public static final String MV_INDICATOR_SUFFIX = "mvindicator"; - - static public String PropertyOrderURI = "urn:exp.labkey.org/#PropertyOrder"; - /** - * A comma-separated list of propertyID that indicates the sort order of the properties attached to an object. - */ - static public SystemProperty PropertyOrder = new SystemProperty(PropertyOrderURI, PropertyType.STRING); - - static - { - BeanObjectFactory.Registry.register(ObjectProperty.class, new ObjectProperty.ObjectPropertyObjectFactory()); - } - - private OntologyManager() - { - } - - /** - * @return map from PropertyURI to value - */ - public static @NotNull Map getProperties(Container container, String parentLSID) - { - Map m = new LinkedHashMap<>(); - Map propVals = getPropertyObjects(container, parentLSID); - if (null != propVals) - { - for (Map.Entry entry : propVals.entrySet()) - { - m.put(entry.getKey(), entry.getValue().value()); - } - } - - return m; - } - - public static final int MAX_PROPS_IN_BATCH = 1000; // Keep this reasonably small so progress indicator is updated regularly - public static final int UPDATE_STATS_BATCH_COUNT = 1000; - - public static void insertTabDelimited(Container c, - User user, - @Nullable Long ownerObjectId, - ImportHelper helper, - Domain domain, - DataIterator rows, - boolean ensureObjects, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - List properties = new ArrayList<>(domain.getProperties().size()); - for (DomainProperty prop : domain.getProperties()) - { - properties.add(prop.getPropertyDescriptor()); - } - insertTabDelimited(c, user, ownerObjectId, helper, properties, rows, ensureObjects, rowCallback); - } - - public interface RowCallback - { - void rowProcessed(Map row, String lsid) throws BatchValidationException; - - default void complete() throws BatchValidationException - {} - - default RowCallback chain(RowCallback other) - { - if (other == NO_OP_ROW_CALLBACK) - { - return this; - } - if (this == NO_OP_ROW_CALLBACK) - { - return other; - } - - RowCallback original = this; - - return new RowCallback() - { - @Override - public void rowProcessed(Map row, String lsid) throws BatchValidationException - { - original.rowProcessed(row, lsid); - other.rowProcessed(row, lsid); - } - - @Override - public void complete() throws BatchValidationException - { - original.complete(); - other.complete(); - } - }; - } - } - - public static final RowCallback NO_OP_ROW_CALLBACK = (row, lsid) -> {}; - - public static void insertTabDelimited(Container c, - User user, - @Nullable Long ownerObjectId, - ImportHelper helper, - List descriptors, - DataIterator rawRows, - boolean ensureObjects, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - MapDataIterator rows = DataIteratorUtil.wrapMap(rawRows, false); - - rowCallback = rowCallback == null ? NO_OP_ROW_CALLBACK : rowCallback; - - CPUTimer total = new CPUTimer("insertTabDelimited"); - CPUTimer before = new CPUTimer("beforeImport"); - CPUTimer ensure = new CPUTimer("ensureObject"); - CPUTimer insert = new CPUTimer("insertProperties"); - - assert total.start(); - assert getExpSchema().getScope().isTransactionActive(); - - // Make sure we have enough rows to handle the overflow of the current row so we don't have to resize the list - List propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); - - ValidatorContext validatorCache = new ValidatorContext(c, user); - - try - { - OntologyObject objInsert = new OntologyObject(); - objInsert.setContainer(c); - if (ownerObjectId != null && ownerObjectId > 0) - objInsert.setOwnerObjectId(ownerObjectId); - - List errors = new ArrayList<>(); - Map> validatorMap = new IntHashMap<>(); - - // cache all the property validators for this upload - for (PropertyDescriptor pd : descriptors) - { - List validators = PropertyService.get().getPropertyValidators(pd); - if (!validators.isEmpty()) - validatorMap.put(pd.getPropertyId(), validators); - } - - int rowCount = 0; - int batchCount = 0; - - while (rows.next()) - { - Map map = rows.getMap(); - // TODO: hack -- should exit and return cancellation status instead of throwing - if (Thread.currentThread().isInterrupted()) - throw new CancellationException(); - - assert before.start(); - - Map modifiableMap = new HashMap<>(map); - String lsid = helper.beforeImportObject(modifiableMap); - map = Collections.unmodifiableMap(modifiableMap); - - if (lsid == null) - { - throw new IllegalStateException("No LSID available"); - } - - assert before.stop(); - - assert ensure.start(); - long objectId; - if (ensureObjects) - objectId = ensureObject(c, lsid, ownerObjectId); - else - { - objInsert.setObjectURI(lsid); - Table.insert(null, getTinfoObject(), objInsert); - objectId = objInsert.getObjectId(); - } - - for (PropertyDescriptor pd : descriptors) - { - Object value = map.get(pd.getPropertyURI()); - if (null == value) - { - if (pd.isRequired()) - throw new BatchValidationException(new ValidationException("Missing value for required property " + pd.getName())); - else - { - continue; - } - } - else - { - if (validatorMap.containsKey(pd.getPropertyId())) - validateProperty(validatorMap.get(pd.getPropertyId()), pd, new ObjectProperty(lsid, c, pd, value), errors, validatorCache); - } - try - { - PropertyRow row = new PropertyRow(objectId, pd, value, pd.getPropertyType()); - propsToInsert.add(row); - } - catch (ConversionException e) - { - throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getPropertyType().getJavaType()))); - } - } - assert ensure.stop(); - - rowCount++; - - if (propsToInsert.size() > MAX_PROPS_IN_BATCH) - { - assert insert.start(); - insertPropertiesBulk(c, propsToInsert, false); - helper.afterBatchInsert(rowCount); - assert insert.stop(); - propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); - - if (++batchCount % UPDATE_STATS_BATCH_COUNT == 0) - { - getExpSchema().getSqlDialect().updateStatistics(getTinfoObject()); - getExpSchema().getSqlDialect().updateStatistics(getTinfoObjectProperty()); - helper.updateStatistics(rowCount); - } - } - - rowCallback.rowProcessed(map, lsid); - } - - if (!errors.isEmpty()) - throw new BatchValidationException(new ValidationException(errors)); - - assert insert.start(); - insertPropertiesBulk(c, propsToInsert, false); - helper.afterBatchInsert(rowCount); - rowCallback.complete(); - assert insert.stop(); - } - catch (SQLException x) - { - SQLException next = x.getNextException(); - if (x instanceof java.sql.BatchUpdateException && null != next) - x = next; - _log.debug("Exception uploading: ", x); - throw x; - } - - assert total.stop(); - _log.debug("\t" + total); - _log.debug("\t" + before); - _log.debug("\t" + ensure); - _log.debug("\t" + insert); - } - - /** - * As an incremental step of QueryUpdateService cleanup, this is a version of insertTabDelimited that works on a - * tableInfo that implements UpdateableTableInfo. Does not support ownerObjectid. - *

    - * This code is made complicated by the fact that while we are trying to move toward a TableInfo/ColumnInfo view - * of the world, validators are attached to PropertyDescriptors. Also, missing value handling is attached - * to PropertyDescriptors. - *

    - * The original version of this method expects a data to be a map PropertyURI->value. This version will also - * accept Name->value. - *

    - * Name->Value is preferred, we are using TableInfo after all. - */ - @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder - public static void insertTabDelimited(TableInfo tableInsert, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator rows, - boolean autoFillDefaultColumns, - Logger logger, - RowCallback rowCallback) - throws SQLException, BatchValidationException - { - saveTabDelimited(tableInsert, c, user, helper, rows, logger, true, autoFillDefaultColumns, rowCallback); - } - - @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder - public static void updateTabDelimited(TableInfo tableInsert, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator rows, - boolean autoFillDefaultColumns, - Logger logger) - throws SQLException, BatchValidationException - { - saveTabDelimited(tableInsert, c, user, helper, rows, logger, false, autoFillDefaultColumns, NO_OP_ROW_CALLBACK); - } - - private static void saveTabDelimited(TableInfo table, - Container c, - User user, - UpdateableTableImportHelper helper, - DataIterator in, - Logger logger, - boolean insert, - boolean autoFillDefaultColumns, - @Nullable RowCallback rowCallback) - throws SQLException, BatchValidationException - { - if (!(table instanceof UpdateableTableInfo)) - throw new IllegalArgumentException(); - - if (rowCallback == null) - { - rowCallback = NO_OP_ROW_CALLBACK; - } - - DbScope scope = table.getSchema().getScope(); - - assert scope.isTransactionActive(); - - Domain d = table.getDomain(); - List properties = null == d ? Collections.emptyList() : d.getProperties(); - - ValidatorContext validatorCache = new ValidatorContext(c, user); - - Connection conn = null; - ParameterMapStatement parameterMap = null; - - Map currentRow = null; - - MapDataIterator rows = DataIteratorUtil.wrapMap(in, false); - try - { - conn = scope.getConnection(); - if (insert) - { - parameterMap = StatementUtils.insertStatement(conn, table, c, user, true, autoFillDefaultColumns); - } - else - { - parameterMap = StatementUtils.updateStatement(conn, table, c, user, false, autoFillDefaultColumns); - } - List errors = new ArrayList<>(); - - Map> validatorMap = new HashMap<>(); - Map propertiesMap = new HashMap<>(); - - // cache all the property validators for this upload - for (DomainProperty dp : properties) - { - propertiesMap.put(dp.getPropertyURI(), dp); - List validators = dp.getValidators(); - if (!validators.isEmpty()) - validatorMap.put(dp.getPropertyURI(), validators); - } - - List columns = table.getColumns(); - PropertyType[] propertyTypes = new PropertyType[columns.size()]; - for (int i = 0; i < columns.size(); i++) - { - String propertyURI = columns.get(i).getPropertyURI(); - DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); - PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); - if (null != pd) - propertyTypes[i] = pd.getPropertyType(); - } - - int rowCount = 0; - - while (rows.next()) - { - - currentRow = new CaseInsensitiveHashMap<>(rows.getMap()); - - // TODO: hack -- should exit and return cancellation status instead of throwing - if (Thread.currentThread().isInterrupted()) - throw new CancellationException(); - - parameterMap.clearParameters(); - - String lsid = helper.beforeImportObject(currentRow); - currentRow.put("lsid", lsid); - - // - // NOTE we validate based on columninfo/propertydescriptor - // However, we bind by name, and there may be parameters that do not correspond to columninfo - // - - for (int i = 0; i < columns.size(); i++) - { - ColumnInfo col = columns.get(i); - if (col.isMvIndicatorColumn() || col.isRawValueColumn()) //TODO col.isNotUpdatableForSomeReasonSoContinue() - continue; - String propertyURI = col.getPropertyURI(); - DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); - PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); - - Object value = currentRow.get(col.getName()); - if (null == value) - value = currentRow.get(propertyURI); - - if (null == value) - { - // TODO col.isNullable() doesn't seem to work here - if (null != pd && pd.isRequired()) - throw new BatchValidationException(new ValidationException("Missing value for required property " + col.getName())); - } - else - { - if (null != pd) - { - try - { - // Use an ObjectProperty to unwrap MvFieldWrapper, do type conversion, etc - ObjectProperty objectProperty = new ObjectProperty(lsid, c, pd, value); - if (!validateProperty(validatorMap.get(propertyURI), pd, objectProperty, errors, validatorCache)) - { - throw new BatchValidationException(new ValidationException(errors)); - } - } - catch (ConversionException e) - { - throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getJavaClass()))); - } - } - } - - // issue 19391: data from R uses "Inf" to represent infinity - if (JdbcType.DOUBLE.equals(col.getJdbcType())) - { - value = "Inf".equals(value) ? "Infinity" : value; - value = "-Inf".equals(value) ? "-Infinity" : value; - } - - try - { - String key = col.getName(); - if (!parameterMap.containsKey(key)) - key = propertyURI; - if (null == propertyTypes[i]) - { - // some built-in columns won't have parameters (createdby, etc) - if (parameterMap.containsKey(key)) - { - assert !(value instanceof MvFieldWrapper); - // Handle type coercion for these built-in columns as well, though we don't need to - // worry about missing values - value = PropertyType.getFromClass(col.getJavaObjectClass()).convert(value); - parameterMap.put(key, value); - } - } - else - { - Pair p = new Pair<>(value, null); - convertValuePair(pd, propertyTypes[i], p); - parameterMap.put(key, p.first); - if (null != p.second) - { - FieldKey mvName = col.getMvColumnName(); - if (mvName != null) - { - String storageName = table.getColumn(mvName).getMetaDataIdentifier().getId(); - parameterMap.put(storageName, p.second); - } - } - } - } - catch (ConversionException e) - { - throw new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), propertyTypes[i].getJavaType())); - } - } - - helper.bindAdditionalParameters(currentRow, parameterMap); - parameterMap.execute(); - if (insert) - { - long rowId = parameterMap.getRowId(); - currentRow.put("rowId", rowId); - } - lsid = helper.afterImportObject(currentRow); - if (lsid == null) - { - throw new IllegalStateException("No LSID available"); - } - rowCallback.rowProcessed(currentRow, lsid); - rowCount++; - } - - - if (!errors.isEmpty()) - throw new BatchValidationException(new ValidationException(errors)); - - rowCallback.complete(); - - helper.afterBatchInsert(rowCount); - if (logger != null) - logger.debug("inserted row " + rowCount + "."); - } - catch (ValidationException e) - { - throw new BatchValidationException(e); - } - catch (SQLException x) - { - SQLException next = x.getNextException(); - if (x instanceof java.sql.BatchUpdateException && null != next) - x = next; - _log.debug("Exception uploading: ", x); - if (null != currentRow) - _log.debug(currentRow.toString()); - throw x; - } - finally - { - if (null != parameterMap) - parameterMap.close(); - if (null != conn) - scope.releaseConnection(conn); - } - } - - // TODO: Consolidate with ColumnValidator - public static boolean validateProperty(List validators, PropertyDescriptor prop, ObjectProperty objectProperty, - List errors, ValidatorContext validatorCache) - { - boolean ret = true; - - Object value = objectProperty.getObjectValue(); - - if (prop.isRequired() && value == null && objectProperty.getMvIndicator() == null) - { - errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is required", prop.getName())); - ret = false; - } - - // Check if the string is too long. Use either the PropertyDescriptor's scale or VARCHAR(4000) for ontology managed values - int stringLengthLimit = prop.getScale() > 0 ? prop.getScale() : getTinfoObjectProperty().getColumn("StringValue").getScale(); - int stringLength = value == null ? 0 : value.toString().length(); - if (value != null && prop.isStringType() && stringLength > stringLengthLimit) - { - String s = stringLength <= 100 ? value.toString() : StringUtilsLabKey.leftSurrogatePairFriendly(value.toString(), 100); - errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is limited to " + stringLengthLimit + " characters, but the value is " + stringLength + " characters. (The value starts with '" + s + "...')", prop.getName())); - ret = false; - } - - // TODO: check date is within postgres date range - - // Don't validate null values, #15683 - if (null != value && validators != null) - { - for (IPropertyValidator validator : validators) - if (!validator.validate(prop, value, errors, validatorCache)) ret = false; - } - return ret; - } - - public interface ImportHelper - { - /** - * may modify map - * - * @return LSID for new or existing Object. Null indicates LSID is still unknown. - */ - String beforeImportObject(Map map) throws SQLException; - - void afterBatchInsert(int currentRow) throws SQLException; - - void updateStatistics(int currentRow) throws SQLException; - } - - - public interface UpdateableTableImportHelper extends ImportHelper - { - /** - * may be used to process attachments, for auditing, etc - * @return the LSID of the inserted row - */ - String afterImportObject(Map map) throws SQLException; - - /** - * may set parameters directly for columns that are not exposed by tableinfo - * e.g. "_key" - *

    - * TODO maybe this can be handled declaratively? see UpdateableTableInfo - */ - void bindAdditionalParameters(Map map, ParameterMapStatement target) throws ValidationException; - } - - @NotNull - private static Pair getPropertyMapCacheKey(@Nullable Container container, @NotNull String objectLSID) - { - return Pair.of(container, objectLSID); - } - - /** - * Get ordered map of property values for an object. The order of the properties in the - * Map corresponds to the PropertyOrder property, if present. - * - * @return map from PropertyURI to ObjectProperty - */ - public static Map getPropertyObjects(@Nullable Container container, @NotNull String objectLSID) - { - Pair cacheKey = getPropertyMapCacheKey(container, objectLSID); - return PROPERTY_MAP_CACHE.get(cacheKey); - } - - public static class PropertyMapCacheLoader implements CacheLoader, Map> - { - @Override - public Map load(@NotNull Pair key, @Nullable Object argument) - { - Container container = key.first; - String objectLSID = key.second; - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), objectLSID); - if (container != null) - { - filter.addCondition(FieldKey.fromParts("Container"), container); - } - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getResultSet()) - { - ResultSetUtil.logData(rs); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - List props = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getArrayList(ObjectProperty.class); - - // check for a "PropertyOrder" value - ObjectProperty propertyOrder = props.stream().filter(op -> PropertyOrderURI.equals(op.getPropertyURI())).findFirst().orElse(null); - if (propertyOrder != null) - { - String order = propertyOrder.getStringValue(); - if (order != null) - { - // CONSIDER: Store as a JSONArray of propertyURI instead of propertyId - String[] parts = order.split(","); - try - { - List propertyIds = Arrays.stream(parts).map(s -> ConvertHelper.convert(s, Integer.class)).toList(); - - // Don't include the "PropertyOrder" property - props = new ArrayList<>(props); - props.remove(propertyOrder); - - // Order by the index found in the PropertyOrder list, otherwise just stick it at the end - Comparator comparator = (op1, op2) -> { - int i1 = propertyIds.indexOf(op1.getPropertyId()); - if (i1 == -1) - i1 = propertyIds.size(); - - int i2 = propertyIds.indexOf(op2.getPropertyId()); - if (i2 == -1) - i2 = propertyIds.size(); - return i1 - i2; - }; - props.sort(comparator); - } - catch (ConversionException e) - { - _log.warn("Failed to parse PropertyOrder integer list: " + order); - } - } - } - - Map m = new LinkedHashMap<>(); - for (ObjectProperty value : props) - { - m.put(value.getPropertyURI(), value); - } - - return unmodifiableMap(m); - } - } - - public static void updateObjectPropertyOrder(User user, Container container, String objectLSID, List properties) - throws ValidationException - { - String ids = null; - if (properties != null && !properties.isEmpty()) - ids = properties.stream().map(pd -> Integer.toString(pd.getPropertyId())).collect(joining(",")); - - updateObjectProperty(user, container, PropertyOrder.getPropertyDescriptor(), objectLSID, ids, null, false); - } - - /** - * Moves the properties of an object from one container to another (used when the object is moving) - * @param targetContainer the container to move the properties to - * @param user the user doing the move - * @param objectLSID the LSID of the object to which the properties are attached - * @return number of properties moved - */ - public static int updateContainer(Container targetContainer, User user, @NotNull String objectLSID) - { - return Table.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); - } - - /** - * Get ordered list of the PropertyURI in {@link #PropertyOrder}, if present. - */ - public static List getObjectPropertyOrder(Container c, String objectLSID) - { - Map props = getPropertyObjects(c, objectLSID); - return new ArrayList<>(props.keySet()); - } - - public static long ensureObject(Container container, String objectURI) - { - return ensureObject(container, objectURI, (Long) null); - } - - public static long ensureObject(Container container, String objectURI, String ownerURI) - { - Long ownerId = null; - if (null != ownerURI) - ownerId = ensureObject(container, ownerURI, (Long) null); - return ensureObject(container, objectURI, ownerId); - } - - public static long ensureObject(Container container, String objectURI, Long ownerId) - { - //TODO: (marki) Transact? - Long objId = OBJECT_ID_CACHE.get(objectURI, container); - - if (null == objId) - { - OntologyObject obj = new OntologyObject(); - obj.setContainer(container); - obj.setObjectURI(objectURI); - if (ownerId != null && ownerId > 0) - obj.setOwnerObjectId(ownerId); - obj = Table.insert(null, getTinfoObject(), obj); - objId = obj.getObjectId(); - OBJECT_ID_CACHE.remove(objectURI); - } - - return objId; - } - - private static class ObjectIdCacheLoader implements CacheLoader - { - @Override - public Long load(@NotNull String objectURI, @Nullable Object argument) - { - Container container = (Container)argument; - OntologyObject obj = getOntologyObject(container, objectURI); - - return obj == null ? null : obj.getObjectId(); - } - } - - public static @Nullable OntologyObject getOntologyObject(Container container, String uri) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), uri); - if (container != null) - { - filter.addCondition(FieldKey.fromParts("Container"), container.getId()); - } - return new TableSelector(getTinfoObject(), filter, null).getObject(OntologyObject.class); - } - - // UNDONE: optimize (see deleteOntologyObjects(Integer[]) - public static void deleteOntologyObjects(Container c, String... uris) - { - if (uris.length == 0) - return; - - try - { - DbSchema schema = getExpSchema(); - String sql = getSqlDialect().execute(getExpSchema(), "deleteObject", "?, ?"); - SqlExecutor executor = new SqlExecutor(schema); - - for (String uri : uris) - { - executor.execute(sql, c.getId(), uri); - } - } - finally - { - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - } - } - - public static int deleteOntologyObjects(DbSchema schema, SQLFragment objectUriSql, @Nullable Container c) - { - SQLFragment objectIdSQL = new SQLFragment("SELECT ObjectId FROM ") - .append(getTinfoObject()).append("\n") - .append(" WHERE "); - if (c != null) - { - objectIdSQL.append(" Container = ?").add(c.getId()); - objectIdSQL.append(" AND "); - } - objectIdSQL.append("ObjectUri IN ("); - objectIdSQL.append(objectUriSql); - objectIdSQL.append(")"); - return deleteOntologyObjectsByObjectIdSql(schema, objectIdSQL); - } - - public static int deleteOntologyObjectsByObjectIdSql(DbSchema schema, SQLFragment objectIdSql) - { - if (!schema.getScope().equals(getExpSchema().getScope())) - throw new UnsupportedOperationException("can only use with same DbScope"); - - SQLFragment sqlDeleteProperties = new SQLFragment(); - sqlDeleteProperties.append("DELETE FROM ").append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (\n"); - sqlDeleteProperties.append(objectIdSql); - sqlDeleteProperties.append(")"); - new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); - - SQLFragment sqlDeleteObjects = new SQLFragment(); - sqlDeleteObjects.append("DELETE FROM ").append(getTinfoObject()).append(" WHERE ObjectId IN ("); - sqlDeleteObjects.append(objectIdSql); - sqlDeleteObjects.append(")"); - return new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); - } - - - public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, long... objectIds) - { - deleteOntologyObjects(c, deleteOwnedObjects, true, true, objectIds); - } - - public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, boolean deleteObjectProperties, boolean deleteObjects, long... objectIds) - { - if (objectIds.length == 0) - return; - - try - { - // if it's a long list, split it up - if (objectIds.length > 1000) - { - int countBatches = objectIds.length / 1000; - int lenBatch = 1 + objectIds.length / (countBatches + 1); - - for (int s = 0; s < objectIds.length; s += lenBatch) - { - long[] sub = new long[Math.min(lenBatch, objectIds.length - s)]; - System.arraycopy(objectIds, s, sub, 0, sub.length); - deleteOntologyObjects(c, deleteOwnedObjects, deleteObjectProperties, deleteObjects, sub); - } - - return; - } - - SQLFragment objectIdInClause = new SQLFragment(); - getExpSchema().getSqlDialect().appendInClauseSql(objectIdInClause, Arrays.stream(objectIds).boxed().toList()); - - if (deleteOwnedObjects) - { - // NOTE: owned objects should never be in a different container than the owner, that would be a problem - SQLFragment sqlDeleteOwnedProperties = new SQLFragment("DELETE FROM ") - .append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND OwnerObjectId ") - .add(c) - .append(objectIdInClause) - .append(")"); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedProperties); - - SQLFragment sqlDeleteOwnedObjects = new SQLFragment("DELETE FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND OwnerObjectId ") - .add(c) - .append(objectIdInClause); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedObjects); - } - - if (deleteObjectProperties) - { - deleteProperties(c, objectIdInClause); - } - - if (deleteObjects) - { - SQLFragment sqlDeleteObjects = new SQLFragment("DELETE FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND ObjectId ") - .add(c) - .append(objectIdInClause); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); - } - } - finally - { - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - } - } - - - public static void deleteOntologyObject(String objectURI, Container container, boolean deleteOwnedObjects) - { - OntologyObject ontologyObject = getOntologyObject(container, objectURI); - - if (null != ontologyObject) - { - deleteOntologyObjects(container, deleteOwnedObjects, true, true, ontologyObject.getObjectId()); - } - } - - - public static OntologyObject getOntologyObject(long id) - { - return new TableSelector(getTinfoObject()).getObject(id, OntologyObject.class); - } - - //todo: review this. this doesn't delete the underlying data objects. should it? - public static void deleteObjectsOfType(String domainURI, Container container) - { - DomainDescriptor dd = null; - if (null != domainURI) - dd = getDomainDescriptor(domainURI, container); - if (null == dd) - { - _log.debug("deleteObjectsOfType called on type not found in database: " + domainURI); - return; - } - - try (Transaction t = getExpSchema().getScope().ensureTransaction()) - { - // until we set a domain on objects themselves, we need to create a list of objects to - // delete based on existing entries in ObjectProperties before we delete the objectProperties - // which we need to do before we delete the objects. - // TODO: Doesn't handle the case when PropertyDescriptors are shared across domains - String selectObjectsToDelete = "SELECT DISTINCT O.ObjectId " + - " FROM " + getTinfoObject() + " O " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON(O.ObjectId = OP.ObjectId) " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (OP.PropertyId = PDM.PropertyId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PD.PropertyId = PDM.PropertyId) " + - " WHERE DD.DomainId = " + dd.getDomainId() + - " AND PD.Container = DD.Container"; - Long[] objIdsToDelete = new SqlSelector(getExpSchema(), selectObjectsToDelete).getArray(Long.class); - - String sep; - StringBuilder sqlIN = null; - Long[] ownerObjIds = null; - - if (objIdsToDelete.length > 0) - { - //also need list of owner objects whose subobjects are going to be deleted - // Seems cheaper but less correct to delete the subobjects then cleanup any owner objects with no children - sep = ""; - sqlIN = new StringBuilder(); - for (Long id : objIdsToDelete) - { - sqlIN.append(sep).append(id); - sep = ", "; - } - - String selectOwnerObjects = "SELECT O.ObjectId FROM " + getTinfoObject() + " O " + - " WHERE ObjectId IN " + - " (SELECT DISTINCT SUBO.OwnerObjectId FROM " + getTinfoObject() + " SUBO " + - " WHERE SUBO.ObjectId IN ( " + sqlIN + " ) )"; - - ownerObjIds = new SqlSelector(getExpSchema(), selectOwnerObjects).getArray(Long.class); - } - - String deleteTypePropsSql = "DELETE FROM " + getTinfoObjectProperty() + - " WHERE PropertyId IN " + - " (SELECT PDM.PropertyId FROM " + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PDM.PropertyId = PD.PropertyId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " WHERE DD.DomainId = " + dd.getDomainId() + - " AND PD.Container = DD.Container " + - " ) "; - new SqlExecutor(getExpSchema()).execute(deleteTypePropsSql); - - if (objIdsToDelete.length > 0) - { - // now cleanup the object table entries from the list we made, but make sure they don't have - // other properties attached to them - String deleteObjSql = "DELETE FROM " + getTinfoObject() + - " WHERE ObjectId IN ( " + sqlIN + " ) " + - " AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + - " WHERE OP.ObjectId = " + getTinfoObject() + ".ObjectId)"; - new SqlExecutor(getExpSchema()).execute(deleteObjSql); - - if (ownerObjIds.length > 0) - { - sep = ""; - sqlIN = new StringBuilder(); - for (Long id : ownerObjIds) - { - sqlIN.append(sep).append(id); - sep = ", "; - } - String deleteOwnerSql = "DELETE FROM " + getTinfoObject() + - " WHERE ObjectId IN ( " + sqlIN + " ) " + - " AND NOT EXISTS (SELECT * FROM " + getTinfoObject() + " SUBO " + - " WHERE SUBO.OwnerObjectId = " + getTinfoObject() + ".ObjectId)"; - new SqlExecutor(getExpSchema()).execute(deleteOwnerSql); - } - } - // whew! - clearCaches(); - t.commit(); - } - } - - public static void deleteDomain(String domainURI, Container container) throws DomainNotFoundException - { - DomainDescriptor dd = getDomainDescriptor(domainURI, container); - String msg; - - if (null == dd) - throw new DomainNotFoundException(domainURI); - - if (!dd.getContainer().getId().equals(container.getId())) - { - // this domain was not created in this folder. Allow if in the project-level root - if (!dd.getProject().getId().equals(container.getId())) - { - msg = "DeleteDomain: Domain can only be deleted in original container or from the project root " - + "\nDomain: " + domainURI + " project " + dd.getProject().getName() + " original container " + dd.getContainer().getPath(); - _log.error(msg); - throw new RuntimeException(msg); - } - } - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - String selectPDsToDelete = "SELECT DISTINCT PDM.PropertyId " + - " FROM " + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + - " WHERE DD.DomainId = ? "; - - Integer[] pdIdsToDelete = new SqlSelector(getExpSchema(), selectPDsToDelete, dd.getDomainId()).getArray(Integer.class); - - String deletePDMs = "DELETE FROM " + getTinfoPropertyDomain() + - " WHERE DomainId = " + - " (SELECT DD.DomainId FROM " + getTinfoDomainDescriptor() + " DD " + - " WHERE DD.DomainId = ? )"; - new SqlExecutor(getExpSchema()).execute(deletePDMs, dd.getDomainId()); - - if (pdIdsToDelete.length > 0) - { - String sep = ""; - StringBuilder sqlIN = new StringBuilder(); - for (Integer id : pdIdsToDelete) - { - PropertyService.get().deleteValidatorsAndFormats(container, id); - - sqlIN.append(sep); - sqlIN.append(id); - sep = ", "; - } - - String deletePDs = "DELETE FROM " + getTinfoPropertyDescriptor() + - " WHERE PropertyId IN ( " + sqlIN + " ) " + - "AND Container = ? " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + - "WHERE OP.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId) " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + - "WHERE PDM.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId)"; - - new SqlExecutor(getExpSchema()).execute(deletePDs, dd.getContainer().getId()); - } - - String deleteDD = "DELETE FROM " + getTinfoDomainDescriptor() + - " WHERE DomainId = ? " + - "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + - "WHERE PDM.DomainId = " + getTinfoDomainDescriptor() + ".DomainId)"; - - new SqlExecutor(getExpSchema()).execute(deleteDD, dd.getDomainId()); - clearCaches(); - - transaction.commit(); - } - } - - - public static void deleteAllObjects(Container c, User user) throws ValidationException - { - Container projectContainer = c.getProject(); - if (null == projectContainer) - projectContainer = c; - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - if (!c.equals(projectContainer)) - { - copyDescriptors(c, projectContainer); - } - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - - // Owned objects should be in same container, so this should work - String deleteObjPropSql = "DELETE FROM " + getTinfoObjectProperty() + " WHERE ObjectId IN (SELECT ObjectId FROM " + getTinfoObject() + " WHERE Container = ?)"; - executor.execute(deleteObjPropSql, c); - String deleteObjSql = "DELETE FROM " + getTinfoObject() + " WHERE Container = ?"; - executor.execute(deleteObjSql, c); - - // delete property validator references on property descriptors - PropertyService.get().deleteValidatorsAndFormats(c); - - // Drop tables directly and allow bulk delete calls below to clean up rows in exp.propertydescriptor, - // exp.domaindescriptor, etc - String selectSQL = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - Collection dds = new SqlSelector(getExpSchema(), selectSQL, c).getCollection(DomainDescriptor.class); - for (DomainDescriptor dd : dds) - { - StorageProvisioner.get().drop(PropertyService.get().getDomain(dd.getDomainId())); - } - - String deletePropDomSqlPD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId IN (SELECT PropertyId FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?)"; - executor.execute(deletePropDomSqlPD, c); - String deletePropDomSqlDD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE DomainId IN (SELECT DomainId FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?)"; - executor.execute(deletePropDomSqlDD, c); - String deleteDomSql = "DELETE FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; - executor.execute(deleteDomSql, c); - // now delete the prop descriptors that are referenced in this container only - String deletePropSql = "DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?"; - executor.execute(deletePropSql, c); - - clearCaches(); - transaction.commit(); - } - } - - private static void copyDescriptors(final Container c, final Container project) throws ValidationException - { - _log.debug("OntologyManager.copyDescriptors " + c.getName() + " " + project.getName()); - - // if c is (was) a project, then nothing to do - if (c.getId().equals(project.getId())) - return; - - // check to see if any Properties defined in this folder are used in other folders. - // if so we will make a copy of all PDs and DDs to ensure no orphans - String sql = " SELECT O.ObjectURI, O.Container, PD.PropertyId, PD.PropertyURI " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + - " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + - " WHERE PD.Container = ? " + - " AND O.Container <> PD.Container "; - - final Map mObjsUsingMyProps = new HashMap<>(); - final StringBuilder sqlIn = new StringBuilder(); - final StringBuilder sep = new StringBuilder(); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c).getResultSet()) - { - ResultSetUtil.logData(rs); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - new SqlSelector(getExpSchema(), sql, c).forEach(rs -> { - String objURI = rs.getString(1); - String objContainer = rs.getString(2); - Integer propId = rs.getInt(3); - String propURI = rs.getString(4); - - sqlIn.append(sep).append(propId); - - if (sep.isEmpty()) - sep.append(", "); - - Map mtemp = getPropertyObjects(ContainerManager.getForId(objContainer), objURI); - - if (null != mtemp) - { - for (Map.Entry entry : mtemp.entrySet()) - { - entry.getValue().setPropertyId(0); - if (entry.getValue().getPropertyURI().equals(propURI)) - mObjsUsingMyProps.put(entry.getKey(), entry.getValue()); - } - } - }); - - // For each property that is referenced outside its container, get the - // domains that it belongs to and the other properties in those domains - // so we can make copies of those domains and properties - // Restrict it to properties and domains also in the same container - - if (!mObjsUsingMyProps.isEmpty()) - { - sql = "SELECT PD.PropertyURI, DD.DomainURI " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + - " ON (PD.PropertyId = PDM2.PropertyId) " + - " WHERE PDM.PropertyId IN (" + sqlIn + ") " + - " OR PD.PropertyId IN (" + sqlIn + ") "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - catch (SQLException x) - { - throw new RuntimeException(x); - } - } - - new SqlSelector(getExpSchema(), sql).forEach(rsMyProps -> { - String propUri = rsMyProps.getString(1); - String domUri = rsMyProps.getString(2); - PropertyDescriptor pd = getPropertyDescriptor(propUri, c); - - if (pd.getContainer().getId().equals(c.getId())) - { - _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - DOMAIN_PROPERTIES_CACHE.clear(); - pd.setContainer(project); - pd.setPropertyId(0); - pd = ensurePropertyDescriptor(pd); - } - - if (null != domUri) - { - DomainDescriptor dd = getDomainDescriptor(domUri, c); - if (dd.getContainer().getId().equals(c.getId())) - { - uncache(dd); - dd = dd.edit() - .setContainer(project) - .setDomainId(0) - .build(); - dd = ensureDomainDescriptor(dd); - ensurePropertyDomain(pd, dd); - } - } - }); - - clearCaches(); - - // now unhook the objects that refer to my properties and rehook them to the properties in their own project - for (ObjectProperty op : mObjsUsingMyProps.values()) - { - deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), c); - insertProperties(op.getContainer(), op.getObjectURI(), op); - } - } - } - - private static void uncache(DomainDescriptor dd) - { - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.remove(dd.getContainer()); - } - - - public static void moveContainer(@NotNull final Container c, @NotNull Container oldParent, @NotNull Container newParent) throws SQLException - { - _log.debug("OntologyManager.moveContainer " + c.getName() + " " + oldParent.getName() + "->" + newParent.getName()); - - final Container oldProject = oldParent.getProject(); - Container newProject = newParent.getProject(); - if (null == newProject) // if container is promoted to a project - newProject = c.getProject(); - - if ((null != oldProject) && oldProject.getId().equals(newProject.getId())) - { - //the folder is being moved within the same project. No problems here - return; - } - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - clearCaches(); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - // update project of any descriptors in folder just moved - TableInfo pdTable = getTinfoPropertyDescriptor(); - String sql = "UPDATE " + pdTable + " SET Project = ? WHERE Container = ?"; - - // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. - // Issue 30477: exclude project level properties descriptors (such as Study) that already exist - sql += " AND PropertyUri NOT IN (SELECT PropertyUri FROM " + pdTable + " WHERE Project = ? AND PropertyUri IN (SELECT PropertyUri FROM " + pdTable + " WHERE Container = ?))"; - - new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - TableInfo ddTable = getTinfoDomainDescriptor(); - sql = "UPDATE " + ddTable + " SET Project = ? WHERE Container = ?"; - - // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. - // Issue 30477: exclude project level domain descriptors (such as Study) that already exist - sql += " AND DomainUri NOT IN (SELECT DomainUri FROM " + ddTable + " WHERE Project = ? AND DomainUri IN (SELECT DomainUri FROM " + ddTable + " WHERE Container = ?))"; - - new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); - - if (null == oldProject) // if container was a project & demoted I'm done - { - transaction.commit(); - return; - } - - // this method makes sure I'm not getting rid of descriptors used by another folder - // it is shared by ContainerDelete - copyDescriptors(c, oldProject); - - // if my objects refer to project-scoped properties I need a copy of those properties - sql = " SELECT O.ObjectURI, PD.PropertyURI, PD.PropertyId, PD.Container " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + - " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + - " WHERE O.Container = ? " + - " AND O.Container <> PD.Container " + - " AND PD.Project NOT IN (?,?) "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - - final Map mMyObjsThatRefProjProps = new HashMap<>(); - final StringBuilder sqlIn = new StringBuilder(); - final StringBuilder sep = new StringBuilder(); - - new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).forEach(rs -> { - String objURI = rs.getString(1); - String propURI = rs.getString(2); - Integer propId = rs.getInt(3); - - sqlIn.append(sep).append(propId); - - if (sep.isEmpty()) - sep.append(", "); - - Map mtemp = getPropertyObjects(c, objURI); - - if (null != mtemp) - { - for (Map.Entry entry : mtemp.entrySet()) - { - if (entry.getValue().getPropertyURI().equals(propURI)) - mMyObjsThatRefProjProps.put(entry.getKey(), entry.getValue()); - } - } - }); - - // this sql gets all properties i ref and the domains they belong to and the - // other properties in those domains - //todo what about materialsource ? - if (!mMyObjsThatRefProjProps.isEmpty()) - { - sql = "SELECT PD.PropertyURI, DD.DomainURI, PD.PropertyId " + - " FROM " + getTinfoPropertyDescriptor() + " PD " + - " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + - " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + - " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + - " ON (PD.PropertyId = PDM2.PropertyId) " + - " WHERE PDM.PropertyId IN (" + sqlIn + " ) "; - - if (_log.isDebugEnabled()) - { - try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) - { - ResultSetUtil.logData(rs, _log); - } - } - - final Container fNewProject = newProject; - - new SqlSelector(getExpSchema(), sql).forEach(rsPropsRefdByMe -> { - String propUri = rsPropsRefdByMe.getString(1); - String domUri = rsPropsRefdByMe.getString(2); - PropertyDescriptor pd = getPropertyDescriptor(propUri, oldProject); - - if (null != pd) - { - // To prevent iterating over a property descriptor update more than once - // we check to make sure both the container and project are equivalent to the updated - // location - if (!pd.getContainer().equals(c) || !pd.getProject().equals(fNewProject)) - { - pd.setContainer(c); - pd.setPropertyId(0); - } - - pd = ensurePropertyDescriptor(pd); - } - - if (null != domUri) - { - DomainDescriptor dd = getDomainDescriptor(domUri, oldProject); - - // To prevent iterating over a domain descriptor update more than once - // we check to make sure both the container and project are equivalent to the updated - // location - if (!dd.getContainer().equals(c) || !dd.getProject().equals(fNewProject)) - { - dd = dd.edit().setContainer(c).setDomainId(0).build(); - } - - dd = ensureDomainDescriptor(dd); - ensurePropertyDomain(pd, dd); - } - }); - - for (ObjectProperty op : mMyObjsThatRefProjProps.values()) - { - deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), oldProject); - // Treat it as new so it's created in the target container as needed - op.setPropertyId(0); - insertProperties(op.getContainer(), op.getObjectURI(), op); - } - clearCaches(); - } - - transaction.commit(); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - private static PropertyDescriptor ensurePropertyDescriptor(String propertyURI, PropertyType type, String name, Container container) - { - PropertyDescriptor pdNew = new PropertyDescriptor(propertyURI, type, name, container); - return ensurePropertyDescriptor(pdNew); - } - - - private static PropertyDescriptor ensurePropertyDescriptor(PropertyDescriptor pdIn) - { - if (null == pdIn.getContainer()) - { - assert false : "Container should be set on PropertyDescriptor"; - pdIn.setContainer(_sharedContainer); - } - - PropertyDescriptor pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); - if (null == pd) - { - assert pdIn.getPropertyId() == 0; - /* return 1 if inserted 0 if not inserted, uses OUT parameter for new PropertyDescriptor */ - PropertyDescriptor[] out = new PropertyDescriptor[1]; - int rowcount = insertPropertyIfNotExists(null, pdIn, out); - pd = out[0]; - if (1 == rowcount && null != pd) - { - _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - return pd; - } - if (null == pd) - { - throw OptimisticConflictException.create(Table.ERROR_DELETED); - } - } - - if (pd.equals(pdIn)) - { - return pd; - } - else - { - List colDiffs = comparePropertyDescriptors(pdIn, pd); - - if (colDiffs.isEmpty()) - { - // if the descriptor differs by container only and the requested descriptor is in the project fldr - if (!pdIn.getContainer().getId().equals(pd.getContainer().getId()) && - pdIn.getContainer().getId().equals(pdIn.getProject().getId())) - { - pdIn.setPropertyId(pd.getPropertyId()); - pd = updatePropertyDescriptor(pdIn); - } - return pd; - } - - // you are allowed to update if you are coming from the project root, or if you are in the container - // in which the descriptor was created - boolean fUpdateIfExists = false; - if (pdIn.getContainer().getId().equals(pd.getContainer().getId()) - || pdIn.getContainer().getId().equals(pdIn.getProject().getId())) - fUpdateIfExists = true; - - - boolean fMajorDifference = false; - if (colDiffs.toString().contains("RangeURI") || colDiffs.toString().contains("PropertyType")) - fMajorDifference = true; - - String errmsg = "ensurePropertyDescriptor: descriptor In different from Found for " + colDiffs + - "\n\t Descriptor In: " + pdIn + - "\n\t Descriptor Found: " + pd; - - if (fUpdateIfExists) - { - //todo: pass list of cols to update - pdIn.setPropertyId(pd.getPropertyId()); - pd = updatePropertyDescriptor(pdIn); - if (fMajorDifference) - _log.debug(errmsg); - } - else - { - if (fMajorDifference) - _log.error(errmsg); - else - _log.debug(errmsg); - } - } - return pd; - } - - - private static int insertPropertyIfNotExists(User user, PropertyDescriptor pd, PropertyDescriptor[] out) - { - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getInsertStmt(conn, user, t, true)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = f.toMap(pd, null); - stmt.putAll(m); - int rowcount = stmt.execute(); - SQLFragment reselect = new SQLFragment("SELECT * FROM exp.propertydescriptor WHERE propertyuri=? AND container=?", pd.getPropertyURI(), pd.getContainer()); - out[0] = (new SqlSelector(getExpSchema(), reselect).getObject(PropertyDescriptor.class)); - return rowcount; - } - catch(SQLException sqlx) - { - throw ExceptionFramework.Spring.translate(getExpSchema().getScope(), "insertPropertyIfNotExists", sqlx); - } - } - - - private static List comparePropertyDescriptors(PropertyDescriptor pdIn, PropertyDescriptor pd) - { - List colDiffs = new ArrayList<>(); - - // if the returned pd is in a different project, it better be the shared project - if (!pd.getProject().equals(pdIn.getProject()) && !pd.getProject().equals(_sharedContainer)) - colDiffs.add("Project"); - - // check the pd values that can't change - if (!pd.getRangeURI().equals(pdIn.getRangeURI())) - colDiffs.add("RangeURI"); - if (!Objects.equals(pd.getPropertyType(), pdIn.getPropertyType())) - colDiffs.add("PropertyType"); - - if (pdIn.getPropertyId() != 0 && pd.getPropertyId() != pdIn.getPropertyId()) - colDiffs.add("PropertyId"); - - if (!Objects.equals(pdIn.getName(), pd.getName())) - colDiffs.add("Name"); - - if (!Objects.equals(pdIn.getConceptURI(), pd.getConceptURI())) - colDiffs.add("ConceptURI"); - - if (!Objects.equals(pdIn.getDescription(), pd.getDescription())) - colDiffs.add("Description"); - - if (!Objects.equals(pdIn.getFormat(), pd.getFormat())) - colDiffs.add("Format"); - - if (!Objects.equals(pdIn.getLabel(), pd.getLabel())) - colDiffs.add("Label"); - - if (pdIn.isHidden() != pd.isHidden()) - colDiffs.add("IsHidden"); - - if (pdIn.isMvEnabled() != pd.isMvEnabled()) - colDiffs.add("IsMvEnabled"); - - if (!Objects.equals(pdIn.getLookupContainer(), pd.getLookupContainer())) - colDiffs.add("LookupContainer"); - - if (!Objects.equals(pdIn.getLookupSchema(), pd.getLookupSchema())) - colDiffs.add("LookupSchema"); - - if (!Objects.equals(pdIn.getLookupQuery(), pd.getLookupQuery())) - colDiffs.add("LookupQuery"); - - if (!Objects.equals(pdIn.getDerivationDataScope(), pd.getDerivationDataScope())) - colDiffs.add("DerivationDataScope"); - - if (!Objects.equals(pdIn.getSourceOntology(), pd.getSourceOntology())) - colDiffs.add("SourceOntology"); - - if (!Objects.equals(pdIn.getConceptImportColumn(), pd.getConceptImportColumn())) - colDiffs.add("ConceptImportColumn"); - - if (!Objects.equals(pdIn.getConceptLabelColumn(), pd.getConceptLabelColumn())) - colDiffs.add("ConceptLabelColumn"); - - if (!Objects.equals(pdIn.getPrincipalConceptCode(), pd.getPrincipalConceptCode())) - colDiffs.add("PrincipalConceptCode"); - - if (!Objects.equals(pdIn.getConceptSubtree(), pd.getConceptSubtree())) - colDiffs.add("ConceptSubtree"); - - if (pdIn.isScannable() != pd.isScannable()) - colDiffs.add("Scannable"); - - return colDiffs; - } - - public static DomainDescriptor ensureDomainDescriptor(String domainURI, String name, Container container) - { - String trimmedName = StringUtils.trimToNull(name); - if (trimmedName == null) - throw new IllegalArgumentException("Non-blank name is required."); - DomainDescriptor dd = new DomainDescriptor.Builder(domainURI, container).setName(trimmedName).build(); - return ensureDomainDescriptor(dd); - } - - /** Inserts or updates the domain as appropriate */ - @NotNull - public static DomainDescriptor ensureDomainDescriptor(DomainDescriptor ddIn) - { - DomainDescriptor dd = null; - // Try to find the previous version of the domain - if (ddIn.getDomainId() > 0) - { - // Try checking the cache first for a value to compare against - dd = getDomainDescriptor(ddIn.getDomainId()); - - // Since we cache mutable objects, get a fresh copy from the DB if the cache returned the same object that - // was passed in so we can do a diff against what's currently in the DB to see if we need to update - if (dd == ddIn) - { - dd = new TableSelector(getTinfoDomainDescriptor()).getObject(ddIn.getDomainId(), DomainDescriptor.class); - } - } - if (dd == null) - { - dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); - } - - if (null == dd) - { - try - { - DbSchema expSchema = getExpSchema(); - // ensureDomainDescriptor() shouldn't fail if there is a race condition, however Table.insert() will throw if row exists, so can't use that - // also a constraint violation will kill any current transaction - // CONSIDER to generalize add an option to check for existing row to Table.insert(ColumnInfo[] keyCols, Object[] keyValues) - String timestamp = expSchema.getSqlDialect().getSqlTypeName(JdbcType.TIMESTAMP); - String templateJson = null==ddIn.getTemplateInfo() ? null : ddIn.getTemplateInfo().toJSON(); - SQLFragment insert = new SQLFragment( - "INSERT INTO ").append(getTinfoDomainDescriptor()) - .append(" (Name, DomainURI, Description, Container, Project, StorageTableName, StorageSchemaName, ModifiedBy, Modified, TemplateInfo, SystemFieldConfig)\n" + - "SELECT ?,?,?,?,?,?,?,CAST(NULL AS INT),CAST(NULL AS " + timestamp + "),?,?\n") - .addAll(ddIn.getName(), ddIn.getDomainURI(), ddIn.getDescription(), ddIn.getContainer(), ddIn.getProject(), ddIn.getStorageTableName(), ddIn.getStorageSchemaName(), templateJson, ddIn.getSystemFieldConfig()) - .append("WHERE NOT EXISTS (SELECT * FROM ").append(getTinfoDomainDescriptor(),"x").append(" WHERE x.DomainURI=? AND x.Project=?)\n") - .add(ddIn.getDomainURI()).add(ddIn.getProject()); - // belt and suspenders approach to avoiding constraint violation exception - if (expSchema.getSqlDialect().isPostgreSQL()) - insert.append(" ON CONFLICT ON CONSTRAINT uq_domaindescriptor DO NOTHING"); - int count; - try (var tx = expSchema.getScope().ensureTransaction()) - { - count = new SqlExecutor(expSchema.getScope()).execute(insert); - tx.commit(); - } - - // alternately we could reselect rowid and then we wouldn't need this separate round trip - dd = fetchDomainDescriptorFromDB(ddIn.getDomainURI(), ddIn.getContainer()); - if (count > 0) - { - if (null == dd) // don't expect this - throw OptimisticConflictException.create(Table.ERROR_DELETED); - // We may have a cached miss that we need to clear - uncache(dd); - return dd; - } - // fall through to update case() - } - catch (RuntimeSQLException x) - { - // might be an optimistic concurrency problem see 16126 - dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); - if (null == dd) - throw x; - } - } - - if (!dd.deepEquals(ddIn)) - { - DomainDescriptor ddToSave = ddIn.edit().setDomainId(dd.getDomainId()).build(); - dd = Table.update(null, getTinfoDomainDescriptor(), ddToSave, ddToSave.getDomainId()); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(ddIn)); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); - DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(ddIn)); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); - } - return dd; - } - - private static void ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd) - { - ensurePropertyDomain(pd, dd, 0); - } - - public static PropertyDescriptor ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) - { - if (null == pd) - throw new IllegalArgumentException("Must supply a PropertyDescriptor"); - if (null == dd) - throw new IllegalArgumentException("Must supply a DomainDescriptor"); - - // Consider: We should check that the pd and dd have been persisted (aka have a non-zero id) - - if (!pd.getContainer().equals(dd.getContainer()) - && !pd.getProject().equals(_sharedContainer)) - throw new IllegalStateException("ensurePropertyDomain: property " + pd.getPropertyURI() + " not in same container as domain " + dd.getDomainURI()); - - SQLFragment sqlInsert = new SQLFragment("INSERT INTO " + getTinfoPropertyDomain() + " ( PropertyId, DomainId, Required, SortOrder ) " + - " SELECT ?, ?, ?, ? WHERE NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + - " WHERE PropertyId=? AND DomainId=?)"); - sqlInsert.add(pd.getPropertyId()); - sqlInsert.add(dd.getDomainId()); - sqlInsert.add(pd.isRequired()); - sqlInsert.add(sortOrder); - sqlInsert.add(pd.getPropertyId()); - sqlInsert.add(dd.getDomainId()); - int count = new SqlExecutor(getExpSchema()).execute(sqlInsert); - // if 0 rows affected, we should do an update to make sure required is correct - if (count == 0) - { - SQLFragment sqlUpdate = new SQLFragment("UPDATE " + getTinfoPropertyDomain() + " SET Required = ?, SortOrder = ? WHERE PropertyId=? AND DomainId= ?"); - sqlUpdate.add(pd.isRequired()); - sqlUpdate.add(sortOrder); - sqlUpdate.add(pd.getPropertyId()); - sqlUpdate.add(dd.getDomainId()); - new SqlExecutor(getExpSchema()).execute(sqlUpdate); - } - DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); - return pd; - } - - - private static void insertPropertiesBulk(Container container, List props, boolean insertNullValues) throws SQLException - { - List> floats = new ArrayList<>(); - List> dates = new ArrayList<>(); - List> strings = new ArrayList<>(); - List> mvIndicators = new ArrayList<>(); - - for (PropertyRow property : props) - { - if (null == property) - continue; - - long objectId = property.getObjectId(); - int propertyId = property.getPropertyId(); - String mvIndicator = property.getMvIndicator(); - assert mvIndicator == null || MvUtil.isMvIndicator(mvIndicator, container) : "Attempt to insert an invalid missing value indicator: " + mvIndicator; - - if (null != property.getFloatValue()) - floats.add(Arrays.asList(objectId, propertyId, property.getFloatValue(), mvIndicator)); - else if (null != property.getDateTimeValue()) - dates.add(Arrays.asList(objectId, propertyId, new java.sql.Timestamp(property.getDateTimeValue().getTime()), mvIndicator)); - else if (null != property.getStringValue()) - strings.add(Arrays.asList(objectId, propertyId, property.getStringValue(), mvIndicator)); - else if (null != mvIndicator) - { - mvIndicators.add(Arrays.asList(objectId, propertyId, property.getTypeTag(), mvIndicator)); - } - else if (insertNullValues) - { - strings.add(Arrays.asList(objectId, propertyId, null, null)); - } - } - - assert getExpSchema().getScope().isTransactionActive(); - - if (!dates.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, DateTimeValue, MvIndicator) VALUES (?,?,'d',?, ?)"; - Table.batchExecute(getExpSchema(), sql, dates); - } - - if (!floats.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, FloatValue, MvIndicator) VALUES (?,?,'f',?, ?)"; - Table.batchExecute(getExpSchema(), sql, floats); - } - - if (!strings.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, StringValue, MvIndicator) VALUES (?,?,'s',?, ?)"; - Table.batchExecute(getExpSchema(), sql, strings); - } - - if (!mvIndicators.isEmpty()) - { - String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, MvIndicator) VALUES (?,?,?,?)"; - Table.batchExecute(getExpSchema(), sql, mvIndicators); - } - - clearPropertyCache(); - } - - public static void deleteProperty(String objectURI, String propertyURI, Container objContainer, Container propContainer) - { - OntologyObject o = getOntologyObject(objContainer, objectURI); - if (o == null) - return; - - PropertyDescriptor pd = getPropertyDescriptor(propertyURI, propContainer); - if (pd == null) - return; - - deleteProperty(o, pd); - } - - public static void deleteProperty(OntologyObject o, PropertyDescriptor pd) - { - deleteProperty(o, pd, true); - } - - public static void deleteProperty(OntologyObject o, PropertyDescriptor pd, boolean deleteCache) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectId"), o.getObjectId()); - filter.addCondition(FieldKey.fromParts("PropertyId"), pd.getPropertyId()); - Table.delete(getTinfoObjectProperty(), filter); - - if (deleteCache) - clearPropertyCache(o.getObjectURI()); - } - - /** - * Delete properties owned by the objects. - */ - public static void deleteProperties(Container objContainer, long objectId) - { - deleteProperties(objContainer, new SQLFragment(" = ?", objectId)); - } - public static void deleteProperties(Container objContainer, SQLFragment objectIdClause) - { - SQLFragment objectUriSql = new SQLFragment("SELECT ObjectURI FROM ") - .append(getTinfoObject(), "o") - .append(" WHERE ObjectId "); - objectUriSql.append(objectIdClause); - - List objectURIs = new SqlSelector(getExpSchema(), objectUriSql).getArrayList(String.class); - - SQLFragment sqlDeleteProperties = new SQLFragment("DELETE FROM ") - .append(getTinfoObjectProperty()) - .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") - .append(getTinfoObject()) - .append(" WHERE Container = ? AND ObjectId ") - .add(objContainer) - .append(objectIdClause) - .append(")"); - - new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); - - for (String uri : objectURIs) - { - clearPropertyCache(uri); - } - } - - /** - * Removes the property from a single domain, and completely deletes it if there are no other references - */ - public static void removePropertyDescriptorFromDomain(DomainProperty domainProp) - { - SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ? AND DomainId = ?", domainProp.getPropertyId(), domainProp.getDomain().getTypeId()); - SqlExecutor executor = new SqlExecutor(getExpSchema()); - DbScope dbScope = getExpSchema().getScope(); - try (Transaction transaction = dbScope.ensureTransaction()) - { - executor.execute(deletePropDomSql); - // Check if there are any other usages - SQLFragment otherUsagesSQL = new SQLFragment("SELECT DomainId FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", domainProp.getPropertyId()); - if (!new SqlSelector(dbScope, otherUsagesSQL).exists()) - { - deletePropertyDescriptor(domainProp.getPropertyDescriptor()); - } - transaction.commit(); - } - } - - /** - * Completely deletes the property from the database - */ - public static void deletePropertyDescriptor(PropertyDescriptor pd) - { - int propId = pd.getPropertyId(); - - SQLFragment deleteObjPropSql = new SQLFragment("DELETE FROM " + getTinfoObjectProperty() + " WHERE PropertyId = ?", propId); - SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", propId); - SQLFragment deletePropSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyId = ?", propId); - - DbScope dbScope = getExpSchema().getScope(); - SqlExecutor executor = new SqlExecutor(getExpSchema()); - try (Transaction transaction = dbScope.ensureTransaction()) - { - executor.execute(deleteObjPropSql); - executor.execute(deletePropDomSql); - executor.execute(deletePropSql); - Pair key = getCacheKey(pd); - _log.debug("Removing property descriptor from cache. Key: " + key + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(key); - DOMAIN_PROPERTIES_CACHE.clear(); - transaction.commit(); - } - } - - /*** - * @deprecated Use {@link #insertProperties(Container, User, String, ObjectProperty...)} so that a user can be - * supplied. - */ - @Deprecated - public static void insertProperties(Container container, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException - { - User user = HttpView.hasCurrentView() ? HttpView.currentContext().getUser() : null; - insertProperties(container, user, ownerObjectLsid, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException - { - insertProperties(container, user, ownerObjectLsid, false, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, ObjectProperty... properties) throws ValidationException - { - insertProperties(container, user, ownerObjectLsid, skipValidation, false, properties); - } - - public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, boolean insertNullValues, ObjectProperty... properties) throws ValidationException - { - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - Long parentId = ownerObjectLsid == null ? null : ensureObject(container, ownerObjectLsid); - HashMap descriptors = new HashMap<>(); - HashMap objects = new HashMap<>(); - List errors = new ArrayList<>(); - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - for (ObjectProperty property : properties) - { - if (null == property) - continue; - - property.setObjectOwnerId(parentId); - - PropertyDescriptor pd = descriptors.get(property.getPropertyURI()); - if (0 == property.getPropertyId()) - { - if (null == pd) - { - PropertyDescriptor pdIn = new PropertyDescriptor(property.getPropertyURI(), property.getPropertyType(), property.getName(), container); - pdIn.setFormat(property.getFormat()); - pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); - - if (null == pd) - pd = ensurePropertyDescriptor(pdIn); - - descriptors.put(property.getPropertyURI(), pd); - } - property.setPropertyId(pd.getPropertyId()); - } - if (0 == property.getObjectId()) - { - Long objectId = objects.get(property.getObjectURI()); - if (null == objectId) - { - // I'm assuming all properties are in the same container - objectId = ensureObject(property.getContainer(), property.getObjectURI(), property.getObjectOwnerId()); - objects.put(property.getObjectURI(), objectId); - } - property.setObjectId(objectId); - } - if (pd == null) - { - pd = getPropertyDescriptor(property.getPropertyId()); - } - if (!skipValidation) - { - validateProperty(PropertyService.get().getPropertyValidators(pd), pd, property, errors, validatorCache); - } - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - - insertPropertiesBulk(container, List.of(properties), insertNullValues); - - transaction.commit(); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - - public static PropertyDescriptor getPropertyDescriptor(long propertyId) - { - return new TableSelector(getTinfoPropertyDescriptor()).getObject(propertyId, PropertyDescriptor.class); - } - - - public static PropertyDescriptor getPropertyDescriptor(String propertyURI, Container c) - { - // cache lookup by project. if not found at project level, check to see if global - Pair key = getCacheKey(propertyURI, c); - PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(key); - if (null != pd) - return pd; - - key = getCacheKey(propertyURI, _sharedContainer); - return PROP_DESCRIPTOR_CACHE.get(key); - } - - private static TableSelector getPropertyDescriptorTableSelector( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter, - @Nullable String sortColumn) - { - final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); - - // To filter by domain kind, we query the exp.DomainProperty table and filter by domainId. - // To construct a PropertyDescriptor, we will need to traverse the lookup to exp.PropertyDescriptor and select all of its columns. - List fields = new ArrayList<>(); - fields.add(FieldKey.fromParts("domainId")); - for (ColumnInfo col : getTinfoPropertyDescriptor().getColumns()) - { - fields.add(new FieldKey(propertyIdKey, col.getName())); - } - var colMap = QueryService.get().getColumns(getTinfoPropertyDomain(), fields); - - var filter = new SimpleFilter(); - if (propertyFilter != null) - { - filter.addAllClauses(propertyFilter); - } - - filter.addCondition(new FieldKey(propertyIdKey, "container"), c.getId()); - - if (!domains.isEmpty()) - { - filter.addInClause(FieldKey.fromParts("domainId"), domains.stream().map(Domain::getTypeId).collect(Collectors.toSet())); - } - - if (searchTerm != null) - { - // Apply Q filter to only some of the text columns - List searchCols = List.of( - colMap.get(new FieldKey(propertyIdKey, "Name")), - colMap.get(new FieldKey(propertyIdKey, "Label")), - colMap.get(new FieldKey(propertyIdKey, "Description")), - colMap.get(new FieldKey(propertyIdKey, "ImportAliases")) - ); - - var clause = CompareType.Q.createFilterClause(new FieldKey(null, "*"), searchTerm); - clause.setSelectColumns(searchCols); - filter.addCondition(clause); - } - - // use propertyId as the default sort - if (sortColumn == null) - sortColumn = "propertyId"; - Sort sort = new Sort(sortColumn); - - return new TableSelector(getTinfoPropertyDomain(), colMap.values(), filter, sort); - } - - public static Set getDomains( - Container c, User user, - @Nullable Set domainIds, - @Nullable Set domainKinds, - @Nullable Set domainNames) - { - Set domains = new HashSet<>(); - if (domainIds != null && !domainIds.isEmpty()) - { - domains.addAll(domainIds.stream().map(id -> PropertyService.get().getDomain(id)).collect(Collectors.toSet())); - } - - Set kinds = emptySet(); - Set names = emptySet(); - if (domainKinds != null && !domainKinds.isEmpty()) - { - kinds = domainKinds; - } - if (domainNames != null && !domainNames.isEmpty()) - { - names = domainNames; - } - if (!kinds.isEmpty() || !names.isEmpty()) - { - domains.addAll(PropertyService.get().getDomains(c, user, kinds, names, true)); - } - - return domains; - } - - public static List getPropertyDescriptors( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter, - @Nullable String sortColumn, - @Nullable Integer maxRows, - @Nullable Long offset) - { - final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); - - TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, - propertyFilter, sortColumn); - - if (maxRows != null) - ts.setMaxRows(maxRows); - if (offset != null) - ts.setOffset(offset); - - // This is a little annoying. We have to remove the "propertyId" lookup parent from - // the map keys for the ObjectFactory to correctly construct the PropertyDescriptor. - List props = new ArrayList<>(); - try (var results = ts.getResults(true)) - { - ObjectFactory of = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - while (results.next()) - { - Map rowMap = results.getFieldKeyRowMap(); - // remove the "propertyId" part from the FieldKey - Map rekey = new CaseInsensitiveHashMap<>(); - for (Map.Entry pair : rowMap.entrySet()) - { - FieldKey key = pair.getKey(); - if (propertyIdKey.equals(key.getParent())) - { - String name = key.getName(); - rekey.put(name, pair.getValue()); - } - } - props.add(of.fromMap(rekey)); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - return props; - } - - public static long getPropertyDescriptorsRowCount( - Container c, User user, - Set domains, - @Nullable String searchTerm, - @Nullable SimpleFilter propertyFilter) - { - - TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, - propertyFilter, null); - - return ts.getRowCount(); - } - - public static List getDomainsForPropertyDescriptor(Container container, PropertyDescriptor pd) - { - return PropertyService.get().getDomains(container) - .stream() - .filter(d -> null != d.getPropertyByURI(pd.getPropertyURI())) - .collect(Collectors.toList()); - } - - private static class DomainDescriptorLoader implements CacheLoader - { - @Override - public DomainDescriptor load(@NotNull Integer key, @Nullable Object argument) - { - return new TableSelector(getTinfoDomainDescriptor()).getObject(key, DomainDescriptor.class); - } - } - - public static DomainDescriptor getDomainDescriptor(int id) - { - return getDomainDescriptor(id, false); - } - - public static DomainDescriptor getDomainDescriptor(int id, boolean forUpdate) - { - if (forUpdate) - return new DomainDescriptorLoader().load(id, null); - - return DOMAIN_DESC_BY_ID_CACHE.get(id); - } - - @Nullable - public static DomainDescriptor getDomainDescriptor(String domainURI, Container c) - { - return getDomainDescriptor(domainURI, c, false); - } - - @Nullable - public static DomainDescriptor getDomainDescriptor(String domainURI, Container c, boolean forUpdate) - { - if (c == null) - return null; - - if (forUpdate) - return getDomainDescriptorForUpdate(domainURI, c); - - // cache lookup by project. if not found at project level, check to see if global - Pair key = getCacheKey(domainURI, c); - DomainDescriptor dd = DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); - if (null != dd) - return dd; - - // Try in the /Shared container too - key = getCacheKey(domainURI, _sharedContainer); - return DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); - } - - @Nullable - private static DomainDescriptor getDomainDescriptorForUpdate(String domainURI, Container c) - { - if (c == null) - return null; - - DomainDescriptor dd = fetchDomainDescriptorFromDB(domainURI, c); - if (dd == null) - dd = fetchDomainDescriptorFromDB(domainURI, _sharedContainer); - return dd; - } - - /** - * Get all the domains in the same project as the specified container. They may not be in use in the container directly - */ - public static Collection getDomainDescriptors(Container container) - { - return getDomainDescriptors(container, null, false); - } - - public static Collection getDomainDescriptors(Container container, User user, boolean includeProjectAndShared) - { - if (container == null) - return Collections.emptyList(); - - if (includeProjectAndShared && user == null) - throw new IllegalArgumentException("Can't include data from other containers without a user to check permissions on"); - - Map dds = getCachedDomainDescriptors(container, user); - - if (includeProjectAndShared) - { - dds = new LinkedHashMap<>(dds); - Container project = container.getProject(); - if (project != null) - { - for (Map.Entry entry : getCachedDomainDescriptors(project, user).entrySet()) - { - dds.putIfAbsent(entry.getKey(), entry.getValue()); - } - } - - if (_sharedContainer.hasPermission(user, ReadPermission.class)) - { - for (Map.Entry entry : getCachedDomainDescriptors(_sharedContainer, user).entrySet()) - { - dds.putIfAbsent(entry.getKey(), entry.getValue()); - } - } - } - - return unmodifiableCollection(dds.values()); - } - - @NotNull - private static Map getCachedDomainDescriptors(@NotNull Container c, @Nullable User user) - { - if (user != null && !c.hasPermission(user, ReadPermission.class)) - return Collections.emptyMap(); - - return DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.get(c); - } - - public static Pair getURICacheKey(DomainDescriptor dd) - { - return getCacheKey(dd.getDomainURI(), dd.getContainer()); - } - - - public static Pair getCacheKey(PropertyDescriptor pd) - { - return getCacheKey(pd.getPropertyURI(), pd.getContainer()); - } - - - public static Pair getCacheKey(String uri, Container c) - { - Container proj = c.getProject(); - GUID projId; - - if (null == proj) - projId = c.getEntityId(); - else - projId = proj.getEntityId(); - - return Pair.of(uri, projId); - } - - //TODO: Cache semantics. This loads the cache but does not fetch cause need to get them all together - public static List getPropertiesForType(String typeURI, Container c) - { - List> propertyURIs = DOMAIN_PROPERTIES_CACHE.get(getCacheKey(typeURI, c)); - if (propertyURIs != null) - { - List result = new ArrayList<>(propertyURIs.size()); - for (Pair propertyURI : propertyURIs) - { - PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(getCacheKey(propertyURI.getKey(), c)); - if (pd == null) - { - return null; - } - // NOTE: cached descriptors may have differing values of isRequired() as that is a per-domain setting - // Descriptors returned from this method will have their required bit set as appropriate for this domain - - // Clone so nobody else messes up our copy - pd = pd.clone(); - pd.setRequired(propertyURI.getValue().booleanValue()); - result.add(pd); - } - return unmodifiableList(result); - } - return null; - } - - public static void deleteType(String domainURI, Container c) throws DomainNotFoundException - { - if (null == domainURI) - return; - - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - try - { - deleteObjectsOfType(domainURI, c); - deleteDomain(domainURI, c); - } - catch (DomainNotFoundException x) - { - // throw exception but do not kill enclosing transaction - transaction.commit(); - throw x; - } - - transaction.commit(); - } - } - - public static PropertyDescriptor insertOrUpdatePropertyDescriptor(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) - throws ChangePropertyDescriptorException - { - validatePropertyDescriptor(pd); - try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) - { - DomainDescriptor dexist = ensureDomainDescriptor(dd); - - if (!dexist.getContainer().equals(pd.getContainer()) - && !pd.getProject().equals(_sharedContainer)) - { - // domain is defined in a different container. - //ToDO define property in the domains container? what security? - throw new ChangePropertyDescriptorException("Attempt to define property for a domain definition that exists in a different folder\n" + - "domain folder = " + dexist.getContainer().getPath() + "\n" + - "property folder = " + pd.getContainer().getPath()); - } - - PropertyDescriptor pexist = ensurePropertyDescriptor(pd); - pexist.setDatabaseDefaultValue(pd.getDatabaseDefaultValue()); - pexist.setNullable(pd.isMvEnabled() || pd.isNullable()); - pexist.setRequired(pd.isRequired()); - - ensurePropertyDomain(pexist, dexist, sortOrder); - - transaction.commit(); - return pexist; - } - } - - - static final String parameters = "propertyuri,name,description,rangeuri,concepturi,label," + - "format,container,project,lookupcontainer,lookupschema,lookupquery,defaultvaluetype,hidden," + - "mvenabled,importaliases,url,shownininsertview,showninupdateview,shownindetailsview,measure,dimension,scale," + - "sourceontology,conceptimportcolumn,conceptlabelcolumn,principalconceptcode,conceptsubtree," + - "recommendedvariable,derivationdatascope,storagecolumnname,facetingbehaviortype,phi,redactedText," + - "excludefromshifting,mvindicatorstoragecolumnname,defaultscale,scannable"; - static final String[] parametersArray = parameters.split(","); - - static ParameterMapStatement getInsertStmt(Connection conn, User user, TableInfo t, boolean ifNotExists) throws SQLException - { - user = null==user ? User.guest : user; - SQLFragment sql = new SQLFragment("INSERT INTO exp.propertydescriptor\n\t\t("); - SQLFragment values = new SQLFragment("\nSELECT\t"); - ColumnInfo c; - String comma = ""; - Parameter container = null; - Parameter propertyuri = null; - for (var p : parametersArray) - { - if (null == (c = t.getColumn(p))) - continue; - sql.append(comma).append(p); - values.append(comma).append("?"); - comma = ","; - Parameter parameter = new Parameter(p, c.getJdbcType()); - values.add(parameter); - if ("container".equals(p)) - container = parameter; - else if ("propertyuri".equals(p)) - propertyuri = parameter; - } - sql.append(", createdby, created, modifiedby, modified)\n"); - values.append(", " + user.getUserId() + ", {fn now()}, " + user.getUserId() + ", {fn now()}"); - sql.append(values); - if (ifNotExists) - { - sql.append("\nWHERE NOT EXISTS (SELECT propertyid FROM exp.propertydescriptor WHERE propertyuri=? AND container=?)\n"); - sql.add(propertyuri).add(container); - } - return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); - } - - static ParameterMapStatement getUpdateStmt(Connection conn, User user, TableInfo t) throws SQLException - { - user = null==user ? User.guest : user; - SQLFragment sql = new SQLFragment("UPDATE exp.propertydescriptor SET "); - ColumnInfo c; - String comma = ""; - for (var p : parametersArray) - { - if (null == (c = t.getColumn(p))) - continue; - sql.append(comma).append(p).append("=?"); - comma = ", "; - sql.add(new Parameter(p, c.getJdbcType())); - } - sql.append(", modifiedby=" + user.getUserId() + ", modified={fn now()}"); - sql.append("\nWHERE propertyid=?"); - sql.add(new Parameter("propertyid", JdbcType.INTEGER)); - return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); - } - - - public static void insertPropertyDescriptors(User user, List pds) throws SQLException - { - if (null == pds || pds.isEmpty()) - return; - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getInsertStmt(conn, user, t, false)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = null; - for (PropertyDescriptor pd : pds) - { - m = f.toMap(pd, m); - stmt.clearParameters(); - stmt.putAll(m); - stmt.addBatch(); - } - stmt.executeBatch(); - } - } - - - public static void updatePropertyDescriptors(User user, List pds) throws SQLException - { - if (null == pds || pds.isEmpty()) - return; - TableInfo t = getTinfoPropertyDescriptor(); - try (Connection conn = t.getSchema().getScope().getConnection(); - ParameterMapStatement stmt = getUpdateStmt(conn, user, t)) - { - ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); - Map m = null; - for (PropertyDescriptor pd : pds) - { - m = f.toMap(pd, m); - stmt.clearParameters(); - stmt.putAll(m); - stmt.addBatch(); - } - stmt.executeBatch(); - } - } - - - public static PropertyDescriptor insertPropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException - { - assert pd.getPropertyId() == 0; - validatePropertyDescriptor(pd); - pd = Table.insert(null, getTinfoPropertyDescriptor(), pd); - _log.debug("Adding property descriptor to cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - return pd; - } - - - //todo: we automatically update a pd to the last one in? - public static PropertyDescriptor updatePropertyDescriptor(PropertyDescriptor pd) - { - assert pd.getPropertyId() != 0; - pd = Table.update(null, getTinfoPropertyDescriptor(), pd, pd.getPropertyId()); - _log.debug("Updating property descriptor in cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); - PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); - // It's possible that the propertyURI has changed, thus breaking our reference - DOMAIN_PROPERTIES_CACHE.clear(); - return pd; - } - - /** - * Insert or update an object property value. - * - * @param user The user inserting the property - currently only used for validating lookup values. - * @param container Insert the property value into this container. - * @param pd The property descriptor. - * @param lsid The object on which to attach the properties. - * @param value The value to insert. - * @param ownerObjectLsid The "owner" object or "parent" object, which isn't necessarily same as the object. For example, samples use the ExpSampleType as the owner object. - * @param insertNullValues When true, a null value will be inserted if the value is null, otherwise any existing property value will be deleted if the value is null. - * @return The inserted ObjectProperty or null - */ - public static ObjectProperty updateObjectProperty(User user, Container container, PropertyDescriptor pd, String lsid, Object value, @Nullable String ownerObjectLsid, boolean insertNullValues) throws ValidationException - { - ObjectProperty oprop; - RemapCache cache = new RemapCache(); - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), container, pd.getContainer()); - - try - { - oprop = new ObjectProperty(lsid, container, pd, value); - } - catch (ConversionException x) - { - // Issue 43529: Assay run property with large lookup doesn't resolve text input by value - // Attempt to resolve lookups by display value and then try creating the ObjectProperty again - if (pd.getLookup() != null) - { - Object remappedValue = getRemappedValueForLookup(user, container, cache, pd.getLookup(), value); - if (remappedValue != null) - value = remappedValue; - } - oprop = new ObjectProperty(lsid, container, pd, value); - } - - if (value != null || insertNullValues) - { - oprop.setPropertyId(pd.getPropertyId()); - OntologyManager.insertProperties(container, user, ownerObjectLsid, false, insertNullValues, oprop); - } - else - { - // We still need to validate blanks - List errors = new ArrayList<>(); - OntologyManager.validateProperty(PropertyService.get().getPropertyValidators(pd), pd, oprop, errors, new ValidatorContext(pd.getContainer(), user)); - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - transaction.commit(); - } - return oprop; - } - - public static Object getRemappedValueForLookup(User user, Container container, RemapCache cache, Lookup lookup, Object value) - { - Container lkContainer = lookup.getContainer() != null ? lookup.getContainer() : container; - return cache.remap(SchemaKey.fromParts(lookup.getSchemaKey()), lookup.getQueryName(), user, lkContainer, ContainerFilter.Type.CurrentPlusProjectAndShared, String.valueOf(value)); - } - - public static List findPropertyUsages(User user, List propertyIds, int maxUsageCount) - { - List ret = new ArrayList<>(propertyIds.size()); - for (int propertyId : propertyIds) - { - var pd = getPropertyDescriptor(propertyId); - if (pd == null) - throw new IllegalArgumentException("property not found: " + propertyId); - - ret.add(findPropertyUsages(user, pd, maxUsageCount)); - } - - return ret; - } - - public static List findPropertyUsages(User user, Container c, List propertyURIs, int maxUsageCount) - { - List ret = new ArrayList<>(propertyURIs.size()); - for (String propertyURI : propertyURIs) - { - var pd = getPropertyDescriptor(propertyURI, c); - if (pd == null) - throw new IllegalArgumentException("property not found: " + propertyURI); - - ret.add(findPropertyUsages(user, pd, maxUsageCount)); - } - - return ret; - } - - public static PropertyUsages findPropertyUsages(@NotNull User user, @NotNull PropertyDescriptor pd, int maxUsageCount) - { - // query exp.ObjectProperty for usages of the property - FieldKey objectId = FieldKey.fromParts("objectId"); - FieldKey objectId_objectURI = FieldKey.fromParts("objectId", "objectURI"); - FieldKey objectId_container = FieldKey.fromParts("objectId", "container"); - List fields = List.of(objectId, objectId_objectURI, objectId_container); - var colMap = QueryService.get().getColumns(getTinfoObjectProperty(), fields); - - int usageCount; - List objects = new ArrayList<>(maxUsageCount); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("propertyId"), pd.getPropertyId(), CompareType.EQUAL); - filter.addCondition(objectId_objectURI, DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX, CompareType.DOES_NOT_CONTAIN); - - TableSelector ts = new TableSelector(getTinfoObjectProperty(), colMap.values(), filter, new Sort("objectId")); - try (var r = ts.getResults(true)) - { - usageCount = r.getSize(); - - for (int i = 0; i < maxUsageCount && r.next(); i++) - { - var row = r.getFieldKeyRowMap(); - long oid = asLong(row.get(objectId)); - String objectURI = (String) row.get(objectId_objectURI); - String container = (String) row.get(objectId_container); - - Identifiable object = LsidManager.get().getObject(objectURI); - if (object != null) - { - Container c = object.getContainer(); - if (c != null && c.hasPermission(user, ReadPermission.class)) - objects.add(object); - } - else - { - Container c = ContainerManager.getForId(container); - if (c != null && c.hasPermission(user, ReadPermission.class)) - { - OntologyObject oo = new OntologyObject(); - oo.setContainer(c); - oo.setObjectId(oid); - oo.setObjectURI(objectURI); - objects.add(new IdentifiableBase(oo)); - } - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return new PropertyUsages(pd.getPropertyId(), pd.getPropertyURI(), usageCount, objects); - } - - public static class PropertyUsages - { - public final int propertyId; - public final String propertyURI; - public final int usageCount; - public final List objects; - - public PropertyUsages(int propertyId, String propertyURI, int usageCount, List objects) - { - this.propertyId = propertyId; - this.propertyURI = propertyURI; - this.usageCount = usageCount; - this.objects = objects; - } - } - - - public static void invalidateDomain(Domain d) - { - // TODO can we please implement a surgical version of this - clearCaches(); - } - - - public static void clearCaches() - { - _log.debug("Clearing caches"); - ExperimentService.get().clearCaches(); - DOMAIN_DESCRIPTORS_BY_URI_CACHE.clear(); - DOMAIN_DESC_BY_ID_CACHE.clear(); - DOMAIN_PROPERTIES_CACHE.clear(); - PROP_DESCRIPTOR_CACHE.clear(); - PROPERTY_MAP_CACHE.clear(); - OBJECT_ID_CACHE.clear(); - DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); - } - - public static void clearPropertyCache(String parentObjectURI) - { - PROPERTY_MAP_CACHE.removeUsingFilter(key -> Objects.equals(key.second, parentObjectURI)); - } - - - public static void clearPropertyCache() - { - PROPERTY_MAP_CACHE.clear(); - } - - public static class ImportPropertyDescriptor - { - public final String domainName; - public final String domainURI; - public final PropertyDescriptor pd; - public final List validators; - public final List formats; - public final String defaultValue; - - private ImportPropertyDescriptor(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) - { - this.domainName = domainName; - this.domainURI = domainURI; - this.pd = pd; - this.validators = null != validators ? validators : Collections.emptyList(); - this.formats = null != formats ? formats : Collections.emptyList(); - this.defaultValue = defaultValue; - } - } - - - public static class ImportPropertyDescriptorsList - { - public final ArrayList properties = new ArrayList<>(); - - void add(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) - { - properties.add(new ImportPropertyDescriptor(domainName, domainURI, pd, validators, formats, defaultValue)); - } - } - - /** - * Updates an existing domain property with an import property descriptor generated - * by _propertyDescriptorFromRowMap below. Properties we don't set are explicitly - * called out - */ - public static void updateDomainPropertyFromDescriptor(DomainProperty p, PropertyDescriptor pd) - { - // don't setName - p.setPropertyURI(pd.getPropertyURI()); - p.setLabel(pd.getLabel()); - p.setConceptURI(pd.getConceptURI()); - p.setRangeURI(pd.getRangeURI()); - // don't setContainer - p.setDescription(pd.getDescription()); - p.setURL((pd.getURL() != null) ? pd.getURL().toString() : null); - p.setImportAliasSet(ColumnRenderPropertiesImpl.convertToSet(pd.getImportAliases())); - p.setRequired(pd.isRequired()); - p.setHidden(pd.isHidden()); - p.setShownInInsertView(pd.isShownInInsertView()); - p.setShownInUpdateView(pd.isShownInUpdateView()); - p.setShownInDetailsView(pd.isShownInDetailsView()); - p.setShownInLookupView(pd.isShownInLookupView()); - p.setDimension(pd.isDimension()); - p.setMeasure(pd.isMeasure()); - p.setRecommendedVariable(pd.isRecommendedVariable()); - p.setDefaultScale(pd.getDefaultScale()); - p.setScale(pd.getScale()); - p.setFormat(pd.getFormat()); - p.setMvEnabled(pd.isMvEnabled()); - - Lookup lookup = new Lookup(); - lookup.setQueryName(pd.getLookupQuery()); - lookup.setSchemaName(pd.getLookupSchema()); - String lookupContainerId = pd.getLookupContainer(); - if (lookupContainerId != null) - { - Container container = ContainerManager.getForId(lookupContainerId); - if (container == null) - lookup = null; - else - lookup.setContainer(container); - } - p.setLookup(lookup); - p.setFacetingBehavior(pd.getFacetingBehaviorType()); - p.setPhi(pd.getPHI()); - p.setRedactedText(pd.getRedactedText()); - p.setExcludeFromShifting(pd.isExcludeFromShifting()); - p.setDefaultValueTypeEnum(pd.getDefaultValueTypeEnum()); - p.setScannable(pd.isScannable()); - p.setDerivationDataScope(pd.getDerivationDataScope()); - } - - @TestWhen(TestWhen.When.BVT) - @TestTimeout(120) - public static class TestCase extends Assert - { - @Test - public void testSchema() - { - assertNotNull(getExpSchema()); - assertNotNull(getTinfoPropertyDescriptor()); - assertNotNull(ExperimentService.get().getTinfoSampleType()); - - assertEquals(11, getTinfoPropertyDescriptor().getColumns("PropertyId,PropertyURI,RangeURI,Name,Description,DerivationDataScope,SourceOntology,ConceptImportColumn,ConceptLabelColumn,PrincipalConceptCode,scannable").size()); - assertEquals(4, getTinfoObject().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId").size()); - assertEquals(11, getTinfoObjectPropertiesView().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId,Name,PropertyURI,RangeURI,TypeTag,StringValue,DateTimeValue,FloatValue").size()); - assertEquals(10, ExperimentService.get().getTinfoSampleType().getColumns("RowId,Name,LSID,MaterialLSIDPrefix,Description,Created,CreatedBy,Modified,ModifiedBy,Container").size()); - } - - @Test - public void testBasicPropertiesObject() throws ValidationException - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - User user = TestContext.get().getUser(); - String parentObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - //First delete in case test case failed before - deleteOntologyObjects(c, parentObjectLsid); - assertNull(getOntologyObject(c, parentObjectLsid)); - assertNull(getOntologyObject(c, childObjectLsid)); - ensureObject(c, childObjectLsid, parentObjectLsid); - OntologyObject oParent = getOntologyObject(c, parentObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - assertNull(oParent.getOwnerObjectId()); - assertEquals(oChild.getContainer(), c); - assertEquals(oParent.getContainer(), c); - - String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - PropertyDescriptor strPd = getPropertyDescriptor(strProp, c); - assertEquals(PropertyType.STRING, strPd.getPropertyType()); - - String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - PropertyDescriptor intPd = getPropertyDescriptor(intProp, c); - assertEquals(PropertyType.INTEGER, intPd.getPropertyType()); - - String longProp = new Lsid("Junit", "OntologyManager", "longProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, longProp, 6L)); - PropertyDescriptor longPd = getPropertyDescriptor(longProp, c); - assertEquals(PropertyType.BIGINT, longPd.getPropertyType()); - - Calendar cal = Calendar.getInstance(); - cal.set(Calendar.MILLISECOND, 0); - String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); - insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); - PropertyDescriptor datePd = getPropertyDescriptor(dateProp, c); - assertEquals(PropertyType.DATE_TIME, datePd.getPropertyType()); - - Map m = getProperties(c, oChild.getObjectURI()); - assertNotNull(m); - assertEquals(4, m.size()); - assertEquals("The String", m.get(strProp)); - assertEquals(5, m.get(intProp)); - assertEquals(6L, m.get(longProp)); - assertEquals(cal.getTime(), m.get(dateProp)); - - // Set property order: date, str, int. Long property will sort to last since it isn't explicitly included. - List propertyOrder = List.of(datePd, strPd, intPd); - updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); - - Map oProps = getPropertyObjects(c, childObjectLsid); - var iter = oProps.entrySet().iterator(); - assertEquals(cal.getTime(), iter.next().getValue().value()); - assertEquals("The String", iter.next().getValue().value()); - assertEquals(5, iter.next().getValue().value()); - assertEquals(6L, iter.next().getValue().value()); - assertFalse(iter.hasNext()); - - // Update property order: int, date, long, str - propertyOrder = List.of(intPd, datePd, longPd, strPd); - updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); - oProps = getPropertyObjects(c, childObjectLsid); - iter = oProps.entrySet().iterator(); - assertEquals(5, iter.next().getValue().value()); - assertEquals(cal.getTime(), iter.next().getValue().value()); - assertEquals(6L, iter.next().getValue().value()); - assertEquals("The String", iter.next().getValue().value()); - assertFalse(iter.hasNext()); - - deleteOntologyObjects(c, parentObjectLsid); - assertNull(getOntologyObject(c, parentObjectLsid)); - assertNull(getOntologyObject(c, childObjectLsid)); - - m = getProperties(c, oChild.getObjectURI()); - assertEquals(0, m.size()); - } - - @Test - public void testContainerDelete() throws ValidationException - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - - Calendar cal = Calendar.getInstance(); - cal.set(Calendar.MILLISECOND, 0); - String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); - - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - - private void defineCrossFolderProperties(Container fldr1a, Container fldr1b) throws SQLException - { - try - { - String fa = fldr1a.getPath(); - String fb = fldr1b.getPath(); - - //object, prop descriptor in folder being moved - String objP1Fa = new Lsid("OntologyObject", "JUnit", fa.replace('/', '.')).toString(); - ensureObject(fldr1a, objP1Fa); - String propP1Fa = fa + "PD1"; - PropertyDescriptor pd1Fa = ensurePropertyDescriptor(propP1Fa, PropertyType.STRING, "PropertyDescriptor 1" + fa, fldr1a); - insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP1Fa, "same fldr")); - - //object in folder not moving, prop desc in folder moving - String objP2Fb = new Lsid("OntologyObject", "JUnit", fb.replace('/', '.')).toString(); - ensureObject(fldr1b, objP2Fb); - insertProperties(fldr1b, null, new ObjectProperty(objP2Fb, fldr1b, propP1Fa, "object in folder not moving, prop desc in folder moving")); - - //object in folder moving, prop desc in folder not moving - String propP2Fb = fb + "PD1"; - ensurePropertyDescriptor(propP2Fb, PropertyType.STRING, "PropertyDescriptor 1" + fb, fldr1b); - insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP2Fb, "object in folder moving, prop desc in folder not moving")); - - // third prop desc in folder that is moving; shares domain with first prop desc - String propP1Fa3 = fa + "PD3"; - PropertyDescriptor pd1Fa3 = ensurePropertyDescriptor(propP1Fa3, PropertyType.STRING, "PropertyDescriptor 3" + fa, fldr1a); - String domP1Fa = fa + "DD1"; - DomainDescriptor dd1 = ensureDomainDescriptor(domP1Fa, "DomDesc 1" + fa, fldr1a); - ensurePropertyDomain(pd1Fa, dd1); - ensurePropertyDomain(pd1Fa3, dd1); - - //second domain desc in folder that is moving - // second prop desc in folder moving, belongs to 2nd domain - String propP1Fa2 = fa + "PD2"; - PropertyDescriptor pd1Fa2 = ensurePropertyDescriptor(propP1Fa2, PropertyType.STRING, "PropertyDescriptor 2" + fa, fldr1a); - String domP1Fa2 = fa + "DD2"; - DomainDescriptor dd2 = ensureDomainDescriptor(domP1Fa2, "DomDesc 2" + fa, fldr1a); - ensurePropertyDomain(pd1Fa2, dd2); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - @Test - public void testContainerMove() throws Exception - { - deleteMoveTestContainers(); - - Container proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); - Container proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - - proj1 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); - proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - - proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); - proj2 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); - doMoveTest(proj1, proj2); - deleteMoveTestContainers(); - } - - private void doMoveTest(Container proj1, Container proj2) throws Exception - { - String p1Path = proj1.getPath() + "/"; - String p2Path = proj2.getPath() + "/"; - if (p1Path.equals("//")) p1Path = "/_ontMgrDemotePromote"; - if (p2Path.equals("//")) p2Path = "/_ontMgrDemotePromote"; - - Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); - Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); - ContainerManager.ensureContainer(p2Path + "Fc", TestContext.get().getUser()); - Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); - Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); - - defineCrossFolderProperties(fldr1a, fldr1b); - //defineCrossFolderProperties(fldr1a, fldr2c); - defineCrossFolderProperties(fldr1aa, fldr1b); - defineCrossFolderProperties(fldr1aaa, fldr1b); - - fldr1a.getProject().getPath(); - String f = fldr1a.getPath(); - String propId = f + "PD1"; - assertNull(getPropertyDescriptor(propId, proj2)); - ContainerManager.move(fldr1a, proj2, TestContext.get().getUser()); - - // if demoting a folder - if (proj1.isRoot()) - { - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD2"; - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj2)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj2)); - - domId = f + "DD2"; - assertNotNull(getDomainDescriptor(domId, proj2)); - } - // if promoting a folder, - else if (proj2.isRoot()) - { - assertNotNull(getPropertyDescriptor(propId, proj1)); - - propId = f + "PD2"; - assertNull(getPropertyDescriptor(propId, proj1)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj1)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj1)); - - domId = f + "DD2"; - assertNull(getDomainDescriptor(domId, proj1)); - } - else - { - assertNotNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD2"; - assertNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - propId = f + "PD3"; - assertNotNull(getPropertyDescriptor(propId, proj1)); - assertNotNull(getPropertyDescriptor(propId, proj2)); - - String domId = f + "DD1"; - assertNotNull(getDomainDescriptor(domId, proj1)); - assertNotNull(getDomainDescriptor(domId, proj2)); - - domId = f + "DD2"; - assertNull(getDomainDescriptor(domId, proj1)); - assertNotNull(getDomainDescriptor(domId, proj2)); - } - } - - @Test - public void testDeleteFoldersWithSharedProps() throws SQLException - { - deleteMoveTestContainers(); - - String projectName = "_ontMgrTestP1"; - Container proj1 = ContainerManager.ensureContainer(projectName, TestContext.get().getUser()); - String p1Path = proj1.getPath() + "/"; - - Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); - Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); - Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); - Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); - - defineCrossFolderProperties(fldr1a, fldr1b); - defineCrossFolderProperties(fldr1aa, fldr1b); - defineCrossFolderProperties(fldr1aaa, fldr1b); - - deleteProjects( projectName); - } - - private void deleteMoveTestContainers() - { - // Remove all projects. Subfolders will be deleted when project is removed. - deleteProjects( - "/_ontMgrTestP1", - "/_ontMgrTestP2", - "/_ontMgrDemotePromoteFa", - "/_ontMgrDemotePromoteFb", - "/_ontMgrDemotePromoteFc", - "/Fa" - ); - } - - private void deleteProjects(String... projectNames) - { - for (String path : projectNames) - { - Container c = ContainerManager.getForPath(path); - - if (null != c) - ContainerManager.deleteAll(c, TestContext.get().getUser()); - } - - for (String path : projectNames) - assertNull("Container " + path + " was not deleted", ContainerManager.getForPath(path)); - } - - @Test - public void testTransactions() throws SQLException - { - try - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - - //Create objects in a transaction & make sure they are all gone. - OntologyObject oParent; - OntologyObject oChild; - String strProp; - String intProp; - - try (Transaction ignored = getExpSchema().getScope().beginTransaction()) - { - ensureObject(c, childObjectLsid, ownerObjectLsid); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - } - - assertEquals(0L, getObjectCount(c)); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNull(oParent); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); - - //Rollback transaction for one new property - try (Transaction ignored = getExpSchema().getScope().beginTransaction()) - { - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - } - - oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - Map m = getProperties(c, childObjectLsid); - assertNotNull(m.get(strProp)); - assertNull(m.get(intProp)); - - try (Transaction transaction = getExpSchema().getScope().beginTransaction()) - { - intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); - insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); - transaction.commit(); - } - - m = getProperties(c, childObjectLsid); - assertNotNull(m.get(strProp)); - assertNotNull(m.get(intProp)); - - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - catch (ValidationException ve) - { - throw new SQLException(ve.getMessage()); - } - } - - @Test - public void testDomains() throws Exception - { - Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); - //Clean up last time's mess - deleteAllObjects(c, TestContext.get().getUser()); - assertEquals(0L, getObjectCount(c)); - String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); - String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); - String child2ObjectLsid = new Lsid("Junit", "OntologyManager", "child2").toString(); - - ensureObject(c, childObjectLsid, ownerObjectLsid); - OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); - assertNotNull(oParent); - OntologyObject oChild = getOntologyObject(c, childObjectLsid); - assertNotNull(oChild); - - String domURIa = new Lsid("Junit", "DD", "Domain1").toString(); - String strPropURI = new Lsid("Junit", "PD", "Domain1.stringProp").toString(); - String intPropURI = new Lsid("Junit", "PD", "Domain1.intProp").toString(); - String longPropURI = new Lsid("Junit", "PD", "Domain1.longProp").toString(); - - DomainDescriptor dd = ensureDomainDescriptor(domURIa, "Domain1", c); - assertNotNull(dd); - - PropertyDescriptor pdStr = new PropertyDescriptor(); - pdStr.setPropertyURI(strPropURI); - pdStr.setRangeURI(PropertyType.STRING.getTypeUri()); - pdStr.setContainer(c); - pdStr.setName("Domain1.stringProp"); - - pdStr = ensurePropertyDescriptor(pdStr); - assertNotNull(pdStr); - - PropertyDescriptor pdInt = ensurePropertyDescriptor(intPropURI, PropertyType.INTEGER, "Domain1.intProp", c); - PropertyDescriptor pdLong = ensurePropertyDescriptor(longPropURI, PropertyType.BIGINT, "Domain1.longProp", c); - - ensurePropertyDomain(pdStr, dd); - ensurePropertyDomain(pdInt, dd); - ensurePropertyDomain(pdLong, dd); - - List pds = getPropertiesForType(domURIa, c); - assertEquals(3, pds.size()); - Map mPds = new HashMap<>(); - for (PropertyDescriptor pd1 : pds) - mPds.put(pd1.getPropertyURI(), pd1); - - assertTrue(mPds.containsKey(strPropURI)); - assertTrue(mPds.containsKey(intPropURI)); - assertTrue(mPds.containsKey(longPropURI)); - - ObjectProperty strProp = new ObjectProperty(childObjectLsid, c, strPropURI, "String value"); - ObjectProperty intProp = new ObjectProperty(childObjectLsid, c, intPropURI, 42); - ObjectProperty longProp = new ObjectProperty(childObjectLsid, c, longPropURI, 52L); - insertProperties(c, ownerObjectLsid, strProp); - insertProperties(c, ownerObjectLsid, intProp); - insertProperties(c, ownerObjectLsid, longProp); - - Map m = getProperties(c, oChild.getObjectURI()); - assertNotNull(m); - assertEquals(3, m.size()); - assertEquals("String value", m.get(strPropURI)); - assertEquals(42, m.get(intPropURI)); - assertEquals(52L, m.get(longPropURI)); - - // test insertTabDelimited - List> rows = List.of( - new CaseInsensitiveMapWrapper<>(Map.of( - "lsid", child2ObjectLsid, - strPropURI, "Second value", - intPropURI, 62, - longPropURI, 72L - ) - )); - ImportHelper helper = new ImportHelper() - { - @Override - public String beforeImportObject(Map map) - { - return (String)map.get("lsid"); - } - - @Override - public void afterBatchInsert(int currentRow) - { } - - @Override - public void updateStatistics(int currentRow) - { } - }; - try (Transaction tx = getExpSchema().getScope().ensureTransaction()) - { - insertTabDelimited(c, TestContext.get().getUser(), oParent.getObjectId(), helper, pds, MapDataIterator.of(rows).getDataIterator(new DataIteratorContext()), false, null); - tx.commit(); - } - - m = getProperties(c, child2ObjectLsid); - assertNotNull(m); - assertEquals(3, m.size()); - assertEquals("Second value", m.get(strPropURI)); - assertEquals(62, m.get(intPropURI)); - assertEquals(72L, m.get(longPropURI)); - - deleteType(domURIa, c); - assertEquals(0L, getObjectCount(c)); - assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); - } - } - - private static long getObjectCount(Container c) - { - return new TableSelector(getTinfoObject(), SimpleFilter.createContainerFilter(c), null).getRowCount(); - } - - /** - * v.first value IN/OUT parameter - * v.second mvIndicator OUT parameter - */ - public static void convertValuePair(PropertyDescriptor pd, PropertyType pt, Pair v) - { - if (v.first == null) - return; - - // Handle field-level QC - if (v.first instanceof MvFieldWrapper mvWrapper) - { - v.second = mvWrapper.getMvIndicator(); - v.first = mvWrapper.getValue(); - } - else if (pd.isMvEnabled()) - { - // Not all callers will have wrapped an MV value if there isn't also - // a real value - if (MvUtil.isMvIndicator(v.first.toString(), pd.getContainer())) - { - v.second = v.first.toString(); - v.first = null; - } - } - - if (null != v.first && null != pt) - v.first = pt.convert(v.first); - } - - @Deprecated // Fold into ObjectProperty? Eliminate insertTabDelimited() methods, the only usage of PropertyRow. - public static class PropertyRow - { - protected long objectId; - protected int propertyId; - protected char typeTag; - protected Double floatValue; - protected String stringValue; - protected Date dateTimeValue; - protected String mvIndicator; - - public PropertyRow() - { - } - - public PropertyRow(long objectId, PropertyDescriptor pd, Object value, PropertyType pt) - { - this.objectId = objectId; - this.propertyId = pd.getPropertyId(); - this.typeTag = pt.getStorageType(); - - Pair p = new Pair<>(value, null); - convertValuePair(pd, pt, p); - mvIndicator = p.second; - - pt.init(this, p.first); - } - - public long getObjectId() - { - return objectId; - } - - public void setObjectId(long objectId) - { - this.objectId = objectId; - } - - public int getPropertyId() - { - return propertyId; - } - - public void setPropertyId(int propertyId) - { - this.propertyId = propertyId; - } - - public char getTypeTag() - { - return typeTag; - } - - public void setTypeTag(char typeTag) - { - this.typeTag = typeTag; - } - - public Double getFloatValue() - { - return floatValue; - } - - public Boolean getBooleanValue() - { - if (floatValue == null) - { - return null; - } - return floatValue.doubleValue() == 1.0; - } - - public void setFloatValue(Double floatValue) - { - this.floatValue = floatValue; - } - - public String getStringValue() - { - return stringValue; - } - - public void setStringValue(String stringValue) - { - this.stringValue = stringValue; - } - - public Date getDateTimeValue() - { - return dateTimeValue; - } - - public void setDateTimeValue(Date dateTimeValue) - { - this.dateTimeValue = dateTimeValue; - } - - public String getMvIndicator() - { - return mvIndicator; - } - - public void setMvIndicator(String mvIndicator) - { - this.mvIndicator = mvIndicator; - } - - public Object getObjectValue() - { - return stringValue != null ? stringValue : floatValue != null ? floatValue : dateTimeValue; - } - - @Override - public String toString() - { - StringBuilder sb = new StringBuilder(); - sb.append("PropertyRow: "); - - sb.append("objectId=").append(objectId); - sb.append(", propertyId=").append(propertyId); - sb.append(", value="); - - if (stringValue != null) - sb.append(stringValue); - else if (floatValue != null) - sb.append(floatValue); - else if (dateTimeValue != null) - sb.append(dateTimeValue); - else - sb.append("null"); - - if (mvIndicator != null) - sb.append(", mvIndicator=").append(mvIndicator); - - return sb.toString(); - } - } - - public static DbSchema getExpSchema() - { - return DbSchema.get("exp", DbSchemaType.Module); - } - - public static SqlDialect getSqlDialect() - { - return getExpSchema().getSqlDialect(); - } - - public static TableInfo getTinfoPropertyDomain() - { - return getExpSchema().getTable("PropertyDomain"); - } - - public static TableInfo getTinfoObject() - { - return getExpSchema().getTable("Object"); - } - - public static TableInfo getTinfoObjectProperty() - { - return getExpSchema().getTable("ObjectProperty"); - } - - public static TableInfo getTinfoPropertyDescriptor() - { - return getExpSchema().getTable("PropertyDescriptor"); - } - - public static TableInfo getTinfoDomainDescriptor() - { - return getExpSchema().getTable("DomainDescriptor"); - } - - public static TableInfo getTinfoObjectPropertiesView() - { - return getExpSchema().getTable("ObjectPropertiesView"); - } - - public static HtmlString doProjectColumnCheck(boolean bFix) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(); - String descriptorTable = getTinfoPropertyDescriptor().toString(); - String uriColumn = "PropertyURI"; - String idColumn = "PropertyID"; - doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); - - descriptorTable = getTinfoDomainDescriptor().toString(); - uriColumn = "DomainURI"; - idColumn = "DomainID"; - doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); - - return builder.getHtmlString(); - } - - private static void doProjectColumnCheck(final String descriptorTable, final String uriColumn, final String idColumn, final HtmlStringBuilder msgBuilder, final boolean bFix) - { - // get all unique combos of Container, project - - String sql = "SELECT Container, Project FROM " + descriptorTable + " GROUP BY Container, Project"; - - new SqlSelector(getExpSchema(), sql).forEach(rs -> { - String containerId = rs.getString("Container"); - String projectId = rs.getString("Project"); - Container container = ContainerManager.getForId(containerId); - if (null == container) - return; // should be handled by container check - String newProjectId = container.getProject() == null ? container.getId() : container.getProject().getId(); - if (!projectId.equals(newProjectId)) - { - if (bFix) - { - fixProjectColumn(descriptorTable, uriColumn, idColumn, container, projectId, newProjectId); - msgBuilder - .unsafeAppend("
       ") - .append("Fixed inconsistent project ids found for ") - .append(descriptorTable).append(" in folder ") - .append(ContainerManager.getForId(containerId).getPath()); - - } - else - msgBuilder - .unsafeAppend("
       ") - .append("ERROR: Inconsistent project ids found for ") - .append(descriptorTable).append(" in folder ").append(container.getPath()); - } - }); - } - - private static void fixProjectColumn(String descriptorTable, String uriColumn, String idColumn, Container container, String projectId, String newProjId) - { - final SqlExecutor executor = new SqlExecutor(getExpSchema()); - - String sql = "UPDATE " + descriptorTable + " SET Project= ? WHERE Project = ? AND Container=? AND " + uriColumn + " NOT IN " + - "(SELECT " + uriColumn + " FROM " + descriptorTable + " WHERE Project = ?)"; - executor.execute(sql, newProjId, projectId, container.getId(), newProjId); - - // now check to see if there is already an existing descriptor in the target (correct) project. - // this can happen if a folder containing a descriptor is moved to another project - // and the OntologyManager's containerMoved handler fails to fire for some reason. (note not in transaction) - // If this is the case, the descriptor is redundant and it should be deleted, after we move the objects that depend on it. - - sql = " SELECT prev." + idColumn + " AS PrevIdCol, cur." + idColumn + " AS CurIdCol FROM " + descriptorTable + " prev " - + " INNER JOIN " + descriptorTable + " cur ON (prev." + uriColumn + "= cur." + uriColumn + " ) " - + " WHERE cur.Project = ? AND prev.Project= ? AND prev.Container = ? "; - final String updsql1 = " UPDATE " + getTinfoObjectProperty() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; - final String updsql2 = " UPDATE " + getTinfoPropertyDomain() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; - final String delSql = " DELETE FROM " + descriptorTable + " WHERE " + idColumn + " = ? "; - - new SqlSelector(getExpSchema(), sql, newProjId, projectId, container).forEach(rs -> { - int prevPropId = rs.getInt(1); - int curPropId = rs.getInt(2); - executor.execute(updsql1, curPropId, prevPropId); - executor.execute(updsql2, curPropId, prevPropId); - executor.execute(delSql, prevPropId); - }); - } - - public static void validatePropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException - { - String name = pd.getName(); - validateValue(name, "Name", null); - validateValue(pd.getPropertyURI(), "PropertyURI", "Please use a shorter field name. Name = " + name); - validateValue(pd.getLabel(), "Label", null); - validateValue(pd.getImportAliases(), "ImportAliases", null); - validateValue(pd.getURL() != null ? pd.getURL().getSource() : null, "URL", null); - validateValue(pd.getConceptURI(), "ConceptURI", null); - validateValue(pd.getRangeURI(), "RangeURI", null); - - // Issue 15484: adding a column ending in 'mvIndicator' is problematic if another column w/ the same - // root exists, or if you later enable mvIndicators on a column w/ the same root - if (pd.getName() != null && pd.getName().toLowerCase().endsWith(MV_INDICATOR_SUFFIX)) - { - throw new ChangePropertyDescriptorException("Field name cannot end with the suffix 'mvIndicator': " + pd.getName()); - } - - if (null != name) - { - for (char ch : name.toCharArray()) - { - if (Character.isWhitespace(ch) && ' ' != ch) - throw new ChangePropertyDescriptorException("Field name cannot contain whitespace other than ' ' (space)"); - } - } - } - - private static void validateValue(String value, String columnName, String extraMessage) throws ChangePropertyDescriptorException - { - int maxLength = getTinfoPropertyDescriptor().getColumn(columnName).getScale(); - if (value != null && value.length() > maxLength) - { - throw new ChangePropertyDescriptorException(columnName + " cannot exceed " + maxLength + " characters, but was " + value.length() + " characters long. " + (extraMessage == null ? "" : extraMessage)); - } - } - - static public boolean checkObjectExistence(String lsid) - { - return new TableSelector(getTinfoObject(), new SimpleFilter(FieldKey.fromParts("ObjectURI"), lsid), null).exists(); - } -} +/* + * Copyright (c) 2005-2018 Fred Hutchinson Cancer Research Center + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.exp; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.cache.BlockingCache; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.data.*; +import org.labkey.api.data.DbScope.Transaction; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.ui.domain.CancellationException; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.test.TestTimeout; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.ResultSetUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.view.HttpView; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableCollection; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.stream.Collectors.joining; +import static org.labkey.api.util.IntegerUtils.asLong; + +/** + * Lots of static methods for dealing with domains and property descriptors. Tends to operate primarily on the bean-style + * classes like {@link PropertyDescriptor} and {@link DomainDescriptor}. When possible, it's usually preferable to use + * {@link PropertyService}, {@link Domain}, and {@link DomainProperty} instead as they tend to provide higher-level + * abstractions. + */ +public class OntologyManager +{ + private static final Logger _log = LogManager.getLogger(OntologyManager.class); + private static final Cache, Map> PROPERTY_MAP_CACHE = DatabaseCache.get(getExpSchema().getScope(), 100000, "Property maps", new PropertyMapCacheLoader()); + private static final BlockingCache OBJECT_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "ObjectIds", new ObjectIdCacheLoader()); + private static final Cache, PropertyDescriptor> PROP_DESCRIPTOR_CACHE = DatabaseCache.get(getExpSchema().getScope(), 40000, "Property descriptors", new CacheLoader<>() + { + @Override + public PropertyDescriptor load(@NotNull Pair key, @Nullable Object argument) + { + PropertyDescriptor ret = null; + String propertyURI = key.first; + Container c = ContainerManager.getForId(key.second); + if (null != c) + { + Container proj = c.getProject(); + if (null == proj) + proj = c; + _log.debug("Loading a property descriptor for key " + key + " using project " + proj); + String sql = " SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyURI = ? AND Project IN (?,?)"; + List pdArray = new SqlSelector(getExpSchema(), sql, propertyURI, proj, _sharedContainer.getId()).getArrayList(PropertyDescriptor.class); + if (!pdArray.isEmpty()) + { + PropertyDescriptor pd = pdArray.get(0); + + // if someone has explicitly inserted a descriptor with the same URI as an existing one, + // and one of the two is in the shared project, use the project-level descriptor. + if (pdArray.size() > 1) + { + _log.debug("Multiple PropertyDescriptors found for " + propertyURI); + if (pd.getProject().equals(_sharedContainer)) + pd = pdArray.get(1); + } + _log.debug("Loaded property descriptor " + pd); + ret = pd; + } + } + return ret; + } + }); + + /** DomainURI, ContainerEntityId -> DomainDescriptor */ + private static final Cache, DomainDescriptor> DOMAIN_DESCRIPTORS_BY_URI_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, CacheManager.UNLIMITED, "Domain descriptors by URI", (key, argument) -> { + String domainURI = key.first; + Container c = ContainerManager.getForId(key.second); + + if (c == null) + { + return null; + } + + return fetchDomainDescriptorFromDB(domainURI, c); + }); + + @Nullable + private static DomainDescriptor fetchDomainDescriptorFromDB(String domainURI, Container c) + { + return fetchDomainDescriptorFromDB(domainURI, c, false); + } + + /** Goes against the DB, bypassing the cache */ + @Nullable + public static DomainDescriptor fetchDomainDescriptorFromDB(String uriOrName, Container c, boolean isName) + { + Container proj = c.getProject(); + if (null == proj) + proj = c; + + String sql = " SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE " + (isName ? "Name" : "DomainURI") + " = ? AND Project IN (?,?) "; + List ddArray = new SqlSelector(getExpSchema(), sql, uriOrName, + proj, + ContainerManager.getSharedContainer().getId()).getArrayList(DomainDescriptor.class); + DomainDescriptor dd = null; + if (!ddArray.isEmpty()) + { + dd = ddArray.get(0); + + // if someone has explicitly inserted a descriptor with the same URI as an existing one , + // and one of the two is in the shared project, use the project-level descriptor. + if (ddArray.size() > 1) + { + _log.debug("Multiple DomainDescriptors found for " + uriOrName); + if (dd.getProject().equals(ContainerManager.getSharedContainer())) + dd = ddArray.get(0); + } + } + return dd; + } + + private static final BlockingCache DOMAIN_DESC_BY_ID_CACHE = DatabaseCache.get(getExpSchema().getScope(),2000, CacheManager.UNLIMITED,"Domain descriptors by ID", new DomainDescriptorLoader()); + private static final BlockingCache, List>> DOMAIN_PROPERTIES_CACHE = DatabaseCache.get(getExpSchema().getScope(), 5000, CacheManager.UNLIMITED, "Domain properties", new CacheLoader<>() + { + @Override + public List> load(@NotNull Pair key, @Nullable Object argument) + { + String typeURI = key.first; + Container c = ContainerManager.getForId(key.second); + if (null == c) + return Collections.emptyList(); + SQLFragment sql = new SQLFragment("SELECT PropertyURI, Required " + + "FROM " + getTinfoPropertyDescriptor() + " PD\n" + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (PD.PropertyId = PDM.PropertyId)\n" + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)\n" + + "WHERE DD.DomainURI = ? AND DD.Project IN (?, ?) ORDER BY PDM.SortOrder, PD.PropertyId"); + + sql.addAll( + typeURI, + // protect against null project, just double-up shared project + c.isRoot() ? c.getId() : (c.getProject() == null ? _sharedContainer.getProject().getId() : c.getProject().getId()), + _sharedContainer.getProject().getId() + ); + + return new SqlSelector(getExpSchema(), sql).mapStream() + .map(map -> Pair.of((String)map.get("PropertyURI"), (Boolean)map.get("Required"))) + .toList(); + } + }); + private static final Cache> DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE = DatabaseCache.get(getExpSchema().getScope(), 2000, "Domain descriptors by container", (c, argument) -> { + String sql = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + + Map dds = new LinkedHashMap<>(); + for (DomainDescriptor dd : new SqlSelector(getExpSchema(), sql, c).getArrayList(DomainDescriptor.class)) + { + dds.putIfAbsent(dd.getDomainURI(), dd); + } + + return unmodifiableMap(dds); + }); + + private static final Container _sharedContainer = ContainerManager.getSharedContainer(); + + public static final String MV_INDICATOR_SUFFIX = "mvindicator"; + + static public String PropertyOrderURI = "urn:exp.labkey.org/#PropertyOrder"; + /** + * A comma-separated list of propertyID that indicates the sort order of the properties attached to an object. + */ + static public SystemProperty PropertyOrder = new SystemProperty(PropertyOrderURI, PropertyType.STRING); + + static + { + BeanObjectFactory.Registry.register(ObjectProperty.class, new ObjectProperty.ObjectPropertyObjectFactory()); + } + + private OntologyManager() + { + } + + /** + * @return map from PropertyURI to value + */ + public static @NotNull Map getProperties(Container container, String parentLSID) + { + Map m = new LinkedHashMap<>(); + Map propVals = getPropertyObjects(container, parentLSID); + if (null != propVals) + { + for (Map.Entry entry : propVals.entrySet()) + { + m.put(entry.getKey(), entry.getValue().value()); + } + } + + return m; + } + + public static final int MAX_PROPS_IN_BATCH = 1000; // Keep this reasonably small so progress indicator is updated regularly + public static final int UPDATE_STATS_BATCH_COUNT = 1000; + + public static void insertTabDelimited(Container c, + User user, + @Nullable Long ownerObjectId, + ImportHelper helper, + Domain domain, + DataIterator rows, + boolean ensureObjects, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + List properties = new ArrayList<>(domain.getProperties().size()); + for (DomainProperty prop : domain.getProperties()) + { + properties.add(prop.getPropertyDescriptor()); + } + insertTabDelimited(c, user, ownerObjectId, helper, properties, rows, ensureObjects, rowCallback); + } + + public interface RowCallback + { + void rowProcessed(Map row, String lsid) throws BatchValidationException; + + default void complete() throws BatchValidationException + {} + + default RowCallback chain(RowCallback other) + { + if (other == NO_OP_ROW_CALLBACK) + { + return this; + } + if (this == NO_OP_ROW_CALLBACK) + { + return other; + } + + RowCallback original = this; + + return new RowCallback() + { + @Override + public void rowProcessed(Map row, String lsid) throws BatchValidationException + { + original.rowProcessed(row, lsid); + other.rowProcessed(row, lsid); + } + + @Override + public void complete() throws BatchValidationException + { + original.complete(); + other.complete(); + } + }; + } + } + + public static final RowCallback NO_OP_ROW_CALLBACK = (row, lsid) -> {}; + + public static void insertTabDelimited(Container c, + User user, + @Nullable Long ownerObjectId, + ImportHelper helper, + List descriptors, + DataIterator rawRows, + boolean ensureObjects, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + MapDataIterator rows = DataIteratorUtil.wrapMap(rawRows, false); + + rowCallback = rowCallback == null ? NO_OP_ROW_CALLBACK : rowCallback; + + CPUTimer total = new CPUTimer("insertTabDelimited"); + CPUTimer before = new CPUTimer("beforeImport"); + CPUTimer ensure = new CPUTimer("ensureObject"); + CPUTimer insert = new CPUTimer("insertProperties"); + + assert total.start(); + assert getExpSchema().getScope().isTransactionActive(); + + // Make sure we have enough rows to handle the overflow of the current row so we don't have to resize the list + List propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); + + ValidatorContext validatorCache = new ValidatorContext(c, user); + + try + { + OntologyObject objInsert = new OntologyObject(); + objInsert.setContainer(c); + if (ownerObjectId != null && ownerObjectId > 0) + objInsert.setOwnerObjectId(ownerObjectId); + + List errors = new ArrayList<>(); + Map> validatorMap = new IntHashMap<>(); + + // cache all the property validators for this upload + for (PropertyDescriptor pd : descriptors) + { + List validators = PropertyService.get().getPropertyValidators(pd); + if (!validators.isEmpty()) + validatorMap.put(pd.getPropertyId(), validators); + } + + int rowCount = 0; + int batchCount = 0; + + while (rows.next()) + { + Map map = rows.getMap(); + // TODO: hack -- should exit and return cancellation status instead of throwing + if (Thread.currentThread().isInterrupted()) + throw new CancellationException(); + + assert before.start(); + + Map modifiableMap = new HashMap<>(map); + String lsid = helper.beforeImportObject(modifiableMap); + map = Collections.unmodifiableMap(modifiableMap); + + if (lsid == null) + { + throw new IllegalStateException("No LSID available"); + } + + assert before.stop(); + + assert ensure.start(); + long objectId; + if (ensureObjects) + objectId = ensureObject(c, lsid, ownerObjectId); + else + { + objInsert.setObjectURI(lsid); + Table.insert(null, getTinfoObject(), objInsert); + objectId = objInsert.getObjectId(); + } + + for (PropertyDescriptor pd : descriptors) + { + Object value = map.get(pd.getPropertyURI()); + if (null == value) + { + if (pd.isRequired()) + throw new BatchValidationException(new ValidationException("Missing value for required property " + pd.getName())); + else + { + continue; + } + } + else + { + if (validatorMap.containsKey(pd.getPropertyId())) + validateProperty(validatorMap.get(pd.getPropertyId()), pd, new ObjectProperty(lsid, c, pd, value), errors, validatorCache); + } + try + { + PropertyRow row = new PropertyRow(objectId, pd, value, pd.getPropertyType()); + propsToInsert.add(row); + } + catch (ConversionException e) + { + throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getPropertyType().getJavaType()))); + } + } + assert ensure.stop(); + + rowCount++; + + if (propsToInsert.size() > MAX_PROPS_IN_BATCH) + { + assert insert.start(); + insertPropertiesBulk(c, propsToInsert, false); + helper.afterBatchInsert(rowCount); + assert insert.stop(); + propsToInsert = new ArrayList<>(MAX_PROPS_IN_BATCH + descriptors.size()); + + if (++batchCount % UPDATE_STATS_BATCH_COUNT == 0) + { + getExpSchema().getSqlDialect().updateStatistics(getTinfoObject()); + getExpSchema().getSqlDialect().updateStatistics(getTinfoObjectProperty()); + helper.updateStatistics(rowCount); + } + } + + rowCallback.rowProcessed(map, lsid); + } + + if (!errors.isEmpty()) + throw new BatchValidationException(new ValidationException(errors)); + + assert insert.start(); + insertPropertiesBulk(c, propsToInsert, false); + helper.afterBatchInsert(rowCount); + rowCallback.complete(); + assert insert.stop(); + } + catch (SQLException x) + { + SQLException next = x.getNextException(); + if (x instanceof java.sql.BatchUpdateException && null != next) + x = next; + _log.debug("Exception uploading: ", x); + throw x; + } + + assert total.stop(); + _log.debug("\t" + total); + _log.debug("\t" + before); + _log.debug("\t" + ensure); + _log.debug("\t" + insert); + } + + /** + * As an incremental step of QueryUpdateService cleanup, this is a version of insertTabDelimited that works on a + * tableInfo that implements UpdateableTableInfo. Does not support ownerObjectid. + *

    + * This code is made complicated by the fact that while we are trying to move toward a TableInfo/ColumnInfo view + * of the world, validators are attached to PropertyDescriptors. Also, missing value handling is attached + * to PropertyDescriptors. + *

    + * The original version of this method expects a data to be a map PropertyURI->value. This version will also + * accept Name->value. + *

    + * Name->Value is preferred, we are using TableInfo after all. + */ + @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder + public static void insertTabDelimited(TableInfo tableInsert, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator rows, + boolean autoFillDefaultColumns, + Logger logger, + RowCallback rowCallback) + throws SQLException, BatchValidationException + { + saveTabDelimited(tableInsert, c, user, helper, rows, logger, true, autoFillDefaultColumns, rowCallback); + } + + @Deprecated // switch to StandardDataIteratorBuilder and TableInsertDataIteratorBuilder + public static void updateTabDelimited(TableInfo tableInsert, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator rows, + boolean autoFillDefaultColumns, + Logger logger) + throws SQLException, BatchValidationException + { + saveTabDelimited(tableInsert, c, user, helper, rows, logger, false, autoFillDefaultColumns, NO_OP_ROW_CALLBACK); + } + + private static void saveTabDelimited(TableInfo table, + Container c, + User user, + UpdateableTableImportHelper helper, + DataIterator in, + Logger logger, + boolean insert, + boolean autoFillDefaultColumns, + @Nullable RowCallback rowCallback) + throws SQLException, BatchValidationException + { + if (!(table instanceof UpdateableTableInfo)) + throw new IllegalArgumentException(); + + if (rowCallback == null) + { + rowCallback = NO_OP_ROW_CALLBACK; + } + + DbScope scope = table.getSchema().getScope(); + + assert scope.isTransactionActive(); + + Domain d = table.getDomain(); + List properties = null == d ? Collections.emptyList() : d.getProperties(); + + ValidatorContext validatorCache = new ValidatorContext(c, user); + + Connection conn = null; + ParameterMapStatement parameterMap = null; + + Map currentRow = null; + + MapDataIterator rows = DataIteratorUtil.wrapMap(in, false); + try + { + conn = scope.getConnection(); + if (insert) + { + parameterMap = StatementUtils.insertStatement(conn, table, c, user, true, autoFillDefaultColumns); + } + else + { + parameterMap = StatementUtils.updateStatement(conn, table, c, user, false, autoFillDefaultColumns); + } + List errors = new ArrayList<>(); + + Map> validatorMap = new HashMap<>(); + Map propertiesMap = new HashMap<>(); + + // cache all the property validators for this upload + for (DomainProperty dp : properties) + { + propertiesMap.put(dp.getPropertyURI(), dp); + List validators = dp.getValidators(); + if (!validators.isEmpty()) + validatorMap.put(dp.getPropertyURI(), validators); + } + + List columns = table.getColumns(); + PropertyType[] propertyTypes = new PropertyType[columns.size()]; + for (int i = 0; i < columns.size(); i++) + { + String propertyURI = columns.get(i).getPropertyURI(); + DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); + PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); + if (null != pd) + propertyTypes[i] = pd.getPropertyType(); + } + + int rowCount = 0; + + while (rows.next()) + { + + currentRow = new CaseInsensitiveHashMap<>(rows.getMap()); + + // TODO: hack -- should exit and return cancellation status instead of throwing + if (Thread.currentThread().isInterrupted()) + throw new CancellationException(); + + parameterMap.clearParameters(); + + String lsid = helper.beforeImportObject(currentRow); + currentRow.put("lsid", lsid); + + // + // NOTE we validate based on columninfo/propertydescriptor + // However, we bind by name, and there may be parameters that do not correspond to columninfo + // + + for (int i = 0; i < columns.size(); i++) + { + ColumnInfo col = columns.get(i); + if (col.isMvIndicatorColumn() || col.isRawValueColumn()) //TODO col.isNotUpdatableForSomeReasonSoContinue() + continue; + String propertyURI = col.getPropertyURI(); + DomainProperty dp = null == propertyURI ? null : propertiesMap.get(propertyURI); + PropertyDescriptor pd = null == dp ? null : dp.getPropertyDescriptor(); + + Object value = currentRow.get(col.getName()); + if (null == value) + value = currentRow.get(propertyURI); + + if (null == value) + { + // TODO col.isNullable() doesn't seem to work here + if (null != pd && pd.isRequired()) + throw new BatchValidationException(new ValidationException("Missing value for required property " + col.getName())); + } + else + { + if (null != pd) + { + try + { + // Use an ObjectProperty to unwrap MvFieldWrapper, do type conversion, etc + ObjectProperty objectProperty = new ObjectProperty(lsid, c, pd, value); + if (!validateProperty(validatorMap.get(propertyURI), pd, objectProperty, errors, validatorCache)) + { + throw new BatchValidationException(new ValidationException(errors)); + } + } + catch (ConversionException e) + { + throw new BatchValidationException(new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), pd.getJavaClass()))); + } + } + } + + // issue 19391: data from R uses "Inf" to represent infinity + if (JdbcType.DOUBLE.equals(col.getJdbcType())) + { + value = "Inf".equals(value) ? "Infinity" : value; + value = "-Inf".equals(value) ? "-Infinity" : value; + } + + try + { + String key = col.getName(); + if (!parameterMap.containsKey(key)) + key = propertyURI; + if (null == propertyTypes[i]) + { + // some built-in columns won't have parameters (createdby, etc) + if (parameterMap.containsKey(key)) + { + assert !(value instanceof MvFieldWrapper); + // Handle type coercion for these built-in columns as well, though we don't need to + // worry about missing values + value = PropertyType.getFromClass(col.getJavaObjectClass()).convert(value); + parameterMap.put(key, value); + } + } + else + { + Pair p = new Pair<>(value, null); + convertValuePair(pd, propertyTypes[i], p); + parameterMap.put(key, p.first); + if (null != p.second) + { + FieldKey mvName = col.getMvColumnName(); + if (mvName != null) + { + String storageName = table.getColumn(mvName).getMetaDataIdentifier().getId(); + parameterMap.put(storageName, p.second); + } + } + } + } + catch (ConversionException e) + { + throw new ValidationException(ConvertHelper.getStandardConversionErrorMessage(value, pd.getName(), propertyTypes[i].getJavaType())); + } + } + + helper.bindAdditionalParameters(currentRow, parameterMap); + parameterMap.execute(); + if (insert) + { + long rowId = parameterMap.getRowId(); + currentRow.put("rowId", rowId); + } + lsid = helper.afterImportObject(currentRow); + if (lsid == null) + { + throw new IllegalStateException("No LSID available"); + } + rowCallback.rowProcessed(currentRow, lsid); + rowCount++; + } + + + if (!errors.isEmpty()) + throw new BatchValidationException(new ValidationException(errors)); + + rowCallback.complete(); + + helper.afterBatchInsert(rowCount); + if (logger != null) + logger.debug("inserted row " + rowCount + "."); + } + catch (ValidationException e) + { + throw new BatchValidationException(e); + } + catch (SQLException x) + { + SQLException next = x.getNextException(); + if (x instanceof java.sql.BatchUpdateException && null != next) + x = next; + _log.debug("Exception uploading: ", x); + if (null != currentRow) + _log.debug(currentRow.toString()); + throw x; + } + finally + { + if (null != parameterMap) + parameterMap.close(); + if (null != conn) + scope.releaseConnection(conn); + } + } + + // TODO: Consolidate with ColumnValidator + public static boolean validateProperty(List validators, PropertyDescriptor prop, ObjectProperty objectProperty, + List errors, ValidatorContext validatorCache) + { + boolean ret = true; + + Object value = objectProperty.getObjectValue(); + + if (prop.isRequired() && value == null && objectProperty.getMvIndicator() == null) + { + errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is required", prop.getName())); + ret = false; + } + + // Check if the string is too long. Use either the PropertyDescriptor's scale or VARCHAR(4000) for ontology managed values + int stringLengthLimit = prop.getScale() > 0 ? prop.getScale() : getTinfoObjectProperty().getColumn("StringValue").getScale(); + int stringLength = value == null ? 0 : value.toString().length(); + if (value != null && prop.isStringType() && stringLength > stringLengthLimit) + { + String s = stringLength <= 100 ? value.toString() : StringUtilsLabKey.leftSurrogatePairFriendly(value.toString(), 100); + errors.add(new PropertyValidationError("Field '" + prop.getName() + "' is limited to " + stringLengthLimit + " characters, but the value is " + stringLength + " characters. (The value starts with '" + s + "...')", prop.getName())); + ret = false; + } + + // TODO: check date is within postgres date range + + // Don't validate null values, #15683 + if (null != value && validators != null) + { + for (IPropertyValidator validator : validators) + if (!validator.validate(prop, value, errors, validatorCache)) ret = false; + } + return ret; + } + + public interface ImportHelper + { + /** + * may modify map + * + * @return LSID for new or existing Object. Null indicates LSID is still unknown. + */ + String beforeImportObject(Map map) throws SQLException; + + void afterBatchInsert(int currentRow) throws SQLException; + + void updateStatistics(int currentRow) throws SQLException; + } + + + public interface UpdateableTableImportHelper extends ImportHelper + { + /** + * may be used to process attachments, for auditing, etc + * @return the LSID of the inserted row + */ + String afterImportObject(Map map) throws SQLException; + + /** + * may set parameters directly for columns that are not exposed by tableinfo + * e.g. "_key" + *

    + * TODO maybe this can be handled declaratively? see UpdateableTableInfo + */ + void bindAdditionalParameters(Map map, ParameterMapStatement target) throws ValidationException; + } + + @NotNull + private static Pair getPropertyMapCacheKey(@Nullable Container container, @NotNull String objectLSID) + { + return Pair.of(container, objectLSID); + } + + /** + * Get ordered map of property values for an object. The order of the properties in the + * Map corresponds to the PropertyOrder property, if present. + * + * @return map from PropertyURI to ObjectProperty + */ + public static Map getPropertyObjects(@Nullable Container container, @NotNull String objectLSID) + { + Pair cacheKey = getPropertyMapCacheKey(container, objectLSID); + return PROPERTY_MAP_CACHE.get(cacheKey); + } + + public static class PropertyMapCacheLoader implements CacheLoader, Map> + { + @Override + public Map load(@NotNull Pair key, @Nullable Object argument) + { + Container container = key.first; + String objectLSID = key.second; + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), objectLSID); + if (container != null) + { + filter.addCondition(FieldKey.fromParts("Container"), container); + } + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getResultSet()) + { + ResultSetUtil.logData(rs); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + List props = new TableSelector(getTinfoObjectPropertiesView(), filter, null).getArrayList(ObjectProperty.class); + + // check for a "PropertyOrder" value + ObjectProperty propertyOrder = props.stream().filter(op -> PropertyOrderURI.equals(op.getPropertyURI())).findFirst().orElse(null); + if (propertyOrder != null) + { + String order = propertyOrder.getStringValue(); + if (order != null) + { + // CONSIDER: Store as a JSONArray of propertyURI instead of propertyId + String[] parts = order.split(","); + try + { + List propertyIds = Arrays.stream(parts).map(s -> ConvertHelper.convert(s, Integer.class)).toList(); + + // Don't include the "PropertyOrder" property + props = new ArrayList<>(props); + props.remove(propertyOrder); + + // Order by the index found in the PropertyOrder list, otherwise just stick it at the end + Comparator comparator = (op1, op2) -> { + int i1 = propertyIds.indexOf(op1.getPropertyId()); + if (i1 == -1) + i1 = propertyIds.size(); + + int i2 = propertyIds.indexOf(op2.getPropertyId()); + if (i2 == -1) + i2 = propertyIds.size(); + return i1 - i2; + }; + props.sort(comparator); + } + catch (ConversionException e) + { + _log.warn("Failed to parse PropertyOrder integer list: " + order); + } + } + } + + Map m = new LinkedHashMap<>(); + for (ObjectProperty value : props) + { + m.put(value.getPropertyURI(), value); + } + + return unmodifiableMap(m); + } + } + + public static void updateObjectPropertyOrder(User user, Container container, String objectLSID, List properties) + throws ValidationException + { + String ids = null; + if (properties != null && !properties.isEmpty()) + ids = properties.stream().map(pd -> Integer.toString(pd.getPropertyId())).collect(joining(",")); + + updateObjectProperty(user, container, PropertyOrder.getPropertyDescriptor(), objectLSID, ids, null, false); + } + + /** + * Moves the properties of an object from one container to another (used when the object is moving) + * @param targetContainer the container to move the properties to + * @param user the user doing the move + * @param objectLSID the LSID of the object to which the properties are attached + * @return number of properties moved + */ + public static int updateContainer(Container targetContainer, User user, @NotNull String objectLSID) + { + return Table.updateContainer(getTinfoObject(), "objectURI", List.of(objectLSID), targetContainer, user, false); + } + + /** + * Get ordered list of the PropertyURI in {@link #PropertyOrder}, if present. + */ + public static List getObjectPropertyOrder(Container c, String objectLSID) + { + Map props = getPropertyObjects(c, objectLSID); + return new ArrayList<>(props.keySet()); + } + + public static long ensureObject(Container container, String objectURI) + { + return ensureObject(container, objectURI, (Long) null); + } + + public static long ensureObject(Container container, String objectURI, String ownerURI) + { + Long ownerId = null; + if (null != ownerURI) + ownerId = ensureObject(container, ownerURI, (Long) null); + return ensureObject(container, objectURI, ownerId); + } + + public static long ensureObject(Container container, String objectURI, Long ownerId) + { + //TODO: (marki) Transact? + Long objId = OBJECT_ID_CACHE.get(objectURI, container); + + if (null == objId) + { + OntologyObject obj = new OntologyObject(); + obj.setContainer(container); + obj.setObjectURI(objectURI); + if (ownerId != null && ownerId > 0) + obj.setOwnerObjectId(ownerId); + obj = Table.insert(null, getTinfoObject(), obj); + objId = obj.getObjectId(); + OBJECT_ID_CACHE.remove(objectURI); + } + + return objId; + } + + private static class ObjectIdCacheLoader implements CacheLoader + { + @Override + public Long load(@NotNull String objectURI, @Nullable Object argument) + { + Container container = (Container)argument; + OntologyObject obj = getOntologyObject(container, objectURI); + + return obj == null ? null : obj.getObjectId(); + } + } + + public static @Nullable OntologyObject getOntologyObject(Container container, String uri) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectURI"), uri); + if (container != null) + { + filter.addCondition(FieldKey.fromParts("Container"), container.getId()); + } + return new TableSelector(getTinfoObject(), filter, null).getObject(OntologyObject.class); + } + + // UNDONE: optimize (see deleteOntologyObjects(Integer[]) + public static void deleteOntologyObjects(Container c, String... uris) + { + if (uris.length == 0) + return; + + try + { + DbSchema schema = getExpSchema(); + String sql = getSqlDialect().execute(getExpSchema(), "deleteObject", "?, ?"); + SqlExecutor executor = new SqlExecutor(schema); + + for (String uri : uris) + { + executor.execute(sql, c.getId(), uri); + } + } + finally + { + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + } + } + + public static int deleteOntologyObjects(DbSchema schema, SQLFragment objectUriSql, @Nullable Container c) + { + SQLFragment objectIdSQL = new SQLFragment("SELECT ObjectId FROM ") + .append(getTinfoObject()).append("\n") + .append(" WHERE "); + if (c != null) + { + objectIdSQL.append(" Container = ?").add(c.getId()); + objectIdSQL.append(" AND "); + } + objectIdSQL.append("ObjectUri IN ("); + objectIdSQL.append(objectUriSql); + objectIdSQL.append(")"); + return deleteOntologyObjectsByObjectIdSql(schema, objectIdSQL); + } + + public static int deleteOntologyObjectsByObjectIdSql(DbSchema schema, SQLFragment objectIdSql) + { + if (!schema.getScope().equals(getExpSchema().getScope())) + throw new UnsupportedOperationException("can only use with same DbScope"); + + SQLFragment sqlDeleteProperties = new SQLFragment(); + sqlDeleteProperties.append("DELETE FROM ").append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (\n"); + sqlDeleteProperties.append(objectIdSql); + sqlDeleteProperties.append(")"); + new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); + + SQLFragment sqlDeleteObjects = new SQLFragment(); + sqlDeleteObjects.append("DELETE FROM ").append(getTinfoObject()).append(" WHERE ObjectId IN ("); + sqlDeleteObjects.append(objectIdSql); + sqlDeleteObjects.append(")"); + return new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); + } + + + public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, long... objectIds) + { + deleteOntologyObjects(c, deleteOwnedObjects, true, true, objectIds); + } + + public static void deleteOntologyObjects(Container c, boolean deleteOwnedObjects, boolean deleteObjectProperties, boolean deleteObjects, long... objectIds) + { + if (objectIds.length == 0) + return; + + try + { + // if it's a long list, split it up + if (objectIds.length > 1000) + { + int countBatches = objectIds.length / 1000; + int lenBatch = 1 + objectIds.length / (countBatches + 1); + + for (int s = 0; s < objectIds.length; s += lenBatch) + { + long[] sub = new long[Math.min(lenBatch, objectIds.length - s)]; + System.arraycopy(objectIds, s, sub, 0, sub.length); + deleteOntologyObjects(c, deleteOwnedObjects, deleteObjectProperties, deleteObjects, sub); + } + + return; + } + + SQLFragment objectIdInClause = new SQLFragment(); + getExpSchema().getSqlDialect().appendInClauseSql(objectIdInClause, Arrays.stream(objectIds).boxed().toList()); + + if (deleteOwnedObjects) + { + // NOTE: owned objects should never be in a different container than the owner, that would be a problem + SQLFragment sqlDeleteOwnedProperties = new SQLFragment("DELETE FROM ") + .append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND OwnerObjectId ") + .add(c) + .append(objectIdInClause) + .append(")"); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedProperties); + + SQLFragment sqlDeleteOwnedObjects = new SQLFragment("DELETE FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND OwnerObjectId ") + .add(c) + .append(objectIdInClause); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteOwnedObjects); + } + + if (deleteObjectProperties) + { + deleteProperties(c, objectIdInClause); + } + + if (deleteObjects) + { + SQLFragment sqlDeleteObjects = new SQLFragment("DELETE FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND ObjectId ") + .add(c) + .append(objectIdInClause); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteObjects); + } + } + finally + { + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + } + } + + + public static void deleteOntologyObject(String objectURI, Container container, boolean deleteOwnedObjects) + { + OntologyObject ontologyObject = getOntologyObject(container, objectURI); + + if (null != ontologyObject) + { + deleteOntologyObjects(container, deleteOwnedObjects, true, true, ontologyObject.getObjectId()); + } + } + + + public static OntologyObject getOntologyObject(long id) + { + return new TableSelector(getTinfoObject()).getObject(id, OntologyObject.class); + } + + //todo: review this. this doesn't delete the underlying data objects. should it? + public static void deleteObjectsOfType(String domainURI, Container container) + { + DomainDescriptor dd = null; + if (null != domainURI) + dd = getDomainDescriptor(domainURI, container); + if (null == dd) + { + _log.debug("deleteObjectsOfType called on type not found in database: " + domainURI); + return; + } + + try (Transaction t = getExpSchema().getScope().ensureTransaction()) + { + // until we set a domain on objects themselves, we need to create a list of objects to + // delete based on existing entries in ObjectProperties before we delete the objectProperties + // which we need to do before we delete the objects. + // TODO: Doesn't handle the case when PropertyDescriptors are shared across domains + String selectObjectsToDelete = "SELECT DISTINCT O.ObjectId " + + " FROM " + getTinfoObject() + " O " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON(O.ObjectId = OP.ObjectId) " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM ON (OP.PropertyId = PDM.PropertyId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PD.PropertyId = PDM.PropertyId) " + + " WHERE DD.DomainId = " + dd.getDomainId() + + " AND PD.Container = DD.Container"; + Long[] objIdsToDelete = new SqlSelector(getExpSchema(), selectObjectsToDelete).getArray(Long.class); + + String sep; + StringBuilder sqlIN = null; + Long[] ownerObjIds = null; + + if (objIdsToDelete.length > 0) + { + //also need list of owner objects whose subobjects are going to be deleted + // Seems cheaper but less correct to delete the subobjects then cleanup any owner objects with no children + sep = ""; + sqlIN = new StringBuilder(); + for (Long id : objIdsToDelete) + { + sqlIN.append(sep).append(id); + sep = ", "; + } + + String selectOwnerObjects = "SELECT O.ObjectId FROM " + getTinfoObject() + " O " + + " WHERE ObjectId IN " + + " (SELECT DISTINCT SUBO.OwnerObjectId FROM " + getTinfoObject() + " SUBO " + + " WHERE SUBO.ObjectId IN ( " + sqlIN + " ) )"; + + ownerObjIds = new SqlSelector(getExpSchema(), selectOwnerObjects).getArray(Long.class); + } + + String deleteTypePropsSql = "DELETE FROM " + getTinfoObjectProperty() + + " WHERE PropertyId IN " + + " (SELECT PDM.PropertyId FROM " + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDescriptor() + " PD ON (PDM.PropertyId = PD.PropertyId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " WHERE DD.DomainId = " + dd.getDomainId() + + " AND PD.Container = DD.Container " + + " ) "; + new SqlExecutor(getExpSchema()).execute(deleteTypePropsSql); + + if (objIdsToDelete.length > 0) + { + // now cleanup the object table entries from the list we made, but make sure they don't have + // other properties attached to them + String deleteObjSql = "DELETE FROM " + getTinfoObject() + + " WHERE ObjectId IN ( " + sqlIN + " ) " + + " AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + + " WHERE OP.ObjectId = " + getTinfoObject() + ".ObjectId)"; + new SqlExecutor(getExpSchema()).execute(deleteObjSql); + + if (ownerObjIds.length > 0) + { + sep = ""; + sqlIN = new StringBuilder(); + for (Long id : ownerObjIds) + { + sqlIN.append(sep).append(id); + sep = ", "; + } + String deleteOwnerSql = "DELETE FROM " + getTinfoObject() + + " WHERE ObjectId IN ( " + sqlIN + " ) " + + " AND NOT EXISTS (SELECT * FROM " + getTinfoObject() + " SUBO " + + " WHERE SUBO.OwnerObjectId = " + getTinfoObject() + ".ObjectId)"; + new SqlExecutor(getExpSchema()).execute(deleteOwnerSql); + } + } + // whew! + clearCaches(); + t.commit(); + } + } + + public static void deleteDomain(String domainURI, Container container) throws DomainNotFoundException + { + DomainDescriptor dd = getDomainDescriptor(domainURI, container); + String msg; + + if (null == dd) + throw new DomainNotFoundException(domainURI); + + if (!dd.getContainer().getId().equals(container.getId())) + { + // this domain was not created in this folder. Allow if in the project-level root + if (!dd.getProject().getId().equals(container.getId())) + { + msg = "DeleteDomain: Domain can only be deleted in original container or from the project root " + + "\nDomain: " + domainURI + " project " + dd.getProject().getName() + " original container " + dd.getContainer().getPath(); + _log.error(msg); + throw new RuntimeException(msg); + } + } + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + String selectPDsToDelete = "SELECT DISTINCT PDM.PropertyId " + + " FROM " + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (PDM.DomainId = DD.DomainId) " + + " WHERE DD.DomainId = ? "; + + Integer[] pdIdsToDelete = new SqlSelector(getExpSchema(), selectPDsToDelete, dd.getDomainId()).getArray(Integer.class); + + String deletePDMs = "DELETE FROM " + getTinfoPropertyDomain() + + " WHERE DomainId = " + + " (SELECT DD.DomainId FROM " + getTinfoDomainDescriptor() + " DD " + + " WHERE DD.DomainId = ? )"; + new SqlExecutor(getExpSchema()).execute(deletePDMs, dd.getDomainId()); + + if (pdIdsToDelete.length > 0) + { + String sep = ""; + StringBuilder sqlIN = new StringBuilder(); + for (Integer id : pdIdsToDelete) + { + PropertyService.get().deleteValidatorsAndFormats(container, id); + + sqlIN.append(sep); + sqlIN.append(id); + sep = ", "; + } + + String deletePDs = "DELETE FROM " + getTinfoPropertyDescriptor() + + " WHERE PropertyId IN ( " + sqlIN + " ) " + + "AND Container = ? " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoObjectProperty() + " OP " + + "WHERE OP.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId) " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + + "WHERE PDM.PropertyId = " + getTinfoPropertyDescriptor() + ".PropertyId)"; + + new SqlExecutor(getExpSchema()).execute(deletePDs, dd.getContainer().getId()); + } + + String deleteDD = "DELETE FROM " + getTinfoDomainDescriptor() + + " WHERE DomainId = ? " + + "AND NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + " PDM " + + "WHERE PDM.DomainId = " + getTinfoDomainDescriptor() + ".DomainId)"; + + new SqlExecutor(getExpSchema()).execute(deleteDD, dd.getDomainId()); + clearCaches(); + + transaction.commit(); + } + } + + + public static void deleteAllObjects(Container c, User user) throws ValidationException + { + Container projectContainer = c.getProject(); + if (null == projectContainer) + projectContainer = c; + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + if (!c.equals(projectContainer)) + { + copyDescriptors(c, projectContainer); + } + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + + // Owned objects should be in same container, so this should work + String deleteObjPropSql = "DELETE FROM " + getTinfoObjectProperty() + " WHERE ObjectId IN (SELECT ObjectId FROM " + getTinfoObject() + " WHERE Container = ?)"; + executor.execute(deleteObjPropSql, c); + String deleteObjSql = "DELETE FROM " + getTinfoObject() + " WHERE Container = ?"; + executor.execute(deleteObjSql, c); + + // delete property validator references on property descriptors + PropertyService.get().deleteValidatorsAndFormats(c); + + // Drop tables directly and allow bulk delete calls below to clean up rows in exp.propertydescriptor, + // exp.domaindescriptor, etc + String selectSQL = "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + Collection dds = new SqlSelector(getExpSchema(), selectSQL, c).getCollection(DomainDescriptor.class); + for (DomainDescriptor dd : dds) + { + StorageProvisioner.get().drop(PropertyService.get().getDomain(dd.getDomainId())); + } + + String deletePropDomSqlPD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId IN (SELECT PropertyId FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?)"; + executor.execute(deletePropDomSqlPD, c); + String deletePropDomSqlDD = "DELETE FROM " + getTinfoPropertyDomain() + " WHERE DomainId IN (SELECT DomainId FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?)"; + executor.execute(deletePropDomSqlDD, c); + String deleteDomSql = "DELETE FROM " + getTinfoDomainDescriptor() + " WHERE Container = ?"; + executor.execute(deleteDomSql, c); + // now delete the prop descriptors that are referenced in this container only + String deletePropSql = "DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE Container = ?"; + executor.execute(deletePropSql, c); + + clearCaches(); + transaction.commit(); + } + } + + private static void copyDescriptors(final Container c, final Container project) throws ValidationException + { + _log.debug("OntologyManager.copyDescriptors " + c.getName() + " " + project.getName()); + + // if c is (was) a project, then nothing to do + if (c.getId().equals(project.getId())) + return; + + // check to see if any Properties defined in this folder are used in other folders. + // if so we will make a copy of all PDs and DDs to ensure no orphans + String sql = " SELECT O.ObjectURI, O.Container, PD.PropertyId, PD.PropertyURI " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + + " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + + " WHERE PD.Container = ? " + + " AND O.Container <> PD.Container "; + + final Map mObjsUsingMyProps = new HashMap<>(); + final StringBuilder sqlIn = new StringBuilder(); + final StringBuilder sep = new StringBuilder(); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c).getResultSet()) + { + ResultSetUtil.logData(rs); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + new SqlSelector(getExpSchema(), sql, c).forEach(rs -> { + String objURI = rs.getString(1); + String objContainer = rs.getString(2); + Integer propId = rs.getInt(3); + String propURI = rs.getString(4); + + sqlIn.append(sep).append(propId); + + if (sep.isEmpty()) + sep.append(", "); + + Map mtemp = getPropertyObjects(ContainerManager.getForId(objContainer), objURI); + + if (null != mtemp) + { + for (Map.Entry entry : mtemp.entrySet()) + { + entry.getValue().setPropertyId(0); + if (entry.getValue().getPropertyURI().equals(propURI)) + mObjsUsingMyProps.put(entry.getKey(), entry.getValue()); + } + } + }); + + // For each property that is referenced outside its container, get the + // domains that it belongs to and the other properties in those domains + // so we can make copies of those domains and properties + // Restrict it to properties and domains also in the same container + + if (!mObjsUsingMyProps.isEmpty()) + { + sql = "SELECT PD.PropertyURI, DD.DomainURI " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + + " ON (PD.PropertyId = PDM2.PropertyId) " + + " WHERE PDM.PropertyId IN (" + sqlIn + ") " + + " OR PD.PropertyId IN (" + sqlIn + ") "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + catch (SQLException x) + { + throw new RuntimeException(x); + } + } + + new SqlSelector(getExpSchema(), sql).forEach(rsMyProps -> { + String propUri = rsMyProps.getString(1); + String domUri = rsMyProps.getString(2); + PropertyDescriptor pd = getPropertyDescriptor(propUri, c); + + if (pd.getContainer().getId().equals(c.getId())) + { + _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + DOMAIN_PROPERTIES_CACHE.clear(); + pd.setContainer(project); + pd.setPropertyId(0); + pd = ensurePropertyDescriptor(pd); + } + + if (null != domUri) + { + DomainDescriptor dd = getDomainDescriptor(domUri, c); + if (dd.getContainer().getId().equals(c.getId())) + { + uncache(dd); + dd = dd.edit() + .setContainer(project) + .setDomainId(0) + .build(); + dd = ensureDomainDescriptor(dd); + ensurePropertyDomain(pd, dd); + } + } + }); + + clearCaches(); + + // now unhook the objects that refer to my properties and rehook them to the properties in their own project + for (ObjectProperty op : mObjsUsingMyProps.values()) + { + deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), c); + insertProperties(op.getContainer(), op.getObjectURI(), op); + } + } + } + + private static void uncache(DomainDescriptor dd) + { + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.remove(dd.getContainer()); + } + + + public static void moveContainer(@NotNull final Container c, @NotNull Container oldParent, @NotNull Container newParent) throws SQLException + { + _log.debug("OntologyManager.moveContainer " + c.getName() + " " + oldParent.getName() + "->" + newParent.getName()); + + final Container oldProject = oldParent.getProject(); + Container newProject = newParent.getProject(); + if (null == newProject) // if container is promoted to a project + newProject = c.getProject(); + + if ((null != oldProject) && oldProject.getId().equals(newProject.getId())) + { + //the folder is being moved within the same project. No problems here + return; + } + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + clearCaches(); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoPropertyDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + // update project of any descriptors in folder just moved + TableInfo pdTable = getTinfoPropertyDescriptor(); + String sql = "UPDATE " + pdTable + " SET Project = ? WHERE Container = ?"; + + // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. + // Issue 30477: exclude project level properties descriptors (such as Study) that already exist + sql += " AND PropertyUri NOT IN (SELECT PropertyUri FROM " + pdTable + " WHERE Project = ? AND PropertyUri IN (SELECT PropertyUri FROM " + pdTable + " WHERE Container = ?))"; + + new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), "SELECT * FROM " + getTinfoDomainDescriptor() + " WHERE Container='" + c.getId() + "'").getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + TableInfo ddTable = getTinfoDomainDescriptor(); + sql = "UPDATE " + ddTable + " SET Project = ? WHERE Container = ?"; + + // TODO The IN clause is a temporary work around solution to avoid unique key violation error when moving study folders. + // Issue 30477: exclude project level domain descriptors (such as Study) that already exist + sql += " AND DomainUri NOT IN (SELECT DomainUri FROM " + ddTable + " WHERE Project = ? AND DomainUri IN (SELECT DomainUri FROM " + ddTable + " WHERE Container = ?))"; + + new SqlExecutor(getExpSchema()).execute(sql, newProject, c, newProject, c); + + if (null == oldProject) // if container was a project & demoted I'm done + { + transaction.commit(); + return; + } + + // this method makes sure I'm not getting rid of descriptors used by another folder + // it is shared by ContainerDelete + copyDescriptors(c, oldProject); + + // if my objects refer to project-scoped properties I need a copy of those properties + sql = " SELECT O.ObjectURI, PD.PropertyURI, PD.PropertyId, PD.Container " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " INNER JOIN " + getTinfoObjectProperty() + " OP ON PD.PropertyId = OP.PropertyId" + + " INNER JOIN " + getTinfoObject() + " O ON (O.ObjectId = OP.ObjectId) " + + " WHERE O.Container = ? " + + " AND O.Container <> PD.Container " + + " AND PD.Project NOT IN (?,?) "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + + final Map mMyObjsThatRefProjProps = new HashMap<>(); + final StringBuilder sqlIn = new StringBuilder(); + final StringBuilder sep = new StringBuilder(); + + new SqlSelector(getExpSchema(), sql, c, _sharedContainer, newProject).forEach(rs -> { + String objURI = rs.getString(1); + String propURI = rs.getString(2); + Integer propId = rs.getInt(3); + + sqlIn.append(sep).append(propId); + + if (sep.isEmpty()) + sep.append(", "); + + Map mtemp = getPropertyObjects(c, objURI); + + if (null != mtemp) + { + for (Map.Entry entry : mtemp.entrySet()) + { + if (entry.getValue().getPropertyURI().equals(propURI)) + mMyObjsThatRefProjProps.put(entry.getKey(), entry.getValue()); + } + } + }); + + // this sql gets all properties i ref and the domains they belong to and the + // other properties in those domains + //todo what about materialsource ? + if (!mMyObjsThatRefProjProps.isEmpty()) + { + sql = "SELECT PD.PropertyURI, DD.DomainURI, PD.PropertyId " + + " FROM " + getTinfoPropertyDescriptor() + " PD " + + " LEFT JOIN (" + getTinfoPropertyDomain() + " PDM " + + " INNER JOIN " + getTinfoPropertyDomain() + " PDM2 ON (PDM.DomainId = PDM2.DomainId) " + + " INNER JOIN " + getTinfoDomainDescriptor() + " DD ON (DD.DomainId = PDM.DomainId)) " + + " ON (PD.PropertyId = PDM2.PropertyId) " + + " WHERE PDM.PropertyId IN (" + sqlIn + " ) "; + + if (_log.isDebugEnabled()) + { + try (ResultSet rs = new SqlSelector(getExpSchema(), sql).getResultSet()) + { + ResultSetUtil.logData(rs, _log); + } + } + + final Container fNewProject = newProject; + + new SqlSelector(getExpSchema(), sql).forEach(rsPropsRefdByMe -> { + String propUri = rsPropsRefdByMe.getString(1); + String domUri = rsPropsRefdByMe.getString(2); + PropertyDescriptor pd = getPropertyDescriptor(propUri, oldProject); + + if (null != pd) + { + // To prevent iterating over a property descriptor update more than once + // we check to make sure both the container and project are equivalent to the updated + // location + if (!pd.getContainer().equals(c) || !pd.getProject().equals(fNewProject)) + { + pd.setContainer(c); + pd.setPropertyId(0); + } + + pd = ensurePropertyDescriptor(pd); + } + + if (null != domUri) + { + DomainDescriptor dd = getDomainDescriptor(domUri, oldProject); + + // To prevent iterating over a domain descriptor update more than once + // we check to make sure both the container and project are equivalent to the updated + // location + if (!dd.getContainer().equals(c) || !dd.getProject().equals(fNewProject)) + { + dd = dd.edit().setContainer(c).setDomainId(0).build(); + } + + dd = ensureDomainDescriptor(dd); + ensurePropertyDomain(pd, dd); + } + }); + + for (ObjectProperty op : mMyObjsThatRefProjProps.values()) + { + deleteProperty(op.getObjectURI(), op.getPropertyURI(), op.getContainer(), oldProject); + // Treat it as new so it's created in the target container as needed + op.setPropertyId(0); + insertProperties(op.getContainer(), op.getObjectURI(), op); + } + clearCaches(); + } + + transaction.commit(); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + private static PropertyDescriptor ensurePropertyDescriptor(String propertyURI, PropertyType type, String name, Container container) + { + PropertyDescriptor pdNew = new PropertyDescriptor(propertyURI, type, name, container); + return ensurePropertyDescriptor(pdNew); + } + + + private static PropertyDescriptor ensurePropertyDescriptor(PropertyDescriptor pdIn) + { + if (null == pdIn.getContainer()) + { + assert false : "Container should be set on PropertyDescriptor"; + pdIn.setContainer(_sharedContainer); + } + + PropertyDescriptor pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); + if (null == pd) + { + assert pdIn.getPropertyId() == 0; + /* return 1 if inserted 0 if not inserted, uses OUT parameter for new PropertyDescriptor */ + PropertyDescriptor[] out = new PropertyDescriptor[1]; + int rowcount = insertPropertyIfNotExists(null, pdIn, out); + pd = out[0]; + if (1 == rowcount && null != pd) + { + _log.debug("Removing property descriptor from cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + return pd; + } + if (null == pd) + { + throw OptimisticConflictException.create(Table.ERROR_DELETED); + } + } + + if (pd.equals(pdIn)) + { + return pd; + } + else + { + List colDiffs = comparePropertyDescriptors(pdIn, pd); + + if (colDiffs.isEmpty()) + { + // if the descriptor differs by container only and the requested descriptor is in the project fldr + if (!pdIn.getContainer().getId().equals(pd.getContainer().getId()) && + pdIn.getContainer().getId().equals(pdIn.getProject().getId())) + { + pdIn.setPropertyId(pd.getPropertyId()); + pd = updatePropertyDescriptor(pdIn); + } + return pd; + } + + // you are allowed to update if you are coming from the project root, or if you are in the container + // in which the descriptor was created + boolean fUpdateIfExists = false; + if (pdIn.getContainer().getId().equals(pd.getContainer().getId()) + || pdIn.getContainer().getId().equals(pdIn.getProject().getId())) + fUpdateIfExists = true; + + + boolean fMajorDifference = false; + if (colDiffs.toString().contains("RangeURI") || colDiffs.toString().contains("PropertyType")) + fMajorDifference = true; + + String errmsg = "ensurePropertyDescriptor: descriptor In different from Found for " + colDiffs + + "\n\t Descriptor In: " + pdIn + + "\n\t Descriptor Found: " + pd; + + if (fUpdateIfExists) + { + //todo: pass list of cols to update + pdIn.setPropertyId(pd.getPropertyId()); + pd = updatePropertyDescriptor(pdIn); + if (fMajorDifference) + _log.debug(errmsg); + } + else + { + if (fMajorDifference) + _log.error(errmsg); + else + _log.debug(errmsg); + } + } + return pd; + } + + + private static int insertPropertyIfNotExists(User user, PropertyDescriptor pd, PropertyDescriptor[] out) + { + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getInsertStmt(conn, user, t, true)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = f.toMap(pd, null); + stmt.putAll(m); + int rowcount = stmt.execute(); + SQLFragment reselect = new SQLFragment("SELECT * FROM exp.propertydescriptor WHERE propertyuri=? AND container=?", pd.getPropertyURI(), pd.getContainer()); + out[0] = (new SqlSelector(getExpSchema(), reselect).getObject(PropertyDescriptor.class)); + return rowcount; + } + catch(SQLException sqlx) + { + throw ExceptionFramework.Spring.translate(getExpSchema().getScope(), "insertPropertyIfNotExists", sqlx); + } + } + + + private static List comparePropertyDescriptors(PropertyDescriptor pdIn, PropertyDescriptor pd) + { + List colDiffs = new ArrayList<>(); + + // if the returned pd is in a different project, it better be the shared project + if (!pd.getProject().equals(pdIn.getProject()) && !pd.getProject().equals(_sharedContainer)) + colDiffs.add("Project"); + + // check the pd values that can't change + if (!pd.getRangeURI().equals(pdIn.getRangeURI())) + colDiffs.add("RangeURI"); + if (!Objects.equals(pd.getPropertyType(), pdIn.getPropertyType())) + colDiffs.add("PropertyType"); + + if (pdIn.getPropertyId() != 0 && pd.getPropertyId() != pdIn.getPropertyId()) + colDiffs.add("PropertyId"); + + if (!Objects.equals(pdIn.getName(), pd.getName())) + colDiffs.add("Name"); + + if (!Objects.equals(pdIn.getConceptURI(), pd.getConceptURI())) + colDiffs.add("ConceptURI"); + + if (!Objects.equals(pdIn.getDescription(), pd.getDescription())) + colDiffs.add("Description"); + + if (!Objects.equals(pdIn.getFormat(), pd.getFormat())) + colDiffs.add("Format"); + + if (!Objects.equals(pdIn.getLabel(), pd.getLabel())) + colDiffs.add("Label"); + + if (pdIn.isHidden() != pd.isHidden()) + colDiffs.add("IsHidden"); + + if (pdIn.isMvEnabled() != pd.isMvEnabled()) + colDiffs.add("IsMvEnabled"); + + if (!Objects.equals(pdIn.getLookupContainer(), pd.getLookupContainer())) + colDiffs.add("LookupContainer"); + + if (!Objects.equals(pdIn.getLookupSchema(), pd.getLookupSchema())) + colDiffs.add("LookupSchema"); + + if (!Objects.equals(pdIn.getLookupQuery(), pd.getLookupQuery())) + colDiffs.add("LookupQuery"); + + if (!Objects.equals(pdIn.getDerivationDataScope(), pd.getDerivationDataScope())) + colDiffs.add("DerivationDataScope"); + + if (!Objects.equals(pdIn.getSourceOntology(), pd.getSourceOntology())) + colDiffs.add("SourceOntology"); + + if (!Objects.equals(pdIn.getConceptImportColumn(), pd.getConceptImportColumn())) + colDiffs.add("ConceptImportColumn"); + + if (!Objects.equals(pdIn.getConceptLabelColumn(), pd.getConceptLabelColumn())) + colDiffs.add("ConceptLabelColumn"); + + if (!Objects.equals(pdIn.getPrincipalConceptCode(), pd.getPrincipalConceptCode())) + colDiffs.add("PrincipalConceptCode"); + + if (!Objects.equals(pdIn.getConceptSubtree(), pd.getConceptSubtree())) + colDiffs.add("ConceptSubtree"); + + if (pdIn.isScannable() != pd.isScannable()) + colDiffs.add("Scannable"); + + return colDiffs; + } + + public static DomainDescriptor ensureDomainDescriptor(String domainURI, String name, Container container) + { + String trimmedName = StringUtils.trimToNull(name); + if (trimmedName == null) + throw new IllegalArgumentException("Non-blank name is required."); + DomainDescriptor dd = new DomainDescriptor.Builder(domainURI, container).setName(trimmedName).build(); + return ensureDomainDescriptor(dd); + } + + /** Inserts or updates the domain as appropriate */ + @NotNull + public static DomainDescriptor ensureDomainDescriptor(DomainDescriptor ddIn) + { + DomainDescriptor dd = null; + // Try to find the previous version of the domain + if (ddIn.getDomainId() > 0) + { + // Try checking the cache first for a value to compare against + dd = getDomainDescriptor(ddIn.getDomainId()); + + // Since we cache mutable objects, get a fresh copy from the DB if the cache returned the same object that + // was passed in so we can do a diff against what's currently in the DB to see if we need to update + if (dd == ddIn) + { + dd = new TableSelector(getTinfoDomainDescriptor()).getObject(ddIn.getDomainId(), DomainDescriptor.class); + } + } + if (dd == null) + { + dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); + } + + if (null == dd) + { + try + { + DbSchema expSchema = getExpSchema(); + // ensureDomainDescriptor() shouldn't fail if there is a race condition, however Table.insert() will throw if row exists, so can't use that + // also a constraint violation will kill any current transaction + // CONSIDER to generalize add an option to check for existing row to Table.insert(ColumnInfo[] keyCols, Object[] keyValues) + String timestamp = expSchema.getSqlDialect().getSqlTypeName(JdbcType.TIMESTAMP); + String templateJson = null==ddIn.getTemplateInfo() ? null : ddIn.getTemplateInfo().toJSON(); + SQLFragment insert = new SQLFragment( + "INSERT INTO ").append(getTinfoDomainDescriptor()) + .append(" (Name, DomainURI, Description, Container, Project, StorageTableName, StorageSchemaName, ModifiedBy, Modified, TemplateInfo, SystemFieldConfig)\n" + + "SELECT ?,?,?,?,?,?,?,CAST(NULL AS INT),CAST(NULL AS " + timestamp + "),?,?\n") + .addAll(ddIn.getName(), ddIn.getDomainURI(), ddIn.getDescription(), ddIn.getContainer(), ddIn.getProject(), ddIn.getStorageTableName(), ddIn.getStorageSchemaName(), templateJson, ddIn.getSystemFieldConfig()) + .append("WHERE NOT EXISTS (SELECT * FROM ").append(getTinfoDomainDescriptor(),"x").append(" WHERE x.DomainURI=? AND x.Project=?)\n") + .add(ddIn.getDomainURI()).add(ddIn.getProject()); + // belt and suspenders approach to avoiding constraint violation exception + if (expSchema.getSqlDialect().isPostgreSQL()) + insert.append(" ON CONFLICT ON CONSTRAINT uq_domaindescriptor DO NOTHING"); + int count; + try (var tx = expSchema.getScope().ensureTransaction()) + { + count = new SqlExecutor(expSchema.getScope()).execute(insert); + tx.commit(); + } + + // alternately we could reselect rowid and then we wouldn't need this separate round trip + dd = fetchDomainDescriptorFromDB(ddIn.getDomainURI(), ddIn.getContainer()); + if (count > 0) + { + if (null == dd) // don't expect this + throw OptimisticConflictException.create(Table.ERROR_DELETED); + // We may have a cached miss that we need to clear + uncache(dd); + return dd; + } + // fall through to update case() + } + catch (RuntimeSQLException x) + { + // might be an optimistic concurrency problem see 16126 + dd = getDomainDescriptor(ddIn.getDomainURI(), ddIn.getContainer()); + if (null == dd) + throw x; + } + } + + if (!dd.deepEquals(ddIn)) + { + DomainDescriptor ddToSave = ddIn.edit().setDomainId(dd.getDomainId()).build(); + dd = Table.update(null, getTinfoDomainDescriptor(), ddToSave, ddToSave.getDomainId()); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(ddIn)); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.remove(getURICacheKey(dd)); + DOMAIN_DESC_BY_ID_CACHE.remove(dd.getDomainId()); + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(ddIn)); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); + } + return dd; + } + + private static void ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd) + { + ensurePropertyDomain(pd, dd, 0); + } + + public static PropertyDescriptor ensurePropertyDomain(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) + { + if (null == pd) + throw new IllegalArgumentException("Must supply a PropertyDescriptor"); + if (null == dd) + throw new IllegalArgumentException("Must supply a DomainDescriptor"); + + // Consider: We should check that the pd and dd have been persisted (aka have a non-zero id) + + if (!pd.getContainer().equals(dd.getContainer()) + && !pd.getProject().equals(_sharedContainer)) + throw new IllegalStateException("ensurePropertyDomain: property " + pd.getPropertyURI() + " not in same container as domain " + dd.getDomainURI()); + + SQLFragment sqlInsert = new SQLFragment("INSERT INTO " + getTinfoPropertyDomain() + " ( PropertyId, DomainId, Required, SortOrder ) " + + " SELECT ?, ?, ?, ? WHERE NOT EXISTS (SELECT * FROM " + getTinfoPropertyDomain() + + " WHERE PropertyId=? AND DomainId=?)"); + sqlInsert.add(pd.getPropertyId()); + sqlInsert.add(dd.getDomainId()); + sqlInsert.add(pd.isRequired()); + sqlInsert.add(sortOrder); + sqlInsert.add(pd.getPropertyId()); + sqlInsert.add(dd.getDomainId()); + int count = new SqlExecutor(getExpSchema()).execute(sqlInsert); + // if 0 rows affected, we should do an update to make sure required is correct + if (count == 0) + { + SQLFragment sqlUpdate = new SQLFragment("UPDATE " + getTinfoPropertyDomain() + " SET Required = ?, SortOrder = ? WHERE PropertyId=? AND DomainId= ?"); + sqlUpdate.add(pd.isRequired()); + sqlUpdate.add(sortOrder); + sqlUpdate.add(pd.getPropertyId()); + sqlUpdate.add(dd.getDomainId()); + new SqlExecutor(getExpSchema()).execute(sqlUpdate); + } + DOMAIN_PROPERTIES_CACHE.remove(getURICacheKey(dd)); + return pd; + } + + + private static void insertPropertiesBulk(Container container, List props, boolean insertNullValues) throws SQLException + { + List> floats = new ArrayList<>(); + List> dates = new ArrayList<>(); + List> strings = new ArrayList<>(); + List> mvIndicators = new ArrayList<>(); + + for (PropertyRow property : props) + { + if (null == property) + continue; + + long objectId = property.getObjectId(); + int propertyId = property.getPropertyId(); + String mvIndicator = property.getMvIndicator(); + assert mvIndicator == null || MvUtil.isMvIndicator(mvIndicator, container) : "Attempt to insert an invalid missing value indicator: " + mvIndicator; + + if (null != property.getFloatValue()) + floats.add(Arrays.asList(objectId, propertyId, property.getFloatValue(), mvIndicator)); + else if (null != property.getDateTimeValue()) + dates.add(Arrays.asList(objectId, propertyId, new java.sql.Timestamp(property.getDateTimeValue().getTime()), mvIndicator)); + else if (null != property.getStringValue()) + strings.add(Arrays.asList(objectId, propertyId, property.getStringValue(), mvIndicator)); + else if (null != mvIndicator) + { + mvIndicators.add(Arrays.asList(objectId, propertyId, property.getTypeTag(), mvIndicator)); + } + else if (insertNullValues) + { + strings.add(Arrays.asList(objectId, propertyId, null, null)); + } + } + + assert getExpSchema().getScope().isTransactionActive(); + + if (!dates.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, DateTimeValue, MvIndicator) VALUES (?,?,'d',?, ?)"; + Table.batchExecute(getExpSchema(), sql, dates); + } + + if (!floats.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, FloatValue, MvIndicator) VALUES (?,?,'f',?, ?)"; + Table.batchExecute(getExpSchema(), sql, floats); + } + + if (!strings.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, StringValue, MvIndicator) VALUES (?,?,'s',?, ?)"; + Table.batchExecute(getExpSchema(), sql, strings); + } + + if (!mvIndicators.isEmpty()) + { + String sql = "INSERT INTO " + getTinfoObjectProperty().toString() + " (ObjectId, PropertyId, TypeTag, MvIndicator) VALUES (?,?,?,?)"; + Table.batchExecute(getExpSchema(), sql, mvIndicators); + } + + clearPropertyCache(); + } + + public static void deleteProperty(String objectURI, String propertyURI, Container objContainer, Container propContainer) + { + OntologyObject o = getOntologyObject(objContainer, objectURI); + if (o == null) + return; + + PropertyDescriptor pd = getPropertyDescriptor(propertyURI, propContainer); + if (pd == null) + return; + + deleteProperty(o, pd); + } + + public static void deleteProperty(OntologyObject o, PropertyDescriptor pd) + { + deleteProperty(o, pd, true); + } + + public static void deleteProperty(OntologyObject o, PropertyDescriptor pd, boolean deleteCache) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("ObjectId"), o.getObjectId()); + filter.addCondition(FieldKey.fromParts("PropertyId"), pd.getPropertyId()); + Table.delete(getTinfoObjectProperty(), filter); + + if (deleteCache) + clearPropertyCache(o.getObjectURI()); + } + + /** + * Delete properties owned by the objects. + */ + public static void deleteProperties(Container objContainer, long objectId) + { + deleteProperties(objContainer, new SQLFragment(" = ?", objectId)); + } + public static void deleteProperties(Container objContainer, SQLFragment objectIdClause) + { + SQLFragment objectUriSql = new SQLFragment("SELECT ObjectURI FROM ") + .append(getTinfoObject(), "o") + .append(" WHERE ObjectId "); + objectUriSql.append(objectIdClause); + + List objectURIs = new SqlSelector(getExpSchema(), objectUriSql).getArrayList(String.class); + + SQLFragment sqlDeleteProperties = new SQLFragment("DELETE FROM ") + .append(getTinfoObjectProperty()) + .append(" WHERE ObjectId IN (SELECT ObjectId FROM ") + .append(getTinfoObject()) + .append(" WHERE Container = ? AND ObjectId ") + .add(objContainer) + .append(objectIdClause) + .append(")"); + + new SqlExecutor(getExpSchema()).execute(sqlDeleteProperties); + + for (String uri : objectURIs) + { + clearPropertyCache(uri); + } + } + + /** + * Removes the property from a single domain, and completely deletes it if there are no other references + */ + public static void removePropertyDescriptorFromDomain(DomainProperty domainProp) + { + SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ? AND DomainId = ?", domainProp.getPropertyId(), domainProp.getDomain().getTypeId()); + SqlExecutor executor = new SqlExecutor(getExpSchema()); + DbScope dbScope = getExpSchema().getScope(); + try (Transaction transaction = dbScope.ensureTransaction()) + { + executor.execute(deletePropDomSql); + // Check if there are any other usages + SQLFragment otherUsagesSQL = new SQLFragment("SELECT DomainId FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", domainProp.getPropertyId()); + if (!new SqlSelector(dbScope, otherUsagesSQL).exists()) + { + deletePropertyDescriptor(domainProp.getPropertyDescriptor()); + } + transaction.commit(); + } + } + + /** + * Completely deletes the property from the database + */ + public static void deletePropertyDescriptor(PropertyDescriptor pd) + { + int propId = pd.getPropertyId(); + + SQLFragment deleteObjPropSql = new SQLFragment("DELETE FROM " + getTinfoObjectProperty() + " WHERE PropertyId = ?", propId); + SQLFragment deletePropDomSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDomain() + " WHERE PropertyId = ?", propId); + SQLFragment deletePropSql = new SQLFragment("DELETE FROM " + getTinfoPropertyDescriptor() + " WHERE PropertyId = ?", propId); + + DbScope dbScope = getExpSchema().getScope(); + SqlExecutor executor = new SqlExecutor(getExpSchema()); + try (Transaction transaction = dbScope.ensureTransaction()) + { + executor.execute(deleteObjPropSql); + executor.execute(deletePropDomSql); + executor.execute(deletePropSql); + Pair key = getCacheKey(pd); + _log.debug("Removing property descriptor from cache. Key: " + key + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(key); + DOMAIN_PROPERTIES_CACHE.clear(); + transaction.commit(); + } + } + + /*** + * @deprecated Use {@link #insertProperties(Container, User, String, ObjectProperty...)} so that a user can be + * supplied. + */ + @Deprecated + public static void insertProperties(Container container, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException + { + User user = HttpView.hasCurrentView() ? HttpView.currentContext().getUser() : null; + insertProperties(container, user, ownerObjectLsid, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, ObjectProperty... properties) throws ValidationException + { + insertProperties(container, user, ownerObjectLsid, false, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, ObjectProperty... properties) throws ValidationException + { + insertProperties(container, user, ownerObjectLsid, skipValidation, false, properties); + } + + public static void insertProperties(Container container, User user, @Nullable String ownerObjectLsid, boolean skipValidation, boolean insertNullValues, ObjectProperty... properties) throws ValidationException + { + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + Long parentId = ownerObjectLsid == null ? null : ensureObject(container, ownerObjectLsid); + HashMap descriptors = new HashMap<>(); + HashMap objects = new HashMap<>(); + List errors = new ArrayList<>(); + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + for (ObjectProperty property : properties) + { + if (null == property) + continue; + + property.setObjectOwnerId(parentId); + + PropertyDescriptor pd = descriptors.get(property.getPropertyURI()); + if (0 == property.getPropertyId()) + { + if (null == pd) + { + PropertyDescriptor pdIn = new PropertyDescriptor(property.getPropertyURI(), property.getPropertyType(), property.getName(), container); + pdIn.setFormat(property.getFormat()); + pd = getPropertyDescriptor(pdIn.getPropertyURI(), pdIn.getContainer()); + + if (null == pd) + pd = ensurePropertyDescriptor(pdIn); + + descriptors.put(property.getPropertyURI(), pd); + } + property.setPropertyId(pd.getPropertyId()); + } + if (0 == property.getObjectId()) + { + Long objectId = objects.get(property.getObjectURI()); + if (null == objectId) + { + // I'm assuming all properties are in the same container + objectId = ensureObject(property.getContainer(), property.getObjectURI(), property.getObjectOwnerId()); + objects.put(property.getObjectURI(), objectId); + } + property.setObjectId(objectId); + } + if (pd == null) + { + pd = getPropertyDescriptor(property.getPropertyId()); + } + if (!skipValidation) + { + validateProperty(PropertyService.get().getPropertyValidators(pd), pd, property, errors, validatorCache); + } + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + + insertPropertiesBulk(container, List.of(properties), insertNullValues); + + transaction.commit(); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + + public static PropertyDescriptor getPropertyDescriptor(long propertyId) + { + return new TableSelector(getTinfoPropertyDescriptor()).getObject(propertyId, PropertyDescriptor.class); + } + + + public static PropertyDescriptor getPropertyDescriptor(String propertyURI, Container c) + { + // cache lookup by project. if not found at project level, check to see if global + Pair key = getCacheKey(propertyURI, c); + PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(key); + if (null != pd) + return pd; + + key = getCacheKey(propertyURI, _sharedContainer); + return PROP_DESCRIPTOR_CACHE.get(key); + } + + private static TableSelector getPropertyDescriptorTableSelector( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter, + @Nullable String sortColumn) + { + final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); + + // To filter by domain kind, we query the exp.DomainProperty table and filter by domainId. + // To construct a PropertyDescriptor, we will need to traverse the lookup to exp.PropertyDescriptor and select all of its columns. + List fields = new ArrayList<>(); + fields.add(FieldKey.fromParts("domainId")); + for (ColumnInfo col : getTinfoPropertyDescriptor().getColumns()) + { + fields.add(new FieldKey(propertyIdKey, col.getName())); + } + var colMap = QueryService.get().getColumns(getTinfoPropertyDomain(), fields); + + var filter = new SimpleFilter(); + if (propertyFilter != null) + { + filter.addAllClauses(propertyFilter); + } + + filter.addCondition(new FieldKey(propertyIdKey, "container"), c.getId()); + + if (!domains.isEmpty()) + { + filter.addInClause(FieldKey.fromParts("domainId"), domains.stream().map(Domain::getTypeId).collect(Collectors.toSet())); + } + + if (searchTerm != null) + { + // Apply Q filter to only some of the text columns + List searchCols = List.of( + colMap.get(new FieldKey(propertyIdKey, "Name")), + colMap.get(new FieldKey(propertyIdKey, "Label")), + colMap.get(new FieldKey(propertyIdKey, "Description")), + colMap.get(new FieldKey(propertyIdKey, "ImportAliases")) + ); + + var clause = CompareType.Q.createFilterClause(new FieldKey(null, "*"), searchTerm); + clause.setSelectColumns(searchCols); + filter.addCondition(clause); + } + + // use propertyId as the default sort + if (sortColumn == null) + sortColumn = "propertyId"; + Sort sort = new Sort(sortColumn); + + return new TableSelector(getTinfoPropertyDomain(), colMap.values(), filter, sort); + } + + public static Set getDomains( + Container c, User user, + @Nullable Set domainIds, + @Nullable Set domainKinds, + @Nullable Set domainNames) + { + Set domains = new HashSet<>(); + if (domainIds != null && !domainIds.isEmpty()) + { + domains.addAll(domainIds.stream().map(id -> PropertyService.get().getDomain(id)).collect(Collectors.toSet())); + } + + Set kinds = emptySet(); + Set names = emptySet(); + if (domainKinds != null && !domainKinds.isEmpty()) + { + kinds = domainKinds; + } + if (domainNames != null && !domainNames.isEmpty()) + { + names = domainNames; + } + if (!kinds.isEmpty() || !names.isEmpty()) + { + domains.addAll(PropertyService.get().getDomains(c, user, kinds, names, true)); + } + + return domains; + } + + public static List getPropertyDescriptors( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter, + @Nullable String sortColumn, + @Nullable Integer maxRows, + @Nullable Long offset) + { + final FieldKey propertyIdKey = FieldKey.fromParts("propertyId"); + + TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, + propertyFilter, sortColumn); + + if (maxRows != null) + ts.setMaxRows(maxRows); + if (offset != null) + ts.setOffset(offset); + + // This is a little annoying. We have to remove the "propertyId" lookup parent from + // the map keys for the ObjectFactory to correctly construct the PropertyDescriptor. + List props = new ArrayList<>(); + try (var results = ts.getResults(true)) + { + ObjectFactory of = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + while (results.next()) + { + Map rowMap = results.getFieldKeyRowMap(); + // remove the "propertyId" part from the FieldKey + Map rekey = new CaseInsensitiveHashMap<>(); + for (Map.Entry pair : rowMap.entrySet()) + { + FieldKey key = pair.getKey(); + if (propertyIdKey.equals(key.getParent())) + { + String name = key.getName(); + rekey.put(name, pair.getValue()); + } + } + props.add(of.fromMap(rekey)); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + return props; + } + + public static long getPropertyDescriptorsRowCount( + Container c, User user, + Set domains, + @Nullable String searchTerm, + @Nullable SimpleFilter propertyFilter) + { + + TableSelector ts = getPropertyDescriptorTableSelector(c, user, domains, searchTerm, + propertyFilter, null); + + return ts.getRowCount(); + } + + public static List getDomainsForPropertyDescriptor(Container container, PropertyDescriptor pd) + { + return PropertyService.get().getDomains(container) + .stream() + .filter(d -> null != d.getPropertyByURI(pd.getPropertyURI())) + .collect(Collectors.toList()); + } + + private static class DomainDescriptorLoader implements CacheLoader + { + @Override + public DomainDescriptor load(@NotNull Integer key, @Nullable Object argument) + { + return new TableSelector(getTinfoDomainDescriptor()).getObject(key, DomainDescriptor.class); + } + } + + public static DomainDescriptor getDomainDescriptor(int id) + { + return getDomainDescriptor(id, false); + } + + public static DomainDescriptor getDomainDescriptor(int id, boolean forUpdate) + { + if (forUpdate) + return new DomainDescriptorLoader().load(id, null); + + return DOMAIN_DESC_BY_ID_CACHE.get(id); + } + + @Nullable + public static DomainDescriptor getDomainDescriptor(String domainURI, Container c) + { + return getDomainDescriptor(domainURI, c, false); + } + + @Nullable + public static DomainDescriptor getDomainDescriptor(String domainURI, Container c, boolean forUpdate) + { + if (c == null) + return null; + + if (forUpdate) + return getDomainDescriptorForUpdate(domainURI, c); + + // cache lookup by project. if not found at project level, check to see if global + Pair key = getCacheKey(domainURI, c); + DomainDescriptor dd = DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); + if (null != dd) + return dd; + + // Try in the /Shared container too + key = getCacheKey(domainURI, _sharedContainer); + return DOMAIN_DESCRIPTORS_BY_URI_CACHE.get(key); + } + + @Nullable + private static DomainDescriptor getDomainDescriptorForUpdate(String domainURI, Container c) + { + if (c == null) + return null; + + DomainDescriptor dd = fetchDomainDescriptorFromDB(domainURI, c); + if (dd == null) + dd = fetchDomainDescriptorFromDB(domainURI, _sharedContainer); + return dd; + } + + /** + * Get all the domains in the same project as the specified container. They may not be in use in the container directly + */ + public static Collection getDomainDescriptors(Container container) + { + return getDomainDescriptors(container, null, false); + } + + public static Collection getDomainDescriptors(Container container, User user, boolean includeProjectAndShared) + { + if (container == null) + return Collections.emptyList(); + + if (includeProjectAndShared && user == null) + throw new IllegalArgumentException("Can't include data from other containers without a user to check permissions on"); + + Map dds = getCachedDomainDescriptors(container, user); + + if (includeProjectAndShared) + { + dds = new LinkedHashMap<>(dds); + Container project = container.getProject(); + if (project != null) + { + for (Map.Entry entry : getCachedDomainDescriptors(project, user).entrySet()) + { + dds.putIfAbsent(entry.getKey(), entry.getValue()); + } + } + + if (_sharedContainer.hasPermission(user, ReadPermission.class)) + { + for (Map.Entry entry : getCachedDomainDescriptors(_sharedContainer, user).entrySet()) + { + dds.putIfAbsent(entry.getKey(), entry.getValue()); + } + } + } + + return unmodifiableCollection(dds.values()); + } + + @NotNull + private static Map getCachedDomainDescriptors(@NotNull Container c, @Nullable User user) + { + if (user != null && !c.hasPermission(user, ReadPermission.class)) + return Collections.emptyMap(); + + return DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.get(c); + } + + public static Pair getURICacheKey(DomainDescriptor dd) + { + return getCacheKey(dd.getDomainURI(), dd.getContainer()); + } + + + public static Pair getCacheKey(PropertyDescriptor pd) + { + return getCacheKey(pd.getPropertyURI(), pd.getContainer()); + } + + + public static Pair getCacheKey(String uri, Container c) + { + Container proj = c.getProject(); + GUID projId; + + if (null == proj) + projId = c.getEntityId(); + else + projId = proj.getEntityId(); + + return Pair.of(uri, projId); + } + + //TODO: Cache semantics. This loads the cache but does not fetch cause need to get them all together + public static List getPropertiesForType(String typeURI, Container c) + { + List> propertyURIs = DOMAIN_PROPERTIES_CACHE.get(getCacheKey(typeURI, c)); + if (propertyURIs != null) + { + List result = new ArrayList<>(propertyURIs.size()); + for (Pair propertyURI : propertyURIs) + { + PropertyDescriptor pd = PROP_DESCRIPTOR_CACHE.get(getCacheKey(propertyURI.getKey(), c)); + if (pd == null) + { + return null; + } + // NOTE: cached descriptors may have differing values of isRequired() as that is a per-domain setting + // Descriptors returned from this method will have their required bit set as appropriate for this domain + + // Clone so nobody else messes up our copy + pd = pd.clone(); + pd.setRequired(propertyURI.getValue().booleanValue()); + result.add(pd); + } + return unmodifiableList(result); + } + return null; + } + + public static void deleteType(String domainURI, Container c) throws DomainNotFoundException + { + if (null == domainURI) + return; + + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + try + { + deleteObjectsOfType(domainURI, c); + deleteDomain(domainURI, c); + } + catch (DomainNotFoundException x) + { + // throw exception but do not kill enclosing transaction + transaction.commit(); + throw x; + } + + transaction.commit(); + } + } + + public static PropertyDescriptor insertOrUpdatePropertyDescriptor(PropertyDescriptor pd, DomainDescriptor dd, int sortOrder) + throws ChangePropertyDescriptorException + { + validatePropertyDescriptor(pd); + try (Transaction transaction = getExpSchema().getScope().ensureTransaction()) + { + DomainDescriptor dexist = ensureDomainDescriptor(dd); + + if (!dexist.getContainer().equals(pd.getContainer()) + && !pd.getProject().equals(_sharedContainer)) + { + // domain is defined in a different container. + //ToDO define property in the domains container? what security? + throw new ChangePropertyDescriptorException("Attempt to define property for a domain definition that exists in a different folder\n" + + "domain folder = " + dexist.getContainer().getPath() + "\n" + + "property folder = " + pd.getContainer().getPath()); + } + + PropertyDescriptor pexist = ensurePropertyDescriptor(pd); + pexist.setDatabaseDefaultValue(pd.getDatabaseDefaultValue()); + pexist.setNullable(pd.isMvEnabled() || pd.isNullable()); + pexist.setRequired(pd.isRequired()); + + ensurePropertyDomain(pexist, dexist, sortOrder); + + transaction.commit(); + return pexist; + } + } + + + static final String parameters = "propertyuri,name,description,rangeuri,concepturi,label," + + "format,container,project,lookupcontainer,lookupschema,lookupquery,defaultvaluetype,hidden," + + "mvenabled,importaliases,url,shownininsertview,showninupdateview,shownindetailsview,measure,dimension,scale," + + "sourceontology,conceptimportcolumn,conceptlabelcolumn,principalconceptcode,conceptsubtree," + + "recommendedvariable,derivationdatascope,storagecolumnname,facetingbehaviortype,phi,redactedText," + + "excludefromshifting,mvindicatorstoragecolumnname,defaultscale,scannable"; + static final String[] parametersArray = parameters.split(","); + + static ParameterMapStatement getInsertStmt(Connection conn, User user, TableInfo t, boolean ifNotExists) throws SQLException + { + user = null==user ? User.guest : user; + SQLFragment sql = new SQLFragment("INSERT INTO exp.propertydescriptor\n\t\t("); + SQLFragment values = new SQLFragment("\nSELECT\t"); + ColumnInfo c; + String comma = ""; + Parameter container = null; + Parameter propertyuri = null; + for (var p : parametersArray) + { + if (null == (c = t.getColumn(p))) + continue; + sql.append(comma).append(p); + values.append(comma).append("?"); + comma = ","; + Parameter parameter = new Parameter(p, c.getJdbcType()); + values.add(parameter); + if ("container".equals(p)) + container = parameter; + else if ("propertyuri".equals(p)) + propertyuri = parameter; + } + sql.append(", createdby, created, modifiedby, modified)\n"); + values.append(", " + user.getUserId() + ", {fn now()}, " + user.getUserId() + ", {fn now()}"); + sql.append(values); + if (ifNotExists) + { + sql.append("\nWHERE NOT EXISTS (SELECT propertyid FROM exp.propertydescriptor WHERE propertyuri=? AND container=?)\n"); + sql.add(propertyuri).add(container); + } + return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); + } + + static ParameterMapStatement getUpdateStmt(Connection conn, User user, TableInfo t) throws SQLException + { + user = null==user ? User.guest : user; + SQLFragment sql = new SQLFragment("UPDATE exp.propertydescriptor SET "); + ColumnInfo c; + String comma = ""; + for (var p : parametersArray) + { + if (null == (c = t.getColumn(p))) + continue; + sql.append(comma).append(p).append("=?"); + comma = ", "; + sql.add(new Parameter(p, c.getJdbcType())); + } + sql.append(", modifiedby=" + user.getUserId() + ", modified={fn now()}"); + sql.append("\nWHERE propertyid=?"); + sql.add(new Parameter("propertyid", JdbcType.INTEGER)); + return new ParameterMapStatement(t.getSchema().getScope(), conn, sql, null); + } + + + public static void insertPropertyDescriptors(User user, List pds) throws SQLException + { + if (null == pds || pds.isEmpty()) + return; + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getInsertStmt(conn, user, t, false)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = null; + for (PropertyDescriptor pd : pds) + { + m = f.toMap(pd, m); + stmt.clearParameters(); + stmt.putAll(m); + stmt.addBatch(); + } + stmt.executeBatch(); + } + } + + + public static void updatePropertyDescriptors(User user, List pds) throws SQLException + { + if (null == pds || pds.isEmpty()) + return; + TableInfo t = getTinfoPropertyDescriptor(); + try (Connection conn = t.getSchema().getScope().getConnection(); + ParameterMapStatement stmt = getUpdateStmt(conn, user, t)) + { + ObjectFactory f = ObjectFactory.Registry.getFactory(PropertyDescriptor.class); + Map m = null; + for (PropertyDescriptor pd : pds) + { + m = f.toMap(pd, m); + stmt.clearParameters(); + stmt.putAll(m); + stmt.addBatch(); + } + stmt.executeBatch(); + } + } + + + public static PropertyDescriptor insertPropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException + { + assert pd.getPropertyId() == 0; + validatePropertyDescriptor(pd); + pd = Table.insert(null, getTinfoPropertyDescriptor(), pd); + _log.debug("Adding property descriptor to cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + return pd; + } + + + //todo: we automatically update a pd to the last one in? + public static PropertyDescriptor updatePropertyDescriptor(PropertyDescriptor pd) + { + assert pd.getPropertyId() != 0; + pd = Table.update(null, getTinfoPropertyDescriptor(), pd, pd.getPropertyId()); + _log.debug("Updating property descriptor in cache. Key: " + getCacheKey(pd) + " descriptor: " + pd); + PROP_DESCRIPTOR_CACHE.remove(getCacheKey(pd)); + // It's possible that the propertyURI has changed, thus breaking our reference + DOMAIN_PROPERTIES_CACHE.clear(); + return pd; + } + + /** + * Insert or update an object property value. + * + * @param user The user inserting the property - currently only used for validating lookup values. + * @param container Insert the property value into this container. + * @param pd The property descriptor. + * @param lsid The object on which to attach the properties. + * @param value The value to insert. + * @param ownerObjectLsid The "owner" object or "parent" object, which isn't necessarily same as the object. For example, samples use the ExpSampleType as the owner object. + * @param insertNullValues When true, a null value will be inserted if the value is null, otherwise any existing property value will be deleted if the value is null. + * @return The inserted ObjectProperty or null + */ + public static ObjectProperty updateObjectProperty(User user, Container container, PropertyDescriptor pd, String lsid, Object value, @Nullable String ownerObjectLsid, boolean insertNullValues) throws ValidationException + { + ObjectProperty oprop; + RemapCache cache = new RemapCache(); + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + OntologyManager.deleteProperty(lsid, pd.getPropertyURI(), container, pd.getContainer()); + + try + { + oprop = new ObjectProperty(lsid, container, pd, value); + } + catch (ConversionException x) + { + // Issue 43529: Assay run property with large lookup doesn't resolve text input by value + // Attempt to resolve lookups by display value and then try creating the ObjectProperty again + if (pd.getLookup() != null) + { + Object remappedValue = getRemappedValueForLookup(user, container, cache, pd.getLookup(), value); + if (remappedValue != null) + value = remappedValue; + } + oprop = new ObjectProperty(lsid, container, pd, value); + } + + if (value != null || insertNullValues) + { + oprop.setPropertyId(pd.getPropertyId()); + OntologyManager.insertProperties(container, user, ownerObjectLsid, false, insertNullValues, oprop); + } + else + { + // We still need to validate blanks + List errors = new ArrayList<>(); + OntologyManager.validateProperty(PropertyService.get().getPropertyValidators(pd), pd, oprop, errors, new ValidatorContext(pd.getContainer(), user)); + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + transaction.commit(); + } + return oprop; + } + + public static Object getRemappedValueForLookup(User user, Container container, RemapCache cache, Lookup lookup, Object value) + { + Container lkContainer = lookup.getContainer() != null ? lookup.getContainer() : container; + return cache.remap(SchemaKey.fromParts(lookup.getSchemaKey()), lookup.getQueryName(), user, lkContainer, ContainerFilter.Type.CurrentPlusProjectAndShared, String.valueOf(value)); + } + + public static List findPropertyUsages(User user, List propertyIds, int maxUsageCount) + { + List ret = new ArrayList<>(propertyIds.size()); + for (int propertyId : propertyIds) + { + var pd = getPropertyDescriptor(propertyId); + if (pd == null) + throw new IllegalArgumentException("property not found: " + propertyId); + + ret.add(findPropertyUsages(user, pd, maxUsageCount)); + } + + return ret; + } + + public static List findPropertyUsages(User user, Container c, List propertyURIs, int maxUsageCount) + { + List ret = new ArrayList<>(propertyURIs.size()); + for (String propertyURI : propertyURIs) + { + var pd = getPropertyDescriptor(propertyURI, c); + if (pd == null) + throw new IllegalArgumentException("property not found: " + propertyURI); + + ret.add(findPropertyUsages(user, pd, maxUsageCount)); + } + + return ret; + } + + public static PropertyUsages findPropertyUsages(@NotNull User user, @NotNull PropertyDescriptor pd, int maxUsageCount) + { + // query exp.ObjectProperty for usages of the property + FieldKey objectId = FieldKey.fromParts("objectId"); + FieldKey objectId_objectURI = FieldKey.fromParts("objectId", "objectURI"); + FieldKey objectId_container = FieldKey.fromParts("objectId", "container"); + List fields = List.of(objectId, objectId_objectURI, objectId_container); + var colMap = QueryService.get().getColumns(getTinfoObjectProperty(), fields); + + int usageCount; + List objects = new ArrayList<>(maxUsageCount); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("propertyId"), pd.getPropertyId(), CompareType.EQUAL); + filter.addCondition(objectId_objectURI, DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX, CompareType.DOES_NOT_CONTAIN); + + TableSelector ts = new TableSelector(getTinfoObjectProperty(), colMap.values(), filter, new Sort("objectId")); + try (var r = ts.getResults(true)) + { + usageCount = r.getSize(); + + for (int i = 0; i < maxUsageCount && r.next(); i++) + { + var row = r.getFieldKeyRowMap(); + long oid = asLong(row.get(objectId)); + String objectURI = (String) row.get(objectId_objectURI); + String container = (String) row.get(objectId_container); + + Identifiable object = LsidManager.get().getObject(objectURI); + if (object != null) + { + Container c = object.getContainer(); + if (c != null && c.hasPermission(user, ReadPermission.class)) + objects.add(object); + } + else + { + Container c = ContainerManager.getForId(container); + if (c != null && c.hasPermission(user, ReadPermission.class)) + { + OntologyObject oo = new OntologyObject(); + oo.setContainer(c); + oo.setObjectId(oid); + oo.setObjectURI(objectURI); + objects.add(new IdentifiableBase(oo)); + } + } + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return new PropertyUsages(pd.getPropertyId(), pd.getPropertyURI(), usageCount, objects); + } + + public static class PropertyUsages + { + public final int propertyId; + public final String propertyURI; + public final int usageCount; + public final List objects; + + public PropertyUsages(int propertyId, String propertyURI, int usageCount, List objects) + { + this.propertyId = propertyId; + this.propertyURI = propertyURI; + this.usageCount = usageCount; + this.objects = objects; + } + } + + + public static void invalidateDomain(Domain d) + { + // TODO can we please implement a surgical version of this + clearCaches(); + } + + + public static void clearCaches() + { + _log.debug("Clearing caches"); + ExperimentService.get().clearCaches(); + DOMAIN_DESCRIPTORS_BY_URI_CACHE.clear(); + DOMAIN_DESC_BY_ID_CACHE.clear(); + DOMAIN_PROPERTIES_CACHE.clear(); + PROP_DESCRIPTOR_CACHE.clear(); + PROPERTY_MAP_CACHE.clear(); + OBJECT_ID_CACHE.clear(); + DOMAIN_DESCRIPTORS_BY_CONTAINER_CACHE.clear(); + } + + public static void clearPropertyCache(String parentObjectURI) + { + PROPERTY_MAP_CACHE.removeUsingFilter(key -> Objects.equals(key.second, parentObjectURI)); + } + + + public static void clearPropertyCache() + { + PROPERTY_MAP_CACHE.clear(); + } + + public static class ImportPropertyDescriptor + { + public final String domainName; + public final String domainURI; + public final PropertyDescriptor pd; + public final List validators; + public final List formats; + public final String defaultValue; + + private ImportPropertyDescriptor(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) + { + this.domainName = domainName; + this.domainURI = domainURI; + this.pd = pd; + this.validators = null != validators ? validators : Collections.emptyList(); + this.formats = null != formats ? formats : Collections.emptyList(); + this.defaultValue = defaultValue; + } + } + + + public static class ImportPropertyDescriptorsList + { + public final ArrayList properties = new ArrayList<>(); + + void add(String domainName, String domainURI, PropertyDescriptor pd, @Nullable List validators, @Nullable List formats, String defaultValue) + { + properties.add(new ImportPropertyDescriptor(domainName, domainURI, pd, validators, formats, defaultValue)); + } + } + + /** + * Updates an existing domain property with an import property descriptor generated + * by _propertyDescriptorFromRowMap below. Properties we don't set are explicitly + * called out + */ + public static void updateDomainPropertyFromDescriptor(DomainProperty p, PropertyDescriptor pd) + { + // don't setName + p.setPropertyURI(pd.getPropertyURI()); + p.setLabel(pd.getLabel()); + p.setConceptURI(pd.getConceptURI()); + p.setRangeURI(pd.getRangeURI()); + // don't setContainer + p.setDescription(pd.getDescription()); + p.setURL((pd.getURL() != null) ? pd.getURL().toString() : null); + p.setImportAliasSet(ColumnRenderPropertiesImpl.convertToSet(pd.getImportAliases())); + p.setRequired(pd.isRequired()); + p.setHidden(pd.isHidden()); + p.setShownInInsertView(pd.isShownInInsertView()); + p.setShownInUpdateView(pd.isShownInUpdateView()); + p.setShownInDetailsView(pd.isShownInDetailsView()); + p.setShownInLookupView(pd.isShownInLookupView()); + p.setDimension(pd.isDimension()); + p.setMeasure(pd.isMeasure()); + p.setRecommendedVariable(pd.isRecommendedVariable()); + p.setDefaultScale(pd.getDefaultScale()); + p.setScale(pd.getScale()); + p.setFormat(pd.getFormat()); + p.setMvEnabled(pd.isMvEnabled()); + + Lookup lookup = new Lookup(); + lookup.setQueryName(pd.getLookupQuery()); + lookup.setSchemaName(pd.getLookupSchema()); + String lookupContainerId = pd.getLookupContainer(); + if (lookupContainerId != null) + { + Container container = ContainerManager.getForId(lookupContainerId); + if (container == null) + lookup = null; + else + lookup.setContainer(container); + } + p.setLookup(lookup); + p.setFacetingBehavior(pd.getFacetingBehaviorType()); + p.setPhi(pd.getPHI()); + p.setRedactedText(pd.getRedactedText()); + p.setExcludeFromShifting(pd.isExcludeFromShifting()); + p.setDefaultValueTypeEnum(pd.getDefaultValueTypeEnum()); + p.setScannable(pd.isScannable()); + p.setDerivationDataScope(pd.getDerivationDataScope()); + } + + @TestWhen(TestWhen.When.BVT) + @TestTimeout(120) + public static class TestCase extends Assert + { + @Test + public void testSchema() + { + assertNotNull(getExpSchema()); + assertNotNull(getTinfoPropertyDescriptor()); + assertNotNull(ExperimentService.get().getTinfoSampleType()); + + assertEquals(11, getTinfoPropertyDescriptor().getColumns("PropertyId,PropertyURI,RangeURI,Name,Description,DerivationDataScope,SourceOntology,ConceptImportColumn,ConceptLabelColumn,PrincipalConceptCode,scannable").size()); + assertEquals(4, getTinfoObject().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId").size()); + assertEquals(11, getTinfoObjectPropertiesView().getColumns("ObjectId,ObjectURI,Container,OwnerObjectId,Name,PropertyURI,RangeURI,TypeTag,StringValue,DateTimeValue,FloatValue").size()); + assertEquals(10, ExperimentService.get().getTinfoSampleType().getColumns("RowId,Name,LSID,MaterialLSIDPrefix,Description,Created,CreatedBy,Modified,ModifiedBy,Container").size()); + } + + @Test + public void testBasicPropertiesObject() throws ValidationException + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + User user = TestContext.get().getUser(); + String parentObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + //First delete in case test case failed before + deleteOntologyObjects(c, parentObjectLsid); + assertNull(getOntologyObject(c, parentObjectLsid)); + assertNull(getOntologyObject(c, childObjectLsid)); + ensureObject(c, childObjectLsid, parentObjectLsid); + OntologyObject oParent = getOntologyObject(c, parentObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + assertNull(oParent.getOwnerObjectId()); + assertEquals(oChild.getContainer(), c); + assertEquals(oParent.getContainer(), c); + + String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + PropertyDescriptor strPd = getPropertyDescriptor(strProp, c); + assertEquals(PropertyType.STRING, strPd.getPropertyType()); + + String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + PropertyDescriptor intPd = getPropertyDescriptor(intProp, c); + assertEquals(PropertyType.INTEGER, intPd.getPropertyType()); + + String longProp = new Lsid("Junit", "OntologyManager", "longProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, longProp, 6L)); + PropertyDescriptor longPd = getPropertyDescriptor(longProp, c); + assertEquals(PropertyType.BIGINT, longPd.getPropertyType()); + + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.MILLISECOND, 0); + String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); + insertProperties(c, user, parentObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); + PropertyDescriptor datePd = getPropertyDescriptor(dateProp, c); + assertEquals(PropertyType.DATE_TIME, datePd.getPropertyType()); + + Map m = getProperties(c, oChild.getObjectURI()); + assertNotNull(m); + assertEquals(4, m.size()); + assertEquals("The String", m.get(strProp)); + assertEquals(5, m.get(intProp)); + assertEquals(6L, m.get(longProp)); + assertEquals(cal.getTime(), m.get(dateProp)); + + // Set property order: date, str, int. Long property will sort to last since it isn't explicitly included. + List propertyOrder = List.of(datePd, strPd, intPd); + updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); + + Map oProps = getPropertyObjects(c, childObjectLsid); + var iter = oProps.entrySet().iterator(); + assertEquals(cal.getTime(), iter.next().getValue().value()); + assertEquals("The String", iter.next().getValue().value()); + assertEquals(5, iter.next().getValue().value()); + assertEquals(6L, iter.next().getValue().value()); + assertFalse(iter.hasNext()); + + // Update property order: int, date, long, str + propertyOrder = List.of(intPd, datePd, longPd, strPd); + updateObjectPropertyOrder(user, c, childObjectLsid, propertyOrder); + oProps = getPropertyObjects(c, childObjectLsid); + iter = oProps.entrySet().iterator(); + assertEquals(5, iter.next().getValue().value()); + assertEquals(cal.getTime(), iter.next().getValue().value()); + assertEquals(6L, iter.next().getValue().value()); + assertEquals("The String", iter.next().getValue().value()); + assertFalse(iter.hasNext()); + + deleteOntologyObjects(c, parentObjectLsid); + assertNull(getOntologyObject(c, parentObjectLsid)); + assertNull(getOntologyObject(c, childObjectLsid)); + + m = getProperties(c, oChild.getObjectURI()); + assertEquals(0, m.size()); + } + + @Test + public void testContainerDelete() throws ValidationException + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + String strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + String intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.MILLISECOND, 0); + String dateProp = new Lsid("Junit", "OntologyManager", "dateProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, dateProp, cal.getTime())); + + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + + private void defineCrossFolderProperties(Container fldr1a, Container fldr1b) throws SQLException + { + try + { + String fa = fldr1a.getPath(); + String fb = fldr1b.getPath(); + + //object, prop descriptor in folder being moved + String objP1Fa = new Lsid("OntologyObject", "JUnit", fa.replace('/', '.')).toString(); + ensureObject(fldr1a, objP1Fa); + String propP1Fa = fa + "PD1"; + PropertyDescriptor pd1Fa = ensurePropertyDescriptor(propP1Fa, PropertyType.STRING, "PropertyDescriptor 1" + fa, fldr1a); + insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP1Fa, "same fldr")); + + //object in folder not moving, prop desc in folder moving + String objP2Fb = new Lsid("OntologyObject", "JUnit", fb.replace('/', '.')).toString(); + ensureObject(fldr1b, objP2Fb); + insertProperties(fldr1b, null, new ObjectProperty(objP2Fb, fldr1b, propP1Fa, "object in folder not moving, prop desc in folder moving")); + + //object in folder moving, prop desc in folder not moving + String propP2Fb = fb + "PD1"; + ensurePropertyDescriptor(propP2Fb, PropertyType.STRING, "PropertyDescriptor 1" + fb, fldr1b); + insertProperties(fldr1a, null, new ObjectProperty(objP1Fa, fldr1a, propP2Fb, "object in folder moving, prop desc in folder not moving")); + + // third prop desc in folder that is moving; shares domain with first prop desc + String propP1Fa3 = fa + "PD3"; + PropertyDescriptor pd1Fa3 = ensurePropertyDescriptor(propP1Fa3, PropertyType.STRING, "PropertyDescriptor 3" + fa, fldr1a); + String domP1Fa = fa + "DD1"; + DomainDescriptor dd1 = ensureDomainDescriptor(domP1Fa, "DomDesc 1" + fa, fldr1a); + ensurePropertyDomain(pd1Fa, dd1); + ensurePropertyDomain(pd1Fa3, dd1); + + //second domain desc in folder that is moving + // second prop desc in folder moving, belongs to 2nd domain + String propP1Fa2 = fa + "PD2"; + PropertyDescriptor pd1Fa2 = ensurePropertyDescriptor(propP1Fa2, PropertyType.STRING, "PropertyDescriptor 2" + fa, fldr1a); + String domP1Fa2 = fa + "DD2"; + DomainDescriptor dd2 = ensureDomainDescriptor(domP1Fa2, "DomDesc 2" + fa, fldr1a); + ensurePropertyDomain(pd1Fa2, dd2); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + @Test + public void testContainerMove() throws Exception + { + deleteMoveTestContainers(); + + Container proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); + Container proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + + proj1 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); + proj2 = ContainerManager.ensureContainer("/_ontMgrTestP2", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + + proj1 = ContainerManager.ensureContainer("/_ontMgrTestP1", TestContext.get().getUser()); + proj2 = ContainerManager.ensureContainer("/", TestContext.get().getUser()); + doMoveTest(proj1, proj2); + deleteMoveTestContainers(); + } + + private void doMoveTest(Container proj1, Container proj2) throws Exception + { + String p1Path = proj1.getPath() + "/"; + String p2Path = proj2.getPath() + "/"; + if (p1Path.equals("//")) p1Path = "/_ontMgrDemotePromote"; + if (p2Path.equals("//")) p2Path = "/_ontMgrDemotePromote"; + + Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); + Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); + ContainerManager.ensureContainer(p2Path + "Fc", TestContext.get().getUser()); + Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); + Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); + + defineCrossFolderProperties(fldr1a, fldr1b); + //defineCrossFolderProperties(fldr1a, fldr2c); + defineCrossFolderProperties(fldr1aa, fldr1b); + defineCrossFolderProperties(fldr1aaa, fldr1b); + + fldr1a.getProject().getPath(); + String f = fldr1a.getPath(); + String propId = f + "PD1"; + assertNull(getPropertyDescriptor(propId, proj2)); + ContainerManager.move(fldr1a, proj2, TestContext.get().getUser()); + + // if demoting a folder + if (proj1.isRoot()) + { + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD2"; + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj2)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj2)); + + domId = f + "DD2"; + assertNotNull(getDomainDescriptor(domId, proj2)); + } + // if promoting a folder, + else if (proj2.isRoot()) + { + assertNotNull(getPropertyDescriptor(propId, proj1)); + + propId = f + "PD2"; + assertNull(getPropertyDescriptor(propId, proj1)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj1)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj1)); + + domId = f + "DD2"; + assertNull(getDomainDescriptor(domId, proj1)); + } + else + { + assertNotNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD2"; + assertNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + propId = f + "PD3"; + assertNotNull(getPropertyDescriptor(propId, proj1)); + assertNotNull(getPropertyDescriptor(propId, proj2)); + + String domId = f + "DD1"; + assertNotNull(getDomainDescriptor(domId, proj1)); + assertNotNull(getDomainDescriptor(domId, proj2)); + + domId = f + "DD2"; + assertNull(getDomainDescriptor(domId, proj1)); + assertNotNull(getDomainDescriptor(domId, proj2)); + } + } + + @Test + public void testDeleteFoldersWithSharedProps() throws SQLException + { + deleteMoveTestContainers(); + + String projectName = "_ontMgrTestP1"; + Container proj1 = ContainerManager.ensureContainer(projectName, TestContext.get().getUser()); + String p1Path = proj1.getPath() + "/"; + + Container fldr1a = ContainerManager.ensureContainer(p1Path + "Fa", TestContext.get().getUser()); + Container fldr1b = ContainerManager.ensureContainer(p1Path + "Fb", TestContext.get().getUser()); + Container fldr1aa = ContainerManager.ensureContainer(p1Path + "Fa/Faa", TestContext.get().getUser()); + Container fldr1aaa = ContainerManager.ensureContainer(p1Path + "Fa/Faa/Faaa", TestContext.get().getUser()); + + defineCrossFolderProperties(fldr1a, fldr1b); + defineCrossFolderProperties(fldr1aa, fldr1b); + defineCrossFolderProperties(fldr1aaa, fldr1b); + + deleteProjects( projectName); + } + + private void deleteMoveTestContainers() + { + // Remove all projects. Subfolders will be deleted when project is removed. + deleteProjects( + "/_ontMgrTestP1", + "/_ontMgrTestP2", + "/_ontMgrDemotePromoteFa", + "/_ontMgrDemotePromoteFb", + "/_ontMgrDemotePromoteFc", + "/Fa" + ); + } + + private void deleteProjects(String... projectNames) + { + for (String path : projectNames) + { + Container c = ContainerManager.getForPath(path); + + if (null != c) + ContainerManager.deleteAll(c, TestContext.get().getUser()); + } + + for (String path : projectNames) + assertNull("Container " + path + " was not deleted", ContainerManager.getForPath(path)); + } + + @Test + public void testTransactions() throws SQLException + { + try + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + + //Create objects in a transaction & make sure they are all gone. + OntologyObject oParent; + OntologyObject oChild; + String strProp; + String intProp; + + try (Transaction ignored = getExpSchema().getScope().beginTransaction()) + { + ensureObject(c, childObjectLsid, ownerObjectLsid); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + } + + assertEquals(0L, getObjectCount(c)); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNull(oParent); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + strProp = new Lsid("Junit", "OntologyManager", "stringProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, strProp, "The String")); + + //Rollback transaction for one new property + try (Transaction ignored = getExpSchema().getScope().beginTransaction()) + { + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + } + + oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + Map m = getProperties(c, childObjectLsid); + assertNotNull(m.get(strProp)); + assertNull(m.get(intProp)); + + try (Transaction transaction = getExpSchema().getScope().beginTransaction()) + { + intProp = new Lsid("Junit", "OntologyManager", "intProp").toString(); + insertProperties(c, ownerObjectLsid, new ObjectProperty(childObjectLsid, c, intProp, 5)); + transaction.commit(); + } + + m = getProperties(c, childObjectLsid); + assertNotNull(m.get(strProp)); + assertNotNull(m.get(intProp)); + + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + catch (ValidationException ve) + { + throw new SQLException(ve.getMessage()); + } + } + + @Test + public void testDomains() throws Exception + { + Container c = ContainerManager.ensureContainer("/_ontologyManagerTest", TestContext.get().getUser()); + //Clean up last time's mess + deleteAllObjects(c, TestContext.get().getUser()); + assertEquals(0L, getObjectCount(c)); + String ownerObjectLsid = new Lsid("Junit", "OntologyManager", "parent").toString(); + String childObjectLsid = new Lsid("Junit", "OntologyManager", "child").toString(); + String child2ObjectLsid = new Lsid("Junit", "OntologyManager", "child2").toString(); + + ensureObject(c, childObjectLsid, ownerObjectLsid); + OntologyObject oParent = getOntologyObject(c, ownerObjectLsid); + assertNotNull(oParent); + OntologyObject oChild = getOntologyObject(c, childObjectLsid); + assertNotNull(oChild); + + String domURIa = new Lsid("Junit", "DD", "Domain1").toString(); + String strPropURI = new Lsid("Junit", "PD", "Domain1.stringProp").toString(); + String intPropURI = new Lsid("Junit", "PD", "Domain1.intProp").toString(); + String longPropURI = new Lsid("Junit", "PD", "Domain1.longProp").toString(); + + DomainDescriptor dd = ensureDomainDescriptor(domURIa, "Domain1", c); + assertNotNull(dd); + + PropertyDescriptor pdStr = new PropertyDescriptor(); + pdStr.setPropertyURI(strPropURI); + pdStr.setRangeURI(PropertyType.STRING.getTypeUri()); + pdStr.setContainer(c); + pdStr.setName("Domain1.stringProp"); + + pdStr = ensurePropertyDescriptor(pdStr); + assertNotNull(pdStr); + + PropertyDescriptor pdInt = ensurePropertyDescriptor(intPropURI, PropertyType.INTEGER, "Domain1.intProp", c); + PropertyDescriptor pdLong = ensurePropertyDescriptor(longPropURI, PropertyType.BIGINT, "Domain1.longProp", c); + + ensurePropertyDomain(pdStr, dd); + ensurePropertyDomain(pdInt, dd); + ensurePropertyDomain(pdLong, dd); + + List pds = getPropertiesForType(domURIa, c); + assertEquals(3, pds.size()); + Map mPds = new HashMap<>(); + for (PropertyDescriptor pd1 : pds) + mPds.put(pd1.getPropertyURI(), pd1); + + assertTrue(mPds.containsKey(strPropURI)); + assertTrue(mPds.containsKey(intPropURI)); + assertTrue(mPds.containsKey(longPropURI)); + + ObjectProperty strProp = new ObjectProperty(childObjectLsid, c, strPropURI, "String value"); + ObjectProperty intProp = new ObjectProperty(childObjectLsid, c, intPropURI, 42); + ObjectProperty longProp = new ObjectProperty(childObjectLsid, c, longPropURI, 52L); + insertProperties(c, ownerObjectLsid, strProp); + insertProperties(c, ownerObjectLsid, intProp); + insertProperties(c, ownerObjectLsid, longProp); + + Map m = getProperties(c, oChild.getObjectURI()); + assertNotNull(m); + assertEquals(3, m.size()); + assertEquals("String value", m.get(strPropURI)); + assertEquals(42, m.get(intPropURI)); + assertEquals(52L, m.get(longPropURI)); + + // test insertTabDelimited + List> rows = List.of( + new CaseInsensitiveMapWrapper<>(Map.of( + "lsid", child2ObjectLsid, + strPropURI, "Second value", + intPropURI, 62, + longPropURI, 72L + ) + )); + ImportHelper helper = new ImportHelper() + { + @Override + public String beforeImportObject(Map map) + { + return (String)map.get("lsid"); + } + + @Override + public void afterBatchInsert(int currentRow) + { } + + @Override + public void updateStatistics(int currentRow) + { } + }; + try (Transaction tx = getExpSchema().getScope().ensureTransaction()) + { + insertTabDelimited(c, TestContext.get().getUser(), oParent.getObjectId(), helper, pds, MapDataIterator.of(rows).getDataIterator(new DataIteratorContext()), false, null); + tx.commit(); + } + + m = getProperties(c, child2ObjectLsid); + assertNotNull(m); + assertEquals(3, m.size()); + assertEquals("Second value", m.get(strPropURI)); + assertEquals(62, m.get(intPropURI)); + assertEquals(72L, m.get(longPropURI)); + + deleteType(domURIa, c); + assertEquals(0L, getObjectCount(c)); + assertTrue(ContainerManager.delete(c, TestContext.get().getUser())); + } + } + + private static long getObjectCount(Container c) + { + return new TableSelector(getTinfoObject(), SimpleFilter.createContainerFilter(c), null).getRowCount(); + } + + /** + * v.first value IN/OUT parameter + * v.second mvIndicator OUT parameter + */ + public static void convertValuePair(PropertyDescriptor pd, PropertyType pt, Pair v) + { + if (v.first == null) + return; + + // Handle field-level QC + if (v.first instanceof MvFieldWrapper mvWrapper) + { + v.second = mvWrapper.getMvIndicator(); + v.first = mvWrapper.getValue(); + } + else if (pd.isMvEnabled()) + { + // Not all callers will have wrapped an MV value if there isn't also + // a real value + if (MvUtil.isMvIndicator(v.first.toString(), pd.getContainer())) + { + v.second = v.first.toString(); + v.first = null; + } + } + + if (null != v.first && null != pt) + v.first = pt.convert(v.first); + } + + @Deprecated // Fold into ObjectProperty? Eliminate insertTabDelimited() methods, the only usage of PropertyRow. + public static class PropertyRow + { + protected long objectId; + protected int propertyId; + protected char typeTag; + protected Double floatValue; + protected String stringValue; + protected Date dateTimeValue; + protected String mvIndicator; + + public PropertyRow() + { + } + + public PropertyRow(long objectId, PropertyDescriptor pd, Object value, PropertyType pt) + { + this.objectId = objectId; + this.propertyId = pd.getPropertyId(); + this.typeTag = pt.getStorageType(); + + Pair p = new Pair<>(value, null); + convertValuePair(pd, pt, p); + mvIndicator = p.second; + + pt.init(this, p.first); + } + + public long getObjectId() + { + return objectId; + } + + public void setObjectId(long objectId) + { + this.objectId = objectId; + } + + public int getPropertyId() + { + return propertyId; + } + + public void setPropertyId(int propertyId) + { + this.propertyId = propertyId; + } + + public char getTypeTag() + { + return typeTag; + } + + public void setTypeTag(char typeTag) + { + this.typeTag = typeTag; + } + + public Double getFloatValue() + { + return floatValue; + } + + public Boolean getBooleanValue() + { + if (floatValue == null) + { + return null; + } + return floatValue.doubleValue() == 1.0; + } + + public void setFloatValue(Double floatValue) + { + this.floatValue = floatValue; + } + + public String getStringValue() + { + return stringValue; + } + + public void setStringValue(String stringValue) + { + this.stringValue = stringValue; + } + + public Date getDateTimeValue() + { + return dateTimeValue; + } + + public void setDateTimeValue(Date dateTimeValue) + { + this.dateTimeValue = dateTimeValue; + } + + public String getMvIndicator() + { + return mvIndicator; + } + + public void setMvIndicator(String mvIndicator) + { + this.mvIndicator = mvIndicator; + } + + public Object getObjectValue() + { + return stringValue != null ? stringValue : floatValue != null ? floatValue : dateTimeValue; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + sb.append("PropertyRow: "); + + sb.append("objectId=").append(objectId); + sb.append(", propertyId=").append(propertyId); + sb.append(", value="); + + if (stringValue != null) + sb.append(stringValue); + else if (floatValue != null) + sb.append(floatValue); + else if (dateTimeValue != null) + sb.append(dateTimeValue); + else + sb.append("null"); + + if (mvIndicator != null) + sb.append(", mvIndicator=").append(mvIndicator); + + return sb.toString(); + } + } + + public static DbSchema getExpSchema() + { + return DbSchema.get("exp", DbSchemaType.Module); + } + + public static SqlDialect getSqlDialect() + { + return getExpSchema().getSqlDialect(); + } + + public static TableInfo getTinfoPropertyDomain() + { + return getExpSchema().getTable("PropertyDomain"); + } + + public static TableInfo getTinfoObject() + { + return getExpSchema().getTable("Object"); + } + + public static TableInfo getTinfoObjectProperty() + { + return getExpSchema().getTable("ObjectProperty"); + } + + public static TableInfo getTinfoPropertyDescriptor() + { + return getExpSchema().getTable("PropertyDescriptor"); + } + + public static TableInfo getTinfoDomainDescriptor() + { + return getExpSchema().getTable("DomainDescriptor"); + } + + public static TableInfo getTinfoObjectPropertiesView() + { + return getExpSchema().getTable("ObjectPropertiesView"); + } + + public static HtmlString doProjectColumnCheck(boolean bFix) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(); + String descriptorTable = getTinfoPropertyDescriptor().toString(); + String uriColumn = "PropertyURI"; + String idColumn = "PropertyID"; + doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); + + descriptorTable = getTinfoDomainDescriptor().toString(); + uriColumn = "DomainURI"; + idColumn = "DomainID"; + doProjectColumnCheck(descriptorTable, uriColumn, idColumn, builder, bFix); + + return builder.getHtmlString(); + } + + private static void doProjectColumnCheck(final String descriptorTable, final String uriColumn, final String idColumn, final HtmlStringBuilder msgBuilder, final boolean bFix) + { + // get all unique combos of Container, project + + String sql = "SELECT Container, Project FROM " + descriptorTable + " GROUP BY Container, Project"; + + new SqlSelector(getExpSchema(), sql).forEach(rs -> { + String containerId = rs.getString("Container"); + String projectId = rs.getString("Project"); + Container container = ContainerManager.getForId(containerId); + if (null == container) + return; // should be handled by container check + String newProjectId = container.getProject() == null ? container.getId() : container.getProject().getId(); + if (!projectId.equals(newProjectId)) + { + if (bFix) + { + fixProjectColumn(descriptorTable, uriColumn, idColumn, container, projectId, newProjectId); + msgBuilder + .unsafeAppend("
       ") + .append("Fixed inconsistent project ids found for ") + .append(descriptorTable).append(" in folder ") + .append(ContainerManager.getForId(containerId).getPath()); + + } + else + msgBuilder + .unsafeAppend("
       ") + .append("ERROR: Inconsistent project ids found for ") + .append(descriptorTable).append(" in folder ").append(container.getPath()); + } + }); + } + + private static void fixProjectColumn(String descriptorTable, String uriColumn, String idColumn, Container container, String projectId, String newProjId) + { + final SqlExecutor executor = new SqlExecutor(getExpSchema()); + + String sql = "UPDATE " + descriptorTable + " SET Project= ? WHERE Project = ? AND Container=? AND " + uriColumn + " NOT IN " + + "(SELECT " + uriColumn + " FROM " + descriptorTable + " WHERE Project = ?)"; + executor.execute(sql, newProjId, projectId, container.getId(), newProjId); + + // now check to see if there is already an existing descriptor in the target (correct) project. + // this can happen if a folder containing a descriptor is moved to another project + // and the OntologyManager's containerMoved handler fails to fire for some reason. (note not in transaction) + // If this is the case, the descriptor is redundant and it should be deleted, after we move the objects that depend on it. + + sql = " SELECT prev." + idColumn + " AS PrevIdCol, cur." + idColumn + " AS CurIdCol FROM " + descriptorTable + " prev " + + " INNER JOIN " + descriptorTable + " cur ON (prev." + uriColumn + "= cur." + uriColumn + " ) " + + " WHERE cur.Project = ? AND prev.Project= ? AND prev.Container = ? "; + final String updsql1 = " UPDATE " + getTinfoObjectProperty() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; + final String updsql2 = " UPDATE " + getTinfoPropertyDomain() + " SET " + idColumn + " = ? WHERE " + idColumn + " = ? "; + final String delSql = " DELETE FROM " + descriptorTable + " WHERE " + idColumn + " = ? "; + + new SqlSelector(getExpSchema(), sql, newProjId, projectId, container).forEach(rs -> { + int prevPropId = rs.getInt(1); + int curPropId = rs.getInt(2); + executor.execute(updsql1, curPropId, prevPropId); + executor.execute(updsql2, curPropId, prevPropId); + executor.execute(delSql, prevPropId); + }); + } + + public static void validatePropertyDescriptor(PropertyDescriptor pd) throws ChangePropertyDescriptorException + { + String name = pd.getName(); + validateValue(name, "Name", null); + validateValue(pd.getPropertyURI(), "PropertyURI", "Please use a shorter field name. Name = " + name); + validateValue(pd.getLabel(), "Label", null); + validateValue(pd.getImportAliases(), "ImportAliases", null); + validateValue(pd.getURL() != null ? pd.getURL().getSource() : null, "URL", null); + validateValue(pd.getConceptURI(), "ConceptURI", null); + validateValue(pd.getRangeURI(), "RangeURI", null); + + // Issue 15484: adding a column ending in 'mvIndicator' is problematic if another column w/ the same + // root exists, or if you later enable mvIndicators on a column w/ the same root + if (pd.getName() != null && pd.getName().toLowerCase().endsWith(MV_INDICATOR_SUFFIX)) + { + throw new ChangePropertyDescriptorException("Field name cannot end with the suffix 'mvIndicator': " + pd.getName()); + } + + if (null != name) + { + for (char ch : name.toCharArray()) + { + if (Character.isWhitespace(ch) && ' ' != ch) + throw new ChangePropertyDescriptorException("Field name cannot contain whitespace other than ' ' (space)"); + } + } + } + + private static void validateValue(String value, String columnName, String extraMessage) throws ChangePropertyDescriptorException + { + int maxLength = getTinfoPropertyDescriptor().getColumn(columnName).getScale(); + if (value != null && value.length() > maxLength) + { + throw new ChangePropertyDescriptorException(columnName + " cannot exceed " + maxLength + " characters, but was " + value.length() + " characters long. " + (extraMessage == null ? "" : extraMessage)); + } + } + + static public boolean checkObjectExistence(String lsid) + { + return new TableSelector(getTinfoObject(), new SimpleFilter(FieldKey.fromParts("ObjectURI"), lsid), null).exists(); + } +} diff --git a/api/src/org/labkey/api/util/MemTracker.java b/api/src/org/labkey/api/util/MemTracker.java index 2ad5d300a0a..6527bc68f12 100644 --- a/api/src/org/labkey/api/util/MemTracker.java +++ b/api/src/org/labkey/api/util/MemTracker.java @@ -1,486 +1,486 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.util; - -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpSession; -import org.apache.commons.collections4.map.AbstractReferenceMap.ReferenceStrength; -import org.apache.commons.collections4.map.ReferenceIdentityMap; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.RequestInfo; -import org.labkey.api.security.User; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.ValidEmail.InvalidEmailException; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewContext; - -import java.security.Principal; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.IdentityHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * Tracks objects that may be expensive, commonly allocated so that we know that they're not being held and creating - * a memory leak. Will not prevent the tracked objects from being garbage collected. - * User: brittp - * Date: Oct 27, 2005 - */ -public class MemTracker -{ - private static final MemTracker _instance = new MemTracker(); - - private final ThreadLocal _requestTracker = new ThreadLocal<>(); - private final List _recentRequests = new LinkedList<>(); - - private static final String UNVIEWED_KEY = "memtracker-unviewed-requests"; - private static final int MAX_UNVIEWED = 100; - - /** Only keep a short history of allocations for the most recent requests */ - private static final int MAX_TRACKED_REQUESTS = 500; - - public synchronized List getNewRequests(long requestId) - { - return _recentRequests.stream().filter(recentRequest -> recentRequest.getId() > requestId).toList(); - } - - static class AllocationInfo - { - @Nullable - private final StackTraceElement[] _stackTrace; - private final long _threadId; - private final long _allocTime; - - AllocationInfo() - { - this(MiniProfiler.getTroubleshootingStackTrace(), Thread.currentThread().getId(), HeartBeat.currentTimeMillis()); - } - - AllocationInfo(@Nullable StackTraceElement[] stackTrace, long threadId, long allocTime) - { - _stackTrace = stackTrace; - _threadId = threadId; - _allocTime = allocTime; - } - - public HtmlString getHtmlStack() - { - if (_stackTrace == null) - { - return HtmlString.of(MiniProfiler.NO_STACK_TRACE_AVAILABLE); - } - HtmlStringBuilder builder = HtmlStringBuilder.of(); - for (int i = 3; i < _stackTrace.length; i++) - { - String line = _stackTrace[i].toString(); - builder.append(line).append(HtmlString.BR).append("\r\n"); - if (line.contains("org.labkey.api.view.ViewServlet.service")) - break; - } - return builder.getHtmlString(); - } - - public long getThreadId() - { - return _threadId; - } - - public long getAllocationTime() - { - return _allocTime; - } - } - - - public static class HeldReference extends AllocationInfo - { - private final Object _reference; - - private HeldReference(Object held, AllocationInfo allocationInfo) - { - super(allocationInfo._stackTrace, allocationInfo._threadId, allocationInfo._allocTime); - _reference = held; - } - - - public String getClassName() - { - if (_reference instanceof Class) - return ((Class) _reference).getName(); - else - return _reference.getClass().getName(); - } - - public String getObjectSummary() - { - String desc = getObjectDescription(); - return desc.length() > 50 ? StringUtilsLabKey.leftSurrogatePairFriendly(desc, 50) + "..." : desc; - } - - public boolean hasShortSummary() - { - return getObjectDescription().length() > 50; - } - - public String getObjectDescription() - { - try - { - String toString = _reference.toString(); - if (toString == null) - return "null"; - return toString; - } - catch (Throwable e) - { - return "toString() failed: " + e.getClass().getName() + (e.getMessage() == null ? "" : (" - " + e.getMessage())); - } - } - - public Object getReference() - { - return _reference; - } - } - - public static MemTracker get() - { - return _instance; - } - - public static MemTracker getInstance() - { - return _instance; - } - - /** - * Create new RequestInfo for the current thread and request. - */ - @NotNull - public RequestInfo startProfiler(HttpServletRequest request, @Nullable String name) - { - String url = request.getRequestURI() + (request.getQueryString() == null ? "" : "?" + request.getQueryString()); - HttpSession session = request.getSession(false); - return startProfiler(url, request.getUserPrincipal(), name, session != null ? session.getId() : null); - } - - /** - * Create new RequestInfo for the current thread. - * Used for profiling background requests that will be merged into a parent profiler. - * @see #merge(RequestInfo) - */ - @NotNull - public RequestInfo startProfiler(@Nullable String name) - { - return startProfiler(null, null, name, null); - } - - /** - * Create new RequestInfo for the current thread and request. - */ - @NotNull - public synchronized RequestInfo startProfiler(String url, Principal user, @Nullable String name, @Nullable String sessionId) - { - RequestInfo req = new RequestInfo(url, user, name, sessionId); - if ((user instanceof User) && ((User) user).isSearchUser()) - req.setIgnored(true); - _requestTracker.set(req); - return req; - } - - @Nullable - public RequestInfo current() - { - return _requestTracker.get(); - } - - /** - * Finish the current profiling session and merge its results into the to RequestInfo. - * Unlike requestComplete, the current timing will not be added to the list of recent requests. - */ - public void merge(@NotNull RequestInfo to) - { - RequestInfo requestInfo = _requestTracker.get(); - if (requestInfo != null) - { - requestInfo.getRoot().stop(); - to.merge(requestInfo); - } - _requestTracker.remove(); - } - - /** - * Mark the current profiling session as ignored. Timings won't be collected. - */ - public synchronized void ignore() - { - RequestInfo requestInfo = _requestTracker.get(); - if (requestInfo != null) - requestInfo.setIgnored(true); - } - - /** - * Finish the current profiling session. - */ - public synchronized void requestComplete(RequestInfo req) - { - RequestInfo requestInfo = _requestTracker.get(); - _requestTracker.remove(); - if (req != requestInfo) - _complete(requestInfo); - _complete(req); - } - - private void _complete(RequestInfo requestInfo) - { - boolean shouldTrack = requestInfo != null && !requestInfo.isIgnored(); - if (requestInfo != null) - { - if (shouldTrack) - { - // Now that we're done, move it into the set of recent requests - _recentRequests.add(requestInfo); - trimOlderRequests(); - if (requestInfo.getUser() != null) - addUnviewed(requestInfo.getUser(), requestInfo.getId()); - } - else - { - // Remove it from the list of unviewed requests - if (requestInfo.getUser() != null) - setViewed(requestInfo.getUser(), requestInfo.getId()); - } - } - } - - private void trimOlderRequests() - { - if (_recentRequests.size() > MAX_TRACKED_REQUESTS) - { - List reqs = _recentRequests.subList(0, _recentRequests.size() - MAX_TRACKED_REQUESTS); - for (RequestInfo r : reqs) - r.cancel(); - reqs.clear(); - } - } - - private void addUnviewed(Principal user, long id) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return; - - HttpSession session = context.getSession(); - if (session == null) - return; - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - unviewed.add(id); - if (unviewed.size() > MAX_UNVIEWED) - { - unviewed.subList(0, unviewed.size() - MAX_UNVIEWED).clear(); - } - } - } - - public List getUnviewed(Principal user) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return Collections.emptyList(); - - HttpSession session = context.getSession(); - if (session == null) - return Collections.emptyList(); - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - - return new LongArrayList(unviewed); - } - } - - public void setViewed(Principal user, long id) - { - ViewContext context = HttpView.getRootContext(); - if (context == null) - return; - - HttpSession session = context.getSession(); - if (session == null) - return; - - synchronized (SessionHelper.getSessionLock(session)) - { - List unviewed = (List)session.getAttribute(UNVIEWED_KEY); - if (unviewed == null) - session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); - - unviewed.remove(id); - } - } - - @Nullable - public synchronized RequestInfo getRequest(long id) - { - // search recent requests backwards looking for the matching id - for (int i = _recentRequests.size() - 1; i > 0; i--) - { - RequestInfo req = _recentRequests.get(i); - if (req.getId() == id) - return req; - } - - return null; - } - - public boolean put(Object object) - { - assert _put(object); - return true; - } - - public boolean remove(Object object) - { - assert _remove(object); - return true; - } - - public void register(MemTrackerListener generator) - { - assert _listeners.add(generator); - } - - public void unregister(MemTrackerListener queue) - { - assert _listeners.remove(queue); - } - - public Set beforeReport() - { - Set ignorableReferences = Collections.newSetFromMap(new IdentityHashMap<>()); - - for (MemTrackerListener generator : _instance._listeners) - generator.beforeReport(ignorableReferences); - - return ignorableReferences; - } - - // Filters out threads that should be ignored (never displayed as an "Active Thread" on the Memory Usage page) - public interface ThreadFilter - { - // Return true to instruct the Memory Usage page to never display this thread as an "Active Thread" - boolean ignore(Thread thread); - } - - private final List _threadFilters = new CopyOnWriteArrayList<>(); - - public void register(ThreadFilter filter) - { - _threadFilters.add(filter); - } - - public boolean shouldDisplay(Thread thread) - { - return _threadFilters.stream().noneMatch(threadFilter -> threadFilter.ignore(thread)); - } - - // - // reference tracking impl - // - - private final Map _references = new ReferenceIdentityMap<>(ReferenceStrength.WEAK, ReferenceStrength.HARD, true); - private final List _listeners = new CopyOnWriteArrayList<>(); - - private synchronized boolean _put(Object object) - { - if (object != null) - _references.put(object, new AllocationInfo()); - MiniProfiler.addObject(object); - return true; - } - - private synchronized boolean _remove(Object object) - { - if (object != null) - _references.remove(object); - return true; - } - - public synchronized List getReferences() - { - List refs = new ArrayList<>(_references.size()); - for (Map.Entry entry : _references.entrySet()) - { - // get a hard reference so we know that we're placing an actual object into our list: - Object obj = entry.getKey(); - if (obj != null) - refs.add(new HeldReference(entry.getKey(), entry.getValue())); - } - refs.sort(Comparator.comparing(HeldReference::getClassName, String.CASE_INSENSITIVE_ORDER)); - return refs; - } - - public static class TestCase extends Assert - { - @Test - public void testIdentity() throws InvalidEmailException - { - MemTracker t = new MemTracker(); - - // test identity - Object a = "I'm me"; - t._put(a); - assertEquals(1, t.getReferences().size()); - t._put(a); - assertEquals(1, t.getReferences().size()); - - // Test with arbitrary class that implements equals() - Object b = new ValidEmail("test@test.com"); - Object c = new ValidEmail("test@test.com"); - assertNotSame(b, c); - assertEquals(b, c); - t._put(b); - assertEquals(2, t.getReferences().size()); - t._put(c); - assertEquals(3, t.getReferences().size()); - - List list = t.getReferences(); - for (HeldReference o : list) - { - assertTrue(o._reference == a || o._reference == b || o._reference == c); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.util; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpSession; +import org.apache.commons.collections4.map.AbstractReferenceMap.ReferenceStrength; +import org.apache.commons.collections4.map.ReferenceIdentityMap; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.RequestInfo; +import org.labkey.api.security.User; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.ValidEmail.InvalidEmailException; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; + +import java.security.Principal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Tracks objects that may be expensive, commonly allocated so that we know that they're not being held and creating + * a memory leak. Will not prevent the tracked objects from being garbage collected. + * User: brittp + * Date: Oct 27, 2005 + */ +public class MemTracker +{ + private static final MemTracker _instance = new MemTracker(); + + private final ThreadLocal _requestTracker = new ThreadLocal<>(); + private final List _recentRequests = new LinkedList<>(); + + private static final String UNVIEWED_KEY = "memtracker-unviewed-requests"; + private static final int MAX_UNVIEWED = 100; + + /** Only keep a short history of allocations for the most recent requests */ + private static final int MAX_TRACKED_REQUESTS = 500; + + public synchronized List getNewRequests(long requestId) + { + return _recentRequests.stream().filter(recentRequest -> recentRequest.getId() > requestId).toList(); + } + + static class AllocationInfo + { + @Nullable + private final StackTraceElement[] _stackTrace; + private final long _threadId; + private final long _allocTime; + + AllocationInfo() + { + this(MiniProfiler.getTroubleshootingStackTrace(), Thread.currentThread().getId(), HeartBeat.currentTimeMillis()); + } + + AllocationInfo(@Nullable StackTraceElement[] stackTrace, long threadId, long allocTime) + { + _stackTrace = stackTrace; + _threadId = threadId; + _allocTime = allocTime; + } + + public HtmlString getHtmlStack() + { + if (_stackTrace == null) + { + return HtmlString.of(MiniProfiler.NO_STACK_TRACE_AVAILABLE); + } + HtmlStringBuilder builder = HtmlStringBuilder.of(); + for (int i = 3; i < _stackTrace.length; i++) + { + String line = _stackTrace[i].toString(); + builder.append(line).append(HtmlString.BR).append("\r\n"); + if (line.contains("org.labkey.api.view.ViewServlet.service")) + break; + } + return builder.getHtmlString(); + } + + public long getThreadId() + { + return _threadId; + } + + public long getAllocationTime() + { + return _allocTime; + } + } + + + public static class HeldReference extends AllocationInfo + { + private final Object _reference; + + private HeldReference(Object held, AllocationInfo allocationInfo) + { + super(allocationInfo._stackTrace, allocationInfo._threadId, allocationInfo._allocTime); + _reference = held; + } + + + public String getClassName() + { + if (_reference instanceof Class) + return ((Class) _reference).getName(); + else + return _reference.getClass().getName(); + } + + public String getObjectSummary() + { + String desc = getObjectDescription(); + return desc.length() > 50 ? StringUtilsLabKey.leftSurrogatePairFriendly(desc, 50) + "..." : desc; + } + + public boolean hasShortSummary() + { + return getObjectDescription().length() > 50; + } + + public String getObjectDescription() + { + try + { + String toString = _reference.toString(); + if (toString == null) + return "null"; + return toString; + } + catch (Throwable e) + { + return "toString() failed: " + e.getClass().getName() + (e.getMessage() == null ? "" : (" - " + e.getMessage())); + } + } + + public Object getReference() + { + return _reference; + } + } + + public static MemTracker get() + { + return _instance; + } + + public static MemTracker getInstance() + { + return _instance; + } + + /** + * Create new RequestInfo for the current thread and request. + */ + @NotNull + public RequestInfo startProfiler(HttpServletRequest request, @Nullable String name) + { + String url = request.getRequestURI() + (request.getQueryString() == null ? "" : "?" + request.getQueryString()); + HttpSession session = request.getSession(false); + return startProfiler(url, request.getUserPrincipal(), name, session != null ? session.getId() : null); + } + + /** + * Create new RequestInfo for the current thread. + * Used for profiling background requests that will be merged into a parent profiler. + * @see #merge(RequestInfo) + */ + @NotNull + public RequestInfo startProfiler(@Nullable String name) + { + return startProfiler(null, null, name, null); + } + + /** + * Create new RequestInfo for the current thread and request. + */ + @NotNull + public synchronized RequestInfo startProfiler(String url, Principal user, @Nullable String name, @Nullable String sessionId) + { + RequestInfo req = new RequestInfo(url, user, name, sessionId); + if ((user instanceof User) && ((User) user).isSearchUser()) + req.setIgnored(true); + _requestTracker.set(req); + return req; + } + + @Nullable + public RequestInfo current() + { + return _requestTracker.get(); + } + + /** + * Finish the current profiling session and merge its results into the to RequestInfo. + * Unlike requestComplete, the current timing will not be added to the list of recent requests. + */ + public void merge(@NotNull RequestInfo to) + { + RequestInfo requestInfo = _requestTracker.get(); + if (requestInfo != null) + { + requestInfo.getRoot().stop(); + to.merge(requestInfo); + } + _requestTracker.remove(); + } + + /** + * Mark the current profiling session as ignored. Timings won't be collected. + */ + public synchronized void ignore() + { + RequestInfo requestInfo = _requestTracker.get(); + if (requestInfo != null) + requestInfo.setIgnored(true); + } + + /** + * Finish the current profiling session. + */ + public synchronized void requestComplete(RequestInfo req) + { + RequestInfo requestInfo = _requestTracker.get(); + _requestTracker.remove(); + if (req != requestInfo) + _complete(requestInfo); + _complete(req); + } + + private void _complete(RequestInfo requestInfo) + { + boolean shouldTrack = requestInfo != null && !requestInfo.isIgnored(); + if (requestInfo != null) + { + if (shouldTrack) + { + // Now that we're done, move it into the set of recent requests + _recentRequests.add(requestInfo); + trimOlderRequests(); + if (requestInfo.getUser() != null) + addUnviewed(requestInfo.getUser(), requestInfo.getId()); + } + else + { + // Remove it from the list of unviewed requests + if (requestInfo.getUser() != null) + setViewed(requestInfo.getUser(), requestInfo.getId()); + } + } + } + + private void trimOlderRequests() + { + if (_recentRequests.size() > MAX_TRACKED_REQUESTS) + { + List reqs = _recentRequests.subList(0, _recentRequests.size() - MAX_TRACKED_REQUESTS); + for (RequestInfo r : reqs) + r.cancel(); + reqs.clear(); + } + } + + private void addUnviewed(Principal user, long id) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return; + + HttpSession session = context.getSession(); + if (session == null) + return; + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + unviewed.add(id); + if (unviewed.size() > MAX_UNVIEWED) + { + unviewed.subList(0, unviewed.size() - MAX_UNVIEWED).clear(); + } + } + } + + public List getUnviewed(Principal user) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return Collections.emptyList(); + + HttpSession session = context.getSession(); + if (session == null) + return Collections.emptyList(); + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + + return new LongArrayList(unviewed); + } + } + + public void setViewed(Principal user, long id) + { + ViewContext context = HttpView.getRootContext(); + if (context == null) + return; + + HttpSession session = context.getSession(); + if (session == null) + return; + + synchronized (SessionHelper.getSessionLock(session)) + { + List unviewed = (List)session.getAttribute(UNVIEWED_KEY); + if (unviewed == null) + session.setAttribute(UNVIEWED_KEY, unviewed = new ArrayList<>()); + + unviewed.remove(id); + } + } + + @Nullable + public synchronized RequestInfo getRequest(long id) + { + // search recent requests backwards looking for the matching id + for (int i = _recentRequests.size() - 1; i > 0; i--) + { + RequestInfo req = _recentRequests.get(i); + if (req.getId() == id) + return req; + } + + return null; + } + + public boolean put(Object object) + { + assert _put(object); + return true; + } + + public boolean remove(Object object) + { + assert _remove(object); + return true; + } + + public void register(MemTrackerListener generator) + { + assert _listeners.add(generator); + } + + public void unregister(MemTrackerListener queue) + { + assert _listeners.remove(queue); + } + + public Set beforeReport() + { + Set ignorableReferences = Collections.newSetFromMap(new IdentityHashMap<>()); + + for (MemTrackerListener generator : _instance._listeners) + generator.beforeReport(ignorableReferences); + + return ignorableReferences; + } + + // Filters out threads that should be ignored (never displayed as an "Active Thread" on the Memory Usage page) + public interface ThreadFilter + { + // Return true to instruct the Memory Usage page to never display this thread as an "Active Thread" + boolean ignore(Thread thread); + } + + private final List _threadFilters = new CopyOnWriteArrayList<>(); + + public void register(ThreadFilter filter) + { + _threadFilters.add(filter); + } + + public boolean shouldDisplay(Thread thread) + { + return _threadFilters.stream().noneMatch(threadFilter -> threadFilter.ignore(thread)); + } + + // + // reference tracking impl + // + + private final Map _references = new ReferenceIdentityMap<>(ReferenceStrength.WEAK, ReferenceStrength.HARD, true); + private final List _listeners = new CopyOnWriteArrayList<>(); + + private synchronized boolean _put(Object object) + { + if (object != null) + _references.put(object, new AllocationInfo()); + MiniProfiler.addObject(object); + return true; + } + + private synchronized boolean _remove(Object object) + { + if (object != null) + _references.remove(object); + return true; + } + + public synchronized List getReferences() + { + List refs = new ArrayList<>(_references.size()); + for (Map.Entry entry : _references.entrySet()) + { + // get a hard reference so we know that we're placing an actual object into our list: + Object obj = entry.getKey(); + if (obj != null) + refs.add(new HeldReference(entry.getKey(), entry.getValue())); + } + refs.sort(Comparator.comparing(HeldReference::getClassName, String.CASE_INSENSITIVE_ORDER)); + return refs; + } + + public static class TestCase extends Assert + { + @Test + public void testIdentity() throws InvalidEmailException + { + MemTracker t = new MemTracker(); + + // test identity + Object a = "I'm me"; + t._put(a); + assertEquals(1, t.getReferences().size()); + t._put(a); + assertEquals(1, t.getReferences().size()); + + // Test with arbitrary class that implements equals() + Object b = new ValidEmail("test@test.com"); + Object c = new ValidEmail("test@test.com"); + assertNotSame(b, c); + assertEquals(b, c); + t._put(b); + assertEquals(2, t.getReferences().size()); + t._put(c); + assertEquals(3, t.getReferences().size()); + + List list = t.getReferences(); + for (HeldReference o : list) + { + assertTrue(o._reference == a || o._reference == b || o._reference == c); + } + } + } +} diff --git a/core/src/org/labkey/core/admin/AdminController.java b/core/src/org/labkey/core/admin/AdminController.java index 7e80a2b2af3..c62f95ec042 100644 --- a/core/src/org/labkey/core/admin/AdminController.java +++ b/core/src/org/labkey/core/admin/AdminController.java @@ -1,12273 +1,12273 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.core.admin; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.util.concurrent.UncheckedExecutionException; -import jakarta.mail.MessagingException; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.map.LRUMap; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.xmlbeans.XmlOptions; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.jfree.chart.ChartFactory; -import org.jfree.chart.ChartUtilities; -import org.jfree.chart.JFreeChart; -import org.jfree.chart.plot.PlotOrientation; -import org.jfree.data.category.DefaultCategoryDataset; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.Constants; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.BaseApiAction; -import org.labkey.api.action.BaseViewAction; -import org.labkey.api.action.ConfirmAction; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasViewContext; -import org.labkey.api.action.IgnoresAllocationTracking; -import org.labkey.api.action.LabKeyError; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.QueryViewAction; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AbstractFolderContext.ExportType; -import org.labkey.api.admin.AdminBean; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.admin.FolderExportContext; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.admin.FolderWriter; -import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.HealthCheck; -import org.labkey.api.admin.HealthCheckRegistry; -import org.labkey.api.admin.ImportOptions; -import org.labkey.api.admin.StaticLoggerGetter; -import org.labkey.api.admin.TableXmlUtils; -import org.labkey.api.admin.sitevalidation.SiteValidationResult; -import org.labkey.api.admin.sitevalidation.SiteValidationResultList; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.cache.CacheStats; -import org.labkey.api.cache.TrackingCache; -import org.labkey.api.cloud.CloudStoreService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.compliance.ComplianceFolderSettings; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.compliance.PhiColumnBehavior; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.ConnectionWrapper; -import org.labkey.api.data.Container; -import org.labkey.api.data.Container.ContainerException; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DataColumn; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DatabaseTableType; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MenuButton; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.NormalContainerType; -import org.labkey.api.data.PHI; -import org.labkey.api.data.RenderContext; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TransactionFilter; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; -import org.labkey.api.data.queryprofiler.QueryProfiler; -import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.files.FileContentService; -import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; -import org.labkey.api.message.settings.MessageConfigService; -import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; -import org.labkey.api.message.settings.MessageConfigService.NotificationOption; -import org.labkey.api.message.settings.MessageConfigService.UserPreference; -import org.labkey.api.miniprofiler.RequestInfo; -import org.labkey.api.module.AllowedBeforeInitialUserIsSet; -import org.labkey.api.module.AllowedDuringUpgrade; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.FolderType; -import org.labkey.api.module.FolderTypeManager; -import org.labkey.api.module.IgnoresForbiddenProjectCheck; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.ModuleLoader.SchemaActions; -import org.labkey.api.module.ModuleLoader.SchemaAndModule; -import org.labkey.api.module.SimpleModule; -import org.labkey.api.moduleeditor.api.ModuleEditorService; -import org.labkey.api.pipeline.DirectoryNotDeletedException; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineStatusFile; -import org.labkey.api.pipeline.PipelineStatusUrls; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.pipeline.view.SetupForm; -import org.labkey.api.products.ProductRegistry; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RuntimeValidationException; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.ExternalScriptEngineDefinition; -import org.labkey.api.reports.LabKeyScriptEngineManager; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.CSRF; -import org.labkey.api.security.Directive; -import org.labkey.api.security.Group; -import org.labkey.api.security.GroupManager; -import org.labkey.api.security.IgnoresTermsOfUse; -import org.labkey.api.security.LoginUrls; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.RequiresLogin; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.RoleAssignment; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicy; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.SecurityUrls; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; -import org.labkey.api.security.impersonation.ImpersonationContext; -import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; -import org.labkey.api.security.impersonation.UserImpersonationContextFactory; -import org.labkey.api.security.permissions.AbstractActionPermissionTest; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.CreateProjectPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.SiteAdminPermission; -import org.labkey.api.security.permissions.TroubleshooterPermission; -import org.labkey.api.security.permissions.UploadFileBasedModulePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.security.roles.FolderAdminRole; -import org.labkey.api.security.roles.ProjectAdminRole; -import org.labkey.api.security.roles.ReaderRole; -import org.labkey.api.security.roles.Role; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.security.roles.SharedViewEditorRole; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.ConceptURIProperties; -import org.labkey.api.settings.DateParsingMode; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; -import org.labkey.api.settings.NetworkDriveProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.settings.OptionalFeatureService.FeatureType; -import org.labkey.api.settings.ProductConfiguration; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; -import org.labkey.api.settings.WriteableLookAndFeelProperties; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.ConfigurationException; -import org.labkey.api.util.DOM; -import org.labkey.api.util.DOM.Renderable; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.DebugInfoDumper; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.FolderDisplayMode; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.HttpsUtil; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.MailHelper; -import org.labkey.api.util.MemTracker; -import org.labkey.api.util.MemTracker.HeldReference; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.SafeToRenderEnum; -import org.labkey.api.util.SessionAppender; -import org.labkey.api.util.StringExpressionFactory; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; -import org.labkey.api.util.SystemMaintenanceJob; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.Tuple3; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UniqueID; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.util.emailTemplate.EmailTemplate; -import org.labkey.api.util.emailTemplate.EmailTemplateService; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.DataView; -import org.labkey.api.view.FolderManagement.FolderManagementViewAction; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; -import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; -import org.labkey.api.view.FolderManagement.TYPE; -import org.labkey.api.view.FolderTab; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.Portal; -import org.labkey.api.view.RedirectException; -import org.labkey.api.view.ShortURLRecord; -import org.labkey.api.view.ShortURLService; -import org.labkey.api.view.TabStripView; -import org.labkey.api.view.URLException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.EmptyView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.view.template.PageConfig.Template; -import org.labkey.api.wiki.WikiRendererType; -import org.labkey.api.wiki.WikiRenderingService; -import org.labkey.api.writer.FileSystemFile; -import org.labkey.api.writer.HtmlWriter; -import org.labkey.api.writer.ZipFile; -import org.labkey.api.writer.ZipUtil; -import org.labkey.bootstrap.ExplodedModuleService; -import org.labkey.core.admin.miniprofiler.MiniProfilerController; -import org.labkey.core.admin.sitevalidation.SiteValidationJob; -import org.labkey.core.admin.sql.SqlScriptController; -import org.labkey.core.login.LoginController; -import org.labkey.core.portal.CollaborationFolderType; -import org.labkey.core.portal.ProjectController; -import org.labkey.core.query.CoreQuerySchema; -import org.labkey.core.query.PostgresUserSchema; -import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; -import org.labkey.core.security.AllowedExternalResourceHosts; -import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; -import org.labkey.core.security.BlockListFilter; -import org.labkey.core.security.SecurityController; -import org.labkey.data.xml.TablesDocument; -import org.labkey.filters.ContentSecurityPolicyFilter; -import org.labkey.security.xml.GroupEnumType; -import org.labkey.vfs.FileLike; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.mvc.Controller; - -import java.awt.*; -import java.beans.Introspector; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.lang.management.BufferPoolMXBean; -import java.lang.management.ClassLoadingMXBean; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryType; -import java.lang.management.MemoryUsage; -import java.lang.management.OperatingSystemMXBean; -import java.lang.management.RuntimeMXBean; -import java.lang.management.ThreadMXBean; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.SQLException; -import java.text.DecimalFormat; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; -import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; -import static org.labkey.api.util.DOM.A; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.Attribute.method; -import static org.labkey.api.util.DOM.Attribute.name; -import static org.labkey.api.util.DOM.Attribute.style; -import static org.labkey.api.util.DOM.Attribute.title; -import static org.labkey.api.util.DOM.Attribute.type; -import static org.labkey.api.util.DOM.Attribute.value; -import static org.labkey.api.util.DOM.BR; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.LI; -import static org.labkey.api.util.DOM.SPAN; -import static org.labkey.api.util.DOM.STYLE; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.UL; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.api.util.DOM.createHtmlFragment; -import static org.labkey.api.util.HtmlString.NBSP; -import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; -import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; -import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; -import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; -import static org.labkey.api.view.FolderManagement.NOT_ROOT; -import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; -import static org.labkey.api.view.FolderManagement.ROOT; -import static org.labkey.api.view.FolderManagement.addTab; - -public class AdminController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( - AdminController.class, - FileListAction.class, - FilesSiteSettingsAction.class, - UpdateFilePathsAction.class - ); - - private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); - private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); - private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; - - private static long _errorMark = 0; - private static long _primaryLogMark = 0; - - public static void registerAdminConsoleLinks() - { - Container root = ContainerManager.getRoot(); - - // Configuration - AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); - AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); - if (!ProductRegistry.getProducts().isEmpty()) - AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); - // TODO move to FileContentModule - if (ModuleLoader.getInstance().hasModule("FileContent")) - AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); - AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); - AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); - AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); - AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); - AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); - AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); - - // Diagnostics - AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); - AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); - AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); - AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); - AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); - AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); - - if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); - AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); - AdminConsole.addLink(Diagnostics, "postgres table sizes", new ActionURL(PostgresTableSizesAction.class, root)); - } - - AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); - AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); - AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); - AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); - AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); - AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); - AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); - AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); - AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); - AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); - AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); - } - - public static void registerManagementTabs() - { - addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); - addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); - addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); - addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { - if (!c.isRoot()) - { - // Show module properties tab only if a module w/ properties to set is present for current folder - for (Module m : c.getActiveModules()) - if (!m.getModuleProperties().isEmpty()) - return true; - } - - return false; - }, ModulePropertiesAction.class); - addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { - // Show Concepts tab only if the experiment module is enabled in this container - return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); - }, AdminController.ConceptsAction.class); - // Show Notifications tab only if we have registered notification providers - addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); - addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); - addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); - addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); - addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); - addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); - addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); - - addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); - addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); - addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); - addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); - - addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); - addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); - } - - public AdminController() - { - setActionResolver(_actionResolver); - } - - @RequiresNoPermission - public static class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getShowAdminURL(); - } - } - - private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) - { - addAdminNavTrail(root, childTitle, action, getContainer()); - } - - private static void addAdminNavTrail(NavTree root, @NotNull Container container) - { - if (container.isRoot()) - root.addChild("Admin Console", getShowAdminURL().setFragment("links")); - } - - private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - addAdminNavTrail(root, container); - root.addChild(childTitle, new ActionURL(action, container)); - } - - public static ActionURL getShowAdminURL() - { - return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); - } - - @Override - protected void beforeAction(Controller action) throws ServletException - { - super.beforeAction(action); - if (action instanceof BaseViewAction viewaction) - viewaction.getPageConfig().setRobotsNone(); - } - - @AdminConsoleAction - public static class ShowAdminAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/admin.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); - if (null != returnUrl) - root.addChild("Return to Project", returnUrl); - root.addChild("Admin Console"); - setHelpTopic("siteManagement"); - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ShowModuleErrorsAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Module Errors", this.getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); - } - } - - public static class AdminUrlsImpl implements AdminUrls - { - @Override - public ActionURL getModuleErrorsURL() - { - return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getAdminConsoleURL() - { - return getShowAdminURL(); - } - - @Override - public ActionURL getModuleStatusURL(URLHelper returnUrl) - { - return AdminController.getModuleStatusURL(returnUrl); - } - - @Override - public ActionURL getCustomizeSiteURL() - { - return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) - { - ActionURL url = getCustomizeSiteURL(); - - if (upgradeInProgress) - url.addParameter("upgradeInProgress", "1"); - - return url; - } - - @Override - public ActionURL getProjectSettingsURL(Container c) - { - return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - ActionURL getLookAndFeelResourcesURL(Container c) - { - return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsMenuURL(Container c) - { - return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getProjectSettingsFileURL(Container c) - { - return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); - } - - @Override - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) - { - return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); - } - - public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) - { - ActionURL url = new ActionURL(CustomizeEmailAction.class, c); - if (selectedTemplate != null) - { - url.addParameter("templateClass", selectedTemplate); - } - if (returnUrl != null) - { - url.addReturnUrl(returnUrl); - } - return url; - } - - public ActionURL getResetLookAndFeelPropertiesURL(Container c) - { - return new ActionURL(ResetPropertiesAction.class, c); - } - - @Override - public ActionURL getMaintenanceURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - @Override - public ActionURL getModulesDetailsURL() - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getDeleteModuleURL(String moduleName) - { - return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); - } - - @Override - public ActionURL getManageFoldersURL(Container c) - { - return new ActionURL(ManageFoldersAction.class, c); - } - - @Override - public ActionURL getFolderTypeURL(Container c) - { - return new ActionURL(FolderTypeAction.class, c); - } - - @Override - public ActionURL getExportFolderURL(Container c) - { - return new ActionURL(ExportFolderAction.class, c); - } - - @Override - public ActionURL getImportFolderURL(Container c) - { - return new ActionURL(ImportFolderAction.class, c); - } - - @Override - public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) - { - return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); - } - - @Override - public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) - { - ActionURL result = new ActionURL(CreateFolderAction.class, c); - if (returnUrl != null) - { - result.addReturnUrl(returnUrl); - } - return result; - } - - public ActionURL getSetFolderPermissionsURL(Container c) - { - return new ActionURL(SetFolderPermissionsAction.class, c); - } - - @Override - public void addAdminNavTrail(NavTree root, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, container); - } - - @Override - public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) - { - AdminController.addAdminNavTrail(root, childTitle, action, container); - } - - @Override - public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) - { - if (container.isRoot()) - addAdminNavTrail(root, "Modules", ModulesAction.class, container); - - root.addChild(childTitle); - } - - @Override - public ActionURL getFileRootsURL(Container c) - { - return new ActionURL(FileRootsAction.class, c); - } - - @Override - public ActionURL getLookAndFeelSettingsURL(Container c) - { - if (c.isRoot()) - return getSiteLookAndFeelSettingsURL(); - else if (c.isProject()) - return getProjectSettingsURL(c); - else - return getFolderSettingsURL(c); - } - - @Override - public ActionURL getSiteLookAndFeelSettingsURL() - { - return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFolderSettingsURL(Container c) - { - return new ActionURL(FolderSettingsAction.class, c); - } - - @Override - public ActionURL getNotificationsURL(Container c) - { - return new ActionURL(NotificationsAction.class, c); - } - - @Override - public ActionURL getModulePropertiesURL(Container c) - { - return new ActionURL(ModulePropertiesAction.class, c); - } - - @Override - public ActionURL getMissingValuesURL(Container c) - { - return new ActionURL(MissingValuesAction.class, c); - } - - public ActionURL getInitialFolderSettingsURL(Container c) - { - return new ActionURL(SetInitialFolderSettingsAction.class, c); - } - - @Override - public ActionURL getMemTrackerURL() - { - return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getFilesSiteSettingsURL() - { - return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSessionLoggingURL() - { - return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getTrackedAllocationsViewerURL() - { - return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); - } - - @Override - public ActionURL getSystemMaintenanceURL() - { - return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); - } - - public static ActionURL getDeprecatedFeaturesURL() - { - return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); - } - } - - public static class MaintenanceBean - { - public HtmlString content; - public ActionURL loginURL; - } - - /** - * During upgrade, startup, or maintenance mode, the user will be redirected to - * MaintenanceAction and only admin users will be allowed to log into the server. - * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off - * and will redirect to the returnUrl or the loginURL. - * See Issue 18758 for more information. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class MaintenanceAction extends SimpleViewAction - { - private String _title = "Maintenance in progress"; - - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - if (!getUser().hasSiteAdminPermission()) - { - getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); - } - getPageConfig().setTemplate(Template.Dialog); - - boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); - boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); - boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); - - HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); - if (upgradeInProgress) - { - _title = "Upgrade in progress"; - content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (startupInProgress) - { - _title = "Startup in progress"; - content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); - } - else if (maintenanceMode) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); - } - - if (content == null) - content = HtmlString.of(_title); - - ActionURL loginURL = null; - if (getUser().isGuest()) - { - URLHelper returnUrl = form.getReturnUrlHelper(); - if (returnUrl != null) - loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); - else - loginURL = urlProvider(LoginUrls.class).getLoginURL(); - } - - MaintenanceBean bean = new MaintenanceBean(); - bean.content = content; - bean.loginURL = loginURL; - - JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); - view.setTitle(_title); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild(_title); - } - } - - /** - * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. - */ - @RequiresNoPermission - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class StartupStatusAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); - result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); - - return new ApiSimpleResponse(result); - } - } - - @RequiresSiteAdmin - @IgnoresTermsOfUse - public static class GetPendingRequestCountAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - JSONObject result = new JSONObject(); - result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); - - return new ApiSimpleResponse(result); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetModulesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(GetModulesForm form, BindException errors) - { - Container c = ContainerManager.getForPath(getContainer().getPath()); - - ApiSimpleResponse response = new ApiSimpleResponse(); - - List> qinfos = new ArrayList<>(); - - FolderType folderType = c.getFolderType(); - List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); - - //note: this has been altered to use Container.getRequiredModules() instead of FolderType - //this is b/c a parent container must consider child workbooks when determining the set of requiredModules - Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); - Set activeModules = c.getActiveModules(getUser()); - - for (Module m : allModules) - { - Map qinfo = new HashMap<>(); - - qinfo.put("name", m.getName()); - qinfo.put("required", requiredModules.contains(m)); - qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); - qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || - m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); - qinfo.put("tabName", m.getTabName(getViewContext())); - qinfo.put("requireSitePermission", m.getRequireSitePermission()); - qinfos.add(qinfo); - } - - response.put("modules", qinfos); - response.put("folderType", folderType.getName()); - - return response; - } - } - - public static class GetModulesForm - { - } - - @RequiresNoPermission - @AllowedDuringUpgrade - // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress - public static class GuidAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - response.getWriter().write(GUID.makeGUID()); - } - } - - /** - * Preform health checks corresponding to the given categories. - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class HealthCheckAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception - { - if (!ModuleLoader.getInstance().isStartupComplete()) - return new ApiSimpleResponse("healthy", false); - - Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); - HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); - - checkResult.getDetails().put("healthy", checkResult.isHealthy()); - - if (getUser().hasRootAdminPermission()) - { - return new ApiSimpleResponse(checkResult.getDetails()); - } - else - { - if (!checkResult.isHealthy()) - { - try (var writer = createResponseWriter()) - { - writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); - } - return null; - } - - return new ApiSimpleResponse("healthy", checkResult.isHealthy()); - } - } - } - - public static class HealthCheckForm - { - private String _categories; // if null, all categories will be checked. - - public String getCategories() - { - return _categories; - } - - @SuppressWarnings("unused") - public void setCategories(String categories) - { - _categories = categories; - } - } - - // No security checks... anyone (even guests) can view the credits page - @RequiresNoPermission - public class CreditsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - VBox views = new VBox(); - List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); - modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); - - addCreditsViews(views, modules, "jars.txt", "JAR"); - addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); - addCreditsViews(views, modules, "source.txt", "Java Source Code"); - addCreditsViews(views, modules, "executables.txt", "Executable"); - - return views; - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Credits", this.getClass()); - } - } - - private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException - { - for (Module module : modules) - { - String wikiSource = getCreditsFile(module, creditsFile); - - if (null != wikiSource) - { - String title = fileType + " Files Distributed with the " + module.getName() + " Module"; - CreditsView credits = new CreditsView(wikiSource, title); - views.addView(credits); - } - } - } - - private static class CreditsView extends WebPartView - { - private Renderable _html; - - CreditsView(@Nullable String wikiSource, String title) - { - super(title); - - wikiSource = StringUtils.trimToEmpty(wikiSource); - - if (StringUtils.isNotEmpty(wikiSource)) - { - WikiRenderingService wikiService = WikiRenderingService.get(); - HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); - _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); - } - } - - @Override - public void renderView(Object model, HtmlWriter out) - { - out.write(_html); - } - } - - private static String getCreditsFile(Module module, String filename) throws IOException - { - // credits files are in /resources/credits - InputStream is = module.getResourceStream("credits/" + filename); - - return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); - } - - private void validateNetworkDrive(NetworkDriveForm form, Errors errors) - { - if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || - isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) - { - errors.reject(ERROR_MSG, "All fields are required"); - } - else if (form.getNetworkDriveLetter().trim().length() > 1) - { - errors.reject(ERROR_MSG, "Network drive letter must be a single character"); - } - else - { - char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); - - if (letter < 'a' || letter > 'z') - { - errors.reject(ERROR_MSG, "Network drive letter must be a letter"); - } - } - } - - public static class ResourceForm - { - private String _resource; - - public String getResource() - { - return _resource; - } - - public void setResource(String resource) - { - _resource = resource; - } - - public ResourceType getResourceType() - { - return ResourceType.valueOf(_resource); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetResourceAction extends FormHandlerAction - { - @Override - public void validateCommand(ResourceForm target, Errors errors) - { - } - - @Override - public boolean handlePost(ResourceForm form, BindException errors) throws Exception - { - form.getResourceType().delete(getContainer(), getUser()); - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - return true; - } - - @Override - public URLHelper getSuccessURL(ResourceForm form) - { - return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResetPropertiesAction extends FormHandlerAction - { - private URLHelper _returnUrl; - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - Container c = getContainer(); - boolean folder = !(c.isRoot() || c.isProject()); - boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); - - WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); - props.clear(hasAdminOpsPerm); - props.save(); - // TODO: Audit log? - - AdminUrls urls = new AdminUrlsImpl(); - - // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision - if (!folder) - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - _returnUrl = urls.getLookAndFeelSettingsURL(c); - - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return _returnUrl; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class CustomizeSiteAction extends FormViewAction - { - @Override - public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) - { - if (form.isUpgradeInProgress()) - getPageConfig().setTemplate(Template.Dialog); - - SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); - setHelpTopic("configAdmin"); - return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Customize Site", this.getClass()); - } - - @Override - public void validateCommand(SiteSettingsForm form, Errors errors) - { - if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) - { - errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); - } - if (form.getMaxBLOBSize() < 0) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); - } - int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); - if (form.getMaxBLOBSize() > hardCap) - { - errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); - } - if (form.getSslPort() < 1 || form.getSslPort() > 65535) - { - errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); - } - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); - } - if (form.getMemoryUsageDumpInterval() < 0) - { - errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); - } - } - - @Override - public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception - { - HttpServletRequest request = getViewContext().getRequest(); - - // We only need to check that SSL is running if the user isn't already using SSL - if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) - { - URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); - Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); - - if (sslResponse != null) - { - errors.reject(ERROR_MSG, sslResponse.first); - return false; - } - } - - if (form.getReadOnlyHttpRequestTimeout() < 0) - { - errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); - } - - WriteableAppProps props = AppProps.getWriteableInstance(); - - props.setPipelineToolsDir(form.getPipelineToolsDirectory()); - props.setNavAccessOpen(form.isNavAccessOpen()); - props.setSSLRequired(form.isSslRequired()); - boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); - props.setSSLPort(form.getSslPort()); - props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); - props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); - props.setMaxBLOBSize(form.getMaxBLOBSize()); - props.setExt3Required(form.isExt3Required()); - props.setExt3APIRequired(form.isExt3APIRequired()); - props.setSelfReportExceptions(form.isSelfReportExceptions()); - - props.setAdminOnlyMessage(form.getAdminOnlyMessage()); - props.setShowRibbonMessage(form.isShowRibbonMessage()); - props.setRibbonMessage(form.getRibbonMessage()); - props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); - - props.setAllowApiKeys(form.isAllowApiKeys()); - props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); - props.setAllowSessionKeys(form.isAllowSessionKeys()); - - try - { - ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); - props.setExceptionReportingLevel(level); - } - catch (IllegalArgumentException ignored) - { - } - - try - { - if (form.getUsageReportingLevel() != null) - { - UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); - props.setUsageReportingLevel(level); - } - } - catch (IllegalArgumentException ignored) - { - } - - props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); - - if (null != form.getBaseServerURL()) - { - if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); - return false; - } - - try - { - props.setBaseServerUrl(form.getBaseServerURL()); - } - catch (URISyntaxException e) - { - errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + - "Please enter a valid base URL containing the protocol, hostname, and port if required. " + - "The webapp context path should not be included. " + - "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); - return false; - } - } - - String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); - if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) - { - errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); - return false; - } - props.setXFrameOption(frameOption); - props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); - - props.save(getViewContext().getUser()); - UsageReportingLevel.reportNow(); - if (sslSettingChanged) - ContentSecurityPolicyFilter.regenerateSubstitutionMap(); - - return true; - } - - @Override - public ActionURL getSuccessURL(SiteSettingsForm form) - { - if (form.isUpgradeInProgress()) - { - return AppProps.getInstance().getHomePageActionURL(); - } - else - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - } - - public static class NetworkDriveForm - { - private String _networkDriveLetter; - private String _networkDrivePath; - private String _networkDriveUser; - private String _networkDrivePassword; - - public String getNetworkDriveLetter() - { - return _networkDriveLetter; - } - - public void setNetworkDriveLetter(String networkDriveLetter) - { - _networkDriveLetter = networkDriveLetter; - } - - public String getNetworkDrivePassword() - { - return _networkDrivePassword; - } - - public void setNetworkDrivePassword(String networkDrivePassword) - { - _networkDrivePassword = networkDrivePassword; - } - - public String getNetworkDrivePath() - { - return _networkDrivePath; - } - - public void setNetworkDrivePath(String networkDrivePath) - { - _networkDrivePath = networkDrivePath; - } - - public String getNetworkDriveUser() - { - return _networkDriveUser; - } - - public void setNetworkDriveUser(String networkDriveUser) - { - _networkDriveUser = networkDriveUser; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @AdminConsoleAction - public class MapNetworkDriveAction extends FormViewAction - { - @Override - public void validateCommand(NetworkDriveForm form, Errors errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); - } - - @Override - public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception - { - NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); - NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); - NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); - NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); - - return true; - } - - @Override - public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) - { - return new ActionURL(FilesSiteSettingsAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("setRoots#map"); - addAdminNavTrail(root, "Map Network Drive", this.getClass()); - } - } - - public static class SiteSettingsBean - { - public final boolean _upgradeInProgress; - public final boolean _showSelfReportExceptions; - - private SiteSettingsBean(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); - } - - public HtmlString getSiteSettingsHelpLink(String fragment) - { - return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); - } - } - - public static class SetRibbonMessageForm - { - private Boolean _show = null; - private String _message = null; - - public Boolean isShow() - { - return _show; - } - - public void setShow(Boolean show) - { - _show = show; - } - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class SetRibbonMessageAction extends MutatingApiAction - { - @Override - public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception - { - if (form.isShow() != null || form.getMessage() != null) - { - WriteableAppProps props = AppProps.getWriteableInstance(); - - if (form.isShow() != null) - props.setShowRibbonMessage(form.isShow()); - - if (form.getMessage() != null) - props.setRibbonMessage(form.getMessage()); - - props.save(getViewContext().getUser()); - } - - return null; - } - } - - @RequiresPermission(AdminPermission.class) - public class ConfigureSiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - public static class SiteValidationForm - { - private List _providers; - private boolean _includeSubfolders = false; - private transient Consumer _logger = s -> { - }; // No-op by default - - public List getProviders() - { - return _providers; - } - - public void setProviders(List providers) - { - _providers = providers; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public Consumer getLogger() - { - return _logger; - } - - public void setLogger(Consumer logger) - { - _logger = logger; - } - } - - @RequiresPermission(AdminPermission.class) - public class SiteValidationAction extends SimpleViewAction - { - @Override - public ModelAndView getView(SiteValidationForm form, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class SiteValidationBackgroundAction extends FormHandlerAction - { - private ActionURL _redirectUrl; - - @Override - public void validateCommand(SiteValidationForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException - { - ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); - PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); - SiteValidationJob job = new SiteValidationJob(vbi, root, form); - PipelineService.get().queueJob(job); - String jobGuid = job.getJobGUID(); - - if (null == jobGuid) - throw new NotFoundException("Unable to determine pipeline job GUID"); - - Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - if (null == jobId) - throw new NotFoundException("Unable to determine pipeline job ID"); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _redirectUrl = urls.urlDetails(getContainer(), jobId); - - return true; - } - - @Override - public URLHelper getSuccessURL(SiteValidationForm form) - { - return _redirectUrl; - } - } - - public static class ViewValidationResultsForm - { - private int _rowId; - - public int getRowId() - { - return _rowId; - } - - public void setRowId(int rowId) - { - _rowId = rowId; - } - } - - @RequiresPermission(AdminPermission.class) - public class ViewValidationResultsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception - { - PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); - if (null == statusFile) - throw new NotFoundException("Status file not found"); - if (!getContainer().equals(statusFile.lookupContainer())) - throw new UnauthorizedException("Wrong container"); - - String logFilePath = statusFile.getFilePath(); - String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; - File htmlFile = new File(htmlFilePath); - - if (!htmlFile.exists()) - throw new NotFoundException("Results file not found"); - return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("siteValidation"); - addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); - } - } - - public interface FileManagementForm - { - String getFolderRootPath(); - - void setFolderRootPath(String folderRootPath); - - String getFileRootOption(); - - void setFileRootOption(String fileRootOption); - - String getConfirmMessage(); - - void setConfirmMessage(String confirmMessage); - - boolean isDisableFileSharing(); - - boolean hasSiteDefaultRoot(); - - String[] getEnabledCloudStore(); - - @SuppressWarnings("unused") - void setEnabledCloudStore(String[] enabledCloudStore); - - boolean isCloudFileRoot(); - - @Nullable - String getCloudRootName(); - - void setCloudRootName(String cloudRootName); - - void setFileRootChanged(boolean changed); - - void setEnabledCloudStoresChanged(boolean changed); - - String getMigrateFilesOption(); - - void setMigrateFilesOption(String migrateFilesOption); - - default boolean isFolderSetup() - { - return false; - } - } - - public enum MigrateFilesOption implements SafeToRenderEnum - { - leave - { - @Override - public String description() - { - return "Source files not copied or moved"; - } - }, - copy - { - @Override - public String description() - { - return "Copy source files to destination"; - } - }, - move - { - @Override - public String description() - { - return "Move source files to destination"; - } - }; - - public abstract String description(); - } - - public static class ProjectSettingsForm extends FolderSettingsForm - { - // Site-only properties - private String _dateParsingMode; - private String _customWelcome; - - // Site & project properties - private boolean _shouldInherit; // new subfolders should inherit parent permissions - private String _systemDescription; - private boolean _systemDescriptionInherited; - private String _systemShortName; - private boolean _systemShortNameInherited; - private String _themeName; - private boolean _themeNameInherited; - private String _folderDisplayMode; - private boolean _folderDisplayModeInherited; - private String _applicationMenuDisplayMode; - private boolean _applicationMenuDisplayModeInherited; - private boolean _helpMenuEnabled; - private boolean _helpMenuEnabledInherited; - private String _logoHref; - private boolean _logoHrefInherited; - private String _companyName; - private boolean _companyNameInherited; - private String _systemEmailAddress; - private boolean _systemEmailAddressInherited; - private String _reportAProblemPath; - private boolean _reportAProblemPathInherited; - private String _supportEmail; - private boolean _supportEmailInherited; - private String _customLogin; - private boolean _customLoginInherited; - - // Site-only properties - - public String getDateParsingMode() - { - return _dateParsingMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setDateParsingMode(String dateParsingMode) - { - _dateParsingMode = dateParsingMode; - } - - public String getCustomWelcome() - { - return _customWelcome; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomWelcome(String customWelcome) - { - _customWelcome = customWelcome; - } - - // Site & project properties - - public boolean getShouldInherit() - { - return _shouldInherit; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setShouldInherit(boolean b) - { - _shouldInherit = b; - } - - public String getSystemDescription() - { - return _systemDescription; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescription(String systemDescription) - { - _systemDescription = systemDescription; - } - - public boolean isSystemDescriptionInherited() - { - return _systemDescriptionInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemDescriptionInherited(boolean systemDescriptionInherited) - { - _systemDescriptionInherited = systemDescriptionInherited; - } - - public String getSystemShortName() - { - return _systemShortName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortName(String systemShortName) - { - _systemShortName = systemShortName; - } - - public boolean isSystemShortNameInherited() - { - return _systemShortNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemShortNameInherited(boolean systemShortNameInherited) - { - _systemShortNameInherited = systemShortNameInherited; - } - - public String getThemeName() - { - return _themeName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeName(String themeName) - { - _themeName = themeName; - } - - public boolean isThemeNameInherited() - { - return _themeNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setThemeNameInherited(boolean themeNameInherited) - { - _themeNameInherited = themeNameInherited; - } - - public String getFolderDisplayMode() - { - return _folderDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayMode(String folderDisplayMode) - { - _folderDisplayMode = folderDisplayMode; - } - - public boolean isFolderDisplayModeInherited() - { - return _folderDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) - { - _folderDisplayModeInherited = folderDisplayModeInherited; - } - - public String getApplicationMenuDisplayMode() - { - return _applicationMenuDisplayMode; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayMode(String displayMode) - { - _applicationMenuDisplayMode = displayMode; - } - - public boolean isApplicationMenuDisplayModeInherited() - { - return _applicationMenuDisplayModeInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) - { - _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; - } - - public boolean isHelpMenuEnabled() - { - return _helpMenuEnabled; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabled(boolean helpMenuEnabled) - { - _helpMenuEnabled = helpMenuEnabled; - } - - public boolean isHelpMenuEnabledInherited() - { - return _helpMenuEnabledInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) - { - _helpMenuEnabledInherited = helpMenuEnabledInherited; - } - - public String getLogoHref() - { - return _logoHref; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHref(String logoHref) - { - _logoHref = logoHref; - } - - public boolean isLogoHrefInherited() - { - return _logoHrefInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setLogoHrefInherited(boolean logoHrefInherited) - { - _logoHrefInherited = logoHrefInherited; - } - - public String getReportAProblemPath() - { - return _reportAProblemPath; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPath(String reportAProblemPath) - { - _reportAProblemPath = reportAProblemPath; - } - - public boolean isReportAProblemPathInherited() - { - return _reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) - { - _reportAProblemPathInherited = reportAProblemPathInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmail(String supportEmail) - { - _supportEmail = supportEmail; - } - - public String getSupportEmail() - { - return _supportEmail; - } - - public boolean isSupportEmailInherited() - { - return _supportEmailInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSupportEmailInherited(boolean supportEmailInherited) - { - _supportEmailInherited = supportEmailInherited; - } - - public String getSystemEmailAddress() - { - return _systemEmailAddress; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddress(String systemEmailAddress) - { - _systemEmailAddress = systemEmailAddress; - } - - public boolean isSystemEmailAddressInherited() - { - return _systemEmailAddressInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) - { - _systemEmailAddressInherited = systemEmailAddressInherited; - } - - public String getCompanyName() - { - return _companyName; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyName(String companyName) - { - _companyName = companyName; - } - - public boolean isCompanyNameInherited() - { - return _companyNameInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCompanyNameInherited(boolean companyNameInherited) - { - _companyNameInherited = companyNameInherited; - } - - public String getCustomLogin() - { - return _customLogin; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLogin(String customLogin) - { - _customLogin = customLogin; - } - - public boolean isCustomLoginInherited() - { - return _customLoginInherited; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setCustomLoginInherited(boolean customLoginInherited) - { - _customLoginInherited = customLoginInherited; - } - } - - public enum FileRootProp implements SafeToRenderEnum - { - disable, - siteDefault, - folderOverride, - cloudRoot - } - - public static class FilesForm extends SetupForm implements FileManagementForm - { - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _cloudRootName; - private String _migrateFilesOption; - private String[] _enabledCloudStore; - private String _fileRootOption; - private String _folderRootPath; - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - } - - @SuppressWarnings("unused") - public static class SiteSettingsForm - { - private boolean _upgradeInProgress = false; - - private String _pipelineToolsDirectory; - private boolean _sslRequired; - private boolean _adminOnlyMode; - private boolean _showRibbonMessage; - private boolean _ext3Required; - private boolean _ext3APIRequired; - private boolean _selfReportExceptions; - private String _adminOnlyMessage; - private String _ribbonMessage; - private int _sslPort; - private int _memoryUsageDumpInterval; - private int _readOnlyHttpRequestTimeout; - private int _maxBLOBSize; - private String _exceptionReportingLevel; - private String _usageReportingLevel; - private String _administratorContactEmail; - - private String _baseServerURL; - private String _callbackPassword; - private boolean _allowApiKeys; - private int _apiKeyExpirationSeconds; - private boolean _allowSessionKeys; - private boolean _navAccessOpen; - - private String _XFrameOption; - private boolean _includeServerHttpHeader; - - public String getPipelineToolsDirectory() - { - return _pipelineToolsDirectory; - } - - public void setPipelineToolsDirectory(String pipelineToolsDirectory) - { - _pipelineToolsDirectory = pipelineToolsDirectory; - } - - public boolean isNavAccessOpen() - { - return _navAccessOpen; - } - - public void setNavAccessOpen(boolean navAccessOpen) - { - _navAccessOpen = navAccessOpen; - } - - public boolean isSslRequired() - { - return _sslRequired; - } - - public void setSslRequired(boolean sslRequired) - { - _sslRequired = sslRequired; - } - - public boolean isExt3Required() - { - return _ext3Required; - } - - public void setExt3Required(boolean ext3Required) - { - _ext3Required = ext3Required; - } - - public boolean isExt3APIRequired() - { - return _ext3APIRequired; - } - - public void setExt3APIRequired(boolean ext3APIRequired) - { - _ext3APIRequired = ext3APIRequired; - } - - public int getSslPort() - { - return _sslPort; - } - - public void setSslPort(int sslPort) - { - _sslPort = sslPort; - } - - public boolean isAdminOnlyMode() - { - return _adminOnlyMode; - } - - public void setAdminOnlyMode(boolean adminOnlyMode) - { - _adminOnlyMode = adminOnlyMode; - } - - public String getAdminOnlyMessage() - { - return _adminOnlyMessage; - } - - public void setAdminOnlyMessage(String adminOnlyMessage) - { - _adminOnlyMessage = adminOnlyMessage; - } - - public boolean isSelfReportExceptions() - { - return _selfReportExceptions; - } - - public void setSelfReportExceptions(boolean selfReportExceptions) - { - _selfReportExceptions = selfReportExceptions; - } - - public String getExceptionReportingLevel() - { - return _exceptionReportingLevel; - } - - public void setExceptionReportingLevel(String exceptionReportingLevel) - { - _exceptionReportingLevel = exceptionReportingLevel; - } - - public String getUsageReportingLevel() - { - return _usageReportingLevel; - } - - public void setUsageReportingLevel(String usageReportingLevel) - { - _usageReportingLevel = usageReportingLevel; - } - - public String getAdministratorContactEmail() - { - return _administratorContactEmail; - } - - public void setAdministratorContactEmail(String administratorContactEmail) - { - _administratorContactEmail = administratorContactEmail; - } - - public boolean isUpgradeInProgress() - { - return _upgradeInProgress; - } - - public void setUpgradeInProgress(boolean upgradeInProgress) - { - _upgradeInProgress = upgradeInProgress; - } - - public int getMemoryUsageDumpInterval() - { - return _memoryUsageDumpInterval; - } - - public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) - { - _memoryUsageDumpInterval = memoryUsageDumpInterval; - } - - public int getReadOnlyHttpRequestTimeout() - { - return _readOnlyHttpRequestTimeout; - } - - public void setReadOnlyHttpRequestTimeout(int timeout) - { - _readOnlyHttpRequestTimeout = timeout; - } - - public int getMaxBLOBSize() - { - return _maxBLOBSize; - } - - public void setMaxBLOBSize(int maxBLOBSize) - { - _maxBLOBSize = maxBLOBSize; - } - - public String getBaseServerURL() - { - return _baseServerURL; - } - - public void setBaseServerURL(String baseServerURL) - { - _baseServerURL = baseServerURL; - } - - public String getCallbackPassword() - { - return _callbackPassword; - } - - public void setCallbackPassword(String callbackPassword) - { - _callbackPassword = callbackPassword; - } - - public boolean isShowRibbonMessage() - { - return _showRibbonMessage; - } - - public void setShowRibbonMessage(boolean showRibbonMessage) - { - _showRibbonMessage = showRibbonMessage; - } - - public String getRibbonMessage() - { - return _ribbonMessage; - } - - public void setRibbonMessage(String ribbonMessage) - { - _ribbonMessage = ribbonMessage; - } - - public boolean isAllowApiKeys() - { - return _allowApiKeys; - } - - public void setAllowApiKeys(boolean allowApiKeys) - { - _allowApiKeys = allowApiKeys; - } - - public int getApiKeyExpirationSeconds() - { - return _apiKeyExpirationSeconds; - } - - public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) - { - _apiKeyExpirationSeconds = apiKeyExpirationSeconds; - } - - public boolean isAllowSessionKeys() - { - return _allowSessionKeys; - } - - public void setAllowSessionKeys(boolean allowSessionKeys) - { - _allowSessionKeys = allowSessionKeys; - } - - public String getXFrameOption() - { - return _XFrameOption; - } - - public void setXFrameOption(String XFrameOption) - { - _XFrameOption = XFrameOption; - } - - public boolean isIncludeServerHttpHeader() - { - return _includeServerHttpHeader; - } - - public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) - { - _includeServerHttpHeader = includeServerHttpHeader; - } - } - - - @AdminConsoleAction - public class ShowThreadsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Log to labkey.log as well as showing through the browser - DebugInfoDumper.dumpThreads(3); - return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpDebugging#threads"); - addAdminNavTrail(root, "Current Threads", this.getClass()); - } - } - - private abstract class AbstractPostgresAction extends QueryViewAction - { - private final String _queryName; - - protected AbstractPostgresAction(String queryName) - { - super(QueryExportForm.class); - _queryName = queryName; - } - - @Override - protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception - { - if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) - { - throw new NotFoundException("Only available with Postgres as the primary database"); - } - - QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); - QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) - { - @Override - public DataView createDataView() - { - // Troubleshooters don't have normal read access to the root container so grant them special access - // for these queries - DataView view = super.createDataView(); - view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); - return view; - } - }; - result.setTitle(_queryName); - result.setFrame(WebPartView.FrameType.PORTAL); - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("postgresActivity"); - addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); - } - - } - - @AdminConsoleAction - public class PostgresStatActivityAction extends AbstractPostgresAction - { - public PostgresStatActivityAction() - { - super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); - } - } - - @AdminConsoleAction - public class PostgresLocksAction extends AbstractPostgresAction - { - public PostgresLocksAction() - { - super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); - } - } - - @AdminConsoleAction - public class PostgresTableSizesAction extends AbstractPostgresAction - { - public PostgresTableSizesAction() - { - super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME); - } - } - - @AdminConsoleAction - public class DumpHeapAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - File destination = DebugInfoDumper.dumpHeap(); - return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("dumpHeap"); - addAdminNavTrail(root, "Heap dump", getClass()); - } - } - - - public static class ThreadsBean - { - public Map> spids; - public List threads; - public Map stackTraces; - - ThreadsBean() - { - stackTraces = Thread.getAllStackTraces(); - threads = new ArrayList<>(stackTraces.keySet()); - threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); - - spids = new HashMap<>(); - - for (Thread t : threads) - { - spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); - } - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class ShowNetworkDriveTestAction extends SimpleViewAction - { - @Override - public void validate(NetworkDriveForm form, BindException errors) - { - validateNetworkDrive(form, errors); - } - - @Override - public ModelAndView getView(NetworkDriveForm form, BindException errors) - { - NetworkDrive testDrive = new NetworkDrive(); - testDrive.setPassword(form.getNetworkDrivePassword()); - testDrive.setPath(form.getNetworkDrivePath()); - testDrive.setUser(form.getNetworkDriveUser()); - TestNetworkDriveBean bean = new TestNetworkDriveBean(); - - if (!errors.hasErrors()) - { - char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); - try - { - String mountError = testDrive.mount(driveLetter); - if (mountError != null) - { - errors.reject(ERROR_MSG, mountError); - } - else - { - File f = new File(driveLetter + ":\\"); - if (!f.exists()) - { - errors.reject(ERROR_MSG, "Could not access network drive"); - } - else - { - String[] fileNames = f.list(); - if (fileNames == null) - fileNames = new String[0]; - Arrays.sort(fileNames); - bean.setFiles(fileNames); - } - } - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - try - { - testDrive.unmount(driveLetter); - } - catch (IOException | InterruptedException e) - { - errors.reject(ERROR_MSG, "Error mounting drive: " + e); - } - } - - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Test Mapping Network Drive"); - } - } - - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetErrorMarkAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - return HtmlView.of("Are you sure you want to reset the site errors?"); - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - File errorLogFile = getErrorLogFile(); - _errorMark = errorLogFile.length(); - - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull URLHelper getSuccessURL(Object o) - { - return getShowAdminURL(); - } - } - - abstract public static class ShowLogAction extends ExportAction - { - @Override - public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException - { - getPageConfig().setNoIndex(); - export(response); - } - - protected abstract void export(HttpServletResponse response) throws IOException; - } - - @AdminConsoleAction - public class ShowErrorsSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); - } - } - - @AdminConsoleAction - public class ShowAllErrorsAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); - } - } - - @AdminConsoleAction(ApplicationAdminPermission.class) - public class ResetPrimaryLogMarkAction extends MutatingApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - File logFile = getPrimaryLogFile(); - _primaryLogMark = logFile.length(); - return null; - } - } - - @AdminConsoleAction - public class ShowPrimaryLogSinceMarkAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowPrimaryLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); - } - } - - @AdminConsoleAction - public class ShowCspReportLogAction extends ShowLogAction - { - @Override - protected void export(HttpServletResponse response) throws IOException - { - PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); - } - } - - private File getErrorLogFile() - { - return new File(getLabKeyLogDir(), "labkey-errors.log"); - } - - private File getPrimaryLogFile() - { - return new File(getLabKeyLogDir(), "labkey.log"); - } - - private File getCspReportLogFile() - { - return new File(getLabKeyLogDir(), "csp-report.log"); - } - - private static ActionURL getActionsURL() - { - return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public class ActionsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new ActionsTabStrip(); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("actionsDiagnostics"); - addAdminNavTrail(root, "Actions", this.getClass()); - } - } - - private static class ActionsTabStrip extends TabStripView - { - @Override - public List getTabList() - { - List tabs = new ArrayList<>(3); - - tabs.add(new TabInfo("Summary", "summary", getActionsURL())); - tabs.add(new TabInfo("Details", "details", getActionsURL())); - tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); - - return tabs; - } - - @Override - public HttpView getTabView(String tabId) - { - if ("exceptions".equals(tabId)) - return new ActionsExceptionsView(); - return new ActionsView(!"details".equals(tabId)); - } - } - - @AdminConsoleAction - public static class ExportActionsAction extends ExportAction - { - @Override - public void export(Object form, HttpServletResponse response, BindException errors) throws Exception - { - try (ActionsTsvWriter writer = new ActionsTsvWriter()) - { - writer.write(response); - } - } - } - - private static ActionURL getQueriesURL(@Nullable String statName) - { - ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); - - if (null != statName) - url.addParameter("stat", statName); - - return url; - } - - - @AdminConsoleAction - public class QueriesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueriesForm form, BindException errors) - { - String buttonHTML = ""; - if (getUser().hasRootAdminPermission()) - buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; - buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

    "; - - return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, - AdminController::getQueryStackTracesURL); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("queryPerf"); - addAdminNavTrail(root, "Queries", this.getClass()); - } - } - - public static class QueriesForm - { - private String _stat = "Count"; - - public String getStat() - { - return _stat; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setStat(String stat) - { - _stat = stat; - } - } - - - private static ActionURL getQueryStackTracesURL(String sqlHash) - { - ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class QueryStackTracesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces"); - } - } - - - private static ActionURL getExecutionPlanURL(String sqlHash) - { - ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); - url.addParameter("sqlHash", sqlHash); - return url; - } - - - @AdminConsoleAction - public class ExecutionPlanAction extends SimpleViewAction - { - private String _sqlHash; - private ExecutionPlanType _type; - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - _sqlHash = form.getSqlHash(); - _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); - if (null == _type) - throw new NotFoundException("Unknown execution plan type"); - - return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Queries", QueriesAction.class); - root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); - root.addChild(_type.getDescription()); - } - } - - - public static class QueryForm - { - private String _sqlHash; - private String _type = "Estimated"; // All dialects support Estimated - private boolean _log = false; - - public String getSqlHash() - { - return _sqlHash; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setSqlHash(String sqlHash) - { - _sqlHash = sqlHash; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setType(String type) - { - _type = type; - } - - public boolean isLog() - { - return _log; - } - - public void setLog(boolean log) - { - _log = log; - } - } - - - private ActionURL getExportQueriesURL() - { - return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); - } - - - @AdminConsoleAction - public static class ExportQueriesAction extends ExportAction - { - @Override - public void export(Object o, HttpServletResponse response, BindException errors) throws Exception - { - try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) - { - writer.setFilenamePrefix("SQL_Queries"); - writer.write(response); - } - } - } - - private static ActionURL getResetQueryStatisticsURL() - { - return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); - } - - - @RequiresPermission(AdminPermission.class) - public static class ResetQueryStatisticsAction extends FormHandlerAction - { - @Override - public void validateCommand(QueriesForm target, Errors errors) - { - } - - @Override - public boolean handlePost(QueriesForm form, BindException errors) throws Exception - { - QueryProfiler.getInstance().resetAllStatistics(); - return true; - } - - @Override - public URLHelper getSuccessURL(QueriesForm form) - { - return getQueriesURL(form.getStat()); - } - } - - - @AdminConsoleAction - public class CachesAction extends SimpleViewAction - { - private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); - private final DecimalFormat percent = new DecimalFormat("0%"); - - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - if (form.isClearCaches()) - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); - throw new RedirectException(redirect); - } - - List> caches = CacheManager.getKnownCaches(); - - if (form.getDebugName() != null) - { - for (TrackingCache cache : caches) - { - if (form.getDebugName().equals(cache.getDebugName())) - { - LOG.info("Purging cache: " + cache.getDebugName()); - cache.clear(); - } - } - ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); - throw new RedirectException(redirect); - } - - List cacheStats = new ArrayList<>(); - List transactionStats = new ArrayList<>(); - - for (TrackingCache cache : caches) - { - cacheStats.add(CacheManager.getCacheStats(cache)); - transactionStats.add(CacheManager.getTransactionCacheStats(cache)); - } - - HtmlStringBuilder html = HtmlStringBuilder.of(); - - html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); - html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); - - html.unsafeAppend("

    \n"); - appendStats(html, "Caches", cacheStats, false); - - html.unsafeAppend("

    \n"); - appendStats(html, "Transaction Caches", transactionStats, true); - - return new HtmlView(html); - } - - private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) - { - List stats = skipUnusedCaches ? - allStats.stream() - .filter(stat->stat.getMaxSize() > 0) - .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : - allStats; - - Collections.sort(stats); - - html.unsafeAppend("

    "); - html.append(title); - html.append(" (").append(stats.size()).unsafeAppend(")

    \n"); - - html.unsafeAppend("\n"); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - html.unsafeAppend(""); - - long size = 0; - long gets = 0; - long misses = 0; - long puts = 0; - long expirations = 0; - long evictions = 0; - long removes = 0; - long clears = 0; - int rowCount = 0; - - for (CacheStats stat : stats) - { - size += stat.getSize(); - gets += stat.getGets(); - misses += stat.getMisses(); - puts += stat.getPuts(); - expirations += stat.getExpirations(); - evictions += stat.getEvictions(); - removes += stat.getRemoves(); - clears += stat.getClears(); - - html.unsafeAppend(""); - - appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); - - Long limit = stat.getLimit(); - long maxSize = stat.getMaxSize(); - - appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); - appendDoubles(html, stat.getMissRatio()); - - html.unsafeAppend("\n"); - - if (null != limit && maxSize >= limit) - html.unsafeAppend(""); - - html.unsafeAppend("\n"); - rowCount++; - } - - double ratio = 0 != gets ? misses / (double)gets : 0; - html.unsafeAppend(""); - - appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); - appendDoubles(html, ratio); - - html.unsafeAppend("\n"); - html.unsafeAppend("
    Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
    ").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
    Total
    \n"); - } - - private static final List PREFIXES_TO_SKIP = List.of( - "java.base/java.lang.Thread.getStackTrace", - "org.labkey.api.cache.CacheManager", - "org.labkey.api.cache.Throttle", - "org.labkey.api.data.DatabaseCache", - "org.labkey.api.module.ModuleResourceCache" - ); - - private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) - { - StringBuilder sb = new StringBuilder(); - - if (creationStackTrace != null) - { - boolean trimming = true; - for (StackTraceElement element : creationStackTrace) - { - // Skip the first few uninteresting stack trace elements to highlight the caller we care about - if (trimming) - { - if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) - continue; - - trimming = false; - } - sb.append(element); - sb.append("\n"); - } - } - - if (!sb.isEmpty()) - { - String message = PageFlowUtil.jsString(sb); - String id = "id" + UniqueID.getServerSessionScopedUID(); - html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); - HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); - } - } - - private void appendLongs(HtmlStringBuilder html, Long... stats) - { - for (Long stat : stats) - { - if (null == stat) - html.unsafeAppend(" "); - else - html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); - } - } - - private void appendDoubles(HtmlStringBuilder html, double... stats) - { - for (double stat : stats) - html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("cachesDiagnostics"); - addAdminNavTrail(root, "Cache Statistics", this.getClass()); - } - } - - @RequiresSiteAdmin - public class EnvironmentVariablesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Environment Variables", this.getClass()); - } - } - - @RequiresSiteAdmin - public class SystemPropertiesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "System Properties", this.getClass()); - } - } - - - public static class ConfigureSystemMaintenanceForm - { - private String _maintenanceTime; - private Set _enable = Collections.emptySet(); - private boolean _enableSystemMaintenance = true; - - public String getMaintenanceTime() - { - return _maintenanceTime; - } - - @SuppressWarnings("unused") - public void setMaintenanceTime(String maintenanceTime) - { - _maintenanceTime = maintenanceTime; - } - - public Set getEnable() - { - return _enable; - } - - @SuppressWarnings("unused") - public void setEnable(Set enable) - { - _enable = enable; - } - - public boolean isEnableSystemMaintenance() - { - return _enableSystemMaintenance; - } - - @SuppressWarnings("unused") - public void setEnableSystemMaintenance(boolean enableSystemMaintenance) - { - _enableSystemMaintenance = enableSystemMaintenance; - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ConfigureSystemMaintenanceAction extends FormViewAction - { - @Override - public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) - { - Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); - - if (null == date) - errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); - } - - @Override - public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) - { - SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); - return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); - } - - @Override - public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) - { - SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); - SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); - - return true; - } - - @Override - public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); - } - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class ResetSystemMaintenanceAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - SystemMaintenance.clearProperties(); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new AdminUrlsImpl().getAdminConsoleURL(); - } - } - - public static class SystemMaintenanceForm - { - private String _taskName; - private boolean _test = false; - - public String getTaskName() - { - return _taskName; - } - - @SuppressWarnings("unused") - public void setTaskName(String taskName) - { - _taskName = taskName; - } - - public boolean isTest() - { - return _test; - } - - public void setTest(boolean test) - { - _test = test; - } - } - - @RequiresSiteAdmin - public class SystemMaintenanceAction extends FormHandlerAction - { - private Long _jobId = null; - private URLHelper _url = null; - - @Override - public void validateCommand(SystemMaintenanceForm form, Errors errors) - { - } - - @Override - public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException - { - // Send the pipeline job details absolute URL back to the test - sendPlainText(_url.getURIString()); - - // Suppress templates, divs, etc. - getPageConfig().setTemplate(Template.None); - return new EmptyView(); - } - - @Override - public boolean handlePost(SystemMaintenanceForm form, BindException errors) - { - String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); - - if (null != jobGuid) - _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); - - PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); - _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SystemMaintenanceForm form) - { - // In the standard case, redirect to the pipeline details URL - // If the test is invoking system maintenance then return the URL instead - return form.isTest() ? null : _url; - } - } - - @AdminConsoleAction - public class AttachmentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getAdminView(getViewContext().getActionURL()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Attachments", getClass()); - } - } - - @AdminConsoleAction - public class FindAttachmentParentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return AttachmentService.get().getFindAttachmentParentsView(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Find Attachment Parents", getClass()); - } - } - - public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCachesURL(boolean clearCaches, boolean gc) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - if (clearCaches) - url.addParameter(MemForm.Params.clearCaches, "1"); - - if (gc) - url.addParameter(MemForm.Params.gc, "1"); - - return url; - } - - public static ActionURL getCacheURL(String debugName) - { - ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); - - url.addParameter(MemForm.Params.debugName, debugName); - - return url; - } - - private static volatile String lastCacheMemUsed = null; - - @AdminConsoleAction - public class MemTrackerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(MemForm form, BindException errors) - { - Set objectsToIgnore = MemTracker.getInstance().beforeReport(); - - boolean gc = form.isGc(); - boolean cc = form.isClearCaches(); - - if (getUser().hasRootAdminPermission() && (gc || cc)) - { - // If both are requested then try to determine and record cache memory usage - if (gc && cc) - { - // gc once to get an accurate free memory read - long before = gc(); - clearCaches(); - // gc again now that we cleared caches - long cacheMemoryUsed = before - gc(); - - // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage - String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; - LOG.info("Estimate of cache memory used: " + cacheMemUsed); - lastCacheMemUsed = cacheMemUsed; - } - else if (cc) - { - clearCaches(); - } - else - { - gc(); - } - - LOG.info("Cache clearing and garbage collecting complete"); - } - - return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); - } - - /** @return estimated current memory usage, post-garbage collection */ - private long gc() - { - LOG.info("Garbage collecting"); - System.gc(); - // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will - return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); - } - - private void clearCaches() - { - LOG.info("Clearing Introspector caches"); - Introspector.flushCaches(); - LOG.info("Purging all caches"); - CacheManager.clearAllKnownCaches(); - LOG.info("Purging SearchService queues"); - SearchService.get().purgeQueues(); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("memTracker"); - addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); - } - } - - public static class MemForm - { - private enum Params {clearCaches, debugName, gc} - - private boolean _clearCaches = false; - private boolean _gc = false; - private String _debugName; - - public boolean isClearCaches() - { - return _clearCaches; - } - - @SuppressWarnings("unused") - public void setClearCaches(boolean clearCaches) - { - _clearCaches = clearCaches; - } - - public boolean isGc() - { - return _gc; - } - - @SuppressWarnings("unused") - public void setGc(boolean gc) - { - _gc = gc; - } - - public String getDebugName() - { - return _debugName; - } - - @SuppressWarnings("unused") - public void setDebugName(String debugName) - { - _debugName = debugName; - } - } - - public static class MemBean - { - public final List> memoryUsages = new ArrayList<>(); - public final List> systemProperties = new ArrayList<>(); - public final List references; - public final List graphNames = new ArrayList<>(); - public final List activeThreads = new LinkedList<>(); - - public boolean assertsEnabled = false; - - private MemBean(HttpServletRequest request, Set objectsToIgnore) - { - MemTracker memTracker = MemTracker.getInstance(); - List all = memTracker.getReferences(); - long threadId = Thread.currentThread().getId(); - - // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found - for (Thread thread : new ThreadsBean().threads) - { - if (thread.getId() == threadId) - continue; - - Thread.State state = thread.getState(); - - if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) - { - boolean labkeyThread = false; - - if (memTracker.shouldDisplay(thread)) - { - for (StackTraceElement element : thread.getStackTrace()) - { - String className = element.getClassName(); - - if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) - { - labkeyThread = true; - break; - } - } - } - - if (labkeyThread) - { - String threadInfo = thread.getName(); - TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); - if (null != uri) - threadInfo += "; processing URL " + uri.toLogString(); - activeThreads.add(threadInfo); - } - } - } - - // ignore recently allocated - long start = ViewServlet.getRequestStartTime(request) - 2000; - references = new ArrayList<>(all.size()); - - for (HeldReference r : all) - { - if (r.getThreadId() == threadId && r.getAllocationTime() >= start) - continue; - - if (objectsToIgnore.contains(r.getReference())) - continue; - - references.add(r); - } - - // memory: - graphNames.add("Heap"); - graphNames.add("Non Heap"); - - MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); - } - - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - if (membean != null) - { - memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); - } - - for (MemoryPoolMXBean pool : pools) - { - if (pool.getType() == MemoryType.NON_HEAP) - { - memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); - graphNames.add(pool.getName()); - } - } - - for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) - { - memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); - graphNames.add(pool.getName()); - } - - DecimalFormat commaf0 = new DecimalFormat("#,##0"); - - - // class loader: - ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); - if (classbean != null) - { - systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); - systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); - systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); - } - - // runtime: - RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); - if (runtimeBean != null) - { - systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); - long upTime = runtimeBean.getUptime(); // round to sec - upTime = upTime - (upTime % 1000); - systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); - systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); - systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); - } - - // threads: - ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); - if (threadBean != null) - { - systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); - systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); - long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); - systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); - } - - // threads: - List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); - for (GarbageCollectorMXBean gcBean : gcBeans) - { - systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); - systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); - } - - String cacheMem = lastCacheMemUsed; - - if (null != cacheMem) - systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); - - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - if (osBean != null) - { - systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); - - DecimalFormat f3 = new DecimalFormat("0.000"); - - if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) - { - systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); - systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); - systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); - systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); - } - } - - //noinspection ConstantConditions - assert assertsEnabled = true; - } - } - - private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) - { - try - { - return getUsage(pool.getUsage()); - } - catch (IllegalArgumentException x) - { - // sometimes we get usage>committed exception with older versions of JRockit - return null; - } - } - - public static class MemoryUsageSummary - { - - public final long _init; - public final long _used; - public final long _committed; - public final long _max; - - public MemoryUsageSummary(MemoryUsage usage) - { - _init = usage.getInit(); - _used = usage.getUsed(); - _committed = usage.getCommitted(); - _max = usage.getMax(); - } - - public MemoryUsageSummary(BufferPoolMXBean pool) - { - _init = -1; - _used = pool.getMemoryUsed(); - _committed = _used; - _max = pool.getTotalCapacity(); - } - } - - private static MemoryUsageSummary getUsage(MemoryUsage usage) - { - if (null == usage) - return null; - - try - { - return new MemoryUsageSummary(usage); - } - catch (IllegalArgumentException x) - { - // sometime we get usage>committed exception with older verions of JRockit - return null; - } - } - - public static class ChartForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - } - - private static class MemoryCategory implements Comparable - { - private final String _type; - private final double _mb; - - public MemoryCategory(String type, double mb) - { - _type = type; - _mb = mb; - } - - @Override - public int compareTo(@NotNull MemoryCategory o) - { - return Double.compare(getMb(), o.getMb()); - } - - public String getType() - { - return _type; - } - - public double getMb() - { - return _mb; - } - } - - @AdminConsoleAction - public static class MemoryChartAction extends ExportAction - { - @Override - public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception - { - MemoryUsage usage = null; - boolean showLegend = false; - String title = form.getType(); - if ("Heap".equals(form.getType())) - { - usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - showLegend = true; - } - else if ("Non Heap".equals(form.getType())) - usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); - else - { - List pools = ManagementFactory.getMemoryPoolMXBeans(); - for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) - { - MemoryPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - usage = pool.getUsage(); - } - } - - Pair divisor = null; - - List types = new ArrayList<>(4); - - if (usage == null) - { - boolean found = false; - for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) - { - BufferPoolMXBean pool = it.next(); - if (form.getType().equals(pool.getName())) - { - long total = pool.getTotalCapacity(); - long used = pool.getMemoryUsed(); - - divisor = getDivisor(total); - - title = "Buffer pool " + title; - - if (total > 0 || used > 0) - { - types.add(new MemoryCategory("Used", used / divisor.first)); - types.add(new MemoryCategory("Max", total / divisor.first)); - } - found = true; - } - } - if (!found) - { - throw new NotFoundException(); - } - } - else - { - if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) - { - divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); - - types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); - types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); - types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); - types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); - } - } - - if (divisor != null) - { - title += " (" + divisor.second + ")"; - } - - DefaultCategoryDataset dataset = new DefaultCategoryDataset(); - - Collections.sort(types); - - for (int i = 0; i < types.size(); i++) - { - double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); - dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); - } - - JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); - chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); - response.setContentType("image/png"); - - ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); - } - - private Pair getDivisor(long l) - { - if (l > 4096L * 1024L * 1024L) - { - return Pair.of(1024L * 1024L * 1024L, "GB"); - } - if (l > 4096L * 1024L) - { - return Pair.of(1024L * 1024L, "MB"); - } - if (l > 4096L) - { - return Pair.of(1024L, "KB"); - } - - return Pair.of(1L, "bytes"); - - } - } - - public static class MemoryStressForm - { - private int _threads = 3; - private int _arraySize = 20_000; - private int _arrayCount = 10_000; - private float _percentChurn = 0.50f; - private int _delay = 20; - private int _iterations = 500; - - public int getThreads() - { - return _threads; - } - - public void setThreads(int threads) - { - _threads = threads; - } - - public int getArraySize() - { - return _arraySize; - } - - public void setArraySize(int arraySize) - { - _arraySize = arraySize; - } - - public int getArrayCount() - { - return _arrayCount; - } - - public void setArrayCount(int arrayCount) - { - _arrayCount = arrayCount; - } - - public float getPercentChurn() - { - return _percentChurn; - } - - public void setPercentChurn(float percentChurn) - { - _percentChurn = percentChurn; - } - - public int getDelay() - { - return _delay; - } - - public void setDelay(int delay) - { - _delay = delay; - } - - public int getIterations() - { - return _iterations; - } - - public void setIterations(int iterations) - { - _iterations = iterations; - } - } - - @RequiresSiteAdmin - public class MemoryStressTestAction extends FormViewAction - { - @Override - public void validateCommand(MemoryStressForm target, Errors errors) - { - - } - - @Override - public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception - { - return new HtmlView( - DOM.LK.FORM(at(method, "POST"), - DOM.LK.ERRORS(errors.getBindingResult()), - DOM.BR(), DOM.BR(), - "This utility action will do a lot of memory allocation to test the memory configuration of the host.", - DOM.BR(), DOM.BR(), - "It spins up threads, all of which allocate a specified number byte arrays of specified length.", - DOM.BR(), - "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", - DOM.BR(), - "They continue for the specified number of allocations.", - DOM.BR(), - "The memory actively held is approximately (threads * array count * array length).", - DOM.BR(), - "The memory turnover is based on the churn percentage, array length, delay, and iterations.", - DOM.BR(), DOM.BR(), - DOM.TABLE( - DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), - DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), - DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), - DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), - DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), - DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) - ), - new ButtonBuilder("Perform stress test").submit(true).build()) - ); - } - - @Override - public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception - { - List threads = new ArrayList<>(); - for (int i = 0; i < memoryStressForm._threads; i++) - { - Thread t = new Thread(() -> - { - Random r = new Random(); - byte[][] arrays = new byte[memoryStressForm._arrayCount][]; - // Initialize the arrays - for (int a = 0; a < arrays.length; a++) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - - for (int iter = 0; iter < memoryStressForm._iterations; iter++) - { - try - { - Thread.sleep(memoryStressForm._delay); - } - catch (InterruptedException ignored) {} - - // Swap the contents based on our desired percent churn - for (int a = 0; a < arrays.length; a++) - { - if (r.nextFloat() <= memoryStressForm._percentChurn) - { - arrays[a] = new byte[memoryStressForm._arraySize]; - } - } - } - }); - t.setUncaughtExceptionHandler((t2, e) -> { - LOG.error("Stress test exception", e); - errors.reject(null, "Stress test exception: " + e); - }); - t.start(); - threads.add(t); - } - - for (Thread thread : threads) - { - thread.join(); - } - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) - { - return new ActionURL(MemTrackerAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); - root.addChild("Memory Stress Test"); - } - } - - public static ActionURL getModuleStatusURL(URLHelper returnUrl) - { - ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); - if (returnUrl != null) - url.addReturnUrl(returnUrl); - return url; - } - - public static class ModuleStatusBean - { - public String verb; - public String verbing; - public ActionURL nextURL; - } - - @RequiresPermission(TroubleshooterPermission.class) - @AllowedDuringUpgrade - @IgnoresAllocationTracking - public static class ModuleStatusAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ReturnUrlForm form, BindException errors) - { - ModuleLoader loader = ModuleLoader.getInstance(); - VBox vbox = new VBox(); - ModuleStatusBean bean = new ModuleStatusBean(); - - if (loader.isNewInstall()) - bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); - else if (form.getReturnUrl() != null) - { - try - { - bean.nextURL = form.getReturnActionURL(); - } - catch (URLException x) - { - // might not be an ActionURL e.g. /labkey/_webdav/home - } - } - if (null == bean.nextURL) - bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - - if (loader.isNewInstall()) - bean.verb = "Install"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verb = "Upgrade"; - else - bean.verb = "Start"; - - if (loader.isNewInstall()) - bean.verbing = "Installing"; - else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) - bean.verbing = "Upgrading"; - else - bean.verbing = "Starting"; - - JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - - getPageConfig().setTemplate(Template.Wizard); - getPageConfig().setTitle(bean.verb + " Modules"); - setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); - - return vbox; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class NewInstallSiteSettingsForm extends FileSettingsForm - { - private String _notificationEmail; - private String _siteName; - - public String getNotificationEmail() - { - return _notificationEmail; - } - - public void setNotificationEmail(String notificationEmail) - { - _notificationEmail = notificationEmail; - } - - public String getSiteName() - { - return _siteName; - } - - public void setSiteName(String siteName) - { - _siteName = siteName; - } - } - - @RequiresSiteAdmin - public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction - { - public NewInstallSiteSettingsAction() - { - super(NewInstallSiteSettingsForm.class); - } - - @Override - public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) - { - super.validateCommand(form, errors); - - if (isBlank(form.getNotificationEmail())) - { - errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); - } - try - { - ValidEmail email = new ValidEmail(form.getNotificationEmail()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - - @Override - public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception - { - boolean success = super.handlePost(form, errors); - if (success) - { - WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); - try - { - lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - lafProps.setSystemShortName(form.getSiteName()); - lafProps.save(); - - // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. - UsageReportingLevel.reportNow(); - - return true; - } - return false; - } - - @Override - public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) - { - if (!reshow) - { - File root = _svc.getSiteDefaultRoot(); - - if (root.exists()) - form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); - form.setSiteName(props.getShortName()); - form.setNotificationEmail(props.getSystemEmailAddress()); - } - - JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Set Defaults"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) - { - return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresSiteAdmin - public static class InstallCompleteAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); - - getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); - getPageConfig().setTitle("Complete"); - getPageConfig().setTemplate(Template.Wizard); - - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static List getInstallUpgradeWizardSteps() - { - List navTrail = new ArrayList<>(); - if (ModuleLoader.getInstance().isNewInstall()) - { - navTrail.add(new NavTree("Account Setup")); - navTrail.add(new NavTree("Install Modules")); - navTrail.add(new NavTree("Set Defaults")); - } - else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) - { - navTrail.add(new NavTree("Upgrade Modules")); - } - else - { - navTrail.add(new NavTree("Start Modules")); - } - navTrail.add(new NavTree("Complete")); - return navTrail; - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DbCheckerAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Check Tools", this.getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DoCheckAction extends SimpleViewAction - { - @Override - public ModelAndView getView(DataCheckForm form, BindException errors) - { - try (var ignore=SpringActionController.ignoreSqlUpdates()) - { - ActionURL currentUrl = getViewContext().cloneActionURL(); - String fixRequested = currentUrl.getParameter("_fix"); - HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
    ")); - - if (null != fixRequested) - { - HtmlString sqlCheck = HtmlString.EMPTY_STRING; - if (fixRequested.equalsIgnoreCase("container")) - sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); - else if (fixRequested.equalsIgnoreCase("descriptor")) - sqlCheck = OntologyManager.doProjectColumnCheck(true); - contentBuilder.append(sqlCheck); - } - else - { - LOG.info("Starting database check"); // Debugging test timeout - LOG.info("Checking container column references"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking Container Column References..."); - HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "container"); - contentBuilder.unsafeAppend("

        ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); - strTemp = OntologyManager.doProjectColumnCheck(false); - if (!strTemp.isEmpty()) - { - contentBuilder.append(strTemp); - currentUrl = getViewContext().cloneActionURL(); - currentUrl.addParameter("_fix", "descriptor"); - contentBuilder.unsafeAppend("

        ") - .append(" click ") - .append(LinkBuilder.simpleLink("here", currentUrl)) - .append(" to attempt recovery."); - } - - LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking Schema consistency with tableXML.") - .unsafeAppend("

    "); - Set schemas = DbSchema.getAllSchemasToTest(); - - for (DbSchema schema : schemas) - { - SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); - List results = schemaResult.getResults(null); - if (results.isEmpty()) - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .append(": OK") - .unsafeAppend("
    "); - } - else - { - contentBuilder.unsafeAppend("") - .append(schema.getDisplayName()) - .unsafeAppend(""); - for (var r : results) - { - HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); - contentBuilder.unsafeAppend("
  • ") - .append(item) - .unsafeAppend("
    • \n"); - } - contentBuilder.unsafeAppend(""); - } - } - - LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Checking Consistency of Provisioned Storage...\n"); - StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); - contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); - for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) - { - for (String error : dr.getErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - } - for (String error : pr.getGlobalErrors()) - { - contentBuilder.unsafeAppend("
    ") - .append(error) - .unsafeAppend("
    "); - } - - LOG.info("Database check complete"); // Debugging test timeout - contentBuilder.unsafeAppend("\n

    ") - .append("Database Consistency checker complete"); - } - - contentBuilder.unsafeAppend("
    "); - - return new HtmlView(contentBuilder); - } - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Database Tools", this.getClass()); - } - } - - public static class DataCheckForm - { - private String _dbSchema = ""; - private boolean _full = false; - - public List modules = ModuleLoader.getInstance().getModules(); - public DataCheckForm(){} - - public List getModules() { return modules; } - public String getDbSchema() { return _dbSchema; } - @SuppressWarnings("unused") - public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } - public boolean getFull() { return _full; } - public void setFull(boolean full) { _full = full; } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class GetSchemaXmlDocAction extends ExportAction - { - @Override - public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception - { - String fullyQualifiedSchemaName = form.getDbSchema(); - if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) - { - throw new NotFoundException("Must specify dbSchema parameter"); - } - - boolean bFull = form.getFull(); - - Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); - TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); - StringWriter sw = new StringWriter(); - - XmlOptions xOpt = new XmlOptions(); - xOpt.setSavePrettyPrint(); - xOpt.setUseDefaultNamespace(); - - tdoc.save(sw, xOpt); - - sw.flush(); - PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderInformationAction extends FolderManagementViewAction - { - @Override - protected HtmlView getTabView() - { - Container c = getContainer(); - User currentUser = getUser(); - - User createdBy = UserManager.getUser(c.getCreatedBy()); - Map propValueMap = new LinkedHashMap<>(); - propValueMap.put("Path", c.getPath()); - propValueMap.put("Name", c.getName()); - propValueMap.put("Displayed Title", c.getTitle()); - propValueMap.put("EntityId", c.getId()); - propValueMap.put("RowId", c.getRowId()); - propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); - propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); - propValueMap.put("Folder Type", c.getFolderType().getName()); - propValueMap.put("Description", c.getDescription()); - - return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); - } - } - - public static class MissingValuesForm - { - private boolean _inheritMvIndicators; - private String[] _mvIndicators; - private String[] _mvLabels; - - public boolean isInheritMvIndicators() - { - return _inheritMvIndicators; - } - - public void setInheritMvIndicators(boolean inheritMvIndicators) - { - _inheritMvIndicators = inheritMvIndicators; - } - - public String[] getMvIndicators() - { - return _mvIndicators; - } - - public void setMvIndicators(String[] mvIndicators) - { - _mvIndicators = mvIndicators; - } - - public String[] getMvLabels() - { - return _mvLabels; - } - - public void setMvLabels(String[] mvLabels) - { - _mvLabels = mvLabels; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MissingValuesAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); - } - - @Override - public void validateCommand(MissingValuesForm form, Errors errors) - { - } - - @Override - public boolean handlePost(MissingValuesForm form, BindException errors) - { - if (form.isInheritMvIndicators()) - { - MvUtil.inheritMvIndicators(getContainer()); - return true; - } - else - { - // Javascript should have enforced any constraints - MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); - return true; - } - } - } - - @SuppressWarnings("unused") - public static class RConfigForm - { - private Integer _reportEngine; - private Integer _pipelineEngine; - private boolean _overrideDefault; - - public Integer getReportEngine() - { - return _reportEngine; - } - - public void setReportEngine(Integer reportEngine) - { - _reportEngine = reportEngine; - } - - public Integer getPipelineEngine() - { - return _pipelineEngine; - } - - public void setPipelineEngine(Integer pipelineEngine) - { - _pipelineEngine = pipelineEngine; - } - - public boolean getOverrideDefault() - { - return _overrideDefault; - } - - public void setOverrideDefault(String overrideDefault) - { - _overrideDefault = "override".equals(overrideDefault); - } - } - - @RequiresPermission(AdminPermission.class) - public static class RConfigurationAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); - } - - @Override - public void validateCommand(RConfigForm form, Errors errors) - { - if (form.getOverrideDefault()) - { - if (form.getReportEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); - if (form.getPipelineEngine() == null) - errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); - } - } - - @Override - public URLHelper getSuccessURL(RConfigForm rConfigForm) - { - return getContainer().getStartURL(getUser()); - } - - @Override - public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception - { - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - if (null != mgr) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - if (rConfigForm.getOverrideDefault()) - { - ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); - ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); - - if (reportEngine != null) - mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - else - { - // need to clear the current scope (if any) - ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); - ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); - - if (reportEngine != null) - mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); - if (pipelineEngine != null) - mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); - } - transaction.commit(); - } - return true; - } - return false; - } - } - - @SuppressWarnings("unused") - public static class ExportFolderForm - { - private String[] _types; - private int _location; - private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. - private String _exportType; - private boolean _includeSubfolders; - private PHI _exportPhiLevel; // Input: max level when viewing form - private boolean _shiftDates; - private boolean _alternateIds; - private boolean _maskClinic; - - public String[] getTypes() - { - return _types; - } - - public void setTypes(String[] types) - { - _types = types; - } - - public int getLocation() - { - return _location; - } - - public void setLocation(int location) - { - _location = location; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - - public ExportType getExportType() - { - if ("study".equals(_exportType)) - return ExportType.STUDY; - else - return ExportType.ALL; - } - - public void setExportType(String exportType) - { - _exportType = exportType; - } - - public boolean isIncludeSubfolders() - { - return _includeSubfolders; - } - - public void setIncludeSubfolders(boolean includeSubfolders) - { - _includeSubfolders = includeSubfolders; - } - - public PHI getExportPhiLevel() - { - return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - public boolean isShiftDates() - { - return _shiftDates; - } - - public void setShiftDates(boolean shiftDates) - { - _shiftDates = shiftDates; - } - - public boolean isAlternateIds() - { - return _alternateIds; - } - - public void setAlternateIds(boolean alternateIds) - { - _alternateIds = alternateIds; - } - - public boolean isMaskClinic() - { - return _maskClinic; - } - - public void setMaskClinic(boolean maskClinic) - { - _maskClinic = maskClinic; - } - } - - public enum ExportOption - { - PipelineRootAsFiles("file root as multiple files") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - else if (root.isCloudRoot()) - { - errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); - } - else - { - File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); - try - { - writer.write(container, ctx, new FileSystemFile(exportDir)); - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - return null; - } - }, - - PipelineRootAsZip("file root as a single zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null || !root.isValid()) - { - throw new NotFoundException("No valid pipeline root found"); - } - Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); - FileUtil.createDirectories(exportDir); - exportFolderToFile(exportDir, container, writer, ctx, errors); - return urlProvider(PipelineUrls.class).urlBrowse(container); - } - }, - DownloadAsZip("browser download as a zip file") - { - @Override - public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception - { - try - { - // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 - // Same pattern as ExportListArchiveAction - Path tempDir = FileUtil.getTempDirectory().toPath(); - Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); - - // No exceptions, so stream the resulting zip file to the browser and delete it - try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) - { - Files.copy(tempZipFile, os); - } - finally - { - Files.delete(tempZipFile); - } - } - catch (ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - return null; - } - }; - - private final String _description; - - ExportOption(String description) - { - _description = description; - } - - public String getDescription() - { - return _description; - } - - public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; - - Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception - { - String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); - - try (ZipFile zip = new ZipFile(exportDir, filename)) - { - writer.write(container, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return exportDir.resolve(filename); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ExportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception - { - // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so - // rendering a view would throw. - return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); - } - - @Override - protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) - { - form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); - - ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); - PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); - PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); - form.setExportPhiLevel(maxAllowedPhiForExport); - - return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ExportFolderForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ExportOption exportOption = null; - if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) - { - exportOption = ExportOption.values()[form.getLocation()]; - } - if (exportOption == null) - { - throw new NotFoundException("Invalid export location: " + form.getLocation()); - } - ContainerManager.checkContainerValidity(container); - - FolderWriterImpl writer = new FolderWriterImpl(); - FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), - form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), - form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); - - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); - AuditLogService.get().addEvent(getUser(), event); - - _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) - { - return _successURL; - } - } - - public static class ImportFolderForm - { - private boolean _createSharedDatasets; - private boolean _validateQueries; - private boolean _failForUndefinedVisits; - private String _sourceTemplateFolder; - private String _sourceTemplateFolderId; - private String _origin; - - public boolean isCreateSharedDatasets() - { - return _createSharedDatasets; - } - - public void setCreateSharedDatasets(boolean createSharedDatasets) - { - _createSharedDatasets = createSharedDatasets; - } - - public boolean isValidateQueries() - { - return _validateQueries; - } - - public boolean isFailForUndefinedVisits() - { - return _failForUndefinedVisits; - } - - public void setFailForUndefinedVisits(boolean failForUndefinedVisits) - { - _failForUndefinedVisits = failForUndefinedVisits; - } - - public void setValidateQueries(boolean validateQueries) - { - _validateQueries = validateQueries; - } - - public String getSourceTemplateFolder() - { - return _sourceTemplateFolder; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolder(String sourceTemplateFolder) - { - _sourceTemplateFolder = sourceTemplateFolder; - } - - public String getSourceTemplateFolderId() - { - return _sourceTemplateFolderId; - } - - @SuppressWarnings("unused") - public void setSourceTemplateFolderId(String sourceTemplateFolderId) - { - _sourceTemplateFolderId = sourceTemplateFolderId; - } - - public String getOrigin() - { - return _origin; - } - - public void setOrigin(String origin) - { - _origin = origin; - } - - public Container getSourceTemplateFolderContainer() - { - if (null == getSourceTemplateFolderId()) - return null; - return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ImportFolderAction extends FolderManagementViewPostAction - { - private ActionURL _successURL; - - @Override - protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) - { - // default the createSharedDatasets and validateQueries to true if this is not a form error reshow - if (!errors.hasErrors()) - { - form.setCreateSharedDatasets(true); - form.setValidateQueries(true); - } - - return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); - } - - @Override - public void validateCommand(ImportFolderForm form, Errors errors) - { - // don't allow import into the root container - if (getContainer().isRoot()) - { - throw new NotFoundException(); - } - } - - @Override - public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception - { - ViewContext context = getViewContext(); - ActionURL url = context.getActionURL(); - User user = getUser(); - Container container = getContainer(); - PipeRoot pipelineRoot; - FileLike pipelineUnzipDir; // Should be local & writable - PipelineUrls pipelineUrlProvider; - - if (form.getOrigin() == null) - { - form.setOrigin("Folder"); - } - - // make sure we have a pipeline url provider to use for the success URL redirect - pipelineUrlProvider = urlProvider(PipelineUrls.class); - if (pipelineUrlProvider == null) - { - errors.reject("folderImport", "Pipeline url provider does not exist."); - return false; - } - - // make sure that the pipeline root is valid for this container - pipelineRoot = PipelineService.get().findPipelineRoot(container); - if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) - { - errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); - return false; - } - - // make sure we are able to delete any existing unzip dir in the pipeline root - try - { - pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); - } - catch (DirectoryNotDeletedException e) - { - errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); - return false; - } - - FolderImportConfig fiConfig; - if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) - { - fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); - } - else - { - fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); - if (fiConfig == null || errors.hasErrors()) - { - return false; - } - } - - // get the folder.xml file from the unzipped import archive - FileLike archiveXml = pipelineUnzipDir.resolveChild("folder.xml"); - if (!archiveXml.exists()) - { - errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); - return false; - } - - ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); - options.setSkipQueryValidation(!form.isValidateQueries()); - options.setCreateSharedDatasets(form.isCreateSharedDatasets()); - options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); - options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); - - // finally, create the study or folder import pipeline job - _successURL = pipelineUrlProvider.urlBegin(container); - PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); - - return !errors.hasErrors(); - } - - private @Nullable FolderImportConfig getFolderFromZipArchive(FileLike pipelineUnzipDir, BindException errors) - { - // user chose to import from a zip file - Map map = getFileMap(); - - // make sure we have a single file selected for import - if (map.size() != 1) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // make sure the file is not empty and that it has a .zip extension - MultipartFile zipFile = map.values().iterator().next(); - String originalFilename = zipFile.getOriginalFilename(); - if (0 == zipFile.getSize() || isBlank(originalFilename) || !originalFilename.toLowerCase().endsWith(".zip")) - { - errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); - return null; - } - - // copy and unzip the uploaded import archive zip file to the pipeline unzip dir - try - { - FileLike pipelineUnzipFile = pipelineUnzipDir.resolveFile(org.labkey.api.util.Path.parse(originalFilename)); - // Check that the resolved file is under the pipelineUnzipDir - if (!pipelineUnzipFile.toNioPathForRead().normalize().startsWith(pipelineUnzipDir.toNioPathForRead().normalize())) - { - errors.reject("folderImport", "Invalid file path - must be within the unzip directory"); - return null; - } - - FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) - FileUtil.createNewFile(pipelineUnzipFile, true); - try (OutputStream os = pipelineUnzipFile.openOutputStream()) - { - FileUtil.copyData(zipFile.getInputStream(), os); - } - ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); - - return new FolderImportConfig( - false, - originalFilename, - pipelineUnzipFile, - pipelineUnzipFile - ); - } - catch (FileNotFoundException e) - { - LOG.debug("Failed to import '" + originalFilename + "'.", e); - errors.reject("folderImport", "File not found."); - return null; - } - catch (IOException e) - { - LOG.debug("Failed to import '" + originalFilename + "'.", e); - errors.reject("folderImport", "Unable to unzip folder archive."); - return null; - } - } - - private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final FileLike pipelineUnzipDir, final BindException errors) throws Exception - { - // user choose to import from a template source folder - Container sourceContainer = form.getSourceTemplateFolderContainer(); - - // In order to support the Advanced import options to import into multiple target folders we need to zip - // the source template folder so that the zip file can be passed to the pipeline processes. - FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, - getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, - PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - FolderWriterImpl writer = new FolderWriterImpl(); - String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); - FileLike implicitZipFile = pipelineUnzipDir.resolveChild(zipFileName); - if (!pipelineUnzipDir.isDirectory()) - pipelineUnzipDir.mkdirs(); - implicitZipFile.createFile(); - try (OutputStream out = implicitZipFile.openOutputStream(); - ZipFile zip = new ZipFile(out, false)) - { - writer.write(sourceContainer, ctx, zip); - } - catch (Container.ContainerException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container - ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); - - return new FolderImportConfig( - StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), - implicitZipFile.getName(), - implicitZipFile, - null - ); - } - - private static class FolderImportConfig { - FileLike pipelineUnzipFile; - String originalFileName; - FileLike archiveFile; - boolean fromTemplateSourceFolder; - - public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, FileLike archiveFile, @Nullable FileLike pipelineUnzipFile) - { - this.originalFileName = originalFileName; - this.archiveFile = archiveFile; - this.fromTemplateSourceFolder = fromTemplateSourceFolder; - this.pipelineUnzipFile = pipelineUnzipFile; - } - } - - @Override - public URLHelper getSuccessURL(ImportFolderForm importFolderForm) - { - return _successURL; - } - } - - private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) - { - // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of - // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers - // to apply during the implicit export. - Set registeredFolderWriters = new HashSet<>(); - FolderSerializationRegistry registry = FolderSerializationRegistry.get(); - if (null == registry) - { - throw new RuntimeException(); - } - Collection registeredWriters = registry.getRegisteredFolderWriters(); - for (FolderWriter writer : registeredWriters) - { - String dataType = writer.getDataType(); - boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); - boolean excludeForTemplate = !writer.includeWithTemplate(); - - if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) - { - registeredFolderWriters.add(dataType); - - // for each Writer also determine if there are related children Writers, if so include them also - Collection> childWriters = writer.getChildren(true, true); - if (!childWriters.isEmpty()) - { - for (org.labkey.api.writer.Writer child : childWriters) - { - dataType = child.getDataType(); - if (dataType != null) - registeredFolderWriters.add(dataType); - } - } - } - } - return registeredFolderWriters; - } - - public static class FolderSettingsForm - { - private String _defaultDateFormat; - private boolean _defaultDateFormatInherited; - private String _defaultDateTimeFormat; - private boolean _defaultDateTimeFormatInherited; - private String _defaultTimeFormat; - private boolean _defaultTimeFormatInherited; - private String _defaultNumberFormat; - private boolean _defaultNumberFormatInherited; - private boolean _restrictedColumnsEnabled; - private boolean _restrictedColumnsEnabledInherited; - - public String getDefaultDateFormat() - { - return _defaultDateFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormat(String defaultDateFormat) - { - _defaultDateFormat = defaultDateFormat; - } - - public boolean isDefaultDateFormatInherited() - { - return _defaultDateFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) - { - _defaultDateFormatInherited = defaultDateFormatInherited; - } - - public String getDefaultDateTimeFormat() - { - return _defaultDateTimeFormat; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormat(String defaultDateTimeFormat) - { - _defaultDateTimeFormat = defaultDateTimeFormat; - } - - public boolean isDefaultDateTimeFormatInherited() - { - return _defaultDateTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) - { - _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; - } - - public String getDefaultTimeFormat() - { - return _defaultTimeFormat; - } - - @SuppressWarnings("UnusedDeclaration") - public void setDefaultTimeFormat(String defaultTimeFormat) - { - _defaultTimeFormat = defaultTimeFormat; - } - - public boolean isDefaultTimeFormatInherited() - { - return _defaultTimeFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) - { - _defaultTimeFormatInherited = defaultTimeFormatInherited; - } - - public String getDefaultNumberFormat() - { - return _defaultNumberFormat; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormat(String defaultNumberFormat) - { - _defaultNumberFormat = defaultNumberFormat; - } - - public boolean isDefaultNumberFormatInherited() - { - return _defaultNumberFormatInherited; - } - - @SuppressWarnings("unused") - public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) - { - _defaultNumberFormatInherited = defaultNumberFormatInherited; - } - - public boolean areRestrictedColumnsEnabled() - { - return _restrictedColumnsEnabled; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) - { - _restrictedColumnsEnabled = restrictedColumnsEnabled; - } - - public boolean isRestrictedColumnsEnabledInherited() - { - return _restrictedColumnsEnabledInherited; - } - - @SuppressWarnings("unused") - public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) - { - _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FolderSettingsAction extends FolderManagementViewPostAction - { - @Override - protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(FolderSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(FolderSettingsForm form, BindException errors) - { - return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - } - } - - // Validate and populate the folder settings; save & log all changes - private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) - { - validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); - validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); - validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); - validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); - - setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); - - if (!errors.hasErrors()) - { - props.save(); - - //write an audit log event - props.writeAuditLogEvent(c, user); - } - - return !errors.hasErrors(); - } - - private interface FormatSaver - { - void save(String format) throws IllegalArgumentException; - } - - private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) - { - String defaultFormat = StringUtils.trimToNull(format); - if (inherited) - { - clearer.run(); - } - else - { - try - { - saver.save(defaultFormat); - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class ModulePropertiesAction extends FolderManagementViewAction - { - @Override - protected JspView getTabView() - { - return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); - } - } - - @SuppressWarnings("unused") - public static class FolderTypeForm - { - private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; - private String _defaultModule; - private String _folderType; - private boolean _wizard; - - public String[] getActiveModules() - { - return _activeModules; - } - - public void setActiveModules(String[] activeModules) - { - _activeModules = activeModules; - } - - public String getDefaultModule() - { - return _defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - _defaultModule = defaultModule; - } - - public String getFolderType() - { - return _folderType; - } - - public void setFolderType(String folderType) - { - _folderType = folderType; - } - - public boolean isWizard() - { - return _wizard; - } - - public void setWizard(boolean wizard) - { - _wizard = wizard; - } - } - - @RequiresPermission(AdminPermission.class) - @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments - public static class FolderTypeAction extends FolderManagementViewPostAction - { - private ActionURL _successURL = null; - - @Override - protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); - } - - @Override - public void validateCommand(FolderTypeForm form, Errors errors) - { - boolean fEmpty = true; - for (String module : form._activeModules) - { - if (module != null) - { - fEmpty = false; - break; - } - } - if (fEmpty && "None".equals(form.getFolderType())) - { - errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); - } - } - - @Override - public boolean handlePost(FolderTypeForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - String[] modules = form.getActiveModules(); - - if (modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) - { - container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - container.setDefaultModule(defaultModule); - } - else - { - FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); - if (container.isContainerTab() && folderType.hasContainerTabs()) - errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); - else - container.setFolderType(folderType, getUser(), errors, activeModules); - } - if (errors.hasErrors()) - return false; - - if (form.isWizard()) - { - _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - } - else - _successURL = container.getFolderType().getStartURL(container, getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) - { - return _successURL; - } - } - - @SuppressWarnings("unused") - public static class FileRootsForm extends SetupForm implements FileManagementForm - { - private String _folderRootPath; - private String _fileRootOption; - private String _cloudRootName; - private boolean _isFolderSetup; - private boolean _fileRootChanged; - private boolean _enabledCloudStoresChanged; - private String _migrateFilesOption; - - // cloud settings - private String[] _enabledCloudStore; - //file management - @Override - public String getFolderRootPath() - { - return _folderRootPath; - } - - @Override - public void setFolderRootPath(String folderRootPath) - { - _folderRootPath = folderRootPath; - } - - @Override - public String getFileRootOption() - { - return _fileRootOption; - } - - @Override - public void setFileRootOption(String fileRootOption) - { - _fileRootOption = fileRootOption; - } - - @Override - public String[] getEnabledCloudStore() - { - return _enabledCloudStore; - } - - @Override - public void setEnabledCloudStore(String[] enabledCloudStore) - { - _enabledCloudStore = enabledCloudStore; - } - - @Override - public boolean isDisableFileSharing() - { - return FileRootProp.disable.name().equals(getFileRootOption()); - } - - @Override - public boolean hasSiteDefaultRoot() - { - return FileRootProp.siteDefault.name().equals(getFileRootOption()); - } - - @Override - public boolean isCloudFileRoot() - { - return FileRootProp.cloudRoot.name().equals(getFileRootOption()); - } - - @Override - @Nullable - public String getCloudRootName() - { - return _cloudRootName; - } - - @Override - public void setCloudRootName(String cloudRootName) - { - _cloudRootName = cloudRootName; - } - - @Override - public boolean isFolderSetup() - { - return _isFolderSetup; - } - - public void setFolderSetup(boolean folderSetup) - { - _isFolderSetup = folderSetup; - } - - public boolean isFileRootChanged() - { - return _fileRootChanged; - } - - @Override - public void setFileRootChanged(boolean changed) - { - _fileRootChanged = changed; - } - - public boolean isEnabledCloudStoresChanged() - { - return _enabledCloudStoresChanged; - } - - @Override - public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) - { - _enabledCloudStoresChanged = enabledCloudStoresChanged; - } - - @Override - public String getMigrateFilesOption() - { - return _migrateFilesOption; - } - - @Override - public void setMigrateFilesOption(String migrateFilesOption) - { - _migrateFilesOption = migrateFilesOption; - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsStandAloneAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - view.setFrame(WebPartView.FrameType.NONE); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); - getPageConfig().setTemplate(PageConfig.Template.Wizard); - getPageConfig().setTitle("Change File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) - .addParameter("folderSetup", true) - .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - /** - * This standalone file root management action can be used on folder types that do not support - * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL - * navigation when a workbook needs it. - */ - @RequiresPermission(AdminPermission.class) - public class ManageFileRootAction extends FormViewAction - { - @Override - public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) - { - JspView view = getFileRootsView(form, errors, getReshow()); - getPageConfig().setTitle("Manage File Root"); - return view; - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = getContainer().getStartURL(getUser()); - - if (getViewContext().getActionURL().getReturnUrl() != null) - { - url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); - } - - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public class FileRootsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) - { - return getFileRootsView(form, errors, getReshow()); - } - - @Override - public void validateCommand(FileRootsForm form, Errors errors) - { - validateCloudFileRoot(form, getContainer(), errors); - } - - @Override - public boolean handlePost(FileRootsForm form, BindException errors) throws Exception - { - return handleFileRootsPost(form, errors); - } - - @Override - public ActionURL getSuccessURL(FileRootsForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); - - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - return url; - } - } - - private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) - { - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - view.setFrame(WebPartView.FrameType.DIV); - try - { - if (!reshow) - setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - - return view; - } - - private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception - { - if (form.isPipelineRootForm()) - { - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - } - else - { - setFileRootFromForm(getViewContext(), form, errors); - setEnabledCloudStores(getViewContext(), form, errors); - return !errors.hasErrors(); - } - } - - public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) - { - FileContentService service = FileContentService.get(); - if (null != service) - { - boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); - String cloudRootName = form.getCloudRootName(); - if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) - { - Path defaultRootPath = service.getDefaultRootPath(container, false); - cloudRootName = service.getDefaultRootInfo(container).getCloudName(); - isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); - } - - if (isOrDefaultsToCloudRoot && null != cloudRootName) - { - if (null != form.getEnabledCloudStore()) - { - for (String storeName : form.getEnabledCloudStore()) - { - if (Strings.CI.equals(cloudRootName, storeName)) - return; - } - } - // Didn't find cloud root in enabled list - errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); - } - } - } - - public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) - { - boolean changed = false; - boolean shouldCopyMove = false; - FileContentService service = FileContentService.get(); - if (null != service) - { - // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. - // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. - MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? - MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : - MigrateFilesOption.leave; - List> sourceInfos = - ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? - Collections.emptyList() : - getCopySourceInfo(service, ctx.getContainer()); - - if (form.isDisableFileSharing()) - { - if (!service.isFileRootDisabled(ctx.getContainer())) - { - service.disableFileRoot(ctx.getContainer()); - changed = true; - } - } - else if (form.hasSiteDefaultRoot()) - { - if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), true); - changed = true; - shouldCopyMove = true; - } - } - else if (form.isCloudFileRoot()) - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String cloudRootName = form.getCloudRootName(); - if (null != cloudRootName && - (!service.isCloudRoot(ctx.getContainer()) || - !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setCloudRoot(ctx.getContainer(), cloudRootName); - try - { - PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); - if (form.isFolderSetup() && !sourceInfos.isEmpty()) - { - // File root was set to cloud storage, remove folder created - Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded - if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) - { - try - { - Files.deleteIfExists(fromPath.getParent()); - } - catch (IOException e) - { - LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); - } - } - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - changed = true; - shouldCopyMove = true; - } - } - else - { - throwIfUnauthorizedFileRootChange(ctx, service, form); - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - URI uri = FileUtil.createUri(root, false); // root is unencoded - Path path = FileUtil.getPath(ctx.getContainer(), uri); - if (null == path || !Files.exists(path)) - { - errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); - } - else - { - Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) - { - service.setIsUseDefaultRoot(ctx.getContainer(), false); - service.setFileRootPath(ctx.getContainer(), root); - changed = true; - shouldCopyMove = true; - } - } - } - else - { - service.setFileRootPath(ctx.getContainer(), null); - changed = true; - } - } - - if (!errors.hasErrors()) - { - if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) - { - // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); - if (null != pipeRoot) - { - try - { - initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); - } - catch (PipelineValidationException e) - { - throw new RuntimeValidationException(e); - } - } - else - { - LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); - } - } - - form.setFileRootChanged(changed); - if (changed && null != ctx.getUser()) - { - setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); - String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); - AuditLogService.get().addEvent(ctx.getUser(), event); - } - } - } - } - - private static List> getCopySourceInfo(FileContentService service, Container container) - { - - List> sourceInfo = new ArrayList<>(); - addCopySourceInfo(service, container, sourceInfo, true); - return sourceInfo; - } - - private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) - { - if (isRoot || service.isUseDefaultRoot(container)) - { - Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); - if (null != sourceFileRootDir) - { - String pathStr = FileUtil.pathToString(sourceFileRootDir); - if (null != pathStr) - sourceInfo.add(new Pair<>(container, pathStr)); - else - throw new RuntimeValidationException("Unexpected error converting path to string"); - } - } - for (Container childContainer : container.getChildren()) - addCopySourceInfo(service, childContainer, sourceInfo, false); - } - - private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, - MigrateFilesOption migrateFilesOption) throws PipelineValidationException - { - CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); - PipelineService.get().queueJob(job); - } - - private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) - { - // test permissions. only site admins are able to turn on a custom file root for a folder - // this is only relevant if the folder is either being switched to a custom file root, - // or if the file root is changed. - if (!service.isUseDefaultRoot(ctx.getContainer())) - { - Path fileRootPath = service.getFileRootPath(ctx.getContainer()); - if (null != fileRootPath) - { - String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); - if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) - { - if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) - throw new UnauthorizedException("Only site admins can change file roots"); - } - } - } - } - - public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) - { - String[] enabledCloudStores = form.getEnabledCloudStore(); - CloudStoreService cloud = CloudStoreService.get(); - if (cloud != null) - { - Set enabled = Collections.emptySet(); - if (enabledCloudStores != null) - enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); - - try - { - // Check if anything changed - boolean changed = false; - Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); - if (enabled.size() != storeNames.size()) - changed = true; - else - if (!enabled.containsAll(storeNames)) - changed = true; - if (changed) - cloud.setEnabledCloudStores(ctx.getContainer(), enabled); - form.setEnabledCloudStoresChanged(changed); - } - catch (UncheckedExecutionException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException - // is what BlobStore hands us if bucket (S3 container) does not exist - if (null != e.getCause()) - errors.reject(ERROR_MSG, e.getCause().getMessage()); - else - throw e; - } - catch (RuntimeException e) - { - LOG.debug("Failed to configure cloud store(s).", e); - errors.reject(ERROR_MSG, e.getMessage()); - } - } - } - - - public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException - { - String rootSetParam = ctx.getActionURL().getParameter("rootSet"); - boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); - String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); - boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); - setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); - } - - public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, - String migrateFilesOption) throws IllegalArgumentException - { - FileContentService service = FileContentService.get(); - String confirmMessage = null; - - String migrateFilesMessage = ""; - if (fileRootChanged && !form.isFolderSetup()) - { - if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) - migrateFilesMessage = ". Existing files not copied or moved."; - else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files copied."; - form.setMigrateFilesOption(migrateFilesOption); - } - else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) - { - migrateFilesMessage = ". Existing files moved."; - form.setMigrateFilesOption(migrateFilesOption); - } - } - - if (service != null) - { - if (service.isFileRootDisabled(container)) - { - form.setFileRootOption(FileRootProp.disable.name()); - if (fileRootChanged) - confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); - } - else if (service.isUseDefaultRoot(container)) - { - form.setFileRootOption(FileRootProp.siteDefault.name()); - Path root = service.getFileRootPath(container); - if (root != null && Files.exists(root) && fileRootChanged) - confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; - } - else if (!service.isCloudRoot(container)) - { - Path root = service.getFileRootPath(container); - - form.setFileRootOption(FileRootProp.folderOverride.name()); - if (root != null) - { - String absolutePath = FileUtil.getAbsolutePath(container, root); - form.setFolderRootPath(absolutePath); - if (Files.exists(root)) - { - if (fileRootChanged) - confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; - } - } - } - else - { - form.setFileRootOption(FileRootProp.cloudRoot.name()); - form.setCloudRootName(service.getCloudRootName(container)); - Path root = service.getFileRootPath(container); - if (root != null && fileRootChanged) - { - confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; - } - } - } - - if (fileRootChanged && confirmMessage != null) - form.setConfirmMessage(confirmMessage); - else if (enabledCloudChanged) - form.setConfirmMessage("The enabled cloud stores changed."); - } - - @RequiresPermission(AdminPermission.class) - public static class ManageFoldersAction extends FolderManagementViewAction - { - @Override - protected HttpView getTabView() - { - return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); - } - } - - public static class NotificationsForm - { - private String _provider; - - public String getProvider() - { - return _provider; - } - - public void setProvider(String provider) - { - _provider = provider; - } - } - - private static final String DATA_REGION_NAME = "Users"; - - @RequiresPermission(AdminPermission.class) - public static class NotificationsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) - { - final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); - DataRegionSelection.clearAll(getViewContext(), key); - - QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); - settings.setAllowChooseView(true); - settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); - - UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); - QueryView queryView = new QueryView(schema, settings, errors) - { - @Override - public List getDisplayColumns() - { - List columns = new ArrayList<>(); - SecurityPolicy policy = getContainer().getPolicy(); - Set assignmentSet = new HashSet<>(); - - for (RoleAssignment assignment : policy.getAssignments()) - { - Group g = SecurityManager.getGroup(assignment.getUserId()); - if (g != null) - assignmentSet.add(g.getName()); - } - - for (DisplayColumn col : super.getDisplayColumns()) - { - if (col.getName().equalsIgnoreCase("Groups")) - columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); - else - columns.add(col); - } - return columns; - } - - @Override - protected void populateButtonBar(DataView dataView, ButtonBar bar) - { - try - { - // add the provider configuration menu items to the admin panel button - MenuButton adminButton = new MenuButton("Update user settings"); - adminButton.setRequiresSelection(true); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); - - bar.add(adminButton); - super.populateButtonBar(dataView, bar); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - }; - queryView.setShadeAlternatingRows(true); - queryView.setShowBorders(true); - queryView.setShowDetailsColumn(false); - queryView.setShowRecordSelectors(true); - queryView.setFrame(WebPartView.FrameType.NONE); - queryView.disableContainerFilterSelection(); - queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); - - VBox defaultsView = new VBox( - HtmlView.unsafe( - "
    Default settings
    " + - "You can change this folder's default settings for email notifications here.") - ); - - PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); - for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) - { - defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); - } - - return new VBox( - new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), - defaultsView, - new VBox( - HtmlView.unsafe( - "
    User settings
    " + - "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + - "notification setting is visible in the appropriately named column.

    " + - "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), - queryView - ) - ); - } - - @Override - public void validateCommand(NotificationsForm form, Errors errors) - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - provider.validateCommand(getViewContext(), errors); - } - - @Override - public boolean handlePost(NotificationsForm form, BindException errors) throws Exception - { - ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); - - if (provider != null) - { - return provider.handlePost(getViewContext(), errors); - } - errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); - return false; - } - } - - public static class NotifyOptionsForm - { - private String _type; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public ConfigTypeProvider getProvider() - { - return MessageConfigService.get().getConfigType(getType()); - } - } - - /** - * Action to populate an Ext store with email notification options for admin settings - */ - @RequiresPermission(AdminPermission.class) - public static class GetEmailOptionsAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(NotifyOptionsForm form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - ConfigTypeProvider provider = form.getProvider(); - if (provider != null) - { - List options = new ArrayList<>(); - - // if the list of options is not for the folder default, add an option to use the folder default - if (getViewContext().get("isDefault") == null) - options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); - - for (NotificationOption option : provider.getOptions()) - { - options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); - } - resp.put("success", true); - if (!options.isEmpty()) - resp.put("options", options); - } - else - resp.put("success", false); - - return resp; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetBulkEmailOptionsAction extends MutatingApiAction - { - @Override - public ApiResponse execute(EmailConfigFormImpl form, BindException errors) - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - ConfigTypeProvider provider = form.getProvider(); - String srcIdentifier = getContainer().getId(); - - Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); - - if (!selections.isEmpty() && provider != null) - { - int newOption = form.getIndividualEmailOption(); - - for (String user : selections) - { - User projectUser = UserManager.getUser(Integer.parseInt(user)); - UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); - - int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; - - //has this projectUser's option changed? if so, update - //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default - if (currentEmailOption != newOption) - { - provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); - } - } - resp.put("success", true); - } - else - { - resp.put("success", false); - resp.put("message", "There were no users selected"); - } - return resp; - } - } - - /** Renders only the groups that are assigned roles in this container */ - private static class FolderGroupColumn extends DataColumn - { - private final Set _assignmentSet; - - public FolderGroupColumn(Set assignmentSet, ColumnInfo col) - { - super(col); - _assignmentSet = assignmentSet; - } - - @Override - public void renderGridCellContents(RenderContext ctx, HtmlWriter out) - { - String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); - - if (value != null) - { - out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) - .filter(_assignmentSet::contains) - .map(HtmlString::of) - .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); - } - } - } - - private static class PanelConfig implements MessageConfigService.PanelInfo - { - private final ActionURL _returnUrl; - private final String _dataRegionSelectionKey; - - public PanelConfig(ActionURL returnUrl, String selectionKey) - { - _returnUrl = returnUrl; - _dataRegionSelectionKey = selectionKey; - } - - @Override - public ActionURL getReturnUrl() - { - return _returnUrl; - } - - @Override - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - } - - public static class ConceptsForm - { - private String _conceptURI; - private String _containerId; - private String _schemaName; - private String _queryName; - - public String getConceptURI() - { - return _conceptURI; - } - - public void setConceptURI(String conceptURI) - { - _conceptURI = conceptURI; - } - - public String getContainerId() - { - return _containerId; - } - - public void setContainerId(String containerId) - { - _containerId = containerId; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConceptsAction extends FolderManagementViewPostAction - { - @Override - protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); - } - - @Override - public void validateCommand(ConceptsForm form, Errors errors) - { - // validate that the required input fields are provided - String missingRequired = "", sep = ""; - if (form.getConceptURI() == null) - { - missingRequired += "conceptURI"; - sep = ", "; - } - if (form.getSchemaName() == null) - { - missingRequired += sep + "schemaName"; - sep = ", "; - } - if (form.getQueryName() == null) - missingRequired += sep + "queryName"; - if (!missingRequired.isEmpty()) - errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); - - // validate that, if provided, the containerId matches an existing container - Container postContainer = null; - if (form.getContainerId() != null) - { - postContainer = ContainerManager.getForId(form.getContainerId()); - if (postContainer == null) - errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); - } - - // validate that the schema and query names provided exist - if (form.getSchemaName() != null && form.getQueryName() != null) - { - Container c = postContainer != null ? postContainer : getContainer(); - UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); - if (schema == null) - errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); - else if (schema.getTable(form.getQueryName()) == null) - errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); - } - } - - @Override - public boolean handlePost(ConceptsForm form, BindException errors) - { - Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); - ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); - - return true; - } - } - - @RequiresPermission(AdminPermission.class) - public class FolderAliasesAction extends FormViewAction - { - @Override - public void validateCommand(FolderAliasesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/folderAliases.jsp"); - } - - @Override - public boolean handlePost(FolderAliasesForm form, BindException errors) - { - List aliases = new ArrayList<>(); - if (form.getAliases() != null) - { - StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); - while (st.hasMoreTokens()) - { - String alias = st.nextToken().trim(); - if (!alias.startsWith("/")) - { - alias = "/" + alias; - } - while (alias.endsWith("/")) - { - alias = alias.substring(0, alias.lastIndexOf('/')); - } - aliases.add(alias); - } - } - ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); - - return true; - } - - @Override - public ActionURL getSuccessURL(FolderAliasesForm form) - { - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); - } - } - - public static class FolderAliasesForm - { - private String _aliases; - - public String getAliases() - { - return _aliases; - } - - @SuppressWarnings("unused") - public void setAliases(String aliases) - { - _aliases = aliases; - } - } - - @RequiresPermission(AdminPermission.class) - public class CustomizeEmailAction extends FormViewAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) - { - JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); - result.setTitle("Email Template"); - return result; - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - - template.setSubject(form.getEmailSubject()); - template.setSenderName(form.getEmailSender()); - template.setReplyToEmail(form.getEmailReplyTo()); - template.setBody(form.getEmailMessage()); - - String[] errorStrings = new String[1]; - if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. - EmailTemplateService.get().saveEmailTemplate(template, getContainer()); - else - errors.reject(ERROR_MSG, errorStrings[0]); - } - - return !errors.hasErrors(); - } - - @Override - public ActionURL getSuccessURL(CustomEmailForm form) - { - ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); - result.replaceParameter("templateClass", form.getTemplateClass()); - if (form.getReturnActionURL() != null) - { - result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - } - return result; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("customEmail"); - addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class DeleteCustomEmailAction extends FormHandlerAction - { - @Override - public void validateCommand(CustomEmailForm target, Errors errors) - { - } - - @Override - public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception - { - if (form.getTemplateClass() != null) - { - EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); - template.setSubject(form.getEmailSubject()); - template.setBody(form.getEmailMessage()); - - EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(CustomEmailForm form) - { - return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); - } - } - - @SuppressWarnings("unused") - public static class CustomEmailForm extends ReturnUrlForm - { - private String _templateClass; - private String _emailSubject; - private String _emailSender; - private String _emailReplyTo; - private String _emailMessage; - private String _templateDescription; - - public void setTemplateClass(String name){_templateClass = name;} - public String getTemplateClass(){return _templateClass;} - public void setEmailSubject(String subject){_emailSubject = subject;} - public String getEmailSubject(){return _emailSubject;} - public void setEmailSender(String sender){_emailSender = sender;} - public String getEmailSender(){return _emailSender;} - public void setEmailMessage(String body){_emailMessage = body;} - public String getEmailMessage(){return _emailMessage;} - public String getEmailReplyTo(){return _emailReplyTo;} - public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} - - public String getTemplateDescription() - { - return _templateDescription; - } - - public void setTemplateDescription(String templateDescription) - { - _templateDescription = templateDescription; - } - } - - private ActionURL getManageFoldersURL() - { - return new AdminUrlsImpl().getManageFoldersURL(getContainer()); - } - - public static class ManageFoldersForm extends ReturnUrlForm - { - private String name; - private String title; - private boolean titleSameAsName; - private String folder; - private String target; - private String folderType; - private String defaultModule; - private String[] activeModules; - private boolean hasLoaded = false; - private boolean showAll; - private boolean confirmed = false; - private boolean addAlias = false; - private String templateSourceId; - private String[] templateWriterTypes; - private boolean templateIncludeSubfolders = false; - private String[] targets; - private PHI _exportPhiLevel = PHI.NotPHI; - - public boolean getHasLoaded() - { - return hasLoaded; - } - - public void setHasLoaded(boolean hasLoaded) - { - this.hasLoaded = hasLoaded; - } - - public String[] getActiveModules() - { - return activeModules; - } - - public void setActiveModules(String[] activeModules) - { - this.activeModules = activeModules; - } - - public String getDefaultModule() - { - return defaultModule; - } - - public void setDefaultModule(String defaultModule) - { - this.defaultModule = defaultModule; - } - - public boolean isShowAll() - { - return showAll; - } - - public void setShowAll(boolean showAll) - { - this.showAll = showAll; - } - - public String getFolder() - { - return folder; - } - - public void setFolder(String folder) - { - this.folder = folder; - } - - public String getName() - { - return name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isTitleSameAsName() - { - return titleSameAsName; - } - - public void setTitleSameAsName(boolean updateTitle) - { - this.titleSameAsName = updateTitle; - } - public void setName(String name) - { - this.name = name; - } - - public boolean isConfirmed() - { - return confirmed; - } - - public void setConfirmed(boolean confirmed) - { - this.confirmed = confirmed; - } - - public String getFolderType() - { - return folderType; - } - - public void setFolderType(String folderType) - { - this.folderType = folderType; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - - public String getTarget() - { - return target; - } - - public void setTarget(String target) - { - this.target = target; - } - - public void setTemplateSourceId(String templateSourceId) - { - this.templateSourceId = templateSourceId; - } - - public String getTemplateSourceId() - { - return templateSourceId; - } - - public Container getTemplateSourceContainer() - { - if (null == getTemplateSourceId()) - return null; - return ContainerManager.getForId(getTemplateSourceId()); - } - - public String[] getTemplateWriterTypes() - { - return templateWriterTypes; - } - - public void setTemplateWriterTypes(String[] templateWriterTypes) - { - this.templateWriterTypes = templateWriterTypes; - } - - public boolean getTemplateIncludeSubfolders() - { - return templateIncludeSubfolders; - } - - public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) - { - this.templateIncludeSubfolders = templateIncludeSubfolders; - } - - public String[] getTargets() - { - return targets; - } - - public void setTargets(String[] targets) - { - this.targets = targets; - } - - public PHI getExportPhiLevel() - { - return _exportPhiLevel; - } - - public void setExportPhiLevel(PHI exportPhiLevel) - { - _exportPhiLevel = exportPhiLevel; - } - - /** - * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, - * but it will work for non-workbook children as well. - */ - public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException - { - if (getTargets() != null) - { - final List targets = new ArrayList<>(); - final List directChildren = ContainerManager.getChildren(currentContainer); - - Arrays.stream(getTargets()).forEach(x -> { - Container c = ContainerManager.getForId(x); - if (c == null) - { - try - { - Integer rowId = ConvertHelper.convert(x, Integer.class); - if (rowId > 0) - c = ContainerManager.getForRowId(rowId); - } - catch (ConversionException e) - { - //ignore - } - } - - if (c != null) - { - if (!c.equals(currentContainer)) - { - if (!directChildren.contains(c)) - { - throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); - } - - if (c.getContainerType().canHaveChildren()) - { - throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); - } - } - - targets.add(c); - } - else - { - throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); - } - }); - - return targets; - } - else - { - return Collections.singletonList(currentContainer); - } - } - } - - public static class RenameContainerForm - { - private String name; - private String title; - private boolean addAlias = true; - - public String getName() - { - return name; - } - - public void setName(String name) - { - this.name = name; - } - - public String getTitle() - { - return title; - } - - public void setTitle(String title) - { - this.title = title; - } - - public boolean isAddAlias() - { - return addAlias; - } - - public void setAddAlias(boolean addAlias) - { - this.addAlias = addAlias; - } - } - - // Note that validation checks occur in ContainerManager.rename() - @RequiresPermission(AdminPermission.class) - public static class RenameContainerAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RenameContainerForm form, BindException errors) - { - Container container = getContainer(); - String name = StringUtils.trimToNull(form.getName()); - String title = StringUtils.trimToNull(form.getTitle()); - - String nameValue = name; - String titleValue = title; - if (name == null && title == null) - { - errors.reject(ERROR_MSG, "Please specify a name or a title."); - return new ApiSimpleResponse("success", false); - } - else if (name != null && title == null) - { - titleValue = name; - } - else if (name == null) - { - nameValue = container.getName(); - } - - boolean addAlias = form.isAddAlias(); - - try - { - Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); - return new ApiSimpleResponse(c.toJSON(getUser())); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - return new ApiSimpleResponse("success", false); - } - } - } - - @RequiresPermission(AdminPermission.class) - public class RenameFolderAction extends FormViewAction - { - private ActionURL _returnUrl; - - @Override - public void validateCommand(ManageFoldersForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - try - { - String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); - Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); - _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); - return true; - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); - } - - return false; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - return _returnUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - String containerType = getContainer().isProject() ? "Project" : "Folder"; - addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); - } - } - - public static class MoveFolderTreeView extends JspView - { - private MoveFolderTreeView(ManageFoldersForm form, BindException errors) - { - super("/org/labkey/core/admin/moveFolder.jsp", form, errors); - } - } - - @RequiresPermission(AdminPermission.class) - @ActionNames("ShowMoveFolderTree,MoveFolder") - public class MoveFolderAction extends FormViewAction - { - boolean showConfirmPage = false; - boolean moveFailed = false; - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - Container c = getContainer(); - - if (c.isRoot()) - throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root - - if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); - - Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); - if (null == newParent) - { - errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); - } - else if (!newParent.hasPermission(getUser(), AdminPermission.class)) - { - throw new UnauthorizedException(); - } - else if (newParent.hasChild(c.getName())) - { - errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception - { - if (showConfirmPage) - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - if (moveFailed) - return new SimpleErrorView(errors); - else - return new MoveFolderTreeView(form, errors); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception - { - Container c = getContainer(); - Container newParent = ContainerManager.getForPath(form.getTarget()); - Container oldProject = c.getProject(); - Container newProject = newParent.isRoot() ? c : newParent.getProject(); - - if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) - { - showConfirmPage = true; - return false; // reshow - } - - try - { - ContainerManager.move(c, newParent, getUser()); - } - catch (ValidationException e) - { - moveFailed = true; - getPageConfig().setTemplate(Template.Dialog); - for (ValidationError validationError : e.getErrors()) - { - errors.addError(new LabKeyError(validationError.getMessage())); - } - if (!errors.hasErrors()) - errors.addError(new LabKeyError("Move failed")); - return false; - } - - if (form.isAddAlias()) - { - List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); - newAliases.add(c.getPath()); - ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); - } - return true; - } - - @Override - public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) - { - Container c = getContainer(); - c = ContainerManager.getForId(c.getId()); // Reload container to populate new location - return new AdminUrlsImpl().getManageFoldersURL(c); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Folder Management", getManageFoldersURL()); - root.addChild("Move Folder"); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ConfirmProjectMoveAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ManageFoldersForm form, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm Project Move"); - } - } - - private static abstract class AbstractCreateFolderAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FORM target, Errors errors) - { - } - - @Override - public ModelAndView getView(FORM form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - if (!reshow) - { - FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); - if (null != folderType) - { - // If a default folder type has been configured by a site admin set that as the default folder type choice - form.setFolderType(folderType.getName()); - } - form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); - } - JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); - vbox.addView(statusView); - - Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder - - setHelpTopic("createProject"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); - getPageConfig().setTemplate(Template.Wizard); - - if (c.isRoot()) - getPageConfig().setTitle("Create Project"); - else - { - String title = "Create Folder"; - - title += " in /"; - if (c == ContainerManager.getHomeContainer()) - title += "Home"; - else - title += c.getName(); - - getPageConfig().setTitle(title); - } - - return vbox; - } - - @Override - public boolean handlePost(FORM form, BindException errors) throws Exception - { - Container parent = getViewContext().getContainerNoTab(); - String folderName = StringUtils.trimToNull(form.getName()); - String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); - StringBuilder error = new StringBuilder(); - Consumer afterCreateHandler = getAfterCreateHandler(form); - - Container container; - - if (Container.isLegalName(folderName, parent.isRoot(), error)) - { - if (parent.hasChild(folderName)) - { - if (parent.isRoot()) - { - error.append("The server already has a project with this name."); - } - else - { - error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); - } - } - else - { - String folderType = form.getFolderType(); - - if (null == folderType) - { - errors.reject(null, "Folder type must be specified"); - return false; - } - - if ("Template".equals(folderType)) // Create folder from selected template - { - Container sourceContainer = form.getTemplateSourceContainer(); - if (null == sourceContainer) - { - errors.reject(null, "Source template folder not selected"); - return false; - } - else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) - { - errors.reject(null, "User does not have administrator permissions to the source container"); - return false; - } - else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) - { - errors.reject(null, "The source folder has a restricted module for which you do not have permission."); - return false; - } - - FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", - form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, - new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); - - container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); - } - else - { - FolderType type = FolderTypeManager.get().getFolderType(folderType); - - if (type == null) - { - errors.reject(null, "Folder type not recognized"); - return false; - } - - String[] modules = form.getActiveModules(); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - if (null == modules || modules.length == 0) - { - errors.reject(null, "At least one module must be selected"); - return false; - } - } - - // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. - Consumer configureContainer = (newContainer) -> - { - afterCreateHandler.accept(newContainer); - newContainer.setFolderType(type, getUser(), errors); - - if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) - { - Set activeModules = new HashSet<>(); - for (String moduleName : modules) - { - Module module = ModuleLoader.getInstance().getModule(moduleName); - if (module != null) - activeModules.add(module); - } - - newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); - Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); - newContainer.setDefaultModule(defaultModule); - } - }; - container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); - } - - _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - } - - errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); - return false; - } - - /** - * Return a Consumer that provides post-creation handling on the new Container - */ - abstract public Consumer getAfterCreateHandler(FORM form); - - @Override - protected String getCommandClassMethodName() - { - return "getAfterCreateHandler"; - } - - @Override - public ActionURL getSuccessURL(FORM form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminPermission.class) - public static class CreateFolderAction extends AbstractCreateFolderAction - { - @Override - public Consumer getAfterCreateHandler(ManageFoldersForm form) - { - // No special handling - return container -> {}; - } - } - - public static class CreateProjectForm extends ManageFoldersForm - { - private boolean _assignProjectAdmin = false; - - public boolean isAssignProjectAdmin() - { - return _assignProjectAdmin; - } - - @SuppressWarnings("unused") - public void setAssignProjectAdmin(boolean assignProjectAdmin) - { - _assignProjectAdmin = assignProjectAdmin; - } - } - - @RequiresPermission(CreateProjectPermission.class) - public static class CreateProjectAction extends AbstractCreateFolderAction - { - @Override - public void validateCommand(CreateProjectForm target, Errors errors) - { - super.validateCommand(target, errors); - if (!getContainer().isRoot()) - errors.reject(ERROR_MSG, "Must be invoked from the root"); - } - - @Override - public Consumer getAfterCreateHandler(CreateProjectForm form) - { - if (form.isAssignProjectAdmin()) - { - return c -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); - policy.addRoleAssignment(getUser(), ProjectAdminRole.class); - User savePolicyUser = getUser(); - if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) - { - // Special case for project creators who don't necessarily yet have permission to save the policy of - // the project they just created - savePolicyUser = User.getAdminServiceUser(); - } - - SecurityPolicyManager.savePolicy(policy, savePolicyUser); - }; - } - else - { - return c -> {}; - } - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetFolderPermissionsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(SetFolderPermissionsForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); - vbox.addView(statusView); - - Container c = getContainer(); - getPageConfig().setTitle("Users / Permissions"); - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - setHelpTopic("createProject"); - - return vbox; - } - - @Override - public boolean handlePost(SetFolderPermissionsForm form, BindException errors) - { - Container c = getContainer(); - String permissionType = form.getPermissionType(); - - if(c.isProject()){ - _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); - } - else - { - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - if (form.isAdvanced()) - { - _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); - } - else - { - _successURL = getContainer().getStartURL(getUser()); - } - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - } - - if(permissionType == null){ - errors.reject(ERROR_MSG, "You must select one of the options for permissions."); - return false; - } - - switch (permissionType) - { - case "CurrentUser" -> { - MutableSecurityPolicy policy = new MutableSecurityPolicy(c); - Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); - policy.addRoleAssignment(getUser(), role); - SecurityPolicyManager.savePolicy(policy, getUser()); - } - case "Inherit" -> SecurityManager.setInheritPermissions(c); - case "CopyExistingProject" -> { - String targetProject = form.getTargetProject(); - if (targetProject == null) - { - errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); - return false; - } - Container source = ContainerManager.getForId(targetProject); - if (source == null) - { - source = ContainerManager.getForPath(targetProject); - } - if (source == null) - { - throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); - } - Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); - - //copy role assignments - SecurityPolicy op = SecurityPolicyManager.getPolicy(source); - MutableSecurityPolicy np = new MutableSecurityPolicy(c); - for (RoleAssignment assignment : op.getAssignments()) - { - int userId = assignment.getUserId(); - UserPrincipal p = SecurityManager.getPrincipal(userId); - Role r = assignment.getRole(); - - if (p instanceof Group g) - { - if (!g.isProjectGroup()) - { - np.addRoleAssignment(p, r, false); - } - else - { - np.addRoleAssignment(groupMap.get(p), r, false); - } - } - else - { - np.addRoleAssignment(p, r, false); - } - } - SecurityPolicyManager.savePolicy(np, getUser()); - } - default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); - } - _successURL.addParameter("wizard", Boolean.TRUE.toString()); - - return true; - } - - @Override - public ActionURL getSuccessURL(SetFolderPermissionsForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - } - } - - public static class SetFolderPermissionsForm - { - private String targetProject; - private String permissionType; - private boolean advanced; - - public String getPermissionType() - { - return permissionType; - } - - @SuppressWarnings("unused") - public void setPermissionType(String permissionType) - { - this.permissionType = permissionType; - } - - public String getTargetProject() - { - return targetProject; - } - - @SuppressWarnings("unused") - public void setTargetProject(String targetProject) - { - this.targetProject = targetProject; - } - - public boolean isAdvanced() - { - return advanced; - } - - @SuppressWarnings("unused") - public void setAdvanced(boolean advanced) - { - this.advanced = advanced; - } - } - - @RequiresPermission(AdminPermission.class) - public static class SetInitialFolderSettingsAction extends FormViewAction - { - private ActionURL _successURL; - - @Override - public void validateCommand(FilesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) - { - VBox vbox = new VBox(); - Container c = getContainer(); - - JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); - vbox.addView(statusView); - - getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); - getPageConfig().setTemplate(Template.Wizard); - - String noun = c.isProject() ? "Project": "Folder"; - getPageConfig().setTitle(noun + " Settings"); - - return vbox; - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) - { - Container c = getContainer(); - String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); - String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; - - if(folderRootPath == null && !fileRootOption.equals("default")) - { - errors.reject(ERROR_MSG, "Error: Must supply a default file location."); - return false; - } - - FileContentService service = FileContentService.get(); - if(fileRootOption.equals("default")) - { - service.setIsUseDefaultRoot(c, true); - } - // Requires AdminOperationsPermission to set file root - else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) - { - if (!service.isValidProjectRoot(folderRootPath)) - { - errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - return false; - } - - service.setIsUseDefaultRoot(c.getProject(), false); - service.setFileRootPath(c.getProject(), folderRootPath); - } - - List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); - if (extraSteps.isEmpty()) - { - _successURL = getContainer().getStartURL(getUser()); - } - else - { - _successURL = new ActionURL(extraSteps.get(0).getHref()); - } - - return true; - } - - @Override - public ActionURL getSuccessURL(FilesForm form) - { - return _successURL; - } - - @Override - public void addNavTrail(NavTree root) - { - getPageConfig().setFocusId("name"); - setHelpTopic("createProject"); - } - } - - @RequiresPermission(DeletePermission.class) - public static class DeleteWorkbooksAction extends SimpleRedirectAction - { - public void validateCommand(ReturnUrlForm target, Errors errors) - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - if (ids.isEmpty()) - { - errors.reject(ERROR_MSG, "No IDs provided"); - } - } - - @Override - public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception - { - Set ids = DataRegionSelection.getSelected(getViewContext(), true); - - ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); - ids.forEach(id -> ret.addParameter("targets", id)); - - ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); - - return ret; - } - } - - //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent - @RequiresPermission(DeletePermission.class) - public static class DeleteFolderAction extends FormViewAction - { - private final List _deleted = new ArrayList<>(); - - @Override - public void validateCommand(ManageFoldersForm form, Errors errors) - { - try - { - List targets = form.getTargetContainers(getContainer()); - for (Container target : targets) - { - if (!ContainerManager.isDeletable(target)) - errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); - - if (target.isProject() && !getUser().hasRootAdminPermission()) - { - throw new UnauthorizedException(); - } - - Class permClass = target.getPermissionNeededToDelete(); - if (!target.hasPermission(getUser(), permClass)) - { - Permission perm = RoleManager.getPermission(permClass); - throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); - } - - if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) - { - throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); - } - - if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) - errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); - } - } - catch (IllegalArgumentException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - } - - @Override - public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) - { - getPageConfig().setTemplate(Template.Dialog); - return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); - } - - @Override - public boolean handlePost(ManageFoldersForm form, BindException errors) - { - List targets = form.getTargetContainers(getContainer()); - - // Must be site/app admin to delete a project - for (Container c : targets) - { - ContainerManager.deleteAll(c, getUser()); - } - - _deleted.addAll(targets); - - return true; - } - - @Override - public ActionURL getSuccessURL(ManageFoldersForm form) - { - // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: - // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders - if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) - { - Container c = getContainer(); - if (c.isProject()) - return AppProps.getInstance().getHomePageActionURL(); - else - return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); - } - else - { - if (form.getReturnUrl() != null) - { - return form.getReturnActionURL(); - } - else - { - return getContainer().getStartURL(getUser()); - } - } - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersAction extends FormViewAction - { - @Override - public void validateCommand(FolderReorderForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) - { - return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); - } - - @Override - public boolean handlePost(FolderReorderForm form, BindException errors) - { - return ReorderFolders(form, errors); - } - - @Override - public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) - { - if (getContainer().isRoot()) - return getShowAdminURL(); - else - return getManageFoldersURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); - addAdminNavTrail(root, title, this.getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public class ReorderFoldersApiAction extends MutatingApiAction - { - @Override - public ApiResponse execute(FolderReorderForm form, BindException errors) - { - return new ApiSimpleResponse("success", ReorderFolders(form, errors)); - } - } - - private boolean ReorderFolders(FolderReorderForm form, BindException errors) - { - Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); - if (form.isResetToAlphabetical()) - ContainerManager.setChildOrderToAlphabetical(parent); - else if (form.getOrder() != null) - { - List children = parent.getChildren(); - String[] order = form.getOrder().split(";"); - Map nameToContainer = new HashMap<>(); - for (Container child : children) - nameToContainer.put(child.getName(), child); - List sorted = new ArrayList<>(children.size()); - for (String childName : order) - { - Container child = nameToContainer.get(childName); - sorted.add(child); - } - - try - { - ContainerManager.setChildOrder(parent, sorted); - } - catch (ContainerException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - } - - return true; - } - - public static class FolderReorderForm - { - private String _order; - private boolean _resetToAlphabetical; - - public String getOrder() - { - return _order; - } - - @SuppressWarnings("unused") - public void setOrder(String order) - { - _order = order; - } - - public boolean isResetToAlphabetical() - { - return _resetToAlphabetical; - } - - @SuppressWarnings("unused") - public void setResetToAlphabetical(boolean resetToAlphabetical) - { - _resetToAlphabetical = resetToAlphabetical; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RevertFolderAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RevertFolderForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - - boolean success = false; - Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); - if (null != revertContainer) - { - if (revertContainer.isContainerTab()) - { - FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - revertContainer.setFolderType(origFolderType, getUser(), errors); - if (!errors.hasErrors()) - success = true; - } - } - } - else if (revertContainer.getFolderType().hasContainerTabs()) - { - try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) - { - List children = revertContainer.getChildren(); - for (Container container : children) - { - if (container.isContainerTab()) - { - FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); - if (null != tab) - { - FolderType origFolderType = tab.getFolderType(); - if (null != origFolderType) - { - container.setFolderType(origFolderType, getUser(), errors); - } - } - } - } - if (!errors.hasErrors()) - { - transaction.commit(); - success = true; - } - } - } - } - return new ApiSimpleResponse("success", success); - } - } - - public static class RevertFolderForm - { - private String _containerPath; - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - } - - public static class EmailTestForm - { - private String _to; - private String _body; - private ConfigurationException _exception; - - public String getTo() - { - return _to; - } - - public void setTo(String to) - { - _to = to; - } - - public String getBody() - { - return _body; - } - - public void setBody(String body) - { - _body = body; - } - - public ConfigurationException getException() - { - return _exception; - } - - public void setException(ConfigurationException exception) - { - _exception = exception; - } - - public String getFrom(Container c) - { - LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); - return props.getSystemEmailAddress(); - } - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class EmailTestAction extends FormViewAction - { - @Override - public void validateCommand(EmailTestForm form, Errors errors) - { - if(null == form.getTo() || form.getTo().isEmpty()) - { - errors.reject(ERROR_MSG, "To field cannot be blank."); - form.setException(new ConfigurationException("To field cannot be blank")); - return; - } - - try - { - ValidEmail email = new ValidEmail(form.getTo()); - } - catch(ValidEmail.InvalidEmailException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - form.setException(new ConfigurationException(e.getMessage())); - } - } - - @Override - public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) - { - JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); - testView.setTitle("Send a Test Email"); - - if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) - { - JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); - emailPropsView.setTitle("Current Email Settings"); - - return new VBox(emailPropsView, testView); - } - else - return testView; - } - - @Override - public boolean handlePost(EmailTestForm form, BindException errors) throws Exception - { - if (errors.hasErrors()) - { - return false; - } - - LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); - try - { - MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); - msg.setSubject("Test email message sent from " + props.getShortName()); - msg.setText(PageFlowUtil.filter(form.getBody())); - - try - { - MailHelper.send(msg, getUser(), getContainer()); - } - catch (ConfigurationException e) - { - form.setException(e); - return false; - } - catch (Exception e) - { - form.setException(new ConfigurationException(e.getMessage())); - return false; - } - } - catch (MessagingException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return false; - } - return true; - } - - @Override - public URLHelper getSuccessURL(EmailTestForm emailTestForm) - { - return new ActionURL(EmailTestAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Test Email Configuration", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class RecreateViewsAction extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(Object o, BindException errors) - { - getPageConfig().setShowHeader(false); - getPageConfig().setTitle("Recreate Views?"); - return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - ModuleLoader.getInstance().recreateViews(); - return true; - } - - @Override - public void validateCommand(Object o, Errors errors) - { - } - - @Override - public @NotNull ActionURL getSuccessURL(Object o) - { - return AppProps.getInstance().getHomePageActionURL(); - } - } - - static public class LoggingForm - { - public boolean isLogging() - { - return logging; - } - - public void setLogging(boolean logging) - { - this.logging = logging; - } - - public boolean logging = false; - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class GetSessionLogEventsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - Integer eventId = null; - try - { - String s = getViewContext().getRequest().getParameter("eventId"); - if (null != s) - eventId = Integer.parseInt(s); - } - catch (NumberFormatException ignored) {} - ApiSimpleResponse res = new ApiSimpleResponse(); - res.put("success", true); - res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); - return res; - } - } - - @RequiresLogin - @AllowedBeforeInitialUserIsSet - @AllowedDuringUpgrade - @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ - public static class GetTrackedAllocationsAction extends ReadOnlyApiAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ApiResponse execute(Object o, BindException errors) - { - long requestId = 0; - try - { - String s = getViewContext().getRequest().getParameter("requestId"); - if (null != s) - requestId = Long.parseLong(s); - } - catch (NumberFormatException ignored) {} - List requests = MemTracker.getInstance().getNewRequests(requestId); - List> jsonRequests = new ArrayList<>(requests.size()); - for (RequestInfo requestInfo : requests) - { - Map m = new HashMap<>(); - m.put("requestId", requestInfo.getId()); - m.put("url", requestInfo.getUrl()); - m.put("date", requestInfo.getDate()); - - - List> sortedObjects = sortByCounts(requestInfo); - - List> jsonObjects = new ArrayList<>(sortedObjects.size()); - for (Map.Entry entry : sortedObjects) - { - Map jsonObject = new HashMap<>(); - jsonObject.put("name", entry.getKey()); - jsonObject.put("count", entry.getValue()); - jsonObjects.add(jsonObject); - } - m.put("objects", jsonObjects); - jsonRequests.add(m); - } - return new ApiSimpleResponse("requests", jsonRequests); - } - - private List> sortByCounts(RequestInfo requestInfo) - { - List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); - objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); - return objects; - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class TrackedAllocationsViewerAction extends SimpleViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getUser().isPlatformDeveloper()) - throw new UnauthorizedException(); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - getPageConfig().setTemplate(Template.Print); - return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresLogin - @AllowedDuringUpgrade - @AllowedBeforeInitialUserIsSet - public static class SessionLoggingAction extends FormViewAction - { - @Override - public void checkPermissions() - { - super.checkPermissions(); - if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) - throw new UnauthorizedException(); - } - - @Override - public boolean handlePost(LoggingForm form, BindException errors) - { - boolean on = SessionAppender.isLogging(getViewContext().getRequest()); - if (form.logging != on) - { - if (!form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging OFF"); - SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); - if (form.logging) - LogManager.getLogger(AdminController.class).info("turn session logging ON"); - } - return true; - } - - @Override - public void validateCommand(LoggingForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) - { - SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); - getPageConfig().setTemplate(Template.Print); - return new LoggingView(); - } - - @Override - public ActionURL getSuccessURL(LoggingForm o) - { - return new ActionURL(SessionLoggingAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); - root.addChild("View Event Log"); - } - } - - static class LoggingView extends JspView - { - LoggingView() - { - super("/org/labkey/core/admin/logging.jsp", null); - } - } - - public static class LogForm - { - private String _message; - private String _level; - - public String getMessage() - { - return _message; - } - - public void setMessage(String message) - { - _message = message; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = level; - } - } - - - // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when - // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. - @RequiresLogin - public static class LogAction extends MutatingApiAction - { - @Override - public ApiResponse execute(LogForm logForm, BindException errors) - { - // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate - // \n to newlines to allow them at the beginning or end of the message as well. - StringBuilder message = new StringBuilder(); - message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); - - Level level = Level.toLevel(logForm.getLevel(), Level.INFO); - CLIENT_LOG.log(level, message); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class ValidateDomainsAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) throws Exception - { - // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file - for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) - { - PipeRoot root = PipelineService.get().findPipelineRoot(project); - if (root != null && root.isValid()) - { - ViewBackgroundInfo info = getViewBackgroundInfo(); - PipelineJob job = new ValidateDomainsPipelineJob(info, root); - PipelineService.get().queueJob(job); - return true; - } - } - return false; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); - } - } - - public static class ModulesForm - { - private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) - private boolean _managedOnly = false; - private boolean _unmanagedOnly = false; - - public double[] getIgnore() - { - return _ignore; - } - - public void setIgnore(double[] ignore) - { - _ignore = ignore; - } - - private Set getIgnoreSet() - { - return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); - } - - public boolean isManagedOnly() - { - return _managedOnly; - } - - @SuppressWarnings("unused") - public void setManagedOnly(boolean managedOnly) - { - _managedOnly = managedOnly; - } - - public boolean isUnmanagedOnly() - { - return _unmanagedOnly; - } - - @SuppressWarnings("unused") - public void setUnmanagedOnly(boolean unmanagedOnly) - { - _unmanagedOnly = unmanagedOnly; - } - } - - public enum ManageFilter - { - ManagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && module.shouldManageVersion(); - } - }, - UnmanagedOnly - { - @Override - public boolean accept(Module module) - { - return null != module && !module.shouldManageVersion(); - } - }, - All - { - @Override - public boolean accept(Module module) - { - return true; - } - }; - - public abstract boolean accept(Module module); - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public class ModulesAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModulesForm form, BindException errors) - { - ModuleLoader ml = ModuleLoader.getInstance(); - boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); - - Collection unknownModules = ml.getUnknownModuleContexts().values(); - Collection knownModules = ml.getAllModuleContexts(); - knownModules.removeAll(unknownModules); - - Set ignoreSet = form.getIgnoreSet(); - HtmlString managedLink = HtmlString.EMPTY_STRING; - HtmlString unmanagedLink = HtmlString.EMPTY_STRING; - - // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release - // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. - if (AppProps.getInstance().isDevMode()) - { - if (ignoreSet.isEmpty() && !form.isManagedOnly()) - { - String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("ignore", "0.00," + lowestSchemaVersion); - url.addParameter("managedOnly", true); - managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); - } - else - { - List ignore = ignoreSet - .stream() - .map(ModuleContext::formatVersion) - .collect(Collectors.toCollection(LinkedList::new)); - - String ignoreString = ignore.isEmpty() ? null : ignore.toString(); - String unmanaged = form.isManagedOnly() ? "unmanaged" : null; - - managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); - } - - if (!form.isUnmanagedOnly()) - { - ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - url.addParameter("unmanagedOnly", true); - unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); - } - else - { - unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); - } - } - - ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); - - HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); - if (hasAdminOpsPerm) - { - deleteInstructions.unsafeAppend("

    ").append( - "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + - "Module files are typically deployed in /modules and /externalModules.") - .unsafeAppend("

    ").append( - LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); - } - - HtmlStringBuilder docLink = HtmlStringBuilder.of(); - docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); - - HtmlStringBuilder knownDescription = HtmlStringBuilder.of() - .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); - HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); - - HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() - .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + - "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + - "is stored is not accessible, or the module file is corrupted.") - .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); - HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + - "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); - HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); - - return new VBox(known, unknown); - } - - private class ModulesView extends WebPartView - { - private final Collection _contexts; - private final HtmlString _descriptionHtml; - private final HtmlString _noModulesDescriptionHtml; - private final Set _ignoreVersions; - private final ManageFilter _manageFilter; - - private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) - { - super(FrameType.PORTAL); - List sorted = new ArrayList<>(contexts); - sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); - - _contexts = sorted; - _descriptionHtml = descriptionHtml; - _noModulesDescriptionHtml = noModulesDescriptionHtml; - _ignoreVersions = ignoreVersions; - _manageFilter = manageFilter; - setTitle(type + " Modules"); - } - - @Override - protected void renderView(Object model, HtmlWriter out) - { - boolean isDevMode = AppProps.getInstance().isDevMode(); - boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); - boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); - final AtomicInteger rowCount = new AtomicInteger(); - ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); - final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); - final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); - - if (_contexts.isEmpty()) - { - out.write(_noModulesDescriptionHtml); - } - else - { - DIV( - DIV(_descriptionHtml), - TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), - TR( - TD(cl("labkey-column-header"),"Name"), - TD(cl("labkey-column-header"),"Release Version"), - TD(cl("labkey-column-header"),"Schema Version"), - TD(cl("labkey-column-header"),"Class"), - TD(cl("labkey-column-header"),"Location"), - TD(cl("labkey-column-header"),"Schemas"), - !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions - null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions - !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions - ), - _contexts.stream() - .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) - .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) - .filter(pair -> _manageFilter.accept(pair.getValue())) - .map(pair -> - { - ModuleContext moduleContext = pair.getKey(); - Module module = pair.getValue(); - List schemas = moduleContext.getSchemaList(); - Double schemaVersion = moduleContext.getSchemaVersion(); - boolean replaceableModule = false; - if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) - { - File zip = module.getZippedPath(); - if (null != zip && zip.getParentFile().equals(externalModulesDir)) - replaceableModule = true; - } - boolean deleteableModule = replaceableModule || null == module; - String className = StringUtils.trimToEmpty(moduleContext.getClassName()); - String fullPathToModule = ""; - String shortPathToModule = ""; - if (null != module) - { - Path p = module.getExplodedPath().toPath(); - if (null != module.getZippedPath()) - p = module.getZippedPath().toPath(); - if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) - if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) - p = Paths.get(module.getSourcePath()); - fullPathToModule = p.toString(); - shortPathToModule = fullPathToModule; - Path rel = relativeRoot.relativize(p); - if (!rel.startsWith("..")) - shortPathToModule = rel.toString(); - } - ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); - - return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), - TD(moduleContext.getName()), - TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), - TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), - TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), - TD(SPAN(at(title,fullPathToModule),shortPathToModule)), - TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), - !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), - null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), - !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) - ); - }) - ) - ).appendTo(out); - } - } - } - - private ActionURL getDeleteURL(String name) - { - ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); - if (null != url) - return url; - url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getUpdateURL(String name) - { - ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); - if (null != url) - return url; - url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); - url.addParameter("name", name); - return url; - } - - private ActionURL getModuleEditorURL(String name) - { - return ModuleEditorService.get().getModuleEditorURL(name); - } - - private ActionURL getCreateURL() - { - ActionURL url = ModuleEditorService.get().getCreateModuleURL(); - if (null != url) - return url; - url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("defaultModules"); - addAdminNavTrail(root, "Modules", getClass()); - } - } - - public static class SchemaVersionTestCase extends Assert - { - @Test - public void verifyMinimumSchemaVersion() - { - List modulesTooLow = ModuleLoader.getInstance().getModules().stream() - .filter(ManageFilter.ManagedOnly::accept) - .filter(m -> null != m.getSchemaVersion()) - .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) - .toList(); - - if (!modulesTooLow.isEmpty()) - fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); - } - - @Test - public void modulesWithSchemaVersionButNoScripts() - { - // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. - List moduleNames = ModuleLoader.getInstance().getModules().stream() - .filter(m -> m.getSchemaVersion() != null) - .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) - .map(m -> m.getName() + ": " + m.getSchemaVersion()) - .toList(); - - if (!moduleNames.isEmpty()) - fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); - } - } - - public static class ModuleForm - { - private String _name; - - public String getName() - { - return _name; - } - - @SuppressWarnings({"UnusedDeclaration"}) - public void setName(String name) - { - _name = name; - } - - @NotNull - private ModuleContext getModuleContext() - { - ModuleLoader ml = ModuleLoader.getInstance(); - ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); - - if (null == ctx) - throw new NotFoundException("Module not found"); - - return ctx; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public class DeleteModuleAction extends ConfirmAction - { - @Override - public void validateCommand(ModuleForm form, Errors errors) - { - } - - @Override - public ModelAndView getConfirmView(ModuleForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Module"); - - ModuleContext ctx = form.getModuleContext(); - Module module = ModuleLoader.getInstance().getModule(ctx.getName()); - boolean hasSchemas = !ctx.getSchemaList().isEmpty(); - boolean hasFiles = false; - if (null != module) - hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); - - HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); - HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); - if (hasSchemas) - { - SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); - List deleteList = schemaActions.deleteList(); - List skipList = schemaActions.skipList(); - - // List all the schemas that will be deleted - if (!deleteList.isEmpty()) - { - description.append(" and delete all data in "); - description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); - } - - // For unknown modules, also list the schemas that won't be deleted - if (!skipList.isEmpty()) - { - skippedSchemas.append(HtmlString.BR); - skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) - .append("Note: Schema \"") - .append(sam.schema()) - .append("\" will not be deleted because it's in use by module \"") - .append(sam.module()) - .append("\"")); - } - } - - return new HtmlView(DIV( - !hasFiles ? null : DIV(cl("labkey-warning-messages"), - "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", - null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), - null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) - ), - BR(), - "Are you sure you want to remove the ", description, "? ", - "This operation cannot be undone!", - skippedSchemas, - BR(), - !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." - )); - } - - @Override - public boolean handlePost(ModuleForm form, BindException errors) - { - ModuleLoader.getInstance().removeModule(form.getModuleContext()); - - return true; - } - - @Override - public @NotNull URLHelper getSuccessURL(ModuleForm form) - { - return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class UpdateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class CreateModuleAction extends SimpleViewAction - { - @Override - public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception - { - return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - public static class OptionalFeatureForm - { - private String feature; - private boolean enabled; - - public String getFeature() - { - return feature; - } - - public void setFeature(String feature) - { - this.feature = feature; - } - - public boolean isEnabled() - { - return enabled; - } - - public void setEnabled(boolean enabled) - { - this.enabled = enabled; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - @ActionNames("OptionalFeature, ExperimentalFeature") - public static class OptionalFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(OptionalFeatureForm form, BindException errors) - { - String feature = StringUtils.trimToNull(form.getFeature()); - if (feature == null) - throw new ApiUsageException("feature is required"); - - OptionalFeatureService svc = OptionalFeatureService.get(); - if (svc == null) - throw new IllegalStateException(); - - Map ret = new HashMap<>(); - ret.put("feature", feature); - - if (isPost()) - { - ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); - svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); - } - - ret.put("enabled", svc.isFeatureEnabled(feature)); - return new ApiSimpleResponse(ret); - } - } - - public static class OptionalFeaturesForm - { - private String _type; - private boolean _showHidden; - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - public @NotNull FeatureType getTypeEnum() - { - return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); - } - - public boolean isShowHidden() - { - return _showHidden; - } - - @SuppressWarnings("unused") - public void setShowHidden(boolean showHidden) - { - _showHidden = showHidden; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class OptionalFeaturesAction extends SimpleViewAction - { - private FeatureType _type; - - @Override - public ModelAndView getView(OptionalFeaturesForm form, BindException errors) - { - _type = form.getTypeEnum(); - JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("experimental"); - addAdminNavTrail(root, _type.name() + " Features", getClass()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ProductFeatureAction extends BaseApiAction - { - @Override - protected ModelAndView handleGet() throws Exception - { - return handlePost(); // 'execute' ensures that only POSTs are mutating - } - - @Override - public ApiResponse execute(ProductConfigForm form, BindException errors) - { - String productKey = StringUtils.trimToNull(form.getProductKey()); - - Map ret = new HashMap<>(); - - if (isPost()) - { - ProductConfiguration.setProductKey(productKey); - } - - ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); - return new ApiSimpleResponse(ret); - } - } - - public static class ProductConfigForm - { - private String productKey; - - public String getProductKey() - { - return productKey; - } - - public void setProductKey(String productKey) - { - this.productKey = productKey; - } - - } - - @AdminConsoleAction - @RequiresPermission(AdminOperationsPermission.class) - public class ProductConfigurationAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Product Configuration", getClass()); - } - - @Override - public ModelAndView getView(Object o, BindException errors) throws Exception - { - JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - } - - - public static class FolderTypesBean - { - private final Collection _allFolderTypes; - private final Collection _enabledFolderTypes; - private final FolderType _defaultFolderType; - - public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) - { - _allFolderTypes = allFolderTypes; - _enabledFolderTypes = enabledFolderTypes; - _defaultFolderType = defaultFolderType; - } - - public Collection getAllFolderTypes() - { - return _allFolderTypes; - } - - public Collection getEnabledFolderTypes() - { - return _enabledFolderTypes; - } - - public FolderType getDefaultFolderType() - { - return _defaultFolderType; - } - } - - @AdminConsoleAction - @RequiresPermission(AdminPermission.class) - public class FolderTypesAction extends FormViewAction - { - @Override - public void validateCommand(Object form, Errors errors) - { - } - - @Override - public ModelAndView getView(Object form, boolean reshow, BindException errors) - { - FolderTypesBean bean; - if (reshow) - { - bean = getOptionsFromRequest(); - } - else - { - FolderTypeManager manager = FolderTypeManager.get(); - var defaultFolderType = manager.getDefaultFolderType(); - // If a default folder type has not yet been configuration use "Collaboration" folder type as the default - defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); - boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); - bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); - } - - return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); - } - - @Override - public boolean handlePost(Object form, BindException errors) - { - FolderTypesBean bean = getOptionsFromRequest(); - var defaultFolderType = bean.getDefaultFolderType(); - if (defaultFolderType == null) - { - errors.reject(ERROR_MSG, "Please select a default folder type."); - return false; - } - var enabledFolderTypes = bean.getEnabledFolderTypes(); - if (!enabledFolderTypes.contains(defaultFolderType)) - { - errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); - return false; - } - - FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); - return true; - } - - private FolderTypesBean getOptionsFromRequest() - { - var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); - List enabledFolderTypes = new ArrayList<>(); - FolderType defaultFolderType = null; - String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); - - for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) - { - boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); - if (enabled) - { - enabledFolderTypes.add(folderType); - } - if (folderType.getName().equals(defaultFolderTypeParam)) - { - defaultFolderType = folderType; - } - } - return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); - } - - @Override - public URLHelper getSuccessURL(Object form) - { - return getShowAdminURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Folder Types", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class CustomizeMenuAction extends MutatingApiAction - { - @Override - public ApiResponse execute(CustomizeMenuForm form, BindException errors) - { - if (null != form.getUrl()) - { - String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); - if (null != errorMessage) - { - errors.reject(ERROR_MSG, errorMessage); - return new ApiSimpleResponse("success", false); - } - } - - setCustomizeMenuForm(form, getContainer(), getUser()); - return new ApiSimpleResponse("success", true); - } - } - - protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; - protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; - protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; - protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; - protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; - protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; - protected static final String CUSTOMMENU_URL = "customMenuUrl"; - protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; - protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; - protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; - protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; - protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; - - public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) - { - CustomizeMenuForm form = new CustomizeMenuForm(); - Map menuProps = webPart.getPropertyMap(); - - String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); - String queryName = menuProps.get(CUSTOMMENU_QUERY); - String columnName = menuProps.get(CUSTOMMENU_COLUMN); - String viewName = menuProps.get(CUSTOMMENU_VIEW); - String folderName = menuProps.get(CUSTOMMENU_FOLDER); - String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; - String urlBottom = menuProps.get(CUSTOMMENU_URL); - String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); - String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); - String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); - boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); - String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); - boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); - String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); - boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); - - form.setSchemaName(schemaName); - form.setQueryName(queryName); - form.setColumnName(columnName); - form.setViewName(viewName); - form.setFolderName(folderName); - form.setTitle(title); - form.setUrl(urlBottom); - form.setRootFolder(rootFolder); - form.setFolderTypes(folderTypes); - form.setChoiceListQuery(choiceListQuery); - form.setIncludeAllDescendants(includeAllDescendants); - form.setCurrentProjectOnly(currentProjectOnly); - - form.setWebPartIndex(webPart.getIndex()); - form.setPageId(webPart.getPageId()); - return form; - } - - private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) - { - Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); - if (null == webPart) - throw new NotFoundException(); - Map menuProps = webPart.getPropertyMap(); - - menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); - menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); - menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); - menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); - menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); - menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); - menuProps.put(CUSTOMMENU_URL, form.getUrl()); - - // If root folder not specified, set as current container - menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); - menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); - menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); - menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); - menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); - - Portal.updatePart(user, webPart); - } - - @RequiresPermission(AdminPermission.class) - public static class AddTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - if(tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived - // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something - // longer than 64 characters. - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - if (name.length() > 50) - name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); - - for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) - { - folderTabMap.put(tab.getName(), tab); - } - - if (pages.containsKey(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (page.getCaption() != null && page.getCaption().equals(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - else if (folderTabMap.containsKey(page.getPageId())) - { - if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); - return; - } - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - validateCommand(form, errors); - - if(errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - String name = form.getTabName(); - String caption = form.getTabName(); - - // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the - // name, and is editable, is allowed to be 64 characters. - if (name.length() > 50) - name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); - - Portal.saveParts(container, name); - Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); - - if (!name.equals(caption)) - { - // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(name); - // Get a mutable copy - page = page.copy(); - page.setCaption(caption); - Portal.updatePortalPage(container, page); - } - - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); - tabURL.addParameter("pageId", name); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ShowTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); - - if (form.getTabPageId() == null) - { - errors.reject(ERROR_MSG, "PageId cannot be blank."); - } - - if (!pages.containsKey(form.getTabPageId())) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - validateCommand(form, errors); - if (errors.hasErrors()) - return response; - - Portal.showPage(tabContainer, form.getTabPageId()); - ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); - tabURL.addParameter("pageId", form.getTabPageId()); - response.put("url", tabURL); - response.put("success", true); - return response; - } - } - - - public static class TabActionForm extends ReturnUrlForm - { - // This class is used for tab related actions (add, rename, show, etc.) - String _tabName; - String _tabPageId; - - public String getTabName() - { - return _tabName; - } - - public void setTabName(String name) - { - _tabName = name; - } - - public String getTabPageId() - { - return _tabPageId; - } - - public void setTabPageId(String tabPageId) - { - _tabPageId = tabPageId; - } - } - - @RequiresPermission(AdminPermission.class) - public class MoveTabAction extends MutatingApiAction - { - @Override - public ApiResponse execute(MoveTabForm form, BindException errors) - { - final Map properties = new HashMap<>(); - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage tab = pages.get(form.getPageId()); - - if (null != tab) - { - int oldIndex = tab.getIndex(); - Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); - - if (null != pageToSwap) - { - properties.put("oldIndex", oldIndex); - properties.put("newIndex", tab.getIndex()); - properties.put("pageId", tab.getPageId()); - properties.put("pageIdToSwap", pageToSwap.getPageId()); - } - else - { - properties.put("error", "Unable to move tab."); - } - } - else - { - properties.put("error", "Requested tab does not exist."); - } - - return new ApiSimpleResponse(properties); - } - } - - public static class MoveTabForm implements HasViewContext - { - private int _direction; - private String _pageId; - private ViewContext _viewContext; - - public int getDirection() - { - // 0 moves left, 1 moves right. - return _direction; - } - - public void setDirection(int direction) - { - _direction = direction; - } - - public String getPageId() - { - return _pageId; - } - - public void setPageId(String pageId) - { - _pageId = pageId; - } - - @Override - public ViewContext getViewContext() - { - return _viewContext; - } - - @Override - public void setViewContext(ViewContext viewContext) - { - _viewContext = viewContext; - } - } - - private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) - { - Map pageMap = new CaseInsensitiveHashMap<>(); - for (Portal.PortalPage pp : Portal.getTabPages(c, true)) - pageMap.put(pp.getPageId(), pp); - - for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) - { - if (pageMap.containsKey(folderTab.getName())) - { - // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to - // be excluded when adjusting the relative indexes. - if (!folderTab.isVisible(c, user)) - pageMap.remove(folderTab.getName()); - } - } - List pagesList = new ArrayList<>(pageMap.values()); - pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); - - int visibleIndex; - for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) - { - if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) - { - break; - } - } - - if (visibleIndex == pagesList.size()) - { - return null; - } - - if (direction == Portal.MOVE_DOWN) - { - if (visibleIndex == pagesList.size() - 1) - { - return page; - } - - Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); - - if (null == nextPage) - return null; - Portal.swapPageIndexes(c, page, nextPage); - return nextPage; - } - else - { - if (visibleIndex < 1) - { - return page; - } - - Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); - - if (null == prevPage) - return null; - Portal.swapPageIndexes(c, page, prevPage); - return prevPage; - } - } - - @RequiresPermission(AdminPermission.class) - public static class RenameTabAction extends MutatingApiAction - { - public void validateCommand(TabActionForm form, Errors errors) - { - Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); - - if (tabContainer.getFolderType() == FolderType.NONE) - { - errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); - } - else - { - String name = form.getTabName(); - if (StringUtils.isEmpty(name)) - { - errors.reject(ERROR_MSG, "A tab name must be specified."); - return; - } - - if (name.length() > 64) - { - errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); - return; - } - - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); - Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); - if (null == pageToChange) - { - errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); - return; - } - - for (Portal.PortalPage page : pages.values()) - { - if (!page.equals(pageToChange)) - { - if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - if (page.getPageId().equalsIgnoreCase(name)) - { - if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - else - errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); - return; - } - } - } - - List folderTabs = tabContainer.getFolderType().getDefaultTabs(); - for (FolderTab folderTab : folderTabs) - { - String folderTabCaption = folderTab.getCaption(getViewContext()); - if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) - { - errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); - return; - } - } - } - } - - @Override - public ApiResponse execute(TabActionForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - validateCommand(form, errors); - - if (errors.hasErrors()) - { - return response; - } - - Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); - CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); - Portal.PortalPage page = pages.get(form.getTabPageId()); - page = page.copy(); - page.setCaption(form.getTabName()); - // Update the page the caption is saved. - Portal.updatePortalPage(container, page); - - response.put("success", true); - return response; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ClearDeletedTabFoldersAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeletedFoldersForm form, BindException errors) - { - if (isBlank(form.getContainerPath())) - throw new NotFoundException(); - Container container = ContainerManager.getForPath(form.getContainerPath()); - for (String tabName : form.getResurrectFolders()) - { - ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); - } - return new ApiSimpleResponse("success", true); - } - } - - @SuppressWarnings("unused") - public static class DeletedFoldersForm - { - private String _containerPath; - private String _newFolderType; - private List _resurrectFolders; - - public List getResurrectFolders() - { - return _resurrectFolders; - } - - public void setResurrectFolders(List resurrectFolders) - { - _resurrectFolders = resurrectFolders; - } - - public String getContainerPath() - { - return _containerPath; - } - - public void setContainerPath(String containerPath) - { - _containerPath = containerPath; - } - - public String getNewFolderType() - { - return _newFolderType; - } - - public void setNewFolderType(String newFolderType) - { - _newFolderType = newFolderType; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetFolderTabsAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object form, BindException errors) throws Exception - { - var data = getContainer() - .getFolderType() - .getAppBar(getViewContext(), getPageConfig()) - .getButtons() - .stream() - .map(this::getProperties) - .toList(); - - return success(data); - } - - private Map getProperties(NavTree navTree) - { - Map props = new HashMap<>(); - props.put("id", navTree.getId()); - props.put("text", navTree.getText()); - props.put("href", navTree.getHref()); - props.put("disabled", navTree.isDisabled()); - return props; - } - } - - @SuppressWarnings("unused") - public static class ShortURLForm - { - private String _shortURL; - private String _fullURL; - private boolean _delete; - - private List _savedShortURLs; - - public void setShortURL(String shortURL) - { - _shortURL = shortURL; - } - - public void setFullURL(String fullURL) - { - _fullURL = fullURL; - } - - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getShortURL() - { - return _shortURL; - } - - public String getFullURL() - { - return _fullURL; - } - - public boolean isDelete() - { - return _delete; - } - } - - public abstract static class AbstractShortURLAdminAction extends FormViewAction - { - @Override - public void validateCommand(ShortURLForm target, Errors errors) {} - - @Override - public boolean handlePost(ShortURLForm form, BindException errors) throws Exception - { - String shortURL = StringUtils.trimToEmpty(form.getShortURL()); - if (StringUtils.isEmpty(shortURL)) - { - errors.addError(new LabKeyError("Short URL must not be blank")); - } - if (shortURL.endsWith(".url")) - shortURL = shortURL.substring(0,shortURL.length()-".url".length()); - if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) - { - errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); - } - URLHelper fullURL = null; - if (!form.isDelete()) - { - String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); - if (trimmedFullURL == null) - { - errors.addError(new LabKeyError("Target URL must not be blank")); - } - else - { - try - { - fullURL = new URLHelper(trimmedFullURL); - } - catch (URISyntaxException e) - { - errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); - } - } - } - if (errors.getErrorCount() > 0) - { - return false; - } - - ShortURLService service = ShortURLService.get(); - if (form.isDelete()) - { - ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); - if (shortURLRecord == null) - { - throw new NotFoundException("No such short URL: " + shortURL); - } - try - { - service.deleteShortURL(shortURLRecord, getUser()); - } - catch (ValidationException e) - { - errors.addError(new LabKeyError("Error deleting short URL:")); - for(ValidationError error: e.getErrors()) - { - errors.addError(new LabKeyError(error.getMessage())); - } - } - - if (errors.getErrorCount() > 0) - { - return false; - } - } - else - { - ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); - MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); - // Add a role assignment to let another group manage the URL. This grants permission to the journal - // to change where the URL redirects you to after they copy the data - SecurityPolicyManager.savePolicy(policy, getUser()); - } - return true; - } - } - - @AdminConsoleAction - public class ShortURLAdminAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); - boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); - newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); - newView.setFrame(WebPartView.FrameType.PORTAL); - - QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); - qSettings.setBaseSort(new Sort("-Created")); - QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); - if (!isAppAdmin) - { - existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - } - existingView.setTitle("Existing Short URLs"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Short URL Admin", getClass()); - } - } - - @RequiresPermission(ApplicationAdminPermission.class) - public class UpdateShortURLAction extends AbstractShortURLAdminAction - { - @Override - public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) - { - var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); - if (shortUrlRecord == null) - { - errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); - return new SimpleErrorView(errors); - } - form.setFullURL(shortUrlRecord.getFullURL()); - - JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); - view.setTitle("Update Short URL"); - view.setFrame(WebPartView.FrameType.PORTAL); - return view; - } - - @Override - public URLHelper getSuccessURL(ShortURLForm form) - { - return new ActionURL(ShortURLAdminAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("shortURL"); - addAdminNavTrail(root, "Update Short URL", getClass()); - } - } - - // API for reporting client-side exceptions. - // UNDONE: Throttle by IP to avoid DOS from buggy clients. - @Marshal(Marshaller.Jackson) - @SuppressWarnings("UnusedDeclaration") - @RequiresLogin // Issue 52520: Prevent bots from submitting reports - @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root - public static class LogClientExceptionAction extends MutatingApiAction - { - @Override - public Object execute(ExceptionForm form, BindException errors) - { - String errorCode = ExceptionUtil.logClientExceptionToMothership( - form.getStackTrace(), - form.getExceptionMessage(), - form.getBrowser(), - null, - form.getRequestURL(), - form.getReferrerURL(), - form.getUsername() - ); - - Map results = new HashMap<>(); - results.put("errorCode", errorCode); - results.put("loggedToMothership", errorCode != null); - - return success(results); - } - } - - @SuppressWarnings("unused") - public static class ExceptionForm - { - private String _exceptionMessage; - private String _stackTrace; - private String _requestURL; - private String _browser; - private String _username; - private String _referrerURL; - private String _file; - private String _line; - private String _platform; - - public String getExceptionMessage() - { - return _exceptionMessage; - } - - public void setExceptionMessage(String exceptionMessage) - { - _exceptionMessage = exceptionMessage; - } - - public String getUsername() - { - return _username; - } - - public void setUsername(String username) - { - _username = username; - } - - public String getStackTrace() - { - return _stackTrace; - } - - public void setStackTrace(String stackTrace) - { - _stackTrace = stackTrace; - } - - public String getRequestURL() - { - return _requestURL; - } - - public void setRequestURL(String requestURL) - { - _requestURL = requestURL; - } - - public String getBrowser() - { - return _browser; - } - - public void setBrowser(String browser) - { - _browser = browser; - } - - public String getReferrerURL() - { - return _referrerURL; - } - - public void setReferrerURL(String referrerURL) - { - _referrerURL = referrerURL; - } - - public String getFile() - { - return _file; - } - - public void setFile(String file) - { - _file = file; - } - - public String getLine() - { - return _line; - } - - public void setLine(String line) - { - _line = line; - } - - public String getPlatform() - { - return _platform; - } - - public void setPlatform(String platform) - { - _platform = platform; - } - } - - - /** generate URLS to seed web-site scanner */ - @SuppressWarnings("UnusedDeclaration") - @RequiresSiteAdmin - public static class SpiderAction extends SimpleViewAction - { - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Spider Initialization"); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - List urls = new ArrayList<>(1000); - - if (getContainer().equals(ContainerManager.getRoot())) - { - for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) - { - urls.add(c.getStartURL(getUser()).toString()); - urls.add(new ActionURL(SpiderAction.class, c).toString()); - } - - Container home = ContainerManager.getHomeContainer(); - for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) - { - ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); - urls.add(url.toString()); - } - } - else - { - DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); - def.getSchemaNames().forEach(name -> - { - QuerySchema q = def.getSchema(name); - if (null == q) - return; - var tableNames = q.getTableNames(); - if (null == tableNames) - return; - tableNames.forEach(table -> - { - try - { - var t = q.getTable(table); - if (null != t) - { - ActionURL grid = t.getGridURL(getContainer()); - if (null != grid) - urls.add(grid.toString()); - else - urls.add(new ActionURL("query", "executeQuery.view", getContainer()) - .addParameter("schemaName", q.getSchemaName()) - .addParameter("query.queryName", t.getName()) - .toString()); - } - } - catch (Exception x) - { - // pass - } - }); - }); - - ModuleLoader.getInstance().getModules().forEach(m -> - { - ActionURL url = m.getTabURL(getContainer(), getUser()); - if (null != url) - urls.add(url.toString()); - }); - } - - return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); - } - } - - @SuppressWarnings("UnusedDeclaration") - @RequiresPermission(TroubleshooterPermission.class) - public static class TestMothershipReportAction extends ReadOnlyApiAction - { - @Override - public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception - { - MothershipReport report; - MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; - if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) - { - report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); - } - else - { - report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), - new SQLException("Intentional exception for testing purposes", "400"), - (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), - target, - ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); - } - - final Map params; - if (report == null) - { - params = new LinkedHashMap<>(); - } - else - { - params = report.getJsonFriendlyParams(); - if (form.isSubmit()) - { - report.setForwardedFor(form.getForwardedFor()); - report.run(); - if (null != report.getUpgradeMessage()) - params.put("upgradeMessage", report.getUpgradeMessage()); - } - } - if (form.isDownload()) - { - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); - } - return new ApiSimpleResponse(params); - } - } - - - static class MothershipReportSelectionForm - { - private String _type = MothershipReport.Type.CheckForUpdates.toString(); - private String _level = UsageReportingLevel.ON.toString(); - private boolean _submit = false; - private boolean _download = false; - private String _forwardedFor = null; - // indicates action is being invoked for dev/test - private boolean _testMode = false; - - public String getType() - { - return _type; - } - - public void setType(String type) - { - _type = type; - } - - public String getLevel() - { - return _level; - } - - public void setLevel(String level) - { - _level = StringUtils.upperCase(level); - } - - public boolean isSubmit() - { - return _submit; - } - - public void setSubmit(boolean submit) - { - _submit = submit; - } - - public String getForwardedFor() - { - return _forwardedFor; - } - - public void setForwardedFor(String forwardedFor) - { - _forwardedFor = forwardedFor; - } - - public boolean isTestMode() - { - return _testMode; - } - - public void setTestMode(boolean testMode) - { - _testMode = testMode; - } - - public boolean isDownload() - { - return _download; - } - - public void setDownload(boolean download) - { - _download = download; - } - } - - - @RequiresPermission(TroubleshooterPermission.class) - public class SuspiciousAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - Collection list = BlockListFilter.reportSuspicious(); - HtmlStringBuilder html = HtmlStringBuilder.of(); - if (list.isEmpty()) - { - html.append("No suspicious activity.\n"); - } - else - { - html.unsafeAppend("") - .unsafeAppend("\n"); - for (BlockListFilter.Suspicious s : list) - { - html.unsafeAppend("\n"); - } - html.unsafeAppend("
    host (user)user-agentcount
    ") - .append(s.host); - if (!isBlank(s.user)) - html.append(HtmlString.NBSP).append("(" + s.user + ")"); - html.unsafeAppend("") - .append(s.userAgent) - .unsafeAppend("") - .append(s.count) - .unsafeAppend("
    "); - } - return new HtmlView(html); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); - } - } - - /** This is a very crude API right now, mostly using default serialization of pre-existing objects - * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways - */ - @Marshal(Marshaller.Jackson) - @RequiresNoPermission - @AllowedBeforeInitialUserIsSet - public static class ConfigurationSummaryAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) - { - if (!getContainer().isRoot()) - throw new NotFoundException("Must be invoked in the root"); - - // requires site-admin, unless there are no users - if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) - throw new UnauthorizedException(); - - return getConfigurationJson(); - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - ObjectMapper result = JsonUtil.createDefaultMapper(); - result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); - return result; - } - - /* returns a jackson serializable object that reports superset of information returned in admin console */ - private JSONObject getConfigurationJson() - { - JSONObject res = new JSONObject(); - - res.put("server", AdminBean.getPropertyMap()); - - final Map> sets = new TreeMap<>(); - new SqlSelector(CoreSchema.getInstance().getScope(), - new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + - "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> - { - String category = (String)m.get("category"); - String name = (String)m.get("name"); - Object value = m.get("value"); - if (!sets.containsKey(category)) - sets.put(category, new TreeMap<>()); - sets.get(category).put(name,value); - } - ); - res.put("siteSettings", sets); - - HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); - res.put("health", result); - - LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); - res.put("scriptEngines", mgr.getEngineDefinitions()); - - return res; - } - } - - @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) - private static class IgnorePasswordMixIn - { - } - - @AdminConsoleAction() - public class AllowListAction extends FormViewAction - { - private AllowListType _type; - - @Override - public void validateCommand(AllowListForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) - { - _type = form.getTypeEnum(); - - form.setExistingValuesList(form.getTypeEnum().getValues()); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); - newView.setTitle("Register New " + form.getTypeEnum().getTitle()); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); - existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - AllowListType allowListType = form.getTypeEnum(); - //handle delete of existing value - if (form.isDelete()) - { - String urlToDelete = form.getExistingValue(); - List values = new ArrayList<>(allowListType.getValues()); - for (String value : values) - { - if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) - { - values.remove(value); - allowListType.setValues(values, getUser()); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated urls - else if (form.isSaveAll()) - { - Set validatedValues = form.validateValues(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(validatedValues.stream().toList(), getUser()); - } - //save new external value - else if (form.isSaveNew()) - { - Set valueSet = form.validateNewValue(errors); - if (errors.hasErrors()) - return false; - - allowListType.setValues(valueSet, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic(_type.getHelpTopic()); - addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); - } - } - - public static class AllowListForm - { - private String _newValue; - private String _existingValue; - private boolean _delete; - private String _existingValues; - private boolean _saveAll; - private boolean _saveNew; - private String _type; - - private List _existingValuesList; - - public String getNewValue() - { - return _newValue; - } - - @SuppressWarnings("unused") - public void setNewValue(String newValue) - { - _newValue = newValue; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public String getExistingValues() - { - return _existingValues; - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public List getExistingValuesList() - { - //for updated urls that comes in as String values from the jsp/html form - if (null != getExistingValues()) - { - // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. - return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); - } - return _existingValuesList; - } - - public void setExistingValuesList(List valuesList) - { - _existingValuesList = valuesList; - } - - public String getType() - { - return _type; - } - - @SuppressWarnings("unused") - public void setType(String type) - { - _type = type; - } - - @NotNull - public AllowListType getTypeEnum() - { - return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); - } - - @JsonIgnore - public Set validateNewValue(BindException errors) - { - String value = StringUtils.trimToEmpty(getNewValue()); - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - return null; - - Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); - checkDuplicatesByAddition(value, valueSet, errors); - return valueSet; - } - - @JsonIgnore - public Set validateValues(BindException errors) - { - List values = getExistingValuesList(); //get values from the form, this includes updated values - Set valueSet = new CaseInsensitiveHashSet(); - - if (null != values && !values.isEmpty()) - { - for (String value : values) - { - getTypeEnum().validateValueFormat(value, errors); - if (errors.hasErrors()) - continue; - - checkDuplicatesByAddition(value, valueSet, errors); - } - } - - return valueSet; - } - - /** - * Adds value to value set unless it is a duplicate, in which case it adds an error - * @param value to check - * @param valueSet of existing values - * @param errors collections of errors observed - */ - @JsonIgnore - private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) - { - String trimValue = StringUtils.trimToEmpty(value); - if (!valueSet.add(trimValue)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); - } - } - - @AdminConsoleAction - public static class DeleteAllValuesAction extends FormHandlerAction - { - @Override - public void validateCommand(AllowListForm form, Errors errors) - { - } - - @Override - public boolean handlePost(AllowListForm form, BindException errors) throws Exception - { - form.getTypeEnum().setValues(Collections.emptyList(), getUser()); - return true; - } - - @Override - public URLHelper getSuccessURL(AllowListForm form) - { - return form.getTypeEnum().getSuccessURL(getContainer()); - } - } - - public static class ExternalSourcesForm - { - private boolean _delete; - private boolean _saveNew; - private boolean _saveAll; - - private String _newDirective; - private String _newHost; - private String _existingValue; - private String _existingValues; - - public boolean isDelete() - { - return _delete; - } - - @SuppressWarnings("unused") - public void setDelete(boolean delete) - { - _delete = delete; - } - - public boolean isSaveNew() - { - return _saveNew; - } - - @SuppressWarnings("unused") - public void setSaveNew(boolean saveNew) - { - _saveNew = saveNew; - } - - public boolean isSaveAll() - { - return _saveAll; - } - - @SuppressWarnings("unused") - public void setSaveAll(boolean saveAll) - { - _saveAll = saveAll; - } - - public String getNewDirective() - { - return _newDirective; - } - - @SuppressWarnings("unused") - public void setNewDirective(String newDirective) - { - _newDirective = newDirective; - } - - public String getNewHost() - { - return _newHost; - } - - @SuppressWarnings("unused") - public void setNewHost(String newHost) - { - _newHost = newHost; - } - - public String getExistingValue() - { - return _existingValue; - } - - @SuppressWarnings("unused") - public void setExistingValue(String existingValue) - { - _existingValue = existingValue; - } - - public List getExistingValues() - { - return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .toList(); - } - - @SuppressWarnings("unused") - public void setExistingValues(String existingValues) - { - _existingValues = existingValues; - } - - private AllowedHost getExistingAllowedHost(BindException errors) - { - return getAllowedHost(getExistingValue(), errors); - } - - private AllowedHost getAllowedHost(String value, BindException errors) - { - String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts - if (parts.length != 2) - { - errors.addError(new LabKeyError("Can't parse allowed host.")); - return null; - } - return validateHost(parts[0], parts[1], errors); - } - - private List getExistingAllowedHosts(BindException errors) - { - List existing = getExistingValues().stream() - .map(value-> getAllowedHost(value, errors)) - .toList(); - - if (errors.hasErrors()) - return null; - - return checkDuplicates(existing, errors); - } - - private List validateNewAllowedHost(BindException errors) throws JsonProcessingException - { - AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); - - if (errors.hasErrors()) - return null; - - List hosts = getSavedAllowedHosts(); - hosts.add(newAllowedHost); - - return checkDuplicates(hosts, errors); - } - - // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons - public static AllowedHost validateHost(String directiveString, String host, BindException errors) - { - AllowedHost ret = null; - - if (StringUtils.isEmpty(directiveString)) - { - errors.addError(new LabKeyError("Directive must not be blank")); - } - else if (StringUtils.isEmpty(host)) - { - errors.addError(new LabKeyError("Host must not be blank")); - } - else if (host.contains(";")) - { - errors.addError(new LabKeyError("Semicolons are not allowed in host names")); - } - else - { - Directive directive = EnumUtils.getEnum(Directive.class, directiveString); - - if (null == directive) - { - errors.addError(new LabKeyError("Unknown directive: " + directiveString)); - } - else - { - ret = new AllowedHost(directive, host.trim()); - } - } - - return ret; - } - - /** - * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons - - * @param hosts a list of AllowedHost objects to check for duplicates - * @param errors errors to populate - * @return hosts if there are no duplicates, otherwise {@code null} - */ - public static @Nullable List checkDuplicates(List hosts, BindException errors) - { - // Not a simple Set check since we want host check to be case-insensitive - MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); - - hosts.forEach(allowedHost -> { - String host = allowedHost.host().trim(); - if (!map.put(allowedHost.directive(), host)) - errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); - }); - - return errors.hasErrors() ? null : hosts; - } - - // Returns a mutable list - public List getSavedAllowedHosts() throws JsonProcessingException - { - return AllowedExternalResourceHosts.readAllowedHosts(); - } - } - - @AdminConsoleAction() - public class ExternalSourcesAction extends FormViewAction - { - @Override - public void validateCommand(ExternalSourcesForm form, Errors errors) - { - } - - @Override - public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) - { - boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); - - JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); - newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); - newView.setFrame(WebPartView.FrameType.PORTAL); - JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); - existingView.setTitle("Existing External Resource Hosts"); - existingView.setFrame(WebPartView.FrameType.PORTAL); - - return new VBox(newView, existingView); - } - - private static final Object HOST_LOCK = new Object(); - - @Override - public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception - { - List allowedHosts = null; - - // Multiple requests could access this in parallel, so synchronize access, Issue 53457 - synchronized (HOST_LOCK) - { - //handle delete of an existing value - if (form.isDelete()) - { - AllowedHost subToDelete = form.getExistingAllowedHost(errors); - if (errors.hasErrors()) - return false; - allowedHosts = form.getSavedAllowedHosts(); - var iter = allowedHosts.listIterator(); - while (iter.hasNext()) - { - AllowedHost sub = iter.next(); - if (sub.equals(subToDelete)) - { - iter.remove(); - break; - } - } - } - //handle updates - clicking on Save button under Existing will save the updated hosts - else if (form.isSaveAll()) - { - allowedHosts = form.getExistingAllowedHosts(errors); - if (errors.hasErrors()) - return false; - } - //save new external value - else if (form.isSaveNew()) - { - allowedHosts = form.validateNewAllowedHost(errors); - } - - if (errors.hasErrors()) - return false; - - AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); - } - - return true; - } - - @Override - public URLHelper getSuccessURL(ExternalSourcesForm form) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("externalHosts"); - addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ProjectSettingsAction extends ProjectSettingsViewPostAction - { - @Override - protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) - { - return new LookAndFeelView(errors); - } - - @Override - public void validateCommand(ProjectSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception - { - return saveProjectSettings(getContainer(), getUser(), form, errors); - } - } - - private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) - { - WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); - boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); - - // Site-only properties - - if (c.isRoot()) - { - DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); - props.setDateParsingMode(dateParsingMode); - - if (hasAdminOpsPerm) - { - String customWelcome = form.getCustomWelcome(); - String welcomeUrl = StringUtils.trimToNull(customWelcome); - if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); - } - else - { - props.setCustomWelcome(welcomeUrl); - } - } - } - - // Site & project properties - - boolean shouldInherit = form.getShouldInherit(); - if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) - { - SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); - } - - setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); - setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); - setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); - setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); - setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); - setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); - - // a few properties on this page should be restricted to operational permissions (i.e. site admin) - if (hasAdminOpsPerm) - { - setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { - String systemEmailAddress = form.getSystemEmailAddress(); - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(systemEmailAddress); - props.setSystemEmailAddress(email); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - }); - - setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { - String customLogin = form.getCustomLogin(); - if (props.isValidUrl(customLogin)) - { - props.setCustomLogin(customLogin); - } - else - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); - } - }); - } - - setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); - setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); - setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); - setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { - String supportEmail = form.getSupportEmail(); - - if (!isBlank(supportEmail)) - { - try - { - // this will throw an InvalidEmailException for invalid email addresses - ValidEmail email = new ValidEmail(supportEmail); - props.setSupportEmail(email.toString()); - } - catch (ValidEmail.InvalidEmailException e) - { - errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" - + e.getBadEmail() + "]. Please enter a valid email address."); - } - } - else - { - // This stores a blank value, not null (which would mean inherit) - props.setSupportEmail(null); - } - }); - - boolean noErrors = !saveFolderSettings(c, user, props, form, errors); - - if (noErrors) - { - // Bump the look & feel revision so browsers retrieve the new theme stylesheet - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - } - - return noErrors; - } - - private static void setProperty(boolean inherited, Runnable clear, Runnable set) - { - if (inherited) - clear.run(); - else - set.run(); - } - - // Same as ProjectSettingsAction, but provides special admin console permissions handling - @AdminConsoleAction(ApplicationAdminPermission.class) - public static class LookAndFeelSettingsAction extends ProjectSettingsAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class UpdateContainerSettingsAction extends MutatingApiAction - { - @Override - public Object execute(FolderSettingsForm form, BindException errors) - { - boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("success", saved && !errors.hasErrors()); - return response; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ResourcesAction extends ProjectSettingsViewPostAction - { - @Override - protected JspView getTabView(Object o, boolean reshow, BindException errors) - { - LookAndFeelBean bean = new LookAndFeelBean(); - return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); - } - - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - Container c = getContainer(); - Map fileMap = getFileMap(); - - for (ResourceType type : ResourceType.values()) - { - MultipartFile file = fileMap.get(type.name()); - - if (file != null && !file.isEmpty()) - { - try - { - type.save(file, c, getUser()); - } - catch (Exception e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - return false; - } - } - } - - // Note that audit logging happens via the attachment code, so we don't log separately here - - // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. - WriteableAppProps.incrementLookAndFeelRevisionAndSave(); - - return true; - } - } - - // Same as ResourcesAction, but provides special admin console permissions handling - @AdminConsoleAction - public static class AdminConsoleResourcesAction extends ResourcesAction - { - @Override - protected TYPE getType() - { - return TYPE.LookAndFeelSettings; - } - } - - @RequiresPermission(AdminPermission.class) - public static class MenuBarAction extends ProjectSettingsViewAction - { - @Override - protected HttpView getTabView() - { - if (getContainer().isRoot()) - return HtmlView.err("Menu bar must be configured for each project separately."); - - WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); - v.setView("menubar", new VBox()); - Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); - - return v; - } - } - - @RequiresPermission(AdminPermission.class) - public static class FilesAction extends ProjectSettingsViewPostAction - { - @Override - protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) - { - Container c = getContainer(); - - if (c.isRoot()) - return HtmlView.err("Files must be configured for each project separately."); - - if (!reshow || form.isPipelineRootForm()) - { - try - { - AdminController.setFormAndConfirmMessage(getViewContext(), form); - } - catch (IllegalArgumentException e) - { - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - } - VBox box = new VBox(); - JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); - String title = "Configure File Root"; - if (CloudStoreService.get() != null) - title += " And Enable Cloud Stores"; - view.setTitle(title); - box.addView(view); - - // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root - if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) - { - SetupForm setupForm = SetupForm.init(c); - setupForm.setShowAdditionalOptionsLink(true); - setupForm.setErrors(errors); - PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); - - if (pipeRoot != null) - { - for (String errorMessage : pipeRoot.validate()) - errors.addError(new LabKeyError(errorMessage)); - } - JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); - pipelineView.setTitle("Configure Data Processing Pipeline"); - box.addView(pipelineView); - } - - return box; - } - - @Override - public void validateCommand(FilesForm form, Errors errors) - { - if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) - { - String root = StringUtils.trimToNull(form.getFolderRootPath()); - if (root != null) - { - File f = new File(root); - if (!f.exists() || !f.isDirectory()) - { - errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); - } - } - else - errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); - } - else if (form.isCloudFileRoot()) - { - AdminController.validateCloudFileRoot(form, getContainer(), errors); - } - } - - @Override - public boolean handlePost(FilesForm form, BindException errors) throws Exception - { - FileContentService service = FileContentService.get(); - if (service != null) - { - if (form.isPipelineRootForm()) - return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); - else - { - AdminController.setFileRootFromForm(getViewContext(), form, errors); - } - } - - // Cloud settings - AdminController.setEnabledCloudStores(getViewContext(), form, errors); - - return !errors.hasErrors(); - } - - @Override - public URLHelper getSuccessURL(FilesForm form) - { - ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); - if (form.isPipelineRootForm()) - { - url.addParameter("piperootSet", true); - } - else - { - if (form.isFileRootChanged()) - url.addParameter("rootSet", form.getMigrateFilesOption()); - if (form.isEnabledCloudStoresChanged()) - url.addParameter("cloudChanged", true); - } - return url; - } - } - - public static class LookAndFeelView extends JspView - { - LookAndFeelView(BindException errors) - { - super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); - } - } - - - public static class LookAndFeelBean - { - public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); - public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); - } - - @RequiresPermission(AdminPermission.class) - public static class AdjustSystemTimestampsAction extends FormViewAction - { - @Override - public void addNavTrail(NavTree root) - { - } - - @Override - public void validateCommand(AdjustTimestampsForm form, Errors errors) - { - if (form.getHourDelta() == null || form.getHourDelta() == 0) - errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); - } - - @Override - public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception - { - return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); - } - - private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) - { - SQLFragment sql = new SQLFragment(); - DbSchema schema = tInfo.getSchema(); - String comma = ""; - List updating = new ArrayList<>(); - - for (String fieldName: fieldNames) - { - ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); - if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) - { - updating.add(fieldName); - if (sql.isEmpty()) - sql.append("UPDATE ").append(tInfo, "").append(" SET "); - sql.append(comma) - .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); - comma = ", "; - } - } - - if (!sql.isEmpty()) - { - logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); - logger.debug(sql.toDebugString()); - int numRows = new SqlExecutor(schema).execute(sql); - logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); - } - } - - @Override - public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception - { - List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); - logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - DbScope scope = DbScope.getLabKeyScope(); - try (DbScope.Transaction t = scope.ensureTransaction()) - { - ModuleLoader.getInstance().getModules().forEach(module -> { - logger.info("==> Beginning update of timestamps for module: " + module.getName()); - module.getSchemaNames().stream().sorted().forEach(schemaName -> { - DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); - scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from - schema.getTableNames().forEach(tableName -> { - TableInfo tInfo = schema.getTable(tableName); - if (tInfo.getTableType() == DatabaseTableType.TABLE) - { - updateFields(tInfo, toUpdate, form.getHourDelta()); - } - }); - }); - logger.info("<== DONE updating timestamps for module: " + module.getName()); - }); - t.commit(); - } - logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); - return true; - } - - @Override - public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) - { - return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); - } - } - - public static class AdjustTimestampsForm - { - private Integer hourDelta; - - public Integer getHourDelta() - { - return hourDelta; - } - - public void setHourDelta(Integer hourDelta) - { - this.hourDelta = hourDelta; - } - } - - @RequiresPermission(TroubleshooterPermission.class) - public class ViewUsageStatistics extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); - } - - @Override - public void addNavTrail(NavTree root) - { - addAdminNavTrail(root, "Usage Statistics", this.getClass()); - } - } - - private static final URI LABKEY_ORG_REPORT_ACTION; - - static - { - LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); - } - - @RequiresNoPermission - @CSRF(CSRF.Method.NONE) - public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction - { - private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); - - // recent reports, to help avoid log spam - private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); - - @Override - public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - var ret = new JSONObject().put("success", true); - - // fail fast - if (!_log.isWarnEnabled()) - return ret; - - var request = getViewContext().getRequest(); - assert null != request; - - var userAgent = request.getHeader("User-Agent"); - if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) - return ret; - - // NOTE User may be "guest", and will always be guest if being relayed to labkey.org - var jsonObj = form.getJsonObject(); - if (null != jsonObj) - { - JSONObject cspReport = jsonObj.optJSONObject("csp-report"); - if (cspReport != null) - { - String blockedUri = cspReport.optString("blocked-uri", null); - - // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org - if (blockedUri != null && - blockedUri.startsWith("https://labkey.org%2C") && - blockedUri.endsWith("undefined") && - !_log.isDebugEnabled()) - { - return ret; - } - - String urlString = cspReport.optString("document-uri", null); - if (urlString != null) - { - String path = new URLHelper(urlString).deleteParameters().getURIString(); - if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) - { - // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. - boolean forwarded = jsonObj.optBoolean("forwarded", false); - if (!forwarded) - { - User user = getUser(); - String email = null; - // If the user is not logged in, we may still be able to snag the email address from our cookie - if (user.isGuest()) - email = LoginController.getEmailFromCookie(getViewContext().getRequest()); - if (null == email) - email = user.getEmail(); - jsonObj.put("user", email); - String ipAddress = request.getHeader("X-FORWARDED-FOR"); - if (ipAddress == null) - ipAddress = request.getRemoteAddr(); - jsonObj.put("ip", ipAddress); - if (isNotBlank(userAgent)) - jsonObj.put("user-agent", userAgent); - String labkeyVersion = request.getParameter("labkeyVersion"); - if (null != labkeyVersion) - jsonObj.put("labkeyVersion", labkeyVersion); - String cspVersion = request.getParameter("cspVersion"); - if (null != cspVersion) - jsonObj.put("cspVersion", cspVersion); - } - - var jsonStr = jsonObj.toString(2); - _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); - - if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) - { - jsonObj.put("forwarded", true); - - // Create an HttpClient - HttpClient client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(10)) - .build(); - - // Create the POST request - HttpRequest remoteRequest = HttpRequest.newBuilder() - .uri(LABKEY_ORG_REPORT_ACTION) - .header("Content-Type", request.getContentType()) // Use whatever the browser set - .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) - .build(); - - // Send the request and get the response - HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() != 200) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); - } - else - { - JSONObject jsonResponse = new JSONObject(response.body()); - boolean success = jsonResponse.optBoolean("success", false); - if (!success) - { - _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); - } - } - } - } - } - } - } - return ret; - } - } - - - public static class TestCase extends AbstractActionPermissionTest - { - @Override - @Test - public void testActionPermissions() - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - AdminController controller = new AdminController(); - - // @RequiresPermission(ReadPermission.class) - assertForReadPermission(user, false, - new GetModulesAction(), - new GetFolderTabsAction(), - new ClearDeletedTabFoldersAction() - ); - - // @RequiresPermission(DeletePermission.class) - assertForUpdateOrDeletePermission(user, - new DeleteFolderAction() - ); - - // @RequiresPermission(AdminPermission.class) - assertForAdminPermission(user, - controller.new CustomizeEmailAction(), - controller.new FolderAliasesAction(), - controller.new MoveFolderAction(), - controller.new MoveTabAction(), - controller.new RenameFolderAction(), - controller.new ReorderFoldersAction(), - controller.new ReorderFoldersApiAction(), - controller.new SiteValidationAction(), - new AddTabAction(), - new ConfirmProjectMoveAction(), - new CreateFolderAction(), - new CustomizeMenuAction(), - new DeleteCustomEmailAction(), - new FilesAction(), - new MenuBarAction(), - new ProjectSettingsAction(), - new RenameContainerAction(), - new RenameTabAction(), - new ResetPropertiesAction(), - new ResetQueryStatisticsAction(), - new ResetResourceAction(), - new ResourcesAction(), - new RevertFolderAction(), - new SetFolderPermissionsAction(), - new SetInitialFolderSettingsAction(), - new ShowTabAction() - ); - - //TODO @RequiresPermission(AdminReadPermission.class) - //controller.new TestMothershipReportAction() - - // @RequiresPermission(AdminOperationsPermission.class) - assertForAdminOperationsPermission(ContainerManager.getRoot(), user, - controller.new DbCheckerAction(), - controller.new DeleteModuleAction(), - controller.new DoCheckAction(), - controller.new EmailTestAction(), - controller.new ShowNetworkDriveTestAction(), - controller.new ValidateDomainsAction(), - new OptionalFeatureAction(), - new GetSchemaXmlDocAction(), - new RecreateViewsAction() - ); - - // @AdminConsoleAction - assertForAdminPermission(ContainerManager.getRoot(), user, - controller.new ActionsAction(), - controller.new CachesAction(), - controller.new ConfigureSystemMaintenanceAction(), - controller.new CustomizeSiteAction(), - controller.new DumpHeapAction(), - controller.new ExecutionPlanAction(), - controller.new FolderTypesAction(), - controller.new MemTrackerAction(), - controller.new ModulesAction(), - controller.new QueriesAction(), - controller.new QueryStackTracesAction(), - controller.new ResetErrorMarkAction(), - controller.new ShortURLAdminAction(), - controller.new ShowAllErrorsAction(), - controller.new ShowErrorsSinceMarkAction(), - controller.new ShowPrimaryLogAction(), - controller.new ShowCspReportLogAction(), - controller.new ShowThreadsAction(), - new ExportActionsAction(), - new ExportQueriesAction(), - new MemoryChartAction(), - new ShowAdminAction() - ); - - // @RequiresSiteAdmin - assertForRequiresSiteAdmin(user, - controller.new EnvironmentVariablesAction(), - controller.new SystemMaintenanceAction(), - controller.new SystemPropertiesAction(), - new GetPendingRequestCountAction(), - new InstallCompleteAction(), - new NewInstallSiteSettingsAction() - ); - - assertForTroubleshooterPermission(ContainerManager.getRoot(), user, - controller.new OptionalFeaturesAction(), - controller.new ShowModuleErrorsAction(), - new ModuleStatusAction() - ); - } - } - - public static class SerializationTest extends PipelineJob.TestSerialization - { - static class TestJob extends PipelineJob - { - ImpersonationContext _impersonationContext; - ImpersonationContext _impersonationContext1; - ImpersonationContext _impersonationContext2; - - @Override - public URLHelper getStatusHref() - { - return null; - } - - @Override - public String getDescription() - { - return "Test Job"; - } - } - - @Test - public void testSerialization() - { - TestJob job = new TestJob(); - TestContext ctx = TestContext.get(); - ViewContext viewContext = new ViewContext(); - viewContext.setContainer(ContainerManager.getSharedContainer()); - viewContext.setUser(ctx.getUser()); - RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( - viewContext.getContainer(), viewContext.getUser(), - Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); - job._impersonationContext = factory.getImpersonationContext(); - - try - { - UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - UserManager.getGuestUser(), null); - job._impersonationContext1 = factory1.getImpersonationContext(); - } - catch (Exception e) - { - LOG.error("Invalid user email for impersonating."); - } - - GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), - GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); - job._impersonationContext2 = factory2.getImpersonationContext(); - testSerialize(job, LOG); - } - } - - public static class WorkbookDeleteTestCase extends Assert - { - private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; - private static final String TEST_EMAIL = "testDelete@myDomain.com"; - - @Test - public void testWorkbookDelete() throws Exception - { - doCleanup(); - - Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); - Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); - - ValidEmail email = new ValidEmail(TEST_EMAIL); - SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); - User nonAdminUser = newUserStatus.getUser(); - MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, ReaderRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - // User lacks any permission, throw unauthorized for parent and workbook: - HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Grant permission, should be able to delete the workbook but not parent: - policy = new MutableSecurityPolicy(project.getPolicy()); - policy.addRoleAssignment(nonAdminUser, EditorRole.class); - SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); - - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); - - // Hitting delete action results in a redirect: - request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); - response = ViewServlet.mockDispatch(request, null); - Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); - - doCleanup(); - } - - protected static void doCleanup() throws Exception - { - Container project = ContainerManager.getForPath(FOLDER_NAME); - if (project != null) - { - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) - { - User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); - UserManager.deleteUser(u.getUserId()); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.admin; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.util.concurrent.UncheckedExecutionException; +import jakarta.mail.MessagingException; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.map.LRUMap; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.xmlbeans.XmlOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.jfree.chart.ChartFactory; +import org.jfree.chart.ChartUtilities; +import org.jfree.chart.JFreeChart; +import org.jfree.chart.plot.PlotOrientation; +import org.jfree.data.category.DefaultCategoryDataset; +import org.json.JSONObject; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.Constants; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.BaseApiAction; +import org.labkey.api.action.BaseViewAction; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasViewContext; +import org.labkey.api.action.IgnoresAllocationTracking; +import org.labkey.api.action.LabKeyError; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.QueryViewAction; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AbstractFolderContext.ExportType; +import org.labkey.api.admin.AdminBean; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.admin.FolderExportContext; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.admin.FolderWriter; +import org.labkey.api.admin.FolderWriterImpl; +import org.labkey.api.admin.HealthCheck; +import org.labkey.api.admin.HealthCheckRegistry; +import org.labkey.api.admin.ImportOptions; +import org.labkey.api.admin.StaticLoggerGetter; +import org.labkey.api.admin.TableXmlUtils; +import org.labkey.api.admin.sitevalidation.SiteValidationResult; +import org.labkey.api.admin.sitevalidation.SiteValidationResultList; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.cache.CacheStats; +import org.labkey.api.cache.TrackingCache; +import org.labkey.api.cloud.CloudStoreService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.CaseInsensitiveHashSetValuedMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.compliance.ComplianceFolderSettings; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.compliance.PhiColumnBehavior; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.ConnectionWrapper; +import org.labkey.api.data.Container; +import org.labkey.api.data.Container.ContainerException; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DataColumn; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DatabaseTableType; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MenuButton; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.NormalContainerType; +import org.labkey.api.data.PHI; +import org.labkey.api.data.RenderContext; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TransactionFilter; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.data.dialect.SqlDialect.ExecutionPlanType; +import org.labkey.api.data.queryprofiler.QueryProfiler; +import org.labkey.api.data.queryprofiler.QueryProfiler.QueryStatTsvWriter; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.files.FileContentService; +import org.labkey.api.message.settings.AbstractConfigTypeProvider.EmailConfigFormImpl; +import org.labkey.api.message.settings.MessageConfigService; +import org.labkey.api.message.settings.MessageConfigService.ConfigTypeProvider; +import org.labkey.api.message.settings.MessageConfigService.NotificationOption; +import org.labkey.api.message.settings.MessageConfigService.UserPreference; +import org.labkey.api.miniprofiler.RequestInfo; +import org.labkey.api.module.AllowedBeforeInitialUserIsSet; +import org.labkey.api.module.AllowedDuringUpgrade; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.FolderType; +import org.labkey.api.module.FolderTypeManager; +import org.labkey.api.module.IgnoresForbiddenProjectCheck; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.ModuleLoader.SchemaActions; +import org.labkey.api.module.ModuleLoader.SchemaAndModule; +import org.labkey.api.module.SimpleModule; +import org.labkey.api.moduleeditor.api.ModuleEditorService; +import org.labkey.api.pipeline.DirectoryNotDeletedException; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineStatusFile; +import org.labkey.api.pipeline.PipelineStatusUrls; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.pipeline.view.SetupForm; +import org.labkey.api.products.ProductRegistry; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RuntimeValidationException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.ExternalScriptEngineDefinition; +import org.labkey.api.reports.LabKeyScriptEngineManager; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.CSRF; +import org.labkey.api.security.Directive; +import org.labkey.api.security.Group; +import org.labkey.api.security.GroupManager; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.LoginUrls; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresLogin; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.RoleAssignment; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicy; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.SecurityUrls; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.impersonation.GroupImpersonationContextFactory; +import org.labkey.api.security.impersonation.ImpersonationContext; +import org.labkey.api.security.impersonation.RoleImpersonationContextFactory; +import org.labkey.api.security.impersonation.UserImpersonationContextFactory; +import org.labkey.api.security.permissions.AbstractActionPermissionTest; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.CreateProjectPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.SiteAdminPermission; +import org.labkey.api.security.permissions.TroubleshooterPermission; +import org.labkey.api.security.permissions.UploadFileBasedModulePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.security.roles.FolderAdminRole; +import org.labkey.api.security.roles.ProjectAdminRole; +import org.labkey.api.security.roles.ReaderRole; +import org.labkey.api.security.roles.Role; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.security.roles.SharedViewEditorRole; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.ConceptURIProperties; +import org.labkey.api.settings.DateParsingMode; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.settings.LookAndFeelPropertiesManager.ResourceType; +import org.labkey.api.settings.NetworkDriveProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.settings.OptionalFeatureService.FeatureType; +import org.labkey.api.settings.ProductConfiguration; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.settings.WriteableFolderLookAndFeelProperties; +import org.labkey.api.settings.WriteableLookAndFeelProperties; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ConfigurationException; +import org.labkey.api.util.DOM; +import org.labkey.api.util.DOM.Renderable; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.DebugInfoDumper; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.FolderDisplayMode; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.HttpsUtil; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.MailHelper; +import org.labkey.api.util.MemTracker; +import org.labkey.api.util.MemTracker.HeldReference; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.SafeToRenderEnum; +import org.labkey.api.util.SessionAppender; +import org.labkey.api.util.StringExpressionFactory; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.SystemMaintenance.SystemMaintenanceProperties; +import org.labkey.api.util.SystemMaintenanceJob; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.Tuple3; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UniqueID; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.util.emailTemplate.EmailTemplate; +import org.labkey.api.util.emailTemplate.EmailTemplateService; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.DataView; +import org.labkey.api.view.FolderManagement.FolderManagementViewAction; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewAction; +import org.labkey.api.view.FolderManagement.ProjectSettingsViewPostAction; +import org.labkey.api.view.FolderManagement.TYPE; +import org.labkey.api.view.FolderTab; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.Portal; +import org.labkey.api.view.RedirectException; +import org.labkey.api.view.ShortURLRecord; +import org.labkey.api.view.ShortURLService; +import org.labkey.api.view.TabStripView; +import org.labkey.api.view.URLException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.EmptyView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.view.template.PageConfig.Template; +import org.labkey.api.wiki.WikiRendererType; +import org.labkey.api.wiki.WikiRenderingService; +import org.labkey.api.writer.FileSystemFile; +import org.labkey.api.writer.HtmlWriter; +import org.labkey.api.writer.ZipFile; +import org.labkey.api.writer.ZipUtil; +import org.labkey.bootstrap.ExplodedModuleService; +import org.labkey.core.admin.miniprofiler.MiniProfilerController; +import org.labkey.core.admin.sitevalidation.SiteValidationJob; +import org.labkey.core.admin.sql.SqlScriptController; +import org.labkey.core.login.LoginController; +import org.labkey.core.portal.CollaborationFolderType; +import org.labkey.core.portal.ProjectController; +import org.labkey.core.query.CoreQuerySchema; +import org.labkey.core.query.PostgresUserSchema; +import org.labkey.core.reports.ExternalScriptEngineDefinitionImpl; +import org.labkey.core.security.AllowedExternalResourceHosts; +import org.labkey.core.security.AllowedExternalResourceHosts.AllowedHost; +import org.labkey.core.security.BlockListFilter; +import org.labkey.core.security.SecurityController; +import org.labkey.data.xml.TablesDocument; +import org.labkey.filters.ContentSecurityPolicyFilter; +import org.labkey.security.xml.GroupEnumType; +import org.labkey.vfs.FileLike; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.Controller; + +import java.awt.*; +import java.beans.Introspector; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.lang.management.BufferPoolMXBean; +import java.lang.management.ClassLoadingMXBean; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryPoolMXBean; +import java.lang.management.MemoryType; +import java.lang.management.MemoryUsage; +import java.lang.management.OperatingSystemMXBean; +import java.lang.management.RuntimeMXBean; +import java.lang.management.ThreadMXBean; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.text.DecimalFormat; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.labkey.api.data.MultiValuedRenderContext.VALUE_DELIMITER_REGEX; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Configuration; +import static org.labkey.api.settings.AdminConsole.SettingsLinkType.Diagnostics; +import static org.labkey.api.util.DOM.A; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.Attribute.method; +import static org.labkey.api.util.DOM.Attribute.name; +import static org.labkey.api.util.DOM.Attribute.style; +import static org.labkey.api.util.DOM.Attribute.title; +import static org.labkey.api.util.DOM.Attribute.type; +import static org.labkey.api.util.DOM.Attribute.value; +import static org.labkey.api.util.DOM.BR; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.LI; +import static org.labkey.api.util.DOM.SPAN; +import static org.labkey.api.util.DOM.STYLE; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.UL; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.api.util.DOM.createHtmlFragment; +import static org.labkey.api.util.HtmlString.NBSP; +import static org.labkey.api.util.logging.LogHelper.getLabKeyLogDir; +import static org.labkey.api.view.FolderManagement.EVERY_CONTAINER; +import static org.labkey.api.view.FolderManagement.FOLDERS_AND_PROJECTS; +import static org.labkey.api.view.FolderManagement.FOLDERS_ONLY; +import static org.labkey.api.view.FolderManagement.NOT_ROOT; +import static org.labkey.api.view.FolderManagement.PROJECTS_ONLY; +import static org.labkey.api.view.FolderManagement.ROOT; +import static org.labkey.api.view.FolderManagement.addTab; + +public class AdminController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver( + AdminController.class, + FileListAction.class, + FilesSiteSettingsAction.class, + UpdateFilePathsAction.class + ); + + private static final Logger LOG = LogHelper.getLogger(AdminController.class, "Admin-related UI and APIs"); + private static final Logger CLIENT_LOG = LogHelper.getLogger(LogAction.class, "Client/browser logging submitted to server"); + private static final String HEAP_MEMORY_KEY = "Total Heap Memory"; + + private static long _errorMark = 0; + private static long _primaryLogMark = 0; + + public static void registerAdminConsoleLinks() + { + Container root = ContainerManager.getRoot(); + + // Configuration + AdminConsole.addLink(Configuration, "authentication", urlProvider(LoginUrls.class).getConfigureURL()); + AdminConsole.addLink(Configuration, "email customization", new ActionURL(CustomizeEmailAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "deprecated features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Deprecated.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "experimental features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Experimental.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "optional features", new ActionURL(OptionalFeaturesAction.class, root).addParameter("type", FeatureType.Optional.name()), TroubleshooterPermission.class); + if (!ProductRegistry.getProducts().isEmpty()) + AdminConsole.addLink(Configuration, "product configuration", new ActionURL(ProductConfigurationAction.class, root), AdminOperationsPermission.class); + // TODO move to FileContentModule + if (ModuleLoader.getInstance().hasModule("FileContent")) + AdminConsole.addLink(Configuration, "files", new ActionURL(FilesSiteSettingsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Configuration, "folder types", new ActionURL(FolderTypesAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "look and feel settings", new ActionURL(LookAndFeelSettingsAction.class, root)); + AdminConsole.addLink(Configuration, "missing value indicators", new AdminUrlsImpl().getMissingValuesURL(root), AdminPermission.class); + AdminConsole.addLink(Configuration, "project display order", new ActionURL(ReorderFoldersAction.class, root), AdminPermission.class); + AdminConsole.addLink(Configuration, "short urls", new ActionURL(ShortURLAdminAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "site settings", new AdminUrlsImpl().getCustomizeSiteURL()); + AdminConsole.addLink(Configuration, "system maintenance", new ActionURL(ConfigureSystemMaintenanceAction.class, root)); + AdminConsole.addLink(Configuration, "allowed external redirect hosts", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.Redirect.name()), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed external resource hosts", new ActionURL(ExternalSourcesAction.class, root), TroubleshooterPermission.class); + AdminConsole.addLink(Configuration, "allowed file extensions", new ActionURL(AllowListAction.class, root).addParameter("type", AllowListType.FileExtension.name()), TroubleshooterPermission.class); + + // Diagnostics + AdminConsole.addLink(Diagnostics, "actions", new ActionURL(ActionsAction.class, root)); + AdminConsole.addLink(Diagnostics, "attachments", new ActionURL(AttachmentsAction.class, root)); + AdminConsole.addLink(Diagnostics, "caches", new ActionURL(CachesAction.class, root)); + AdminConsole.addLink(Diagnostics, "check database", new ActionURL(DbCheckerAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "credits", new ActionURL(CreditsAction.class, root)); + AdminConsole.addLink(Diagnostics, "dump heap", new ActionURL(DumpHeapAction.class, root)); + AdminConsole.addLink(Diagnostics, "environment variables", new ActionURL(EnvironmentVariablesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "memory usage", new ActionURL(MemTrackerAction.class, root)); + + if (CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + AdminConsole.addLink(Diagnostics, "postgres activity", new ActionURL(PostgresStatActivityAction.class, root)); + AdminConsole.addLink(Diagnostics, "postgres locks", new ActionURL(PostgresLocksAction.class, root)); + AdminConsole.addLink(Diagnostics, "postgres table sizes", new ActionURL(PostgresTableSizesAction.class, root)); + } + + AdminConsole.addLink(Diagnostics, "profiler", new ActionURL(MiniProfilerController.ManageAction.class, root)); + AdminConsole.addLink(Diagnostics, "queries", getQueriesURL(null)); + AdminConsole.addLink(Diagnostics, "reset site errors", new ActionURL(ResetErrorMarkAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "running threads", new ActionURL(ShowThreadsAction.class, root)); + AdminConsole.addLink(Diagnostics, "site validation", new ActionURL(ConfigureSiteValidationAction.class, root), AdminPermission.class); + AdminConsole.addLink(Diagnostics, "sql scripts", new ActionURL(SqlScriptController.ScriptsAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "suspicious activity", new ActionURL(SuspiciousAction.class, root)); + AdminConsole.addLink(Diagnostics, "system properties", new ActionURL(SystemPropertiesAction.class, root), SiteAdminPermission.class); + AdminConsole.addLink(Diagnostics, "test email configuration", new ActionURL(EmailTestAction.class, root), AdminOperationsPermission.class); + AdminConsole.addLink(Diagnostics, "view all site errors", new ActionURL(ShowAllErrorsAction.class, root)); + AdminConsole.addLink(Diagnostics, "view all site errors since reset", new ActionURL(ShowErrorsSinceMarkAction.class, root)); + AdminConsole.addLink(Diagnostics, "view csp report log file", new ActionURL(ShowCspReportLogAction.class, root)); + AdminConsole.addLink(Diagnostics, "view primary site log file", new ActionURL(ShowPrimaryLogAction.class, root)); + } + + public static void registerManagementTabs() + { + addTab(TYPE.FolderManagement, "Folder Tree", "folderTree", EVERY_CONTAINER, ManageFoldersAction.class); + addTab(TYPE.FolderManagement, "Folder Type", "folderType", NOT_ROOT, FolderTypeAction.class); + addTab(TYPE.FolderManagement, "Missing Values", "mvIndicators", EVERY_CONTAINER, MissingValuesAction.class); + addTab(TYPE.FolderManagement, "Module Properties", "props", c -> { + if (!c.isRoot()) + { + // Show module properties tab only if a module w/ properties to set is present for current folder + for (Module m : c.getActiveModules()) + if (!m.getModuleProperties().isEmpty()) + return true; + } + + return false; + }, ModulePropertiesAction.class); + addTab(TYPE.FolderManagement, "Concepts", "concepts", c -> { + // Show Concepts tab only if the experiment module is enabled in this container + return c.getActiveModules().contains(ModuleLoader.getInstance().getModule(ExperimentService.MODULE_NAME)); + }, AdminController.ConceptsAction.class); + // Show Notifications tab only if we have registered notification providers + addTab(TYPE.FolderManagement, "Notifications", "notifications", c -> NOT_ROOT.test(c) && !MessageConfigService.get().getConfigTypes().isEmpty(), NotificationsAction.class); + addTab(TYPE.FolderManagement, "Export", "export", NOT_ROOT, ExportFolderAction.class); + addTab(TYPE.FolderManagement, "Import", "import", NOT_ROOT, ImportFolderAction.class); + addTab(TYPE.FolderManagement, "Files", "files", FOLDERS_AND_PROJECTS, FileRootsAction.class); + addTab(TYPE.FolderManagement, "Formats", "settings", FOLDERS_ONLY, FolderSettingsAction.class); + addTab(TYPE.FolderManagement, "Information", "info", NOT_ROOT, FolderInformationAction.class); + addTab(TYPE.FolderManagement, "R Config", "rConfig", NOT_ROOT, RConfigurationAction.class); + + addTab(TYPE.ProjectSettings, "Properties", "properties", PROJECTS_ONLY, ProjectSettingsAction.class); + addTab(TYPE.ProjectSettings, "Resources", "resources", PROJECTS_ONLY, ResourcesAction.class); + addTab(TYPE.ProjectSettings, "Menu Bar", "menubar", PROJECTS_ONLY, MenuBarAction.class); + addTab(TYPE.ProjectSettings, "Files", "files", PROJECTS_ONLY, FilesAction.class); + + addTab(TYPE.LookAndFeelSettings, "Properties", "properties", ROOT, LookAndFeelSettingsAction.class); + addTab(TYPE.LookAndFeelSettings, "Resources", "resources", ROOT, AdminConsoleResourcesAction.class); + } + + public AdminController() + { + setActionResolver(_actionResolver); + } + + @RequiresNoPermission + public static class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getShowAdminURL(); + } + } + + private void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action) + { + addAdminNavTrail(root, childTitle, action, getContainer()); + } + + private static void addAdminNavTrail(NavTree root, @NotNull Container container) + { + if (container.isRoot()) + root.addChild("Admin Console", getShowAdminURL().setFragment("links")); + } + + private static void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + addAdminNavTrail(root, container); + root.addChild(childTitle, new ActionURL(action, container)); + } + + public static ActionURL getShowAdminURL() + { + return new ActionURL(ShowAdminAction.class, ContainerManager.getRoot()); + } + + @Override + protected void beforeAction(Controller action) throws ServletException + { + super.beforeAction(action); + if (action instanceof BaseViewAction viewaction) + viewaction.getPageConfig().setRobotsNone(); + } + + @AdminConsoleAction + public static class ShowAdminAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/admin.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + URLHelper returnUrl = getViewContext().getActionURL().getReturnUrl(); + if (null != returnUrl) + root.addChild("Return to Project", returnUrl); + root.addChild("Admin Console"); + setHelpTopic("siteManagement"); + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ShowModuleErrorsAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Module Errors", this.getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/moduleErrors.jsp"); + } + } + + public static class AdminUrlsImpl implements AdminUrls + { + @Override + public ActionURL getModuleErrorsURL() + { + return new ActionURL(ShowModuleErrorsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getAdminConsoleURL() + { + return getShowAdminURL(); + } + + @Override + public ActionURL getModuleStatusURL(URLHelper returnUrl) + { + return AdminController.getModuleStatusURL(returnUrl); + } + + @Override + public ActionURL getCustomizeSiteURL() + { + return new ActionURL(CustomizeSiteAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getCustomizeSiteURL(boolean upgradeInProgress) + { + ActionURL url = getCustomizeSiteURL(); + + if (upgradeInProgress) + url.addParameter("upgradeInProgress", "1"); + + return url; + } + + @Override + public ActionURL getProjectSettingsURL(Container c) + { + return new ActionURL(ProjectSettingsAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + ActionURL getLookAndFeelResourcesURL(Container c) + { + return c.isRoot() ? new ActionURL(AdminConsoleResourcesAction.class, c) : new ActionURL(ResourcesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsMenuURL(Container c) + { + return new ActionURL(MenuBarAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getProjectSettingsFileURL(Container c) + { + return new ActionURL(FilesAction.class, LookAndFeelProperties.getSettingsContainer(c)); + } + + @Override + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable Class selectedTemplate, @Nullable URLHelper returnUrl) + { + return getCustomizeEmailURL(c, selectedTemplate == null ? null : selectedTemplate.getName(), returnUrl); + } + + public ActionURL getCustomizeEmailURL(@NotNull Container c, @Nullable String selectedTemplate, @Nullable URLHelper returnUrl) + { + ActionURL url = new ActionURL(CustomizeEmailAction.class, c); + if (selectedTemplate != null) + { + url.addParameter("templateClass", selectedTemplate); + } + if (returnUrl != null) + { + url.addReturnUrl(returnUrl); + } + return url; + } + + public ActionURL getResetLookAndFeelPropertiesURL(Container c) + { + return new ActionURL(ResetPropertiesAction.class, c); + } + + @Override + public ActionURL getMaintenanceURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(MaintenanceAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + @Override + public ActionURL getModulesDetailsURL() + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getDeleteModuleURL(String moduleName) + { + return new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()).addParameter("name", moduleName); + } + + @Override + public ActionURL getManageFoldersURL(Container c) + { + return new ActionURL(ManageFoldersAction.class, c); + } + + @Override + public ActionURL getFolderTypeURL(Container c) + { + return new ActionURL(FolderTypeAction.class, c); + } + + @Override + public ActionURL getExportFolderURL(Container c) + { + return new ActionURL(ExportFolderAction.class, c); + } + + @Override + public ActionURL getImportFolderURL(Container c) + { + return new ActionURL(ImportFolderAction.class, c); + } + + @Override + public ActionURL getCreateProjectURL(@Nullable ActionURL returnUrl) + { + return getCreateFolderURL(ContainerManager.getRoot(), returnUrl); + } + + @Override + public ActionURL getCreateFolderURL(Container c, @Nullable ActionURL returnUrl) + { + ActionURL result = new ActionURL(CreateFolderAction.class, c); + if (returnUrl != null) + { + result.addReturnUrl(returnUrl); + } + return result; + } + + public ActionURL getSetFolderPermissionsURL(Container c) + { + return new ActionURL(SetFolderPermissionsAction.class, c); + } + + @Override + public void addAdminNavTrail(NavTree root, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, container); + } + + @Override + public void addAdminNavTrail(NavTree root, String childTitle, @NotNull Class action, @NotNull Container container) + { + AdminController.addAdminNavTrail(root, childTitle, action, container); + } + + @Override + public void addModulesNavTrail(NavTree root, String childTitle, @NotNull Container container) + { + if (container.isRoot()) + addAdminNavTrail(root, "Modules", ModulesAction.class, container); + + root.addChild(childTitle); + } + + @Override + public ActionURL getFileRootsURL(Container c) + { + return new ActionURL(FileRootsAction.class, c); + } + + @Override + public ActionURL getLookAndFeelSettingsURL(Container c) + { + if (c.isRoot()) + return getSiteLookAndFeelSettingsURL(); + else if (c.isProject()) + return getProjectSettingsURL(c); + else + return getFolderSettingsURL(c); + } + + @Override + public ActionURL getSiteLookAndFeelSettingsURL() + { + return new ActionURL(LookAndFeelSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFolderSettingsURL(Container c) + { + return new ActionURL(FolderSettingsAction.class, c); + } + + @Override + public ActionURL getNotificationsURL(Container c) + { + return new ActionURL(NotificationsAction.class, c); + } + + @Override + public ActionURL getModulePropertiesURL(Container c) + { + return new ActionURL(ModulePropertiesAction.class, c); + } + + @Override + public ActionURL getMissingValuesURL(Container c) + { + return new ActionURL(MissingValuesAction.class, c); + } + + public ActionURL getInitialFolderSettingsURL(Container c) + { + return new ActionURL(SetInitialFolderSettingsAction.class, c); + } + + @Override + public ActionURL getMemTrackerURL() + { + return new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getFilesSiteSettingsURL() + { + return new ActionURL(FilesSiteSettingsAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSessionLoggingURL() + { + return new ActionURL(SessionLoggingAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getTrackedAllocationsViewerURL() + { + return new ActionURL(TrackedAllocationsViewerAction.class, ContainerManager.getRoot()); + } + + @Override + public ActionURL getSystemMaintenanceURL() + { + return new ActionURL(ConfigureSystemMaintenanceAction.class, ContainerManager.getRoot()); + } + + public static ActionURL getDeprecatedFeaturesURL() + { + return new ActionURL(OptionalFeaturesAction.class, ContainerManager.getRoot()).addParameter("type", FeatureType.Deprecated.name()); + } + } + + public static class MaintenanceBean + { + public HtmlString content; + public ActionURL loginURL; + } + + /** + * During upgrade, startup, or maintenance mode, the user will be redirected to + * MaintenanceAction and only admin users will be allowed to log into the server. + * The maintenance.jsp page checks startup is complete or adminOnly mode is turned off + * and will redirect to the returnUrl or the loginURL. + * See Issue 18758 for more information. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class MaintenanceAction extends SimpleViewAction + { + private String _title = "Maintenance in progress"; + + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + if (!getUser().hasSiteAdminPermission()) + { + getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); + } + getPageConfig().setTemplate(Template.Dialog); + + boolean upgradeInProgress = ModuleLoader.getInstance().isUpgradeInProgress(); + boolean startupInProgress = ModuleLoader.getInstance().isStartupInProgress(); + boolean maintenanceMode = AppProps.getInstance().isUserRequestedAdminOnlyMode(); + + HtmlString content = HtmlString.of("This site is currently undergoing maintenance, only site admins may login at this time."); + if (upgradeInProgress) + { + _title = "Upgrade in progress"; + content = HtmlString.of("Upgrade in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (startupInProgress) + { + _title = "Startup in progress"; + content = HtmlString.of("Startup in progress: only site admins may login at this time. Your browser will be redirected when startup is complete."); + } + else if (maintenanceMode) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + content = wikiService.getFormattedHtml(WikiRendererType.RADEOX, ModuleLoader.getInstance().getAdminOnlyMessage(), "Admin only message"); + } + + if (content == null) + content = HtmlString.of(_title); + + ActionURL loginURL = null; + if (getUser().isGuest()) + { + URLHelper returnUrl = form.getReturnUrlHelper(); + if (returnUrl != null) + loginURL = urlProvider(LoginUrls.class).getLoginURL(ContainerManager.getRoot(), returnUrl); + else + loginURL = urlProvider(LoginUrls.class).getLoginURL(); + } + + MaintenanceBean bean = new MaintenanceBean(); + bean.content = content; + bean.loginURL = loginURL; + + JspView view = new JspView<>("/org/labkey/core/admin/maintenance.jsp", bean, errors); + view.setTitle(_title); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild(_title); + } + } + + /** + * Similar to SqlScriptController.GetModuleStatusAction except that Guest is allowed to check that the startup is complete. + */ + @RequiresNoPermission + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class StartupStatusAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("startupComplete", ModuleLoader.getInstance().isStartupComplete()); + result.put("adminOnly", AppProps.getInstance().isUserRequestedAdminOnlyMode()); + + return new ApiSimpleResponse(result); + } + } + + @RequiresSiteAdmin + @IgnoresTermsOfUse + public static class GetPendingRequestCountAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + JSONObject result = new JSONObject(); + result.put("pendingRequestCount", TransactionFilter.getPendingRequestCount() - 1 /* Exclude this request */); + + return new ApiSimpleResponse(result); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetModulesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(GetModulesForm form, BindException errors) + { + Container c = ContainerManager.getForPath(getContainer().getPath()); + + ApiSimpleResponse response = new ApiSimpleResponse(); + + List> qinfos = new ArrayList<>(); + + FolderType folderType = c.getFolderType(); + List allModules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + allModules.sort(Comparator.comparing(module -> module.getTabName(getViewContext()), String.CASE_INSENSITIVE_ORDER)); + + //note: this has been altered to use Container.getRequiredModules() instead of FolderType + //this is b/c a parent container must consider child workbooks when determining the set of requiredModules + Set requiredModules = c.getRequiredModules(); //folderType.getActiveModules() != null ? folderType.getActiveModules() : new HashSet(); + Set activeModules = c.getActiveModules(getUser()); + + for (Module m : allModules) + { + Map qinfo = new HashMap<>(); + + qinfo.put("name", m.getName()); + qinfo.put("required", requiredModules.contains(m)); + qinfo.put("active", activeModules.contains(m) || requiredModules.contains(m)); + qinfo.put("enabled", (m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE || + m.getTabDisplayMode() == Module.TabDisplayMode.DISPLAY_USER_PREFERENCE_DEFAULT) && !requiredModules.contains(m)); + qinfo.put("tabName", m.getTabName(getViewContext())); + qinfo.put("requireSitePermission", m.getRequireSitePermission()); + qinfos.add(qinfo); + } + + response.put("modules", qinfos); + response.put("folderType", folderType.getName()); + + return response; + } + } + + public static class GetModulesForm + { + } + + @RequiresNoPermission + @AllowedDuringUpgrade + // This action is invoked by HttpsUtil.checkSslRedirectConfiguration(), often while upgrade is in progress + public static class GuidAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + response.getWriter().write(GUID.makeGUID()); + } + } + + /** + * Preform health checks corresponding to the given categories. + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class HealthCheckAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(HealthCheckForm form, BindException errors) throws Exception + { + if (!ModuleLoader.getInstance().isStartupComplete()) + return new ApiSimpleResponse("healthy", false); + + Collection categories = form.getCategories() == null ? Collections.singleton(HealthCheckRegistry.DEFAULT_CATEGORY) : Arrays.asList(form.getCategories().split(",")); + HealthCheck.Result checkResult = HealthCheckRegistry.get().checkHealth(categories); + + checkResult.getDetails().put("healthy", checkResult.isHealthy()); + + if (getUser().hasRootAdminPermission()) + { + return new ApiSimpleResponse(checkResult.getDetails()); + } + else + { + if (!checkResult.isHealthy()) + { + try (var writer = createResponseWriter()) + { + writer.writeResponse(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "Server isn't ready yet"); + } + return null; + } + + return new ApiSimpleResponse("healthy", checkResult.isHealthy()); + } + } + } + + public static class HealthCheckForm + { + private String _categories; // if null, all categories will be checked. + + public String getCategories() + { + return _categories; + } + + @SuppressWarnings("unused") + public void setCategories(String categories) + { + _categories = categories; + } + } + + // No security checks... anyone (even guests) can view the credits page + @RequiresNoPermission + public class CreditsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + VBox views = new VBox(); + List modules = new ArrayList<>(ModuleLoader.getInstance().getModules()); + modules.sort(Comparator.comparing(Module::getName, String.CASE_INSENSITIVE_ORDER)); + + addCreditsViews(views, modules, "jars.txt", "JAR"); + addCreditsViews(views, modules, "scripts.txt", "Script, Icon and Font"); + addCreditsViews(views, modules, "source.txt", "Java Source Code"); + addCreditsViews(views, modules, "executables.txt", "Executable"); + + return views; + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Credits", this.getClass()); + } + } + + private void addCreditsViews(VBox views, List modules, String creditsFile, String fileType) throws IOException + { + for (Module module : modules) + { + String wikiSource = getCreditsFile(module, creditsFile); + + if (null != wikiSource) + { + String title = fileType + " Files Distributed with the " + module.getName() + " Module"; + CreditsView credits = new CreditsView(wikiSource, title); + views.addView(credits); + } + } + } + + private static class CreditsView extends WebPartView + { + private Renderable _html; + + CreditsView(@Nullable String wikiSource, String title) + { + super(title); + + wikiSource = StringUtils.trimToEmpty(wikiSource); + + if (StringUtils.isNotEmpty(wikiSource)) + { + WikiRenderingService wikiService = WikiRenderingService.get(); + HtmlString html = wikiService.getFormattedHtml(WikiRendererType.RADEOX, wikiSource, "Credits page"); + _html = DOM.createHtmlFragment(STYLE(at(type, "text/css"), "tr.table-odd td { background-color: #EEEEEE; }"), html); + } + } + + @Override + public void renderView(Object model, HtmlWriter out) + { + out.write(_html); + } + } + + private static String getCreditsFile(Module module, String filename) throws IOException + { + // credits files are in /resources/credits + InputStream is = module.getResourceStream("credits/" + filename); + + return null == is ? null : PageFlowUtil.getStreamContentsAsString(is); + } + + private void validateNetworkDrive(NetworkDriveForm form, Errors errors) + { + if (isBlank(form.getNetworkDriveUser()) || isBlank(form.getNetworkDrivePath()) || + isBlank(form.getNetworkDrivePassword()) || isBlank(form.getNetworkDriveLetter())) + { + errors.reject(ERROR_MSG, "All fields are required"); + } + else if (form.getNetworkDriveLetter().trim().length() > 1) + { + errors.reject(ERROR_MSG, "Network drive letter must be a single character"); + } + else + { + char letter = form.getNetworkDriveLetter().trim().toLowerCase().charAt(0); + + if (letter < 'a' || letter > 'z') + { + errors.reject(ERROR_MSG, "Network drive letter must be a letter"); + } + } + } + + public static class ResourceForm + { + private String _resource; + + public String getResource() + { + return _resource; + } + + public void setResource(String resource) + { + _resource = resource; + } + + public ResourceType getResourceType() + { + return ResourceType.valueOf(_resource); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetResourceAction extends FormHandlerAction + { + @Override + public void validateCommand(ResourceForm target, Errors errors) + { + } + + @Override + public boolean handlePost(ResourceForm form, BindException errors) throws Exception + { + form.getResourceType().delete(getContainer(), getUser()); + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + return true; + } + + @Override + public URLHelper getSuccessURL(ResourceForm form) + { + return new AdminUrlsImpl().getLookAndFeelResourcesURL(getContainer()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResetPropertiesAction extends FormHandlerAction + { + private URLHelper _returnUrl; + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + Container c = getContainer(); + boolean folder = !(c.isRoot() || c.isProject()); + boolean hasAdminOpsPerm = c.hasPermission(getUser(), AdminOperationsPermission.class); + + WriteableFolderLookAndFeelProperties props = folder ? LookAndFeelProperties.getWriteableFolderInstance(c) : LookAndFeelProperties.getWriteableInstance(c); + props.clear(hasAdminOpsPerm); + props.save(); + // TODO: Audit log? + + AdminUrls urls = new AdminUrlsImpl(); + + // Folder-level settings are just display formats and measure/dimension flags -- no need to increment L&F revision + if (!folder) + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + _returnUrl = urls.getLookAndFeelSettingsURL(c); + + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return _returnUrl; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class CustomizeSiteAction extends FormViewAction + { + @Override + public ModelAndView getView(SiteSettingsForm form, boolean reshow, BindException errors) + { + if (form.isUpgradeInProgress()) + getPageConfig().setTemplate(Template.Dialog); + + SiteSettingsBean bean = new SiteSettingsBean(form.isUpgradeInProgress()); + setHelpTopic("configAdmin"); + return new JspView<>("/org/labkey/core/admin/customizeSite.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Customize Site", this.getClass()); + } + + @Override + public void validateCommand(SiteSettingsForm form, Errors errors) + { + if (form.isShowRibbonMessage() && StringUtils.isEmpty(form.getRibbonMessage())) + { + errors.reject(ERROR_MSG, "Cannot enable the ribbon message without providing a message to show"); + } + if (form.getMaxBLOBSize() < 0) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be negative"); + } + int hardCap = Math.max(WriteableAppProps.SOFT_MAX_BLOB_SIZE, AppProps.getInstance().getMaxBLOBSize()); + if (form.getMaxBLOBSize() > hardCap) + { + errors.reject(ERROR_MSG, "Maximum BLOB size cannot be set higher than " + hardCap + " bytes"); + } + if (form.getSslPort() < 1 || form.getSslPort() > 65535) + { + errors.reject(ERROR_MSG, "HTTPS port must be between 1 and 65,535"); + } + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "HTTP timeout must be non-negative"); + } + if (form.getMemoryUsageDumpInterval() < 0) + { + errors.reject(ERROR_MSG, "Memory logging frequency must be non-negative"); + } + } + + @Override + public boolean handlePost(SiteSettingsForm form, BindException errors) throws Exception + { + HttpServletRequest request = getViewContext().getRequest(); + + // We only need to check that SSL is running if the user isn't already using SSL + if (form.isSslRequired() && !(request.isSecure() && (form.getSslPort() == request.getServerPort()))) + { + URL testURL = new URL("https", request.getServerName(), form.getSslPort(), AppProps.getInstance().getContextPath()); + Pair sslResponse = HttpsUtil.testHttpsUrl(testURL, "Ensure that the web server is configured for SSL and the port is correct. If SSL is enabled, try saving these settings while connected via SSL."); + + if (sslResponse != null) + { + errors.reject(ERROR_MSG, sslResponse.first); + return false; + } + } + + if (form.getReadOnlyHttpRequestTimeout() < 0) + { + errors.reject(ERROR_MSG, "Read only HTTP request timeout must be non-negative"); + } + + WriteableAppProps props = AppProps.getWriteableInstance(); + + props.setPipelineToolsDir(form.getPipelineToolsDirectory()); + props.setNavAccessOpen(form.isNavAccessOpen()); + props.setSSLRequired(form.isSslRequired()); + boolean sslSettingChanged = AppProps.getInstance().isSSLRequired() != form.isSslRequired(); + props.setSSLPort(form.getSslPort()); + props.setMemoryUsageDumpInterval(form.getMemoryUsageDumpInterval()); + props.setReadOnlyHttpRequestTimeout(form.getReadOnlyHttpRequestTimeout()); + props.setMaxBLOBSize(form.getMaxBLOBSize()); + props.setExt3Required(form.isExt3Required()); + props.setExt3APIRequired(form.isExt3APIRequired()); + props.setSelfReportExceptions(form.isSelfReportExceptions()); + + props.setAdminOnlyMessage(form.getAdminOnlyMessage()); + props.setShowRibbonMessage(form.isShowRibbonMessage()); + props.setRibbonMessage(form.getRibbonMessage()); + props.setUserRequestedAdminOnlyMode(form.isAdminOnlyMode()); + + props.setAllowApiKeys(form.isAllowApiKeys()); + props.setApiKeyExpirationSeconds(form.getApiKeyExpirationSeconds()); + props.setAllowSessionKeys(form.isAllowSessionKeys()); + + try + { + ExceptionReportingLevel level = ExceptionReportingLevel.valueOf(form.getExceptionReportingLevel()); + props.setExceptionReportingLevel(level); + } + catch (IllegalArgumentException ignored) + { + } + + try + { + if (form.getUsageReportingLevel() != null) + { + UsageReportingLevel level = UsageReportingLevel.valueOf(form.getUsageReportingLevel()); + props.setUsageReportingLevel(level); + } + } + catch (IllegalArgumentException ignored) + { + } + + props.setAdministratorContactEmail(form.getAdministratorContactEmail() == null ? null : form.getAdministratorContactEmail().trim()); + + if (null != form.getBaseServerURL()) + { + if (form.isSslRequired() && !form.getBaseServerURL().startsWith("https")) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL. SSL connection is required. Consider https://."); + return false; + } + + try + { + props.setBaseServerUrl(form.getBaseServerURL()); + } + catch (URISyntaxException e) + { + errors.reject(ERROR_MSG, "Invalid Base Server URL, \"" + e.getMessage() + "\"." + + "Please enter a valid base URL containing the protocol, hostname, and port if required. " + + "The webapp context path should not be included. " + + "For example: \"https://www.example.com\" or \"http://www.labkey.org:8080\" and not \"http://www.example.com/labkey/\""); + return false; + } + } + + String frameOption = StringUtils.trimToEmpty(form.getXFrameOption()); + if (!frameOption.equals("DENY") && !frameOption.equals("SAMEORIGIN") && !frameOption.equals("ALLOW")) + { + errors.reject(ERROR_MSG, "XFrameOption must equal DENY, or SAMEORIGIN, or ALLOW"); + return false; + } + props.setXFrameOption(frameOption); + props.setIncludeServerHttpHeader(form.isIncludeServerHttpHeader()); + + props.save(getViewContext().getUser()); + UsageReportingLevel.reportNow(); + if (sslSettingChanged) + ContentSecurityPolicyFilter.regenerateSubstitutionMap(); + + return true; + } + + @Override + public ActionURL getSuccessURL(SiteSettingsForm form) + { + if (form.isUpgradeInProgress()) + { + return AppProps.getInstance().getHomePageActionURL(); + } + else + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + } + + public static class NetworkDriveForm + { + private String _networkDriveLetter; + private String _networkDrivePath; + private String _networkDriveUser; + private String _networkDrivePassword; + + public String getNetworkDriveLetter() + { + return _networkDriveLetter; + } + + public void setNetworkDriveLetter(String networkDriveLetter) + { + _networkDriveLetter = networkDriveLetter; + } + + public String getNetworkDrivePassword() + { + return _networkDrivePassword; + } + + public void setNetworkDrivePassword(String networkDrivePassword) + { + _networkDrivePassword = networkDrivePassword; + } + + public String getNetworkDrivePath() + { + return _networkDrivePath; + } + + public void setNetworkDrivePath(String networkDrivePath) + { + _networkDrivePath = networkDrivePath; + } + + public String getNetworkDriveUser() + { + return _networkDriveUser; + } + + public void setNetworkDriveUser(String networkDriveUser) + { + _networkDriveUser = networkDriveUser; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @AdminConsoleAction + public class MapNetworkDriveAction extends FormViewAction + { + @Override + public void validateCommand(NetworkDriveForm form, Errors errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mapNetworkDrive.jsp", null, errors); + } + + @Override + public boolean handlePost(NetworkDriveForm form, BindException errors) throws Exception + { + NetworkDriveProps.setNetworkDriveLetter(form.getNetworkDriveLetter().trim()); + NetworkDriveProps.setNetworkDrivePath(form.getNetworkDrivePath().trim()); + NetworkDriveProps.setNetworkDriveUser(form.getNetworkDriveUser().trim()); + NetworkDriveProps.setNetworkDrivePassword(form.getNetworkDrivePassword().trim()); + + return true; + } + + @Override + public URLHelper getSuccessURL(NetworkDriveForm siteSettingsForm) + { + return new ActionURL(FilesSiteSettingsAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("setRoots#map"); + addAdminNavTrail(root, "Map Network Drive", this.getClass()); + } + } + + public static class SiteSettingsBean + { + public final boolean _upgradeInProgress; + public final boolean _showSelfReportExceptions; + + private SiteSettingsBean(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + _showSelfReportExceptions = MothershipReport.isShowSelfReportExceptions(); + } + + public HtmlString getSiteSettingsHelpLink(String fragment) + { + return new HelpTopic("configAdmin", fragment).getSimpleLinkHtml("more info..."); + } + } + + public static class SetRibbonMessageForm + { + private Boolean _show = null; + private String _message = null; + + public Boolean isShow() + { + return _show; + } + + public void setShow(Boolean show) + { + _show = show; + } + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class SetRibbonMessageAction extends MutatingApiAction + { + @Override + public Object execute(SetRibbonMessageForm form, BindException errors) throws Exception + { + if (form.isShow() != null || form.getMessage() != null) + { + WriteableAppProps props = AppProps.getWriteableInstance(); + + if (form.isShow() != null) + props.setShowRibbonMessage(form.isShow()); + + if (form.getMessage() != null) + props.setRibbonMessage(form.getMessage()); + + props.save(getViewContext().getUser()); + } + + return null; + } + } + + @RequiresPermission(AdminPermission.class) + public class ConfigureSiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/configureSiteValidation.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "Configure " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + public static class SiteValidationForm + { + private List _providers; + private boolean _includeSubfolders = false; + private transient Consumer _logger = s -> { + }; // No-op by default + + public List getProviders() + { + return _providers; + } + + public void setProviders(List providers) + { + _providers = providers; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public Consumer getLogger() + { + return _logger; + } + + public void setLogger(Consumer logger) + { + _logger = logger; + } + } + + @RequiresPermission(AdminPermission.class) + public class SiteValidationAction extends SimpleViewAction + { + @Override + public ModelAndView getView(SiteValidationForm form, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/sitevalidation/siteValidation.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, (getContainer().isRoot() ? "Site" : "Folder") + " Validation", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class SiteValidationBackgroundAction extends FormHandlerAction + { + private ActionURL _redirectUrl; + + @Override + public void validateCommand(SiteValidationForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SiteValidationForm form, BindException errors) throws PipelineValidationException + { + ViewBackgroundInfo vbi = new ViewBackgroundInfo(getContainer(), getUser(), null); + PipeRoot root = PipelineService.get().findPipelineRoot(getContainer()); + SiteValidationJob job = new SiteValidationJob(vbi, root, form); + PipelineService.get().queueJob(job); + String jobGuid = job.getJobGUID(); + + if (null == jobGuid) + throw new NotFoundException("Unable to determine pipeline job GUID"); + + Long jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + if (null == jobId) + throw new NotFoundException("Unable to determine pipeline job ID"); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _redirectUrl = urls.urlDetails(getContainer(), jobId); + + return true; + } + + @Override + public URLHelper getSuccessURL(SiteValidationForm form) + { + return _redirectUrl; + } + } + + public static class ViewValidationResultsForm + { + private int _rowId; + + public int getRowId() + { + return _rowId; + } + + public void setRowId(int rowId) + { + _rowId = rowId; + } + } + + @RequiresPermission(AdminPermission.class) + public class ViewValidationResultsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ViewValidationResultsForm form, BindException errors) throws Exception + { + PipelineStatusFile statusFile = PipelineService.get().getStatusFile(form.getRowId()); + if (null == statusFile) + throw new NotFoundException("Status file not found"); + if (!getContainer().equals(statusFile.lookupContainer())) + throw new UnauthorizedException("Wrong container"); + + String logFilePath = statusFile.getFilePath(); + String htmlFilePath = FileUtil.getBaseName(logFilePath) + ".html"; + File htmlFile = new File(htmlFilePath); + + if (!htmlFile.exists()) + throw new NotFoundException("Results file not found"); + return new HtmlView(HtmlString.unsafe(PageFlowUtil.getFileContentsAsString(htmlFile))); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("siteValidation"); + addAdminNavTrail(root, "View " + (getContainer().isRoot() ? "Site" : "Folder") + " Validation Results", getClass()); + } + } + + public interface FileManagementForm + { + String getFolderRootPath(); + + void setFolderRootPath(String folderRootPath); + + String getFileRootOption(); + + void setFileRootOption(String fileRootOption); + + String getConfirmMessage(); + + void setConfirmMessage(String confirmMessage); + + boolean isDisableFileSharing(); + + boolean hasSiteDefaultRoot(); + + String[] getEnabledCloudStore(); + + @SuppressWarnings("unused") + void setEnabledCloudStore(String[] enabledCloudStore); + + boolean isCloudFileRoot(); + + @Nullable + String getCloudRootName(); + + void setCloudRootName(String cloudRootName); + + void setFileRootChanged(boolean changed); + + void setEnabledCloudStoresChanged(boolean changed); + + String getMigrateFilesOption(); + + void setMigrateFilesOption(String migrateFilesOption); + + default boolean isFolderSetup() + { + return false; + } + } + + public enum MigrateFilesOption implements SafeToRenderEnum + { + leave + { + @Override + public String description() + { + return "Source files not copied or moved"; + } + }, + copy + { + @Override + public String description() + { + return "Copy source files to destination"; + } + }, + move + { + @Override + public String description() + { + return "Move source files to destination"; + } + }; + + public abstract String description(); + } + + public static class ProjectSettingsForm extends FolderSettingsForm + { + // Site-only properties + private String _dateParsingMode; + private String _customWelcome; + + // Site & project properties + private boolean _shouldInherit; // new subfolders should inherit parent permissions + private String _systemDescription; + private boolean _systemDescriptionInherited; + private String _systemShortName; + private boolean _systemShortNameInherited; + private String _themeName; + private boolean _themeNameInherited; + private String _folderDisplayMode; + private boolean _folderDisplayModeInherited; + private String _applicationMenuDisplayMode; + private boolean _applicationMenuDisplayModeInherited; + private boolean _helpMenuEnabled; + private boolean _helpMenuEnabledInherited; + private String _logoHref; + private boolean _logoHrefInherited; + private String _companyName; + private boolean _companyNameInherited; + private String _systemEmailAddress; + private boolean _systemEmailAddressInherited; + private String _reportAProblemPath; + private boolean _reportAProblemPathInherited; + private String _supportEmail; + private boolean _supportEmailInherited; + private String _customLogin; + private boolean _customLoginInherited; + + // Site-only properties + + public String getDateParsingMode() + { + return _dateParsingMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setDateParsingMode(String dateParsingMode) + { + _dateParsingMode = dateParsingMode; + } + + public String getCustomWelcome() + { + return _customWelcome; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomWelcome(String customWelcome) + { + _customWelcome = customWelcome; + } + + // Site & project properties + + public boolean getShouldInherit() + { + return _shouldInherit; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setShouldInherit(boolean b) + { + _shouldInherit = b; + } + + public String getSystemDescription() + { + return _systemDescription; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescription(String systemDescription) + { + _systemDescription = systemDescription; + } + + public boolean isSystemDescriptionInherited() + { + return _systemDescriptionInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemDescriptionInherited(boolean systemDescriptionInherited) + { + _systemDescriptionInherited = systemDescriptionInherited; + } + + public String getSystemShortName() + { + return _systemShortName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortName(String systemShortName) + { + _systemShortName = systemShortName; + } + + public boolean isSystemShortNameInherited() + { + return _systemShortNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemShortNameInherited(boolean systemShortNameInherited) + { + _systemShortNameInherited = systemShortNameInherited; + } + + public String getThemeName() + { + return _themeName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeName(String themeName) + { + _themeName = themeName; + } + + public boolean isThemeNameInherited() + { + return _themeNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setThemeNameInherited(boolean themeNameInherited) + { + _themeNameInherited = themeNameInherited; + } + + public String getFolderDisplayMode() + { + return _folderDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayMode(String folderDisplayMode) + { + _folderDisplayMode = folderDisplayMode; + } + + public boolean isFolderDisplayModeInherited() + { + return _folderDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setFolderDisplayModeInherited(boolean folderDisplayModeInherited) + { + _folderDisplayModeInherited = folderDisplayModeInherited; + } + + public String getApplicationMenuDisplayMode() + { + return _applicationMenuDisplayMode; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayMode(String displayMode) + { + _applicationMenuDisplayMode = displayMode; + } + + public boolean isApplicationMenuDisplayModeInherited() + { + return _applicationMenuDisplayModeInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setApplicationMenuDisplayModeInherited(boolean applicationMenuDisplayModeInherited) + { + _applicationMenuDisplayModeInherited = applicationMenuDisplayModeInherited; + } + + public boolean isHelpMenuEnabled() + { + return _helpMenuEnabled; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabled(boolean helpMenuEnabled) + { + _helpMenuEnabled = helpMenuEnabled; + } + + public boolean isHelpMenuEnabledInherited() + { + return _helpMenuEnabledInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setHelpMenuEnabledInherited(boolean helpMenuEnabledInherited) + { + _helpMenuEnabledInherited = helpMenuEnabledInherited; + } + + public String getLogoHref() + { + return _logoHref; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHref(String logoHref) + { + _logoHref = logoHref; + } + + public boolean isLogoHrefInherited() + { + return _logoHrefInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setLogoHrefInherited(boolean logoHrefInherited) + { + _logoHrefInherited = logoHrefInherited; + } + + public String getReportAProblemPath() + { + return _reportAProblemPath; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPath(String reportAProblemPath) + { + _reportAProblemPath = reportAProblemPath; + } + + public boolean isReportAProblemPathInherited() + { + return _reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setReportAProblemPathInherited(boolean reportAProblemPathInherited) + { + _reportAProblemPathInherited = reportAProblemPathInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmail(String supportEmail) + { + _supportEmail = supportEmail; + } + + public String getSupportEmail() + { + return _supportEmail; + } + + public boolean isSupportEmailInherited() + { + return _supportEmailInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSupportEmailInherited(boolean supportEmailInherited) + { + _supportEmailInherited = supportEmailInherited; + } + + public String getSystemEmailAddress() + { + return _systemEmailAddress; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddress(String systemEmailAddress) + { + _systemEmailAddress = systemEmailAddress; + } + + public boolean isSystemEmailAddressInherited() + { + return _systemEmailAddressInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSystemEmailAddressInherited(boolean systemEmailAddressInherited) + { + _systemEmailAddressInherited = systemEmailAddressInherited; + } + + public String getCompanyName() + { + return _companyName; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyName(String companyName) + { + _companyName = companyName; + } + + public boolean isCompanyNameInherited() + { + return _companyNameInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCompanyNameInherited(boolean companyNameInherited) + { + _companyNameInherited = companyNameInherited; + } + + public String getCustomLogin() + { + return _customLogin; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLogin(String customLogin) + { + _customLogin = customLogin; + } + + public boolean isCustomLoginInherited() + { + return _customLoginInherited; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setCustomLoginInherited(boolean customLoginInherited) + { + _customLoginInherited = customLoginInherited; + } + } + + public enum FileRootProp implements SafeToRenderEnum + { + disable, + siteDefault, + folderOverride, + cloudRoot + } + + public static class FilesForm extends SetupForm implements FileManagementForm + { + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _cloudRootName; + private String _migrateFilesOption; + private String[] _enabledCloudStore; + private String _fileRootOption; + private String _folderRootPath; + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + } + + @SuppressWarnings("unused") + public static class SiteSettingsForm + { + private boolean _upgradeInProgress = false; + + private String _pipelineToolsDirectory; + private boolean _sslRequired; + private boolean _adminOnlyMode; + private boolean _showRibbonMessage; + private boolean _ext3Required; + private boolean _ext3APIRequired; + private boolean _selfReportExceptions; + private String _adminOnlyMessage; + private String _ribbonMessage; + private int _sslPort; + private int _memoryUsageDumpInterval; + private int _readOnlyHttpRequestTimeout; + private int _maxBLOBSize; + private String _exceptionReportingLevel; + private String _usageReportingLevel; + private String _administratorContactEmail; + + private String _baseServerURL; + private String _callbackPassword; + private boolean _allowApiKeys; + private int _apiKeyExpirationSeconds; + private boolean _allowSessionKeys; + private boolean _navAccessOpen; + + private String _XFrameOption; + private boolean _includeServerHttpHeader; + + public String getPipelineToolsDirectory() + { + return _pipelineToolsDirectory; + } + + public void setPipelineToolsDirectory(String pipelineToolsDirectory) + { + _pipelineToolsDirectory = pipelineToolsDirectory; + } + + public boolean isNavAccessOpen() + { + return _navAccessOpen; + } + + public void setNavAccessOpen(boolean navAccessOpen) + { + _navAccessOpen = navAccessOpen; + } + + public boolean isSslRequired() + { + return _sslRequired; + } + + public void setSslRequired(boolean sslRequired) + { + _sslRequired = sslRequired; + } + + public boolean isExt3Required() + { + return _ext3Required; + } + + public void setExt3Required(boolean ext3Required) + { + _ext3Required = ext3Required; + } + + public boolean isExt3APIRequired() + { + return _ext3APIRequired; + } + + public void setExt3APIRequired(boolean ext3APIRequired) + { + _ext3APIRequired = ext3APIRequired; + } + + public int getSslPort() + { + return _sslPort; + } + + public void setSslPort(int sslPort) + { + _sslPort = sslPort; + } + + public boolean isAdminOnlyMode() + { + return _adminOnlyMode; + } + + public void setAdminOnlyMode(boolean adminOnlyMode) + { + _adminOnlyMode = adminOnlyMode; + } + + public String getAdminOnlyMessage() + { + return _adminOnlyMessage; + } + + public void setAdminOnlyMessage(String adminOnlyMessage) + { + _adminOnlyMessage = adminOnlyMessage; + } + + public boolean isSelfReportExceptions() + { + return _selfReportExceptions; + } + + public void setSelfReportExceptions(boolean selfReportExceptions) + { + _selfReportExceptions = selfReportExceptions; + } + + public String getExceptionReportingLevel() + { + return _exceptionReportingLevel; + } + + public void setExceptionReportingLevel(String exceptionReportingLevel) + { + _exceptionReportingLevel = exceptionReportingLevel; + } + + public String getUsageReportingLevel() + { + return _usageReportingLevel; + } + + public void setUsageReportingLevel(String usageReportingLevel) + { + _usageReportingLevel = usageReportingLevel; + } + + public String getAdministratorContactEmail() + { + return _administratorContactEmail; + } + + public void setAdministratorContactEmail(String administratorContactEmail) + { + _administratorContactEmail = administratorContactEmail; + } + + public boolean isUpgradeInProgress() + { + return _upgradeInProgress; + } + + public void setUpgradeInProgress(boolean upgradeInProgress) + { + _upgradeInProgress = upgradeInProgress; + } + + public int getMemoryUsageDumpInterval() + { + return _memoryUsageDumpInterval; + } + + public void setMemoryUsageDumpInterval(int memoryUsageDumpInterval) + { + _memoryUsageDumpInterval = memoryUsageDumpInterval; + } + + public int getReadOnlyHttpRequestTimeout() + { + return _readOnlyHttpRequestTimeout; + } + + public void setReadOnlyHttpRequestTimeout(int timeout) + { + _readOnlyHttpRequestTimeout = timeout; + } + + public int getMaxBLOBSize() + { + return _maxBLOBSize; + } + + public void setMaxBLOBSize(int maxBLOBSize) + { + _maxBLOBSize = maxBLOBSize; + } + + public String getBaseServerURL() + { + return _baseServerURL; + } + + public void setBaseServerURL(String baseServerURL) + { + _baseServerURL = baseServerURL; + } + + public String getCallbackPassword() + { + return _callbackPassword; + } + + public void setCallbackPassword(String callbackPassword) + { + _callbackPassword = callbackPassword; + } + + public boolean isShowRibbonMessage() + { + return _showRibbonMessage; + } + + public void setShowRibbonMessage(boolean showRibbonMessage) + { + _showRibbonMessage = showRibbonMessage; + } + + public String getRibbonMessage() + { + return _ribbonMessage; + } + + public void setRibbonMessage(String ribbonMessage) + { + _ribbonMessage = ribbonMessage; + } + + public boolean isAllowApiKeys() + { + return _allowApiKeys; + } + + public void setAllowApiKeys(boolean allowApiKeys) + { + _allowApiKeys = allowApiKeys; + } + + public int getApiKeyExpirationSeconds() + { + return _apiKeyExpirationSeconds; + } + + public void setApiKeyExpirationSeconds(int apiKeyExpirationSeconds) + { + _apiKeyExpirationSeconds = apiKeyExpirationSeconds; + } + + public boolean isAllowSessionKeys() + { + return _allowSessionKeys; + } + + public void setAllowSessionKeys(boolean allowSessionKeys) + { + _allowSessionKeys = allowSessionKeys; + } + + public String getXFrameOption() + { + return _XFrameOption; + } + + public void setXFrameOption(String XFrameOption) + { + _XFrameOption = XFrameOption; + } + + public boolean isIncludeServerHttpHeader() + { + return _includeServerHttpHeader; + } + + public void setIncludeServerHttpHeader(boolean includeServerHttpHeader) + { + _includeServerHttpHeader = includeServerHttpHeader; + } + } + + + @AdminConsoleAction + public class ShowThreadsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Log to labkey.log as well as showing through the browser + DebugInfoDumper.dumpThreads(3); + return new JspView<>("/org/labkey/core/admin/threads.jsp", new ThreadsBean()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpDebugging#threads"); + addAdminNavTrail(root, "Current Threads", this.getClass()); + } + } + + private abstract class AbstractPostgresAction extends QueryViewAction + { + private final String _queryName; + + protected AbstractPostgresAction(String queryName) + { + super(QueryExportForm.class); + _queryName = queryName; + } + + @Override + protected QueryView createQueryView(QueryExportForm form, BindException errors, boolean forExport, @Nullable String dataRegion) throws Exception + { + if (!CoreSchema.getInstance().getSqlDialect().isPostgreSQL()) + { + throw new NotFoundException("Only available with Postgres as the primary database"); + } + + QuerySettings qSettings = new QuerySettings(getViewContext(), "query", _queryName); + QueryView result = new QueryView(new PostgresUserSchema(getUser(), getContainer()), qSettings, errors) + { + @Override + public DataView createDataView() + { + // Troubleshooters don't have normal read access to the root container so grant them special access + // for these queries + DataView view = super.createDataView(); + view.getRenderContext().getViewContext().addContextualRole(ReaderRole.class); + return view; + } + }; + result.setTitle(_queryName); + result.setFrame(WebPartView.FrameType.PORTAL); + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("postgresActivity"); + addAdminNavTrail(root, "Postgres " + _queryName, this.getClass()); + } + + } + + @AdminConsoleAction + public class PostgresStatActivityAction extends AbstractPostgresAction + { + public PostgresStatActivityAction() + { + super(PostgresUserSchema.POSTGRES_STAT_ACTIVITY_TABLE_NAME); + } + } + + @AdminConsoleAction + public class PostgresLocksAction extends AbstractPostgresAction + { + public PostgresLocksAction() + { + super(PostgresUserSchema.POSTGRES_LOCKS_TABLE_NAME); + } + } + + @AdminConsoleAction + public class PostgresTableSizesAction extends AbstractPostgresAction + { + public PostgresTableSizesAction() + { + super(PostgresUserSchema.POSTGRES_TABLE_SIZES_TABLE_NAME); + } + } + + @AdminConsoleAction + public class DumpHeapAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + File destination = DebugInfoDumper.dumpHeap(); + return new HtmlView(HtmlString.of("Heap dumped to " + destination.getAbsolutePath())); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("dumpHeap"); + addAdminNavTrail(root, "Heap dump", getClass()); + } + } + + + public static class ThreadsBean + { + public Map> spids; + public List threads; + public Map stackTraces; + + ThreadsBean() + { + stackTraces = Thread.getAllStackTraces(); + threads = new ArrayList<>(stackTraces.keySet()); + threads.sort(Comparator.comparing(Thread::getName, String.CASE_INSENSITIVE_ORDER)); + + spids = new HashMap<>(); + + for (Thread t : threads) + { + spids.put(t, ConnectionWrapper.getSPIDsForThread(t)); + } + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class ShowNetworkDriveTestAction extends SimpleViewAction + { + @Override + public void validate(NetworkDriveForm form, BindException errors) + { + validateNetworkDrive(form, errors); + } + + @Override + public ModelAndView getView(NetworkDriveForm form, BindException errors) + { + NetworkDrive testDrive = new NetworkDrive(); + testDrive.setPassword(form.getNetworkDrivePassword()); + testDrive.setPath(form.getNetworkDrivePath()); + testDrive.setUser(form.getNetworkDriveUser()); + TestNetworkDriveBean bean = new TestNetworkDriveBean(); + + if (!errors.hasErrors()) + { + char driveLetter = form.getNetworkDriveLetter().trim().charAt(0); + try + { + String mountError = testDrive.mount(driveLetter); + if (mountError != null) + { + errors.reject(ERROR_MSG, mountError); + } + else + { + File f = new File(driveLetter + ":\\"); + if (!f.exists()) + { + errors.reject(ERROR_MSG, "Could not access network drive"); + } + else + { + String[] fileNames = f.list(); + if (fileNames == null) + fileNames = new String[0]; + Arrays.sort(fileNames); + bean.setFiles(fileNames); + } + } + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + try + { + testDrive.unmount(driveLetter); + } + catch (IOException | InterruptedException e) + { + errors.reject(ERROR_MSG, "Error mounting drive: " + e); + } + } + + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/testNetworkDrive.jsp", bean, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Test Mapping Network Drive"); + } + } + + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetErrorMarkAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + return HtmlView.of("Are you sure you want to reset the site errors?"); + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + File errorLogFile = getErrorLogFile(); + _errorMark = errorLogFile.length(); + + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull URLHelper getSuccessURL(Object o) + { + return getShowAdminURL(); + } + } + + abstract public static class ShowLogAction extends ExportAction + { + @Override + public final void export(Object o, HttpServletResponse response, BindException errors) throws IOException + { + getPageConfig().setNoIndex(); + export(response); + } + + protected abstract void export(HttpServletResponse response) throws IOException; + } + + @AdminConsoleAction + public class ShowErrorsSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _errorMark, getErrorLogFile()); + } + } + + @AdminConsoleAction + public class ShowAllErrorsAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getErrorLogFile()); + } + } + + @AdminConsoleAction(ApplicationAdminPermission.class) + public class ResetPrimaryLogMarkAction extends MutatingApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + File logFile = getPrimaryLogFile(); + _primaryLogMark = logFile.length(); + return null; + } + } + + @AdminConsoleAction + public class ShowPrimaryLogSinceMarkAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, _primaryLogMark, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowPrimaryLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getPrimaryLogFile()); + } + } + + @AdminConsoleAction + public class ShowCspReportLogAction extends ShowLogAction + { + @Override + protected void export(HttpServletResponse response) throws IOException + { + PageFlowUtil.streamLogFile(response, 0, getCspReportLogFile()); + } + } + + private File getErrorLogFile() + { + return new File(getLabKeyLogDir(), "labkey-errors.log"); + } + + private File getPrimaryLogFile() + { + return new File(getLabKeyLogDir(), "labkey.log"); + } + + private File getCspReportLogFile() + { + return new File(getLabKeyLogDir(), "csp-report.log"); + } + + private static ActionURL getActionsURL() + { + return new ActionURL(ActionsAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public class ActionsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new ActionsTabStrip(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("actionsDiagnostics"); + addAdminNavTrail(root, "Actions", this.getClass()); + } + } + + private static class ActionsTabStrip extends TabStripView + { + @Override + public List getTabList() + { + List tabs = new ArrayList<>(3); + + tabs.add(new TabInfo("Summary", "summary", getActionsURL())); + tabs.add(new TabInfo("Details", "details", getActionsURL())); + tabs.add(new TabInfo("Exceptions", "exceptions", getActionsURL())); + + return tabs; + } + + @Override + public HttpView getTabView(String tabId) + { + if ("exceptions".equals(tabId)) + return new ActionsExceptionsView(); + return new ActionsView(!"details".equals(tabId)); + } + } + + @AdminConsoleAction + public static class ExportActionsAction extends ExportAction + { + @Override + public void export(Object form, HttpServletResponse response, BindException errors) throws Exception + { + try (ActionsTsvWriter writer = new ActionsTsvWriter()) + { + writer.write(response); + } + } + } + + private static ActionURL getQueriesURL(@Nullable String statName) + { + ActionURL url = new ActionURL(QueriesAction.class, ContainerManager.getRoot()); + + if (null != statName) + url.addParameter("stat", statName); + + return url; + } + + + @AdminConsoleAction + public class QueriesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueriesForm form, BindException errors) + { + String buttonHTML = ""; + if (getUser().hasRootAdminPermission()) + buttonHTML += PageFlowUtil.button("Reset All Statistics").href(getResetQueryStatisticsURL()).usePost() + " "; + buttonHTML += PageFlowUtil.button("Export").href(getExportQueriesURL()) + "

    "; + + return QueryProfiler.getInstance().getReportView(form.getStat(), buttonHTML, AdminController::getQueriesURL, + AdminController::getQueryStackTracesURL); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("queryPerf"); + addAdminNavTrail(root, "Queries", this.getClass()); + } + } + + public static class QueriesForm + { + private String _stat = "Count"; + + public String getStat() + { + return _stat; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setStat(String stat) + { + _stat = stat; + } + } + + + private static ActionURL getQueryStackTracesURL(String sqlHash) + { + ActionURL url = new ActionURL(QueryStackTracesAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class QueryStackTracesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return QueryProfiler.getInstance().getStackTraceView(form.getSqlHash(), AdminController::getExecutionPlanURL); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces"); + } + } + + + private static ActionURL getExecutionPlanURL(String sqlHash) + { + ActionURL url = new ActionURL(ExecutionPlanAction.class, ContainerManager.getRoot()); + url.addParameter("sqlHash", sqlHash); + return url; + } + + + @AdminConsoleAction + public class ExecutionPlanAction extends SimpleViewAction + { + private String _sqlHash; + private ExecutionPlanType _type; + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + _sqlHash = form.getSqlHash(); + _type = EnumUtils.getEnum(ExecutionPlanType.class, form.getType()); + if (null == _type) + throw new NotFoundException("Unknown execution plan type"); + + return QueryProfiler.getInstance().getExecutionPlanView(form.getSqlHash(), _type, form.isLog()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Queries", QueriesAction.class); + root.addChild("Query Stack Traces", getQueryStackTracesURL(_sqlHash)); + root.addChild(_type.getDescription()); + } + } + + + public static class QueryForm + { + private String _sqlHash; + private String _type = "Estimated"; // All dialects support Estimated + private boolean _log = false; + + public String getSqlHash() + { + return _sqlHash; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setSqlHash(String sqlHash) + { + _sqlHash = sqlHash; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setType(String type) + { + _type = type; + } + + public boolean isLog() + { + return _log; + } + + public void setLog(boolean log) + { + _log = log; + } + } + + + private ActionURL getExportQueriesURL() + { + return new ActionURL(ExportQueriesAction.class, ContainerManager.getRoot()); + } + + + @AdminConsoleAction + public static class ExportQueriesAction extends ExportAction + { + @Override + public void export(Object o, HttpServletResponse response, BindException errors) throws Exception + { + try (QueryStatTsvWriter writer = new QueryStatTsvWriter()) + { + writer.setFilenamePrefix("SQL_Queries"); + writer.write(response); + } + } + } + + private static ActionURL getResetQueryStatisticsURL() + { + return new ActionURL(ResetQueryStatisticsAction.class, ContainerManager.getRoot()); + } + + + @RequiresPermission(AdminPermission.class) + public static class ResetQueryStatisticsAction extends FormHandlerAction + { + @Override + public void validateCommand(QueriesForm target, Errors errors) + { + } + + @Override + public boolean handlePost(QueriesForm form, BindException errors) throws Exception + { + QueryProfiler.getInstance().resetAllStatistics(); + return true; + } + + @Override + public URLHelper getSuccessURL(QueriesForm form) + { + return getQueriesURL(form.getStat()); + } + } + + + @AdminConsoleAction + public class CachesAction extends SimpleViewAction + { + private final DecimalFormat commaf0 = new DecimalFormat("#,##0"); + private final DecimalFormat percent = new DecimalFormat("0%"); + + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + if (form.isClearCaches()) + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("clearCaches"); + throw new RedirectException(redirect); + } + + List> caches = CacheManager.getKnownCaches(); + + if (form.getDebugName() != null) + { + for (TrackingCache cache : caches) + { + if (form.getDebugName().equals(cache.getDebugName())) + { + LOG.info("Purging cache: " + cache.getDebugName()); + cache.clear(); + } + } + ActionURL redirect = getViewContext().cloneActionURL().deleteParameter("debugName"); + throw new RedirectException(redirect); + } + + List cacheStats = new ArrayList<>(); + List transactionStats = new ArrayList<>(); + + for (TrackingCache cache : caches) + { + cacheStats.add(CacheManager.getCacheStats(cache)); + transactionStats.add(CacheManager.getTransactionCacheStats(cache)); + } + + HtmlStringBuilder html = HtmlStringBuilder.of(); + + html.append(LinkBuilder.labkeyLink("Clear Caches and Refresh", getCachesURL(true, false))); + html.append(LinkBuilder.labkeyLink("Refresh", getCachesURL(false, false))); + + html.unsafeAppend("

    \n"); + appendStats(html, "Caches", cacheStats, false); + + html.unsafeAppend("

    \n"); + appendStats(html, "Transaction Caches", transactionStats, true); + + return new HtmlView(html); + } + + private void appendStats(HtmlStringBuilder html, String title, List allStats, boolean skipUnusedCaches) + { + List stats = skipUnusedCaches ? + allStats.stream() + .filter(stat->stat.getMaxSize() > 0) + .collect(Collectors.toCollection((Supplier>) ArrayList::new)) : + allStats; + + Collections.sort(stats); + + html.unsafeAppend("

    "); + html.append(title); + html.append(" (").append(stats.size()).unsafeAppend(")

    \n"); + + html.unsafeAppend("\n"); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + html.unsafeAppend(""); + + long size = 0; + long gets = 0; + long misses = 0; + long puts = 0; + long expirations = 0; + long evictions = 0; + long removes = 0; + long clears = 0; + int rowCount = 0; + + for (CacheStats stat : stats) + { + size += stat.getSize(); + gets += stat.getGets(); + misses += stat.getMisses(); + puts += stat.getPuts(); + expirations += stat.getExpirations(); + evictions += stat.getEvictions(); + removes += stat.getRemoves(); + clears += stat.getClears(); + + html.unsafeAppend(""); + + appendDescription(html, stat.getDescription(), stat.getCreationStackTrace()); + + Long limit = stat.getLimit(); + long maxSize = stat.getMaxSize(); + + appendLongs(html, limit, maxSize, stat.getSize(), stat.getGets(), stat.getMisses(), stat.getPuts(), stat.getExpirations(), stat.getEvictions(), stat.getRemoves(), stat.getClears()); + appendDoubles(html, stat.getMissRatio()); + + html.unsafeAppend("\n"); + + if (null != limit && maxSize >= limit) + html.unsafeAppend(""); + + html.unsafeAppend("\n"); + rowCount++; + } + + double ratio = 0 != gets ? misses / (double)gets : 0; + html.unsafeAppend(""); + + appendLongs(html, null, null, size, gets, misses, puts, expirations, evictions, removes, clears); + appendDoubles(html, ratio); + + html.unsafeAppend("\n"); + html.unsafeAppend("
    Debug NameLimitMax SizeCurrent SizeGetsMissesPutsExpirationsEvictionsRemovesClearsMiss PercentageClear
    ").append(LinkBuilder.labkeyLink("Clear", getCacheURL(stat.getDescription()))).unsafeAppend("This cache has been limited
    Total
    \n"); + } + + private static final List PREFIXES_TO_SKIP = List.of( + "java.base/java.lang.Thread.getStackTrace", + "org.labkey.api.cache.CacheManager", + "org.labkey.api.cache.Throttle", + "org.labkey.api.data.DatabaseCache", + "org.labkey.api.module.ModuleResourceCache" + ); + + private void appendDescription(HtmlStringBuilder html, String description, @Nullable StackTraceElement[] creationStackTrace) + { + StringBuilder sb = new StringBuilder(); + + if (creationStackTrace != null) + { + boolean trimming = true; + for (StackTraceElement element : creationStackTrace) + { + // Skip the first few uninteresting stack trace elements to highlight the caller we care about + if (trimming) + { + if (PREFIXES_TO_SKIP.stream().anyMatch(prefix->element.toString().startsWith(prefix))) + continue; + + trimming = false; + } + sb.append(element); + sb.append("\n"); + } + } + + if (!sb.isEmpty()) + { + String message = PageFlowUtil.jsString(sb); + String id = "id" + UniqueID.getServerSessionScopedUID(); + html.append(DOM.createHtmlFragment(TD(A(at(href, "#").id(id), description)))); + HttpView.currentPageConfig().addHandler(id, "click", "alert(" + message + ");return false;"); + } + } + + private void appendLongs(HtmlStringBuilder html, Long... stats) + { + for (Long stat : stats) + { + if (null == stat) + html.unsafeAppend(" "); + else + html.unsafeAppend("").append(commaf0.format(stat)).unsafeAppend(""); + } + } + + private void appendDoubles(HtmlStringBuilder html, double... stats) + { + for (double stat : stats) + html.unsafeAppend("").append(percent.format(stat)).unsafeAppend(""); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("cachesDiagnostics"); + addAdminNavTrail(root, "Cache Statistics", this.getClass()); + } + } + + @RequiresSiteAdmin + public class EnvironmentVariablesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/properties.jsp", System.getenv()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Environment Variables", this.getClass()); + } + } + + @RequiresSiteAdmin + public class SystemPropertiesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView>("/org/labkey/core/admin/properties.jsp", new HashMap(System.getProperties())); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "System Properties", this.getClass()); + } + } + + + public static class ConfigureSystemMaintenanceForm + { + private String _maintenanceTime; + private Set _enable = Collections.emptySet(); + private boolean _enableSystemMaintenance = true; + + public String getMaintenanceTime() + { + return _maintenanceTime; + } + + @SuppressWarnings("unused") + public void setMaintenanceTime(String maintenanceTime) + { + _maintenanceTime = maintenanceTime; + } + + public Set getEnable() + { + return _enable; + } + + @SuppressWarnings("unused") + public void setEnable(Set enable) + { + _enable = enable; + } + + public boolean isEnableSystemMaintenance() + { + return _enableSystemMaintenance; + } + + @SuppressWarnings("unused") + public void setEnableSystemMaintenance(boolean enableSystemMaintenance) + { + _enableSystemMaintenance = enableSystemMaintenance; + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ConfigureSystemMaintenanceAction extends FormViewAction + { + @Override + public void validateCommand(ConfigureSystemMaintenanceForm form, Errors errors) + { + Date date = SystemMaintenance.parseSystemMaintenanceTime(form.getMaintenanceTime()); + + if (null == date) + errors.reject(ERROR_MSG, "Invalid format for system maintenance time"); + } + + @Override + public ModelAndView getView(ConfigureSystemMaintenanceForm form, boolean reshow, BindException errors) + { + SystemMaintenanceProperties prop = SystemMaintenance.getProperties(); + return new JspView<>("/org/labkey/core/admin/systemMaintenance.jsp", prop, errors); + } + + @Override + public boolean handlePost(ConfigureSystemMaintenanceForm form, BindException errors) + { + SystemMaintenance.setTimeDisabled(!form.isEnableSystemMaintenance()); + SystemMaintenance.setProperties(form.getEnable(), form.getMaintenanceTime()); + + return true; + } + + @Override + public URLHelper getSuccessURL(ConfigureSystemMaintenanceForm form) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Configure System Maintenance", this.getClass()); + } + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class ResetSystemMaintenanceAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + SystemMaintenance.clearProperties(); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new AdminUrlsImpl().getAdminConsoleURL(); + } + } + + public static class SystemMaintenanceForm + { + private String _taskName; + private boolean _test = false; + + public String getTaskName() + { + return _taskName; + } + + @SuppressWarnings("unused") + public void setTaskName(String taskName) + { + _taskName = taskName; + } + + public boolean isTest() + { + return _test; + } + + public void setTest(boolean test) + { + _test = test; + } + } + + @RequiresSiteAdmin + public class SystemMaintenanceAction extends FormHandlerAction + { + private Long _jobId = null; + private URLHelper _url = null; + + @Override + public void validateCommand(SystemMaintenanceForm form, Errors errors) + { + } + + @Override + public ModelAndView getSuccessView(SystemMaintenanceForm form) throws IOException + { + // Send the pipeline job details absolute URL back to the test + sendPlainText(_url.getURIString()); + + // Suppress templates, divs, etc. + getPageConfig().setTemplate(Template.None); + return new EmptyView(); + } + + @Override + public boolean handlePost(SystemMaintenanceForm form, BindException errors) + { + String jobGuid = new SystemMaintenanceJob(form.getTaskName(), getUser()).call(); + + if (null != jobGuid) + _jobId = PipelineService.get().getJobId(getUser(), getContainer(), jobGuid); + + PipelineStatusUrls urls = urlProvider(PipelineStatusUrls.class); + _url = null != _jobId ? urls.urlDetails(getContainer(), _jobId) : urls.urlBegin(getContainer()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SystemMaintenanceForm form) + { + // In the standard case, redirect to the pipeline details URL + // If the test is invoking system maintenance then return the URL instead + return form.isTest() ? null : _url; + } + } + + @AdminConsoleAction + public class AttachmentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getAdminView(getViewContext().getActionURL()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Attachments", getClass()); + } + } + + @AdminConsoleAction + public class FindAttachmentParentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return AttachmentService.get().getFindAttachmentParentsView(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Find Attachment Parents", getClass()); + } + } + + public static ActionURL getMemTrackerURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(MemTrackerAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCachesURL(boolean clearCaches, boolean gc) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + if (clearCaches) + url.addParameter(MemForm.Params.clearCaches, "1"); + + if (gc) + url.addParameter(MemForm.Params.gc, "1"); + + return url; + } + + public static ActionURL getCacheURL(String debugName) + { + ActionURL url = new ActionURL(CachesAction.class, ContainerManager.getRoot()); + + url.addParameter(MemForm.Params.debugName, debugName); + + return url; + } + + private static volatile String lastCacheMemUsed = null; + + @AdminConsoleAction + public class MemTrackerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(MemForm form, BindException errors) + { + Set objectsToIgnore = MemTracker.getInstance().beforeReport(); + + boolean gc = form.isGc(); + boolean cc = form.isClearCaches(); + + if (getUser().hasRootAdminPermission() && (gc || cc)) + { + // If both are requested then try to determine and record cache memory usage + if (gc && cc) + { + // gc once to get an accurate free memory read + long before = gc(); + clearCaches(); + // gc again now that we cleared caches + long cacheMemoryUsed = before - gc(); + + // Difference could be < 0 if JVM or other threads have performed gc, in which case we can't guesstimate cache memory usage + String cacheMemUsed = cacheMemoryUsed > 0 ? FileUtils.byteCountToDisplaySize(cacheMemoryUsed) : "Unknown"; + LOG.info("Estimate of cache memory used: " + cacheMemUsed); + lastCacheMemUsed = cacheMemUsed; + } + else if (cc) + { + clearCaches(); + } + else + { + gc(); + } + + LOG.info("Cache clearing and garbage collecting complete"); + } + + return new JspView<>("/org/labkey/core/admin/memTracker.jsp", new MemBean(getViewContext().getRequest(), objectsToIgnore)); + } + + /** @return estimated current memory usage, post-garbage collection */ + private long gc() + { + LOG.info("Garbage collecting"); + System.gc(); + // This is more reliable than relying on just free memory size, as the VM can grow/shrink the heap at will + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } + + private void clearCaches() + { + LOG.info("Clearing Introspector caches"); + Introspector.flushCaches(); + LOG.info("Purging all caches"); + CacheManager.clearAllKnownCaches(); + LOG.info("Purging SearchService queues"); + SearchService.get().purgeQueues(); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("memTracker"); + addAdminNavTrail(root, "Memory usage -- " + DateUtil.formatDateTime(getContainer()), this.getClass()); + } + } + + public static class MemForm + { + private enum Params {clearCaches, debugName, gc} + + private boolean _clearCaches = false; + private boolean _gc = false; + private String _debugName; + + public boolean isClearCaches() + { + return _clearCaches; + } + + @SuppressWarnings("unused") + public void setClearCaches(boolean clearCaches) + { + _clearCaches = clearCaches; + } + + public boolean isGc() + { + return _gc; + } + + @SuppressWarnings("unused") + public void setGc(boolean gc) + { + _gc = gc; + } + + public String getDebugName() + { + return _debugName; + } + + @SuppressWarnings("unused") + public void setDebugName(String debugName) + { + _debugName = debugName; + } + } + + public static class MemBean + { + public final List> memoryUsages = new ArrayList<>(); + public final List> systemProperties = new ArrayList<>(); + public final List references; + public final List graphNames = new ArrayList<>(); + public final List activeThreads = new LinkedList<>(); + + public boolean assertsEnabled = false; + + private MemBean(HttpServletRequest request, Set objectsToIgnore) + { + MemTracker memTracker = MemTracker.getInstance(); + List all = memTracker.getReferences(); + long threadId = Thread.currentThread().getId(); + + // Attempt to detect other threads running labkey code -- mem tracker page will warn if any are found + for (Thread thread : new ThreadsBean().threads) + { + if (thread.getId() == threadId) + continue; + + Thread.State state = thread.getState(); + + if (state == Thread.State.RUNNABLE || state == Thread.State.BLOCKED) + { + boolean labkeyThread = false; + + if (memTracker.shouldDisplay(thread)) + { + for (StackTraceElement element : thread.getStackTrace()) + { + String className = element.getClassName(); + + if (className.startsWith("org.labkey") || className.startsWith("org.fhcrc")) + { + labkeyThread = true; + break; + } + } + } + + if (labkeyThread) + { + String threadInfo = thread.getName(); + TransactionFilter.RequestTracker uri = TransactionFilter.getRequestSummary(thread); + if (null != uri) + threadInfo += "; processing URL " + uri.toLogString(); + activeThreads.add(threadInfo); + } + } + } + + // ignore recently allocated + long start = ViewServlet.getRequestStartTime(request) - 2000; + references = new ArrayList<>(all.size()); + + for (HeldReference r : all) + { + if (r.getThreadId() == threadId && r.getAllocationTime() >= start) + continue; + + if (objectsToIgnore.contains(r.getReference())) + continue; + + references.add(r); + } + + // memory: + graphNames.add("Heap"); + graphNames.add("Non Heap"); + + MemoryMXBean membean = ManagementFactory.getMemoryMXBean(); + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, HEAP_MEMORY_KEY, getUsage(membean.getHeapMemoryUsage()))); + } + + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + if (membean != null) + { + memoryUsages.add(Tuple3.of(true, "Total Non-heap Memory", getUsage(membean.getNonHeapMemoryUsage()))); + } + + for (MemoryPoolMXBean pool : pools) + { + if (pool.getType() == MemoryType.NON_HEAP) + { + memoryUsages.add(Tuple3.of(false, pool.getName() + " " + pool.getType(), getUsage(pool))); + graphNames.add(pool.getName()); + } + } + + for (BufferPoolMXBean pool : ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class)) + { + memoryUsages.add(Tuple3.of(true, "Buffer pool " + pool.getName(), new MemoryUsageSummary(pool))); + graphNames.add(pool.getName()); + } + + DecimalFormat commaf0 = new DecimalFormat("#,##0"); + + + // class loader: + ClassLoadingMXBean classbean = ManagementFactory.getClassLoadingMXBean(); + if (classbean != null) + { + systemProperties.add(new Pair<>("Loaded Class Count", commaf0.format(classbean.getLoadedClassCount()))); + systemProperties.add(new Pair<>("Unloaded Class Count", commaf0.format(classbean.getUnloadedClassCount()))); + systemProperties.add(new Pair<>("Total Loaded Class Count", commaf0.format(classbean.getTotalLoadedClassCount()))); + } + + // runtime: + RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean(); + if (runtimeBean != null) + { + systemProperties.add(new Pair<>("VM Start Time", DateUtil.formatIsoDateShortTime(new Date(runtimeBean.getStartTime())))); + long upTime = runtimeBean.getUptime(); // round to sec + upTime = upTime - (upTime % 1000); + systemProperties.add(new Pair<>("VM Uptime", DateUtil.formatDuration(upTime))); + systemProperties.add(new Pair<>("VM Version", runtimeBean.getVmVersion())); + systemProperties.add(new Pair<>("VM Classpath", runtimeBean.getClassPath())); + } + + // threads: + ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + if (threadBean != null) + { + systemProperties.add(new Pair<>("Thread Count", threadBean.getThreadCount())); + systemProperties.add(new Pair<>("Peak Thread Count", threadBean.getPeakThreadCount())); + long[] deadlockedThreads = threadBean.findMonitorDeadlockedThreads(); + systemProperties.add(new Pair<>("Deadlocked Thread Count", deadlockedThreads != null ? deadlockedThreads.length : 0)); + } + + // threads: + List gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + for (GarbageCollectorMXBean gcBean : gcBeans) + { + systemProperties.add(new Pair<>(gcBean.getName() + " GC count", gcBean.getCollectionCount())); + systemProperties.add(new Pair<>(gcBean.getName() + " GC time", DateUtil.formatDuration(gcBean.getCollectionTime()))); + } + + String cacheMem = lastCacheMemUsed; + + if (null != cacheMem) + systemProperties.add(new Pair<>("Most Recent Estimated Cache Memory Usage", cacheMem)); + + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + if (osBean != null) + { + systemProperties.add(new Pair<>("CPU count", osBean.getAvailableProcessors())); + + DecimalFormat f3 = new DecimalFormat("0.000"); + + if (osBean instanceof com.sun.management.OperatingSystemMXBean sunOsBean) + { + systemProperties.add(new Pair<>("Total OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getTotalMemorySize()))); + systemProperties.add(new Pair<>("Free OS memory", FileUtils.byteCountToDisplaySize(sunOsBean.getFreeMemorySize()))); + systemProperties.add(new Pair<>("OS CPU load", f3.format(sunOsBean.getCpuLoad()))); + systemProperties.add(new Pair<>("JVM CPU load", f3.format(sunOsBean.getProcessCpuLoad()))); + } + } + + //noinspection ConstantConditions + assert assertsEnabled = true; + } + } + + private static MemoryUsageSummary getUsage(MemoryPoolMXBean pool) + { + try + { + return getUsage(pool.getUsage()); + } + catch (IllegalArgumentException x) + { + // sometimes we get usage>committed exception with older versions of JRockit + return null; + } + } + + public static class MemoryUsageSummary + { + + public final long _init; + public final long _used; + public final long _committed; + public final long _max; + + public MemoryUsageSummary(MemoryUsage usage) + { + _init = usage.getInit(); + _used = usage.getUsed(); + _committed = usage.getCommitted(); + _max = usage.getMax(); + } + + public MemoryUsageSummary(BufferPoolMXBean pool) + { + _init = -1; + _used = pool.getMemoryUsed(); + _committed = _used; + _max = pool.getTotalCapacity(); + } + } + + private static MemoryUsageSummary getUsage(MemoryUsage usage) + { + if (null == usage) + return null; + + try + { + return new MemoryUsageSummary(usage); + } + catch (IllegalArgumentException x) + { + // sometime we get usage>committed exception with older verions of JRockit + return null; + } + } + + public static class ChartForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + } + + private static class MemoryCategory implements Comparable + { + private final String _type; + private final double _mb; + + public MemoryCategory(String type, double mb) + { + _type = type; + _mb = mb; + } + + @Override + public int compareTo(@NotNull MemoryCategory o) + { + return Double.compare(getMb(), o.getMb()); + } + + public String getType() + { + return _type; + } + + public double getMb() + { + return _mb; + } + } + + @AdminConsoleAction + public static class MemoryChartAction extends ExportAction + { + @Override + public void export(ChartForm form, HttpServletResponse response, BindException errors) throws Exception + { + MemoryUsage usage = null; + boolean showLegend = false; + String title = form.getType(); + if ("Heap".equals(form.getType())) + { + usage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + showLegend = true; + } + else if ("Non Heap".equals(form.getType())) + usage = ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage(); + else + { + List pools = ManagementFactory.getMemoryPoolMXBeans(); + for (Iterator it = pools.iterator(); it.hasNext() && usage == null;) + { + MemoryPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + usage = pool.getUsage(); + } + } + + Pair divisor = null; + + List types = new ArrayList<>(4); + + if (usage == null) + { + boolean found = false; + for (Iterator it = ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class).iterator(); it.hasNext() && !found;) + { + BufferPoolMXBean pool = it.next(); + if (form.getType().equals(pool.getName())) + { + long total = pool.getTotalCapacity(); + long used = pool.getMemoryUsed(); + + divisor = getDivisor(total); + + title = "Buffer pool " + title; + + if (total > 0 || used > 0) + { + types.add(new MemoryCategory("Used", used / divisor.first)); + types.add(new MemoryCategory("Max", total / divisor.first)); + } + found = true; + } + } + if (!found) + { + throw new NotFoundException(); + } + } + else + { + if (usage.getInit() > 0 || usage.getUsed() > 0 || usage.getCommitted() > 0 || usage.getMax() > 0) + { + divisor = getDivisor(Math.max(usage.getInit(), Math.max(usage.getUsed(), Math.max(usage.getCommitted(), usage.getMax())))); + + types.add(new MemoryCategory("Init", (double) usage.getInit() / divisor.first)); + types.add(new MemoryCategory("Used", (double) usage.getUsed() / divisor.first)); + types.add(new MemoryCategory("Committed", (double) usage.getCommitted() / divisor.first)); + types.add(new MemoryCategory("Max", (double) usage.getMax() / divisor.first)); + } + } + + if (divisor != null) + { + title += " (" + divisor.second + ")"; + } + + DefaultCategoryDataset dataset = new DefaultCategoryDataset(); + + Collections.sort(types); + + for (int i = 0; i < types.size(); i++) + { + double mbPastPrevious = i > 0 ? types.get(i).getMb() - types.get(i - 1).getMb() : types.get(i).getMb(); + dataset.addValue(mbPastPrevious, types.get(i).getType(), ""); + } + + JFreeChart chart = ChartFactory.createStackedBarChart(title, null, null, dataset, PlotOrientation.HORIZONTAL, showLegend, false, false); + chart.getTitle().setFont(new Font("SansSerif", Font.BOLD, 14)); + response.setContentType("image/png"); + + ChartUtilities.writeChartAsPNG(response.getOutputStream(), chart, showLegend ? 800 : 398, showLegend ? 100 : 70); + } + + private Pair getDivisor(long l) + { + if (l > 4096L * 1024L * 1024L) + { + return Pair.of(1024L * 1024L * 1024L, "GB"); + } + if (l > 4096L * 1024L) + { + return Pair.of(1024L * 1024L, "MB"); + } + if (l > 4096L) + { + return Pair.of(1024L, "KB"); + } + + return Pair.of(1L, "bytes"); + + } + } + + public static class MemoryStressForm + { + private int _threads = 3; + private int _arraySize = 20_000; + private int _arrayCount = 10_000; + private float _percentChurn = 0.50f; + private int _delay = 20; + private int _iterations = 500; + + public int getThreads() + { + return _threads; + } + + public void setThreads(int threads) + { + _threads = threads; + } + + public int getArraySize() + { + return _arraySize; + } + + public void setArraySize(int arraySize) + { + _arraySize = arraySize; + } + + public int getArrayCount() + { + return _arrayCount; + } + + public void setArrayCount(int arrayCount) + { + _arrayCount = arrayCount; + } + + public float getPercentChurn() + { + return _percentChurn; + } + + public void setPercentChurn(float percentChurn) + { + _percentChurn = percentChurn; + } + + public int getDelay() + { + return _delay; + } + + public void setDelay(int delay) + { + _delay = delay; + } + + public int getIterations() + { + return _iterations; + } + + public void setIterations(int iterations) + { + _iterations = iterations; + } + } + + @RequiresSiteAdmin + public class MemoryStressTestAction extends FormViewAction + { + @Override + public void validateCommand(MemoryStressForm target, Errors errors) + { + + } + + @Override + public ModelAndView getView(MemoryStressForm memoryStressForm, boolean reshow, BindException errors) throws Exception + { + return new HtmlView( + DOM.LK.FORM(at(method, "POST"), + DOM.LK.ERRORS(errors.getBindingResult()), + DOM.BR(), DOM.BR(), + "This utility action will do a lot of memory allocation to test the memory configuration of the host.", + DOM.BR(), DOM.BR(), + "It spins up threads, all of which allocate a specified number byte arrays of specified length.", + DOM.BR(), + "The threads sleep for the delay period, and then replace the specified percent of arrays with new ones.", + DOM.BR(), + "They continue for the specified number of allocations.", + DOM.BR(), + "The memory actively held is approximately (threads * array count * array length).", + DOM.BR(), + "The memory turnover is based on the churn percentage, array length, delay, and iterations.", + DOM.BR(), DOM.BR(), + DOM.TABLE( + DOM.TR(DOM.TD("Thread count"), DOM.TD(DOM.INPUT(at(name, "threads", value, memoryStressForm._threads)))), + DOM.TR(DOM.TD("Byte array count"), DOM.TD(DOM.INPUT(at(name, "arrayCount", value, memoryStressForm._arrayCount)))), + DOM.TR(DOM.TD("Byte array size"), DOM.TD(DOM.INPUT(at(name, "arraySize", value, memoryStressForm._arraySize)))), + DOM.TR(DOM.TD("Iterations"), DOM.TD(DOM.INPUT(at(name, "iterations", value, memoryStressForm._iterations)))), + DOM.TR(DOM.TD("Delay between iterations (ms)"), DOM.TD(DOM.INPUT(at(name, "delay", value, memoryStressForm._delay)))), + DOM.TR(DOM.TD("Percent churn per iteration (0.0 - 1.0)"), DOM.TD(DOM.INPUT(at(name, "percentChurn", value, memoryStressForm._percentChurn)))) + ), + new ButtonBuilder("Perform stress test").submit(true).build()) + ); + } + + @Override + public boolean handlePost(MemoryStressForm memoryStressForm, BindException errors) throws Exception + { + List threads = new ArrayList<>(); + for (int i = 0; i < memoryStressForm._threads; i++) + { + Thread t = new Thread(() -> + { + Random r = new Random(); + byte[][] arrays = new byte[memoryStressForm._arrayCount][]; + // Initialize the arrays + for (int a = 0; a < arrays.length; a++) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + + for (int iter = 0; iter < memoryStressForm._iterations; iter++) + { + try + { + Thread.sleep(memoryStressForm._delay); + } + catch (InterruptedException ignored) {} + + // Swap the contents based on our desired percent churn + for (int a = 0; a < arrays.length; a++) + { + if (r.nextFloat() <= memoryStressForm._percentChurn) + { + arrays[a] = new byte[memoryStressForm._arraySize]; + } + } + } + }); + t.setUncaughtExceptionHandler((t2, e) -> { + LOG.error("Stress test exception", e); + errors.reject(null, "Stress test exception: " + e); + }); + t.start(); + threads.add(t); + } + + for (Thread thread : threads) + { + thread.join(); + } + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(MemoryStressForm memoryStressForm) + { + return new ActionURL(MemTrackerAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Memory Usage", MemTrackerAction.class); + root.addChild("Memory Stress Test"); + } + } + + public static ActionURL getModuleStatusURL(URLHelper returnUrl) + { + ActionURL url = new ActionURL(ModuleStatusAction.class, ContainerManager.getRoot()); + if (returnUrl != null) + url.addReturnUrl(returnUrl); + return url; + } + + public static class ModuleStatusBean + { + public String verb; + public String verbing; + public ActionURL nextURL; + } + + @RequiresPermission(TroubleshooterPermission.class) + @AllowedDuringUpgrade + @IgnoresAllocationTracking + public static class ModuleStatusAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ReturnUrlForm form, BindException errors) + { + ModuleLoader loader = ModuleLoader.getInstance(); + VBox vbox = new VBox(); + ModuleStatusBean bean = new ModuleStatusBean(); + + if (loader.isNewInstall()) + bean.nextURL = new ActionURL(NewInstallSiteSettingsAction.class, ContainerManager.getRoot()); + else if (form.getReturnUrl() != null) + { + try + { + bean.nextURL = form.getReturnActionURL(); + } + catch (URLException x) + { + // might not be an ActionURL e.g. /labkey/_webdav/home + } + } + if (null == bean.nextURL) + bean.nextURL = new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + + if (loader.isNewInstall()) + bean.verb = "Install"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verb = "Upgrade"; + else + bean.verb = "Start"; + + if (loader.isNewInstall()) + bean.verbing = "Installing"; + else if (loader.isUpgradeRequired() || loader.isUpgradeInProgress()) + bean.verbing = "Upgrading"; + else + bean.verbing = "Starting"; + + JspView statusView = new JspView<>("/org/labkey/core/admin/moduleStatus.jsp", bean, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + + getPageConfig().setTemplate(Template.Wizard); + getPageConfig().setTitle(bean.verb + " Modules"); + setHelpTopic(ModuleLoader.getInstance().isNewInstall() ? "config" : "upgrade"); + + return vbox; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class NewInstallSiteSettingsForm extends FileSettingsForm + { + private String _notificationEmail; + private String _siteName; + + public String getNotificationEmail() + { + return _notificationEmail; + } + + public void setNotificationEmail(String notificationEmail) + { + _notificationEmail = notificationEmail; + } + + public String getSiteName() + { + return _siteName; + } + + public void setSiteName(String siteName) + { + _siteName = siteName; + } + } + + @RequiresSiteAdmin + public static class NewInstallSiteSettingsAction extends AbstractFileSiteSettingsAction + { + public NewInstallSiteSettingsAction() + { + super(NewInstallSiteSettingsForm.class); + } + + @Override + public void validateCommand(NewInstallSiteSettingsForm form, Errors errors) + { + super.validateCommand(form, errors); + + if (isBlank(form.getNotificationEmail())) + { + errors.reject(SpringActionController.ERROR_MSG, "Notification email address may not be blank."); + } + try + { + ValidEmail email = new ValidEmail(form.getNotificationEmail()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + + @Override + public boolean handlePost(NewInstallSiteSettingsForm form, BindException errors) throws Exception + { + boolean success = super.handlePost(form, errors); + if (success) + { + WriteableLookAndFeelProperties lafProps = LookAndFeelProperties.getWriteableInstance(ContainerManager.getRoot()); + try + { + lafProps.setSystemEmailAddress(new ValidEmail(form.getNotificationEmail())); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + lafProps.setSystemShortName(form.getSiteName()); + lafProps.save(); + + // Send an immediate report now that they've set up their account and defaults, and then every 24 hours after. + UsageReportingLevel.reportNow(); + + return true; + } + return false; + } + + @Override + public ModelAndView getView(NewInstallSiteSettingsForm form, boolean reshow, BindException errors) + { + if (!reshow) + { + File root = _svc.getSiteDefaultRoot(); + + if (root.exists()) + form.setRootPath(FileUtil.getAbsoluteCaseSensitiveFile(root).getAbsolutePath()); + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(ContainerManager.getRoot()); + form.setSiteName(props.getShortName()); + form.setNotificationEmail(props.getSystemEmailAddress()); + } + + JspView view = new JspView<>("/org/labkey/core/admin/newInstallSiteSettings.jsp", form, errors); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Set Defaults"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public URLHelper getSuccessURL(NewInstallSiteSettingsForm form) + { + return new ActionURL(InstallCompleteAction.class, ContainerManager.getRoot()); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresSiteAdmin + public static class InstallCompleteAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + JspView view = new JspView<>("/org/labkey/core/admin/installComplete.jsp"); + + getPageConfig().setNavTrail(getInstallUpgradeWizardSteps()); + getPageConfig().setTitle("Complete"); + getPageConfig().setTemplate(Template.Wizard); + + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static List getInstallUpgradeWizardSteps() + { + List navTrail = new ArrayList<>(); + if (ModuleLoader.getInstance().isNewInstall()) + { + navTrail.add(new NavTree("Account Setup")); + navTrail.add(new NavTree("Install Modules")); + navTrail.add(new NavTree("Set Defaults")); + } + else if (ModuleLoader.getInstance().isUpgradeRequired() || ModuleLoader.getInstance().isUpgradeInProgress()) + { + navTrail.add(new NavTree("Upgrade Modules")); + } + else + { + navTrail.add(new NavTree("Start Modules")); + } + navTrail.add(new NavTree("Complete")); + return navTrail; + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DbCheckerAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/checkDatabase.jsp", new DataCheckForm()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Check Tools", this.getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DoCheckAction extends SimpleViewAction + { + @Override + public ModelAndView getView(DataCheckForm form, BindException errors) + { + try (var ignore=SpringActionController.ignoreSqlUpdates()) + { + ActionURL currentUrl = getViewContext().cloneActionURL(); + String fixRequested = currentUrl.getParameter("_fix"); + HtmlStringBuilder contentBuilder = HtmlStringBuilder.of(HtmlString.unsafe("
    ")); + + if (null != fixRequested) + { + HtmlString sqlCheck = HtmlString.EMPTY_STRING; + if (fixRequested.equalsIgnoreCase("container")) + sqlCheck = DbSchema.checkAllContainerCols(getUser(), true); + else if (fixRequested.equalsIgnoreCase("descriptor")) + sqlCheck = OntologyManager.doProjectColumnCheck(true); + contentBuilder.append(sqlCheck); + } + else + { + LOG.info("Starting database check"); // Debugging test timeout + LOG.info("Checking container column references"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Container Column References..."); + HtmlString strTemp = DbSchema.checkAllContainerCols(getUser(), false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "container"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking PropertyDescriptor and DomainDescriptor consistency"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking PropertyDescriptor and DomainDescriptor consistency..."); + strTemp = OntologyManager.doProjectColumnCheck(false); + if (!strTemp.isEmpty()) + { + contentBuilder.append(strTemp); + currentUrl = getViewContext().cloneActionURL(); + currentUrl.addParameter("_fix", "descriptor"); + contentBuilder.unsafeAppend("

        ") + .append(" click ") + .append(LinkBuilder.simpleLink("here", currentUrl)) + .append(" to attempt recovery."); + } + + LOG.info("Checking Schema consistency with tableXML"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Schema consistency with tableXML.") + .unsafeAppend("

    "); + Set schemas = DbSchema.getAllSchemasToTest(); + + for (DbSchema schema : schemas) + { + SiteValidationResultList schemaResult = TableXmlUtils.compareXmlToMetaData(schema, form.getFull(), false, true); + List results = schemaResult.getResults(null); + if (results.isEmpty()) + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .append(": OK") + .unsafeAppend("
    "); + } + else + { + contentBuilder.unsafeAppend("") + .append(schema.getDisplayName()) + .unsafeAppend(""); + for (var r : results) + { + HtmlString item = r.getMessage().isEmpty() ? NBSP : r.getMessage(); + contentBuilder.unsafeAppend("
  • ") + .append(item) + .unsafeAppend("
    • \n"); + } + contentBuilder.unsafeAppend(""); + } + } + + LOG.info("Checking consistency of provisioned storage"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Checking Consistency of Provisioned Storage...\n"); + StorageProvisioner.ProvisioningReport pr = StorageProvisioner.get().getProvisioningReport(); + contentBuilder.append(String.format("%d domains use Storage Provisioner", pr.getProvisionedDomains().size())); + for (StorageProvisioner.ProvisioningReport.DomainReport dr : pr.getProvisionedDomains()) + { + for (String error : dr.getErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + } + for (String error : pr.getGlobalErrors()) + { + contentBuilder.unsafeAppend("
    ") + .append(error) + .unsafeAppend("
    "); + } + + LOG.info("Database check complete"); // Debugging test timeout + contentBuilder.unsafeAppend("\n

    ") + .append("Database Consistency checker complete"); + } + + contentBuilder.unsafeAppend("
    "); + + return new HtmlView(contentBuilder); + } + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Database Tools", this.getClass()); + } + } + + public static class DataCheckForm + { + private String _dbSchema = ""; + private boolean _full = false; + + public List modules = ModuleLoader.getInstance().getModules(); + public DataCheckForm(){} + + public List getModules() { return modules; } + public String getDbSchema() { return _dbSchema; } + @SuppressWarnings("unused") + public void setDbSchema(String dbSchema){ _dbSchema = dbSchema; } + public boolean getFull() { return _full; } + public void setFull(boolean full) { _full = full; } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class GetSchemaXmlDocAction extends ExportAction + { + @Override + public void export(DataCheckForm form, HttpServletResponse response, BindException errors) throws Exception + { + String fullyQualifiedSchemaName = form.getDbSchema(); + if (null == fullyQualifiedSchemaName || fullyQualifiedSchemaName.isEmpty()) + { + throw new NotFoundException("Must specify dbSchema parameter"); + } + + boolean bFull = form.getFull(); + + Pair scopeAndSchemaName = DbSchema.getDbScopeAndSchemaName(fullyQualifiedSchemaName); + TablesDocument tdoc = TableXmlUtils.createXmlDocumentFromDatabaseMetaData(scopeAndSchemaName.first, scopeAndSchemaName.second, bFull); + StringWriter sw = new StringWriter(); + + XmlOptions xOpt = new XmlOptions(); + xOpt.setSavePrettyPrint(); + xOpt.setUseDefaultNamespace(); + + tdoc.save(sw, xOpt); + + sw.flush(); + PageFlowUtil.streamFileBytes(response, fullyQualifiedSchemaName + ".xml", sw.toString().getBytes(StringUtilsLabKey.DEFAULT_CHARSET), true); + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderInformationAction extends FolderManagementViewAction + { + @Override + protected HtmlView getTabView() + { + Container c = getContainer(); + User currentUser = getUser(); + + User createdBy = UserManager.getUser(c.getCreatedBy()); + Map propValueMap = new LinkedHashMap<>(); + propValueMap.put("Path", c.getPath()); + propValueMap.put("Name", c.getName()); + propValueMap.put("Displayed Title", c.getTitle()); + propValueMap.put("EntityId", c.getId()); + propValueMap.put("RowId", c.getRowId()); + propValueMap.put("Created", DateUtil.formatDateTime(c, c.getCreated())); + propValueMap.put("Created By", (createdBy != null ? createdBy.getDisplayName(currentUser) : "<" + c.getCreatedBy() + ">")); + propValueMap.put("Folder Type", c.getFolderType().getName()); + propValueMap.put("Description", c.getDescription()); + + return new HtmlView(PageFlowUtil.getDataRegionHtmlForPropertyObjects(propValueMap)); + } + } + + public static class MissingValuesForm + { + private boolean _inheritMvIndicators; + private String[] _mvIndicators; + private String[] _mvLabels; + + public boolean isInheritMvIndicators() + { + return _inheritMvIndicators; + } + + public void setInheritMvIndicators(boolean inheritMvIndicators) + { + _inheritMvIndicators = inheritMvIndicators; + } + + public String[] getMvIndicators() + { + return _mvIndicators; + } + + public void setMvIndicators(String[] mvIndicators) + { + _mvIndicators = mvIndicators; + } + + public String[] getMvLabels() + { + return _mvLabels; + } + + public void setMvLabels(String[] mvLabels) + { + _mvLabels = mvLabels; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MissingValuesAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(MissingValuesForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/mvIndicators.jsp", form, errors); + } + + @Override + public void validateCommand(MissingValuesForm form, Errors errors) + { + } + + @Override + public boolean handlePost(MissingValuesForm form, BindException errors) + { + if (form.isInheritMvIndicators()) + { + MvUtil.inheritMvIndicators(getContainer()); + return true; + } + else + { + // Javascript should have enforced any constraints + MvUtil.assignMvIndicators(getContainer(), form.getMvIndicators(), form.getMvLabels()); + return true; + } + } + } + + @SuppressWarnings("unused") + public static class RConfigForm + { + private Integer _reportEngine; + private Integer _pipelineEngine; + private boolean _overrideDefault; + + public Integer getReportEngine() + { + return _reportEngine; + } + + public void setReportEngine(Integer reportEngine) + { + _reportEngine = reportEngine; + } + + public Integer getPipelineEngine() + { + return _pipelineEngine; + } + + public void setPipelineEngine(Integer pipelineEngine) + { + _pipelineEngine = pipelineEngine; + } + + public boolean getOverrideDefault() + { + return _overrideDefault; + } + + public void setOverrideDefault(String overrideDefault) + { + _overrideDefault = "override".equals(overrideDefault); + } + } + + @RequiresPermission(AdminPermission.class) + public static class RConfigurationAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(RConfigForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/rConfiguration.jsp", form, errors); + } + + @Override + public void validateCommand(RConfigForm form, Errors errors) + { + if (form.getOverrideDefault()) + { + if (form.getReportEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid report engine configuration"); + if (form.getPipelineEngine() == null) + errors.reject(ERROR_MSG, "Please select a valid pipeline engine configuration"); + } + } + + @Override + public URLHelper getSuccessURL(RConfigForm rConfigForm) + { + return getContainer().getStartURL(getUser()); + } + + @Override + public boolean handlePost(RConfigForm rConfigForm, BindException errors) throws Exception + { + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + if (null != mgr) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + if (rConfigForm.getOverrideDefault()) + { + ExternalScriptEngineDefinition reportEngine = mgr.getEngineDefinition(rConfigForm.getReportEngine(), ExternalScriptEngineDefinition.Type.R); + ExternalScriptEngineDefinition pipelineEngine = mgr.getEngineDefinition(rConfigForm.getPipelineEngine(), ExternalScriptEngineDefinition.Type.R); + + if (reportEngine != null) + mgr.setEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.setEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + else + { + // need to clear the current scope (if any) + ExternalScriptEngineDefinition reportEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.report, false); + ExternalScriptEngineDefinition pipelineEngine = mgr.getScopedEngine(getContainer(), "r", LabKeyScriptEngineManager.EngineContext.pipeline, false); + + if (reportEngine != null) + mgr.removeEngineScope(getContainer(), reportEngine, LabKeyScriptEngineManager.EngineContext.report); + if (pipelineEngine != null) + mgr.removeEngineScope(getContainer(), pipelineEngine, LabKeyScriptEngineManager.EngineContext.pipeline); + } + transaction.commit(); + } + return true; + } + return false; + } + } + + @SuppressWarnings("unused") + public static class ExportFolderForm + { + private String[] _types; + private int _location; + private String _format = "new"; // As of 14.3, this is the only supported format. But leave in place for the future. + private String _exportType; + private boolean _includeSubfolders; + private PHI _exportPhiLevel; // Input: max level when viewing form + private boolean _shiftDates; + private boolean _alternateIds; + private boolean _maskClinic; + + public String[] getTypes() + { + return _types; + } + + public void setTypes(String[] types) + { + _types = types; + } + + public int getLocation() + { + return _location; + } + + public void setLocation(int location) + { + _location = location; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + + public ExportType getExportType() + { + if ("study".equals(_exportType)) + return ExportType.STUDY; + else + return ExportType.ALL; + } + + public void setExportType(String exportType) + { + _exportType = exportType; + } + + public boolean isIncludeSubfolders() + { + return _includeSubfolders; + } + + public void setIncludeSubfolders(boolean includeSubfolders) + { + _includeSubfolders = includeSubfolders; + } + + public PHI getExportPhiLevel() + { + return null != _exportPhiLevel ? _exportPhiLevel : PHI.NotPHI; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + public boolean isShiftDates() + { + return _shiftDates; + } + + public void setShiftDates(boolean shiftDates) + { + _shiftDates = shiftDates; + } + + public boolean isAlternateIds() + { + return _alternateIds; + } + + public void setAlternateIds(boolean alternateIds) + { + _alternateIds = alternateIds; + } + + public boolean isMaskClinic() + { + return _maskClinic; + } + + public void setMaskClinic(boolean maskClinic) + { + _maskClinic = maskClinic; + } + } + + public enum ExportOption + { + PipelineRootAsFiles("file root as multiple files") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + else if (root.isCloudRoot()) + { + errors.reject(ERROR_MSG, "Cannot export as individual files when root is in the cloud"); + } + else + { + File exportDir = root.resolvePath(PipelineService.EXPORT_DIR); + try + { + writer.write(container, ctx, new FileSystemFile(exportDir)); + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + return null; + } + }, + + PipelineRootAsZip("file root as a single zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null || !root.isValid()) + { + throw new NotFoundException("No valid pipeline root found"); + } + Path exportDir = root.resolveToNioPath(PipelineService.EXPORT_DIR); + FileUtil.createDirectories(exportDir); + exportFolderToFile(exportDir, container, writer, ctx, errors); + return urlProvider(PipelineUrls.class).urlBrowse(container); + } + }, + DownloadAsZip("browser download as a zip file") + { + @Override + public ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception + { + try + { + // Export to a temporary file first so exceptions are displayed by the standard error page, Issue #44152 + // Same pattern as ExportListArchiveAction + Path tempDir = FileUtil.getTempDirectory().toPath(); + Path tempZipFile = exportFolderToFile(tempDir, container, writer, ctx, errors); + + // No exceptions, so stream the resulting zip file to the browser and delete it + try (OutputStream os = ZipFile.getOutputStream(response, tempZipFile.getFileName().toString())) + { + Files.copy(tempZipFile, os); + } + finally + { + Files.delete(tempZipFile); + } + } + catch (ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + return null; + } + }; + + private final String _description; + + ExportOption(String description) + { + _description = description; + } + + public String getDescription() + { + return _description; + } + + public abstract ActionURL initiateExport(Container container, BindException errors, FolderWriterImpl writer, FolderExportContext ctx, HttpServletResponse response) throws Exception; + + Path exportFolderToFile(Path exportDir, Container container, FolderWriterImpl writer, FolderExportContext ctx, BindException errors) throws Exception + { + String filename = FileUtil.makeFileNameWithTimestamp(container.getName(), "folder.zip"); + + try (ZipFile zip = new ZipFile(exportDir, filename)) + { + writer.write(container, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return exportDir.resolve(filename); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ExportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + public ModelAndView getView(ExportFolderForm exportFolderForm, boolean reshow, BindException errors) throws Exception + { + // In export-to-browser do nothing (leave the export page in place). We just exported to the response, so + // rendering a view would throw. + return reshow && !errors.hasErrors() ? null : super.getView(exportFolderForm, reshow, errors); + } + + @Override + protected HttpView getTabView(ExportFolderForm form, boolean reshow, BindException errors) + { + form.setExportType(PageFlowUtil.filter(getViewContext().getActionURL().getParameter("exportType"))); + + ComplianceFolderSettings settings = ComplianceService.get().getFolderSettings(getContainer(), User.getAdminServiceUser()); + PhiColumnBehavior columnBehavior = null==settings ? PhiColumnBehavior.show : settings.getPhiColumnBehavior(); + PHI maxAllowedPhiForExport = PhiColumnBehavior.show == columnBehavior ? PHI.Restricted : ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser()); + form.setExportPhiLevel(maxAllowedPhiForExport); + + return new JspView<>("/org/labkey/core/admin/exportFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ExportFolderForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ExportFolderForm form, BindException errors) throws Exception + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ExportOption exportOption = null; + if (form.getLocation() >= 0 && form.getLocation() < ExportOption.values().length) + { + exportOption = ExportOption.values()[form.getLocation()]; + } + if (exportOption == null) + { + throw new NotFoundException("Invalid export location: " + form.getLocation()); + } + ContainerManager.checkContainerValidity(container); + + FolderWriterImpl writer = new FolderWriterImpl(); + FolderExportContext ctx = new FolderExportContext(getUser(), container, PageFlowUtil.set(form.getTypes()), + form.getFormat(), form.isIncludeSubfolders(), form.getExportPhiLevel(), form.isShiftDates(), + form.isAlternateIds(), form.isMaskClinic(), new StaticLoggerGetter(FolderWriterImpl.LOG)); + + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, "Folder export initiated to " + exportOption.getDescription() + " " + (form.isIncludeSubfolders() ? "including" : "excluding") + " subfolders."); + AuditLogService.get().addEvent(getUser(), event); + + _successURL = exportOption.initiateExport(container, errors, writer, ctx, getViewContext().getResponse()); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(ExportFolderForm exportFolderForm) + { + return _successURL; + } + } + + public static class ImportFolderForm + { + private boolean _createSharedDatasets; + private boolean _validateQueries; + private boolean _failForUndefinedVisits; + private String _sourceTemplateFolder; + private String _sourceTemplateFolderId; + private String _origin; + + public boolean isCreateSharedDatasets() + { + return _createSharedDatasets; + } + + public void setCreateSharedDatasets(boolean createSharedDatasets) + { + _createSharedDatasets = createSharedDatasets; + } + + public boolean isValidateQueries() + { + return _validateQueries; + } + + public boolean isFailForUndefinedVisits() + { + return _failForUndefinedVisits; + } + + public void setFailForUndefinedVisits(boolean failForUndefinedVisits) + { + _failForUndefinedVisits = failForUndefinedVisits; + } + + public void setValidateQueries(boolean validateQueries) + { + _validateQueries = validateQueries; + } + + public String getSourceTemplateFolder() + { + return _sourceTemplateFolder; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolder(String sourceTemplateFolder) + { + _sourceTemplateFolder = sourceTemplateFolder; + } + + public String getSourceTemplateFolderId() + { + return _sourceTemplateFolderId; + } + + @SuppressWarnings("unused") + public void setSourceTemplateFolderId(String sourceTemplateFolderId) + { + _sourceTemplateFolderId = sourceTemplateFolderId; + } + + public String getOrigin() + { + return _origin; + } + + public void setOrigin(String origin) + { + _origin = origin; + } + + public Container getSourceTemplateFolderContainer() + { + if (null == getSourceTemplateFolderId()) + return null; + return ContainerManager.getForId(getSourceTemplateFolderId().replace(',', ' ').trim()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ImportFolderAction extends FolderManagementViewPostAction + { + private ActionURL _successURL; + + @Override + protected HttpView getTabView(ImportFolderForm form, boolean reshow, BindException errors) + { + // default the createSharedDatasets and validateQueries to true if this is not a form error reshow + if (!errors.hasErrors()) + { + form.setCreateSharedDatasets(true); + form.setValidateQueries(true); + } + + return new JspView<>("/org/labkey/core/admin/importFolder.jsp", form, errors); + } + + @Override + public void validateCommand(ImportFolderForm form, Errors errors) + { + // don't allow import into the root container + if (getContainer().isRoot()) + { + throw new NotFoundException(); + } + } + + @Override + public boolean handlePost(ImportFolderForm form, BindException errors) throws Exception + { + ViewContext context = getViewContext(); + ActionURL url = context.getActionURL(); + User user = getUser(); + Container container = getContainer(); + PipeRoot pipelineRoot; + FileLike pipelineUnzipDir; // Should be local & writable + PipelineUrls pipelineUrlProvider; + + if (form.getOrigin() == null) + { + form.setOrigin("Folder"); + } + + // make sure we have a pipeline url provider to use for the success URL redirect + pipelineUrlProvider = urlProvider(PipelineUrls.class); + if (pipelineUrlProvider == null) + { + errors.reject("folderImport", "Pipeline url provider does not exist."); + return false; + } + + // make sure that the pipeline root is valid for this container + pipelineRoot = PipelineService.get().findPipelineRoot(container); + if (!PipelineService.get().hasValidPipelineRoot(container) || pipelineRoot == null) + { + errors.reject("folderImport", "Pipeline root not set or does not exist on disk."); + return false; + } + + // make sure we are able to delete any existing unzip dir in the pipeline root + try + { + pipelineUnzipDir = pipelineRoot.deleteImportDirectory(null); + } + catch (DirectoryNotDeletedException e) + { + errors.reject("studyImport", "Import failed: Could not delete the directory \"" + PipelineService.UNZIP_DIR + "\""); + return false; + } + + FolderImportConfig fiConfig; + if (!StringUtils.isEmpty(form.getSourceTemplateFolder())) + { + fiConfig = getFolderImportConfigFromTemplateFolder(form, pipelineUnzipDir, errors); + } + else + { + fiConfig = getFolderFromZipArchive(pipelineUnzipDir, errors); + if (fiConfig == null || errors.hasErrors()) + { + return false; + } + } + + // get the folder.xml file from the unzipped import archive + FileLike archiveXml = pipelineUnzipDir.resolveChild("folder.xml"); + if (!archiveXml.exists()) + { + errors.reject("folderImport", "This archive doesn't contain a folder.xml file."); + return false; + } + + ImportOptions options = new ImportOptions(getContainer().getId(), user.getUserId()); + options.setSkipQueryValidation(!form.isValidateQueries()); + options.setCreateSharedDatasets(form.isCreateSharedDatasets()); + options.setFailForUndefinedVisits(form.isFailForUndefinedVisits()); + options.setActivity(ComplianceService.get().getCurrentActivity(getViewContext())); + + // finally, create the study or folder import pipeline job + _successURL = pipelineUrlProvider.urlBegin(container); + PipelineService.get().runFolderImportJob(container, user, url, archiveXml, fiConfig.originalFileName, pipelineRoot, options); + + return !errors.hasErrors(); + } + + private @Nullable FolderImportConfig getFolderFromZipArchive(FileLike pipelineUnzipDir, BindException errors) + { + // user chose to import from a zip file + Map map = getFileMap(); + + // make sure we have a single file selected for import + if (map.size() != 1) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // make sure the file is not empty and that it has a .zip extension + MultipartFile zipFile = map.values().iterator().next(); + String originalFilename = zipFile.getOriginalFilename(); + if (0 == zipFile.getSize() || isBlank(originalFilename) || !originalFilename.toLowerCase().endsWith(".zip")) + { + errors.reject("folderImport", "You must select a valid zip archive (folder or study)."); + return null; + } + + // copy and unzip the uploaded import archive zip file to the pipeline unzip dir + try + { + FileLike pipelineUnzipFile = pipelineUnzipDir.resolveFile(org.labkey.api.util.Path.parse(originalFilename)); + // Check that the resolved file is under the pipelineUnzipDir + if (!pipelineUnzipFile.toNioPathForRead().normalize().startsWith(pipelineUnzipDir.toNioPathForRead().normalize())) + { + errors.reject("folderImport", "Invalid file path - must be within the unzip directory"); + return null; + } + + FileUtil.createDirectories(pipelineUnzipFile.getParent()); // Non-pipeline import sometimes fails here on Windows (shrug) + FileUtil.createNewFile(pipelineUnzipFile, true); + try (OutputStream os = pipelineUnzipFile.openOutputStream()) + { + FileUtil.copyData(zipFile.getInputStream(), os); + } + ZipUtil.unzipToDirectory(pipelineUnzipFile, pipelineUnzipDir); + + return new FolderImportConfig( + false, + originalFilename, + pipelineUnzipFile, + pipelineUnzipFile + ); + } + catch (FileNotFoundException e) + { + LOG.debug("Failed to import '" + originalFilename + "'.", e); + errors.reject("folderImport", "File not found."); + return null; + } + catch (IOException e) + { + LOG.debug("Failed to import '" + originalFilename + "'.", e); + errors.reject("folderImport", "Unable to unzip folder archive."); + return null; + } + } + + private FolderImportConfig getFolderImportConfigFromTemplateFolder(final ImportFolderForm form, final FileLike pipelineUnzipDir, final BindException errors) throws Exception + { + // user choose to import from a template source folder + Container sourceContainer = form.getSourceTemplateFolderContainer(); + + // In order to support the Advanced import options to import into multiple target folders we need to zip + // the source template folder so that the zip file can be passed to the pipeline processes. + FolderExportContext ctx = new FolderExportContext(getUser(), sourceContainer, + getRegisteredFolderWritersForImplicitExport(sourceContainer), "new", false, + PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + FolderWriterImpl writer = new FolderWriterImpl(); + String zipFileName = FileUtil.makeFileNameWithTimestamp(sourceContainer.getName(), "folder.zip"); + FileLike implicitZipFile = pipelineUnzipDir.resolveChild(zipFileName); + if (!pipelineUnzipDir.isDirectory()) + pipelineUnzipDir.mkdirs(); + implicitZipFile.createFile(); + try (OutputStream out = implicitZipFile.openOutputStream(); + ZipFile zip = new ZipFile(out, false)) + { + writer.write(sourceContainer, ctx, zip); + } + catch (Container.ContainerException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + // To support the simple import option unzip the zip file to the pipeline unzip dir of the current container + ZipUtil.unzipToDirectory(implicitZipFile, pipelineUnzipDir); + + return new FolderImportConfig( + StringUtils.isNotEmpty(form.getSourceTemplateFolderId()), + implicitZipFile.getName(), + implicitZipFile, + null + ); + } + + private static class FolderImportConfig { + FileLike pipelineUnzipFile; + String originalFileName; + FileLike archiveFile; + boolean fromTemplateSourceFolder; + + public FolderImportConfig(boolean fromTemplateSourceFolder, String originalFileName, FileLike archiveFile, @Nullable FileLike pipelineUnzipFile) + { + this.originalFileName = originalFileName; + this.archiveFile = archiveFile; + this.fromTemplateSourceFolder = fromTemplateSourceFolder; + this.pipelineUnzipFile = pipelineUnzipFile; + } + } + + @Override + public URLHelper getSuccessURL(ImportFolderForm importFolderForm) + { + return _successURL; + } + } + + private Set getRegisteredFolderWritersForImplicitExport(Container sourceContainer) + { + // this method is very similar to CoreController.GetRegisteredFolderWritersAction.execute() method, but instead of + // of building up a map of Writer object names to display in the UI, we are instead adding them to the list of Writers + // to apply during the implicit export. + Set registeredFolderWriters = new HashSet<>(); + FolderSerializationRegistry registry = FolderSerializationRegistry.get(); + if (null == registry) + { + throw new RuntimeException(); + } + Collection registeredWriters = registry.getRegisteredFolderWriters(); + for (FolderWriter writer : registeredWriters) + { + String dataType = writer.getDataType(); + boolean excludeForDataspace = sourceContainer.isDataspace() && "Study".equals(dataType); + boolean excludeForTemplate = !writer.includeWithTemplate(); + + if (dataType != null && writer.show(sourceContainer) && !excludeForDataspace && !excludeForTemplate) + { + registeredFolderWriters.add(dataType); + + // for each Writer also determine if there are related children Writers, if so include them also + Collection> childWriters = writer.getChildren(true, true); + if (!childWriters.isEmpty()) + { + for (org.labkey.api.writer.Writer child : childWriters) + { + dataType = child.getDataType(); + if (dataType != null) + registeredFolderWriters.add(dataType); + } + } + } + } + return registeredFolderWriters; + } + + public static class FolderSettingsForm + { + private String _defaultDateFormat; + private boolean _defaultDateFormatInherited; + private String _defaultDateTimeFormat; + private boolean _defaultDateTimeFormatInherited; + private String _defaultTimeFormat; + private boolean _defaultTimeFormatInherited; + private String _defaultNumberFormat; + private boolean _defaultNumberFormatInherited; + private boolean _restrictedColumnsEnabled; + private boolean _restrictedColumnsEnabledInherited; + + public String getDefaultDateFormat() + { + return _defaultDateFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormat(String defaultDateFormat) + { + _defaultDateFormat = defaultDateFormat; + } + + public boolean isDefaultDateFormatInherited() + { + return _defaultDateFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateFormatInherited(boolean defaultDateFormatInherited) + { + _defaultDateFormatInherited = defaultDateFormatInherited; + } + + public String getDefaultDateTimeFormat() + { + return _defaultDateTimeFormat; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormat(String defaultDateTimeFormat) + { + _defaultDateTimeFormat = defaultDateTimeFormat; + } + + public boolean isDefaultDateTimeFormatInherited() + { + return _defaultDateTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultDateTimeFormatInherited(boolean defaultDateTimeFormatInherited) + { + _defaultDateTimeFormatInherited = defaultDateTimeFormatInherited; + } + + public String getDefaultTimeFormat() + { + return _defaultTimeFormat; + } + + @SuppressWarnings("UnusedDeclaration") + public void setDefaultTimeFormat(String defaultTimeFormat) + { + _defaultTimeFormat = defaultTimeFormat; + } + + public boolean isDefaultTimeFormatInherited() + { + return _defaultTimeFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultTimeFormatInherited(boolean defaultTimeFormatInherited) + { + _defaultTimeFormatInherited = defaultTimeFormatInherited; + } + + public String getDefaultNumberFormat() + { + return _defaultNumberFormat; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormat(String defaultNumberFormat) + { + _defaultNumberFormat = defaultNumberFormat; + } + + public boolean isDefaultNumberFormatInherited() + { + return _defaultNumberFormatInherited; + } + + @SuppressWarnings("unused") + public void setDefaultNumberFormatInherited(boolean defaultNumberFormatInherited) + { + _defaultNumberFormatInherited = defaultNumberFormatInherited; + } + + public boolean areRestrictedColumnsEnabled() + { + return _restrictedColumnsEnabled; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabled(boolean restrictedColumnsEnabled) + { + _restrictedColumnsEnabled = restrictedColumnsEnabled; + } + + public boolean isRestrictedColumnsEnabledInherited() + { + return _restrictedColumnsEnabledInherited; + } + + @SuppressWarnings("unused") + public void setRestrictedColumnsEnabledInherited(boolean restrictedColumnsEnabledInherited) + { + _restrictedColumnsEnabledInherited = restrictedColumnsEnabledInherited; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FolderSettingsAction extends FolderManagementViewPostAction + { + @Override + protected LookAndFeelView getTabView(FolderSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(FolderSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(FolderSettingsForm form, BindException errors) + { + return saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + } + } + + // Validate and populate the folder settings; save & log all changes + private static boolean saveFolderSettings(Container c, User user, WriteableFolderLookAndFeelProperties props, FolderSettingsForm form, BindException errors) + { + validateAndSaveFormat(form.getDefaultDateFormat(), form.isDefaultDateFormatInherited(), props::clearDefaultDateFormat, props::setDefaultDateFormat, errors, "date display format"); + validateAndSaveFormat(form.getDefaultDateTimeFormat(), form.isDefaultDateTimeFormatInherited(), props::clearDefaultDateTimeFormat, props::setDefaultDateTimeFormat, errors, "date-time display format"); + validateAndSaveFormat(form.getDefaultTimeFormat(), form.isDefaultTimeFormatInherited(), props::clearDefaultTimeFormat, props::setDefaultTimeFormat, errors, "time display format"); + validateAndSaveFormat(form.getDefaultNumberFormat(), form.isDefaultNumberFormatInherited(), props::clearDefaultNumberFormat, props::setDefaultNumberFormat, errors, "number display format"); + + setProperty(form.isRestrictedColumnsEnabledInherited(), props::clearRestrictedColumnsEnabled, () -> props.setRestrictedColumnsEnabled(form.areRestrictedColumnsEnabled())); + + if (!errors.hasErrors()) + { + props.save(); + + //write an audit log event + props.writeAuditLogEvent(c, user); + } + + return !errors.hasErrors(); + } + + private interface FormatSaver + { + void save(String format) throws IllegalArgumentException; + } + + private static void validateAndSaveFormat(String format, boolean inherited, Runnable clearer, FormatSaver saver, BindException errors, String what) + { + String defaultFormat = StringUtils.trimToNull(format); + if (inherited) + { + clearer.run(); + } + else + { + try + { + saver.save(defaultFormat); + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, "Invalid " + what + ": " + e.getMessage()); + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class ModulePropertiesAction extends FolderManagementViewAction + { + @Override + protected JspView getTabView() + { + return new JspView<>("/org/labkey/core/project/modulePropertiesAdmin.jsp"); + } + } + + @SuppressWarnings("unused") + public static class FolderTypeForm + { + private String[] _activeModules = new String[ModuleLoader.getInstance().getModules().size()]; + private String _defaultModule; + private String _folderType; + private boolean _wizard; + + public String[] getActiveModules() + { + return _activeModules; + } + + public void setActiveModules(String[] activeModules) + { + _activeModules = activeModules; + } + + public String getDefaultModule() + { + return _defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + _defaultModule = defaultModule; + } + + public String getFolderType() + { + return _folderType; + } + + public void setFolderType(String folderType) + { + _folderType = folderType; + } + + public boolean isWizard() + { + return _wizard; + } + + public void setWizard(boolean wizard) + { + _wizard = wizard; + } + } + + @RequiresPermission(AdminPermission.class) + @IgnoresTermsOfUse // At the moment, compliance configuration is very sensitive to active modules, so allow those adjustments + public static class FolderTypeAction extends FolderManagementViewPostAction + { + private ActionURL _successURL = null; + + @Override + protected JspView getTabView(FolderTypeForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/folderType.jsp", form, errors); + } + + @Override + public void validateCommand(FolderTypeForm form, Errors errors) + { + boolean fEmpty = true; + for (String module : form._activeModules) + { + if (module != null) + { + fEmpty = false; + break; + } + } + if (fEmpty && "None".equals(form.getFolderType())) + { + errors.reject(SpringActionController.ERROR_MSG, "Error: Please select at least one module to display."); + } + } + + @Override + public boolean handlePost(FolderTypeForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + String[] modules = form.getActiveModules(); + + if (modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + if (null == StringUtils.trimToNull(form.getFolderType()) || FolderType.NONE.getName().equals(form.getFolderType())) + { + container.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + container.setDefaultModule(defaultModule); + } + else + { + FolderType folderType = FolderTypeManager.get().getFolderType(form.getFolderType()); + if (container.isContainerTab() && folderType.hasContainerTabs()) + errors.reject(null, "You cannot set a tab folder to a folder type that also has tab folders"); + else + container.setFolderType(folderType, getUser(), errors, activeModules); + } + if (errors.hasErrors()) + return false; + + if (form.isWizard()) + { + _successURL = urlProvider(SecurityUrls.class).getContainerURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + } + else + _successURL = container.getFolderType().getStartURL(container, getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(FolderTypeForm folderTypeForm) + { + return _successURL; + } + } + + @SuppressWarnings("unused") + public static class FileRootsForm extends SetupForm implements FileManagementForm + { + private String _folderRootPath; + private String _fileRootOption; + private String _cloudRootName; + private boolean _isFolderSetup; + private boolean _fileRootChanged; + private boolean _enabledCloudStoresChanged; + private String _migrateFilesOption; + + // cloud settings + private String[] _enabledCloudStore; + //file management + @Override + public String getFolderRootPath() + { + return _folderRootPath; + } + + @Override + public void setFolderRootPath(String folderRootPath) + { + _folderRootPath = folderRootPath; + } + + @Override + public String getFileRootOption() + { + return _fileRootOption; + } + + @Override + public void setFileRootOption(String fileRootOption) + { + _fileRootOption = fileRootOption; + } + + @Override + public String[] getEnabledCloudStore() + { + return _enabledCloudStore; + } + + @Override + public void setEnabledCloudStore(String[] enabledCloudStore) + { + _enabledCloudStore = enabledCloudStore; + } + + @Override + public boolean isDisableFileSharing() + { + return FileRootProp.disable.name().equals(getFileRootOption()); + } + + @Override + public boolean hasSiteDefaultRoot() + { + return FileRootProp.siteDefault.name().equals(getFileRootOption()); + } + + @Override + public boolean isCloudFileRoot() + { + return FileRootProp.cloudRoot.name().equals(getFileRootOption()); + } + + @Override + @Nullable + public String getCloudRootName() + { + return _cloudRootName; + } + + @Override + public void setCloudRootName(String cloudRootName) + { + _cloudRootName = cloudRootName; + } + + @Override + public boolean isFolderSetup() + { + return _isFolderSetup; + } + + public void setFolderSetup(boolean folderSetup) + { + _isFolderSetup = folderSetup; + } + + public boolean isFileRootChanged() + { + return _fileRootChanged; + } + + @Override + public void setFileRootChanged(boolean changed) + { + _fileRootChanged = changed; + } + + public boolean isEnabledCloudStoresChanged() + { + return _enabledCloudStoresChanged; + } + + @Override + public void setEnabledCloudStoresChanged(boolean enabledCloudStoresChanged) + { + _enabledCloudStoresChanged = enabledCloudStoresChanged; + } + + @Override + public String getMigrateFilesOption() + { + return _migrateFilesOption; + } + + @Override + public void setMigrateFilesOption(String migrateFilesOption) + { + _migrateFilesOption = migrateFilesOption; + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsStandAloneAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + view.setFrame(WebPartView.FrameType.NONE); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(getContainer(), getContainer().getParent())); + getPageConfig().setTemplate(PageConfig.Template.Wizard); + getPageConfig().setTitle("Change File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new ActionURL(FileRootsStandAloneAction.class, getContainer()) + .addParameter("folderSetup", true) + .addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + /** + * This standalone file root management action can be used on folder types that do not support + * the normal 'Manage Folder' UI. Not currently linked in the UI, but available for direct URL + * navigation when a workbook needs it. + */ + @RequiresPermission(AdminPermission.class) + public class ManageFileRootAction extends FormViewAction + { + @Override + public ModelAndView getView(FileRootsForm form, boolean reShow, BindException errors) + { + JspView view = getFileRootsView(form, errors, getReshow()); + getPageConfig().setTitle("Manage File Root"); + return view; + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = getContainer().getStartURL(getUser()); + + if (getViewContext().getActionURL().getReturnUrl() != null) + { + url.addReturnUrl(getViewContext().getActionURL().getReturnUrl()); + } + + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public class FileRootsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(FileRootsForm form, boolean reshow, BindException errors) + { + return getFileRootsView(form, errors, getReshow()); + } + + @Override + public void validateCommand(FileRootsForm form, Errors errors) + { + validateCloudFileRoot(form, getContainer(), errors); + } + + @Override + public boolean handlePost(FileRootsForm form, BindException errors) throws Exception + { + return handleFileRootsPost(form, errors); + } + + @Override + public ActionURL getSuccessURL(FileRootsForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getFileRootsURL(getContainer()); + + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + return url; + } + } + + private JspView getFileRootsView(FileRootsForm form, BindException errors, boolean reshow) + { + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + view.setFrame(WebPartView.FrameType.DIV); + try + { + if (!reshow) + setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + + return view; + } + + private boolean handleFileRootsPost(FileRootsForm form, BindException errors) throws Exception + { + if (form.isPipelineRootForm()) + { + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + } + else + { + setFileRootFromForm(getViewContext(), form, errors); + setEnabledCloudStores(getViewContext(), form, errors); + return !errors.hasErrors(); + } + } + + public static void validateCloudFileRoot(FileManagementForm form, Container container, Errors errors) + { + FileContentService service = FileContentService.get(); + if (null != service) + { + boolean isOrDefaultsToCloudRoot = form.isCloudFileRoot(); + String cloudRootName = form.getCloudRootName(); + if (!isOrDefaultsToCloudRoot && form.hasSiteDefaultRoot()) + { + Path defaultRootPath = service.getDefaultRootPath(container, false); + cloudRootName = service.getDefaultRootInfo(container).getCloudName(); + isOrDefaultsToCloudRoot = (null != defaultRootPath && FileUtil.hasCloudScheme(defaultRootPath)); + } + + if (isOrDefaultsToCloudRoot && null != cloudRootName) + { + if (null != form.getEnabledCloudStore()) + { + for (String storeName : form.getEnabledCloudStore()) + { + if (Strings.CI.equals(cloudRootName, storeName)) + return; + } + } + // Didn't find cloud root in enabled list + errors.reject(ERROR_MSG, "Cannot disable cloud store used as File Root."); + } + } + } + + public static void setFileRootFromForm(ViewContext ctx, FileManagementForm form, BindException errors) + { + boolean changed = false; + boolean shouldCopyMove = false; + FileContentService service = FileContentService.get(); + if (null != service) + { + // If we need to copy/move files based on the FileRoot change, we need to check children that use the default and move them, too. + // And we need to capture the source roots for each of those, because changing this parent file root changes the child source roots. + MigrateFilesOption migrateFilesOption = null != form.getMigrateFilesOption() ? + MigrateFilesOption.valueOf(form.getMigrateFilesOption()) : + MigrateFilesOption.leave; + List> sourceInfos = + ((MigrateFilesOption.leave.equals(migrateFilesOption) && !form.isFolderSetup()) || form.isDisableFileSharing()) ? + Collections.emptyList() : + getCopySourceInfo(service, ctx.getContainer()); + + if (form.isDisableFileSharing()) + { + if (!service.isFileRootDisabled(ctx.getContainer())) + { + service.disableFileRoot(ctx.getContainer()); + changed = true; + } + } + else if (form.hasSiteDefaultRoot()) + { + if (service.isFileRootDisabled(ctx.getContainer()) || !service.isUseDefaultRoot(ctx.getContainer())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), true); + changed = true; + shouldCopyMove = true; + } + } + else if (form.isCloudFileRoot()) + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String cloudRootName = form.getCloudRootName(); + if (null != cloudRootName && + (!service.isCloudRoot(ctx.getContainer()) || + !cloudRootName.equalsIgnoreCase(service.getCloudRootName(ctx.getContainer())))) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setCloudRoot(ctx.getContainer(), cloudRootName); + try + { + PipelineService.get().setPipelineRoot(ctx.getUser(), ctx.getContainer(), PipelineService.PRIMARY_ROOT, false); + if (form.isFolderSetup() && !sourceInfos.isEmpty()) + { + // File root was set to cloud storage, remove folder created + Path fromPath = FileUtil.stringToPath(sourceInfos.get(0).first, sourceInfos.get(0).second); // sourceInfos paths should be encoded + if (FileContentService.FILES_LINK.equals(FileUtil.getFileName(fromPath))) + { + try + { + Files.deleteIfExists(fromPath.getParent()); + } + catch (IOException e) + { + LOG.warn("Could not delete directory '" + FileUtil.pathToString(fromPath.getParent()) + "'"); + } + } + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + changed = true; + shouldCopyMove = true; + } + } + else + { + throwIfUnauthorizedFileRootChange(ctx, service, form); + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + URI uri = FileUtil.createUri(root, false); // root is unencoded + Path path = FileUtil.getPath(ctx.getContainer(), uri); + if (null == path || !Files.exists(path)) + { + errors.reject(ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + ctx.getRequest().getServerName() + "."); + } + else + { + Path currentFileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null == currentFileRootPath || !root.equalsIgnoreCase(currentFileRootPath.toAbsolutePath().toString())) + { + service.setIsUseDefaultRoot(ctx.getContainer(), false); + service.setFileRootPath(ctx.getContainer(), root); + changed = true; + shouldCopyMove = true; + } + } + } + else + { + service.setFileRootPath(ctx.getContainer(), null); + changed = true; + } + } + + if (!errors.hasErrors()) + { + if (changed && shouldCopyMove && !MigrateFilesOption.leave.equals(migrateFilesOption)) + { + // Make sure we have pipeRoot before starting jobs, even though each subfolder needs to get its own + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(ctx.getContainer()); + if (null != pipeRoot) + { + try + { + initiateCopyFilesPipelineJobs(ctx, sourceInfos, pipeRoot, migrateFilesOption); + } + catch (PipelineValidationException e) + { + throw new RuntimeValidationException(e); + } + } + else + { + LOG.warn("Change File Root: Can't copy or move files with no pipeline root"); + } + } + + form.setFileRootChanged(changed); + if (changed && null != ctx.getUser()) + { + setFormAndConfirmMessage(ctx.getContainer(), form, true, false, migrateFilesOption.name()); + String comment = (ctx.getContainer().isProject() ? "Project " : "Folder ") + ctx.getContainer().getPath() + ": " + form.getConfirmMessage(); + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, ctx.getContainer(), comment); + AuditLogService.get().addEvent(ctx.getUser(), event); + } + } + } + } + + private static List> getCopySourceInfo(FileContentService service, Container container) + { + + List> sourceInfo = new ArrayList<>(); + addCopySourceInfo(service, container, sourceInfo, true); + return sourceInfo; + } + + private static void addCopySourceInfo(FileContentService service, Container container, List> sourceInfo, boolean isRoot) + { + if (isRoot || service.isUseDefaultRoot(container)) + { + Path sourceFileRootDir = service.getFileRootPath(container, FileContentService.ContentType.files); + if (null != sourceFileRootDir) + { + String pathStr = FileUtil.pathToString(sourceFileRootDir); + if (null != pathStr) + sourceInfo.add(new Pair<>(container, pathStr)); + else + throw new RuntimeValidationException("Unexpected error converting path to string"); + } + } + for (Container childContainer : container.getChildren()) + addCopySourceInfo(service, childContainer, sourceInfo, false); + } + + private static void initiateCopyFilesPipelineJobs(ViewContext ctx, @NotNull List> sourceInfos, PipeRoot pipeRoot, + MigrateFilesOption migrateFilesOption) throws PipelineValidationException + { + CopyFileRootPipelineJob job = new CopyFileRootPipelineJob(ctx.getContainer(), ctx.getUser(), sourceInfos, pipeRoot, migrateFilesOption); + PipelineService.get().queueJob(job); + } + + private static void throwIfUnauthorizedFileRootChange(ViewContext ctx, FileContentService service, FileManagementForm form) + { + // test permissions. only site admins are able to turn on a custom file root for a folder + // this is only relevant if the folder is either being switched to a custom file root, + // or if the file root is changed. + if (!service.isUseDefaultRoot(ctx.getContainer())) + { + Path fileRootPath = service.getFileRootPath(ctx.getContainer()); + if (null != fileRootPath) + { + String absolutePath = FileUtil.getAbsolutePath(ctx.getContainer(), fileRootPath); + if (Strings.CI.equals(absolutePath, form.getFolderRootPath())) + { + if (!ctx.getUser().hasRootPermission(AdminOperationsPermission.class)) + throw new UnauthorizedException("Only site admins can change file roots"); + } + } + } + } + + public static void setEnabledCloudStores(ViewContext ctx, FileManagementForm form, BindException errors) + { + String[] enabledCloudStores = form.getEnabledCloudStore(); + CloudStoreService cloud = CloudStoreService.get(); + if (cloud != null) + { + Set enabled = Collections.emptySet(); + if (enabledCloudStores != null) + enabled = new HashSet<>(Arrays.asList(enabledCloudStores)); + + try + { + // Check if anything changed + boolean changed = false; + Collection storeNames = cloud.getEnabledCloudStores(ctx.getContainer()); + if (enabled.size() != storeNames.size()) + changed = true; + else + if (!enabled.containsAll(storeNames)) + changed = true; + if (changed) + cloud.setEnabledCloudStores(ctx.getContainer(), enabled); + form.setEnabledCloudStoresChanged(changed); + } + catch (UncheckedExecutionException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + // UncheckedExecutionException with cause org.jclouds.blobstore.ContainerNotFoundException + // is what BlobStore hands us if bucket (S3 container) does not exist + if (null != e.getCause()) + errors.reject(ERROR_MSG, e.getCause().getMessage()); + else + throw e; + } + catch (RuntimeException e) + { + LOG.debug("Failed to configure cloud store(s).", e); + errors.reject(ERROR_MSG, e.getMessage()); + } + } + } + + + public static void setFormAndConfirmMessage(ViewContext ctx, FileManagementForm form) throws IllegalArgumentException + { + String rootSetParam = ctx.getActionURL().getParameter("rootSet"); + boolean fileRootChanged = null != rootSetParam && !"false".equalsIgnoreCase(rootSetParam); + String cloudChangedParam = ctx.getActionURL().getParameter("cloudChanged"); + boolean enabledCloudChanged = "true".equalsIgnoreCase(cloudChangedParam); + setFormAndConfirmMessage(ctx.getContainer(), form, fileRootChanged, enabledCloudChanged, rootSetParam); + } + + public static void setFormAndConfirmMessage(Container container, FileManagementForm form, boolean fileRootChanged, boolean enabledCloudChanged, + String migrateFilesOption) throws IllegalArgumentException + { + FileContentService service = FileContentService.get(); + String confirmMessage = null; + + String migrateFilesMessage = ""; + if (fileRootChanged && !form.isFolderSetup()) + { + if (MigrateFilesOption.leave.name().equals(migrateFilesOption)) + migrateFilesMessage = ". Existing files not copied or moved."; + else if (MigrateFilesOption.copy.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files copied."; + form.setMigrateFilesOption(migrateFilesOption); + } + else if (MigrateFilesOption.move.name().equals(migrateFilesOption)) + { + migrateFilesMessage = ". Existing files moved."; + form.setMigrateFilesOption(migrateFilesOption); + } + } + + if (service != null) + { + if (service.isFileRootDisabled(container)) + { + form.setFileRootOption(FileRootProp.disable.name()); + if (fileRootChanged) + confirmMessage = "File sharing has been disabled for this " + container.getContainerNoun(); + } + else if (service.isUseDefaultRoot(container)) + { + form.setFileRootOption(FileRootProp.siteDefault.name()); + Path root = service.getFileRootPath(container); + if (root != null && Files.exists(root) && fileRootChanged) + confirmMessage = "The file root is set to a default of: " + FileUtil.getAbsolutePath(container, root) + migrateFilesMessage; + } + else if (!service.isCloudRoot(container)) + { + Path root = service.getFileRootPath(container); + + form.setFileRootOption(FileRootProp.folderOverride.name()); + if (root != null) + { + String absolutePath = FileUtil.getAbsolutePath(container, root); + form.setFolderRootPath(absolutePath); + if (Files.exists(root)) + { + if (fileRootChanged) + confirmMessage = "The file root is set to: " + absolutePath + migrateFilesMessage; + } + } + } + else + { + form.setFileRootOption(FileRootProp.cloudRoot.name()); + form.setCloudRootName(service.getCloudRootName(container)); + Path root = service.getFileRootPath(container); + if (root != null && fileRootChanged) + { + confirmMessage = "The file root is set to: " + FileUtil.getCloudRootPathString(form.getCloudRootName()) + migrateFilesMessage; + } + } + } + + if (fileRootChanged && confirmMessage != null) + form.setConfirmMessage(confirmMessage); + else if (enabledCloudChanged) + form.setConfirmMessage("The enabled cloud stores changed."); + } + + @RequiresPermission(AdminPermission.class) + public static class ManageFoldersAction extends FolderManagementViewAction + { + @Override + protected HttpView getTabView() + { + return new JspView<>("/org/labkey/core/admin/manageFolders.jsp"); + } + } + + public static class NotificationsForm + { + private String _provider; + + public String getProvider() + { + return _provider; + } + + public void setProvider(String provider) + { + _provider = provider; + } + } + + private static final String DATA_REGION_NAME = "Users"; + + @RequiresPermission(AdminPermission.class) + public static class NotificationsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(NotificationsForm form, boolean reshow, BindException errors) + { + final String key = DataRegionSelection.getSelectionKey("core", CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME, null, DATA_REGION_NAME); + DataRegionSelection.clearAll(getViewContext(), key); + + QuerySettings settings = new QuerySettings(getViewContext(), DATA_REGION_NAME, CoreQuerySchema.USERS_MSG_SETTINGS_TABLE_NAME); + settings.setAllowChooseView(true); + settings.getBaseSort().insertSortColumn(FieldKey.fromParts("DisplayName")); + + UserSchema schema = QueryService.get().getUserSchema(getViewContext().getUser(), getViewContext().getContainer(), SchemaKey.fromParts(CoreQuerySchema.NAME)); + QueryView queryView = new QueryView(schema, settings, errors) + { + @Override + public List getDisplayColumns() + { + List columns = new ArrayList<>(); + SecurityPolicy policy = getContainer().getPolicy(); + Set assignmentSet = new HashSet<>(); + + for (RoleAssignment assignment : policy.getAssignments()) + { + Group g = SecurityManager.getGroup(assignment.getUserId()); + if (g != null) + assignmentSet.add(g.getName()); + } + + for (DisplayColumn col : super.getDisplayColumns()) + { + if (col.getName().equalsIgnoreCase("Groups")) + columns.add(new FolderGroupColumn(assignmentSet, col.getColumnInfo())); + else + columns.add(col); + } + return columns; + } + + @Override + protected void populateButtonBar(DataView dataView, ButtonBar bar) + { + try + { + // add the provider configuration menu items to the admin panel button + MenuButton adminButton = new MenuButton("Update user settings"); + adminButton.setRequiresSelection(true); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + adminButton.addMenuItem("For " + provider.getName().toLowerCase(), "userSettings_"+provider.getName()+"(LABKEY.DataRegions.Users.getSelectionCount())" ); + + bar.add(adminButton); + super.populateButtonBar(dataView, bar); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + }; + queryView.setShadeAlternatingRows(true); + queryView.setShowBorders(true); + queryView.setShowDetailsColumn(false); + queryView.setShowRecordSelectors(true); + queryView.setFrame(WebPartView.FrameType.NONE); + queryView.disableContainerFilterSelection(); + queryView.setButtonBarPosition(DataRegion.ButtonBarPosition.TOP); + + VBox defaultsView = new VBox( + HtmlView.unsafe( + "
    Default settings
    " + + "You can change this folder's default settings for email notifications here.") + ); + + PanelConfig config = new PanelConfig(getViewContext().getActionURL().clone(), key); + for (ConfigTypeProvider provider : MessageConfigService.get().getConfigTypes()) + { + defaultsView.addView(new JspView<>("/org/labkey/core/admin/view/notifySettings.jsp", provider.createConfigForm(getViewContext(), config))); + } + + return new VBox( + new JspView<>("/org/labkey/core/admin/view/folderSettingsHeader.jsp", null, errors), + defaultsView, + new VBox( + HtmlView.unsafe( + "
    User settings
    " + + "The list below contains all users with read access to this folder who are able to receive notifications. Each user's current
    " + + "notification setting is visible in the appropriately named column.

    " + + "To bulk edit individual settings: select one or more users, click the 'Update user settings' menu, and select the notification type."), + queryView + ) + ); + } + + @Override + public void validateCommand(NotificationsForm form, Errors errors) + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + provider.validateCommand(getViewContext(), errors); + } + + @Override + public boolean handlePost(NotificationsForm form, BindException errors) throws Exception + { + ConfigTypeProvider provider = MessageConfigService.get().getConfigType(form.getProvider()); + + if (provider != null) + { + return provider.handlePost(getViewContext(), errors); + } + errors.reject(SpringActionController.ERROR_MSG, "Unable to find the selected config provider"); + return false; + } + } + + public static class NotifyOptionsForm + { + private String _type; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public ConfigTypeProvider getProvider() + { + return MessageConfigService.get().getConfigType(getType()); + } + } + + /** + * Action to populate an Ext store with email notification options for admin settings + */ + @RequiresPermission(AdminPermission.class) + public static class GetEmailOptionsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(NotifyOptionsForm form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + + ConfigTypeProvider provider = form.getProvider(); + if (provider != null) + { + List options = new ArrayList<>(); + + // if the list of options is not for the folder default, add an option to use the folder default + if (getViewContext().get("isDefault") == null) + options.add(PageFlowUtil.map("id", -1, "label", "Folder default")); + + for (NotificationOption option : provider.getOptions()) + { + options.add(PageFlowUtil.map("id", option.getEmailOptionId(), "label", option.getEmailOption())); + } + resp.put("success", true); + if (!options.isEmpty()) + resp.put("options", options); + } + else + resp.put("success", false); + + return resp; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetBulkEmailOptionsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(EmailConfigFormImpl form, BindException errors) + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + ConfigTypeProvider provider = form.getProvider(); + String srcIdentifier = getContainer().getId(); + + Set selections = DataRegionSelection.getSelected(getViewContext(), form.getDataRegionSelectionKey(), true); + + if (!selections.isEmpty() && provider != null) + { + int newOption = form.getIndividualEmailOption(); + + for (String user : selections) + { + User projectUser = UserManager.getUser(Integer.parseInt(user)); + UserPreference pref = provider.getPreference(getContainer(), projectUser, srcIdentifier); + + int currentEmailOption = pref != null ? pref.getEmailOptionId() : -1; + + //has this projectUser's option changed? if so, update + //creating new record in EmailPrefs table if there isn't one, or deleting if set back to folder default + if (currentEmailOption != newOption) + { + provider.savePreference(getUser(), getContainer(), projectUser, newOption, srcIdentifier); + } + } + resp.put("success", true); + } + else + { + resp.put("success", false); + resp.put("message", "There were no users selected"); + } + return resp; + } + } + + /** Renders only the groups that are assigned roles in this container */ + private static class FolderGroupColumn extends DataColumn + { + private final Set _assignmentSet; + + public FolderGroupColumn(Set assignmentSet, ColumnInfo col) + { + super(col); + _assignmentSet = assignmentSet; + } + + @Override + public void renderGridCellContents(RenderContext ctx, HtmlWriter out) + { + String value = (String)ctx.get(getBoundColumn().getDisplayField().getFieldKey()); + + if (value != null) + { + out.write(Arrays.stream(value.split(VALUE_DELIMITER_REGEX)) + .filter(_assignmentSet::contains) + .map(HtmlString::of) + .collect(LabKeyCollectors.joining(HtmlString.unsafe(",
    ")))); + } + } + } + + private static class PanelConfig implements MessageConfigService.PanelInfo + { + private final ActionURL _returnUrl; + private final String _dataRegionSelectionKey; + + public PanelConfig(ActionURL returnUrl, String selectionKey) + { + _returnUrl = returnUrl; + _dataRegionSelectionKey = selectionKey; + } + + @Override + public ActionURL getReturnUrl() + { + return _returnUrl; + } + + @Override + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + } + + public static class ConceptsForm + { + private String _conceptURI; + private String _containerId; + private String _schemaName; + private String _queryName; + + public String getConceptURI() + { + return _conceptURI; + } + + public void setConceptURI(String conceptURI) + { + _conceptURI = conceptURI; + } + + public String getContainerId() + { + return _containerId; + } + + public void setContainerId(String containerId) + { + _containerId = containerId; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConceptsAction extends FolderManagementViewPostAction + { + @Override + protected HttpView getTabView(ConceptsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/manageConcepts.jsp", form, errors); + } + + @Override + public void validateCommand(ConceptsForm form, Errors errors) + { + // validate that the required input fields are provided + String missingRequired = "", sep = ""; + if (form.getConceptURI() == null) + { + missingRequired += "conceptURI"; + sep = ", "; + } + if (form.getSchemaName() == null) + { + missingRequired += sep + "schemaName"; + sep = ", "; + } + if (form.getQueryName() == null) + missingRequired += sep + "queryName"; + if (!missingRequired.isEmpty()) + errors.reject(SpringActionController.ERROR_MSG, "Missing required field(s): " + missingRequired + "."); + + // validate that, if provided, the containerId matches an existing container + Container postContainer = null; + if (form.getContainerId() != null) + { + postContainer = ContainerManager.getForId(form.getContainerId()); + if (postContainer == null) + errors.reject(SpringActionController.ERROR_MSG, "Container does not exist for containerId provided."); + } + + // validate that the schema and query names provided exist + if (form.getSchemaName() != null && form.getQueryName() != null) + { + Container c = postContainer != null ? postContainer : getContainer(); + UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); + if (schema == null) + errors.reject(SpringActionController.ERROR_MSG, "UserSchema '" + form.getSchemaName() + "' not found."); + else if (schema.getTable(form.getQueryName()) == null) + errors.reject(SpringActionController.ERROR_MSG, "Table '" + form.getSchemaName() + "." + form.getQueryName() + "' not found."); + } + } + + @Override + public boolean handlePost(ConceptsForm form, BindException errors) + { + Lookup lookup = new Lookup(ContainerManager.getForId(form.getContainerId()), form.getSchemaName(), form.getQueryName()); + ConceptURIProperties.setLookup(getContainer(), form.getConceptURI(), lookup); + + return true; + } + } + + @RequiresPermission(AdminPermission.class) + public class FolderAliasesAction extends FormViewAction + { + @Override + public void validateCommand(FolderAliasesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderAliasesForm form, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/folderAliases.jsp"); + } + + @Override + public boolean handlePost(FolderAliasesForm form, BindException errors) + { + List aliases = new ArrayList<>(); + if (form.getAliases() != null) + { + StringTokenizer st = new StringTokenizer(form.getAliases(), "\n\r", false); + while (st.hasMoreTokens()) + { + String alias = st.nextToken().trim(); + if (!alias.startsWith("/")) + { + alias = "/" + alias; + } + while (alias.endsWith("/")) + { + alias = alias.substring(0, alias.lastIndexOf('/')); + } + aliases.add(alias); + } + } + ContainerManager.saveAliasesForContainer(getContainer(), aliases, getUser()); + + return true; + } + + @Override + public ActionURL getSuccessURL(FolderAliasesForm form) + { + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Aliases: " + getContainer().getPath(), this.getClass()); + } + } + + public static class FolderAliasesForm + { + private String _aliases; + + public String getAliases() + { + return _aliases; + } + + @SuppressWarnings("unused") + public void setAliases(String aliases) + { + _aliases = aliases; + } + } + + @RequiresPermission(AdminPermission.class) + public class CustomizeEmailAction extends FormViewAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(CustomEmailForm form, boolean reshow, BindException errors) + { + JspView result = new JspView<>("/org/labkey/core/admin/customizeEmail.jsp", form, errors); + result.setTitle("Email Template"); + return result; + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + + template.setSubject(form.getEmailSubject()); + template.setSenderName(form.getEmailSender()); + template.setReplyToEmail(form.getEmailReplyTo()); + template.setBody(form.getEmailMessage()); + + String[] errorStrings = new String[1]; + if (template.isValid(errorStrings)) // TODO: Pass in errors collection directly? Should also build a list of all validation errors and display them all. + EmailTemplateService.get().saveEmailTemplate(template, getContainer()); + else + errors.reject(ERROR_MSG, errorStrings[0]); + } + + return !errors.hasErrors(); + } + + @Override + public ActionURL getSuccessURL(CustomEmailForm form) + { + ActionURL result = new ActionURL(CustomizeEmailAction.class, getContainer()); + result.replaceParameter("templateClass", form.getTemplateClass()); + if (form.getReturnActionURL() != null) + { + result.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + } + return result; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("customEmail"); + addAdminNavTrail(root, "Customize " + (getContainer().isRoot() ? "Site-Wide" : StringUtils.capitalize(getContainer().getContainerNoun()) + "-Level") + " Email", this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class DeleteCustomEmailAction extends FormHandlerAction + { + @Override + public void validateCommand(CustomEmailForm target, Errors errors) + { + } + + @Override + public boolean handlePost(CustomEmailForm form, BindException errors) throws Exception + { + if (form.getTemplateClass() != null) + { + EmailTemplate template = EmailTemplateService.get().createTemplate(form.getTemplateClass()); + template.setSubject(form.getEmailSubject()); + template.setBody(form.getEmailMessage()); + + EmailTemplateService.get().deleteEmailTemplate(template, getContainer()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(CustomEmailForm form) + { + return new AdminUrlsImpl().getCustomizeEmailURL(getContainer(), form.getTemplateClass(), form.getReturnUrlHelper()); + } + } + + @SuppressWarnings("unused") + public static class CustomEmailForm extends ReturnUrlForm + { + private String _templateClass; + private String _emailSubject; + private String _emailSender; + private String _emailReplyTo; + private String _emailMessage; + private String _templateDescription; + + public void setTemplateClass(String name){_templateClass = name;} + public String getTemplateClass(){return _templateClass;} + public void setEmailSubject(String subject){_emailSubject = subject;} + public String getEmailSubject(){return _emailSubject;} + public void setEmailSender(String sender){_emailSender = sender;} + public String getEmailSender(){return _emailSender;} + public void setEmailMessage(String body){_emailMessage = body;} + public String getEmailMessage(){return _emailMessage;} + public String getEmailReplyTo(){return _emailReplyTo;} + public void setEmailReplyTo(String emailReplyTo){_emailReplyTo = emailReplyTo;} + + public String getTemplateDescription() + { + return _templateDescription; + } + + public void setTemplateDescription(String templateDescription) + { + _templateDescription = templateDescription; + } + } + + private ActionURL getManageFoldersURL() + { + return new AdminUrlsImpl().getManageFoldersURL(getContainer()); + } + + public static class ManageFoldersForm extends ReturnUrlForm + { + private String name; + private String title; + private boolean titleSameAsName; + private String folder; + private String target; + private String folderType; + private String defaultModule; + private String[] activeModules; + private boolean hasLoaded = false; + private boolean showAll; + private boolean confirmed = false; + private boolean addAlias = false; + private String templateSourceId; + private String[] templateWriterTypes; + private boolean templateIncludeSubfolders = false; + private String[] targets; + private PHI _exportPhiLevel = PHI.NotPHI; + + public boolean getHasLoaded() + { + return hasLoaded; + } + + public void setHasLoaded(boolean hasLoaded) + { + this.hasLoaded = hasLoaded; + } + + public String[] getActiveModules() + { + return activeModules; + } + + public void setActiveModules(String[] activeModules) + { + this.activeModules = activeModules; + } + + public String getDefaultModule() + { + return defaultModule; + } + + public void setDefaultModule(String defaultModule) + { + this.defaultModule = defaultModule; + } + + public boolean isShowAll() + { + return showAll; + } + + public void setShowAll(boolean showAll) + { + this.showAll = showAll; + } + + public String getFolder() + { + return folder; + } + + public void setFolder(String folder) + { + this.folder = folder; + } + + public String getName() + { + return name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isTitleSameAsName() + { + return titleSameAsName; + } + + public void setTitleSameAsName(boolean updateTitle) + { + this.titleSameAsName = updateTitle; + } + public void setName(String name) + { + this.name = name; + } + + public boolean isConfirmed() + { + return confirmed; + } + + public void setConfirmed(boolean confirmed) + { + this.confirmed = confirmed; + } + + public String getFolderType() + { + return folderType; + } + + public void setFolderType(String folderType) + { + this.folderType = folderType; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + + public String getTarget() + { + return target; + } + + public void setTarget(String target) + { + this.target = target; + } + + public void setTemplateSourceId(String templateSourceId) + { + this.templateSourceId = templateSourceId; + } + + public String getTemplateSourceId() + { + return templateSourceId; + } + + public Container getTemplateSourceContainer() + { + if (null == getTemplateSourceId()) + return null; + return ContainerManager.getForId(getTemplateSourceId()); + } + + public String[] getTemplateWriterTypes() + { + return templateWriterTypes; + } + + public void setTemplateWriterTypes(String[] templateWriterTypes) + { + this.templateWriterTypes = templateWriterTypes; + } + + public boolean getTemplateIncludeSubfolders() + { + return templateIncludeSubfolders; + } + + public void setTemplateIncludeSubfolders(boolean templateIncludeSubfolders) + { + this.templateIncludeSubfolders = templateIncludeSubfolders; + } + + public String[] getTargets() + { + return targets; + } + + public void setTargets(String[] targets) + { + this.targets = targets; + } + + public PHI getExportPhiLevel() + { + return _exportPhiLevel; + } + + public void setExportPhiLevel(PHI exportPhiLevel) + { + _exportPhiLevel = exportPhiLevel; + } + + /** + * Note: this is designed to allow code to specify a set of children to delete in bulk. The main use-case is workbooks, + * but it will work for non-workbook children as well. + */ + public List getTargetContainers(final Container currentContainer) throws IllegalArgumentException + { + if (getTargets() != null) + { + final List targets = new ArrayList<>(); + final List directChildren = ContainerManager.getChildren(currentContainer); + + Arrays.stream(getTargets()).forEach(x -> { + Container c = ContainerManager.getForId(x); + if (c == null) + { + try + { + Integer rowId = ConvertHelper.convert(x, Integer.class); + if (rowId > 0) + c = ContainerManager.getForRowId(rowId); + } + catch (ConversionException e) + { + //ignore + } + } + + if (c != null) + { + if (!c.equals(currentContainer)) + { + if (!directChildren.contains(c)) + { + throw new IllegalArgumentException("Folder " + c.getPath() + " is not a direct child of the current folder: " + currentContainer.getPath()); + } + + if (c.getContainerType().canHaveChildren()) + { + throw new IllegalArgumentException("Multi-folder delete is not supported for containers of type: " + c.getContainerType().getName()); + } + } + + targets.add(c); + } + else + { + throw new IllegalArgumentException("Unable to find folder with ID or RowId of: " + x); + } + }); + + return targets; + } + else + { + return Collections.singletonList(currentContainer); + } + } + } + + public static class RenameContainerForm + { + private String name; + private String title; + private boolean addAlias = true; + + public String getName() + { + return name; + } + + public void setName(String name) + { + this.name = name; + } + + public String getTitle() + { + return title; + } + + public void setTitle(String title) + { + this.title = title; + } + + public boolean isAddAlias() + { + return addAlias; + } + + public void setAddAlias(boolean addAlias) + { + this.addAlias = addAlias; + } + } + + // Note that validation checks occur in ContainerManager.rename() + @RequiresPermission(AdminPermission.class) + public static class RenameContainerAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RenameContainerForm form, BindException errors) + { + Container container = getContainer(); + String name = StringUtils.trimToNull(form.getName()); + String title = StringUtils.trimToNull(form.getTitle()); + + String nameValue = name; + String titleValue = title; + if (name == null && title == null) + { + errors.reject(ERROR_MSG, "Please specify a name or a title."); + return new ApiSimpleResponse("success", false); + } + else if (name != null && title == null) + { + titleValue = name; + } + else if (name == null) + { + nameValue = container.getName(); + } + + boolean addAlias = form.isAddAlias(); + + try + { + Container c = ContainerManager.rename(container, getUser(), nameValue, titleValue, addAlias); + return new ApiSimpleResponse(c.toJSON(getUser())); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + return new ApiSimpleResponse("success", false); + } + } + } + + @RequiresPermission(AdminPermission.class) + public class RenameFolderAction extends FormViewAction + { + private ActionURL _returnUrl; + + @Override + public void validateCommand(ManageFoldersForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/core/admin/renameFolder.jsp", form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + try + { + String title = form.isTitleSameAsName() ? null : StringUtils.trimToNull(form.getTitle()); + Container c = ContainerManager.rename(getContainer(), getUser(), form.getName(), title, form.isAddAlias()); + _returnUrl = new AdminUrlsImpl().getManageFoldersURL(c); + return true; + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : "Failed to rename folder. An error has occurred."); + } + + return false; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + return _returnUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + String containerType = getContainer().isProject() ? "Project" : "Folder"; + addAdminNavTrail(root, "Change " + containerType + " Name Settings", this.getClass()); + } + } + + public static class MoveFolderTreeView extends JspView + { + private MoveFolderTreeView(ManageFoldersForm form, BindException errors) + { + super("/org/labkey/core/admin/moveFolder.jsp", form, errors); + } + } + + @RequiresPermission(AdminPermission.class) + @ActionNames("ShowMoveFolderTree,MoveFolder") + public class MoveFolderAction extends FormViewAction + { + boolean showConfirmPage = false; + boolean moveFailed = false; + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + Container c = getContainer(); + + if (c.isRoot()) + throw new NotFoundException("Can't move the root folder."); // Don't show move tree from root + + if (c.equals(ContainerManager.getSharedContainer()) || c.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Moving /Shared or /home is not possible."); + + Container newParent = isBlank(form.getTarget()) ? null : ContainerManager.getForPath(form.getTarget()); + if (null == newParent) + { + errors.reject(ERROR_MSG, "Target '" + form.getTarget() + "' folder does not exist."); + } + else if (!newParent.hasPermission(getUser(), AdminPermission.class)) + { + throw new UnauthorizedException(); + } + else if (newParent.hasChild(c.getName())) + { + errors.reject(ERROR_MSG, "Error: The selected folder already has a folder with that name. Please select a different location (or Cancel)."); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) throws Exception + { + if (showConfirmPage) + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + if (moveFailed) + return new SimpleErrorView(errors); + else + return new MoveFolderTreeView(form, errors); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) throws Exception + { + Container c = getContainer(); + Container newParent = ContainerManager.getForPath(form.getTarget()); + Container oldProject = c.getProject(); + Container newProject = newParent.isRoot() ? c : newParent.getProject(); + + if (!oldProject.getId().equals(newProject.getId()) && !form.isConfirmed()) + { + showConfirmPage = true; + return false; // reshow + } + + try + { + ContainerManager.move(c, newParent, getUser()); + } + catch (ValidationException e) + { + moveFailed = true; + getPageConfig().setTemplate(Template.Dialog); + for (ValidationError validationError : e.getErrors()) + { + errors.addError(new LabKeyError(validationError.getMessage())); + } + if (!errors.hasErrors()) + errors.addError(new LabKeyError("Move failed")); + return false; + } + + if (form.isAddAlias()) + { + List newAliases = new ArrayList<>(ContainerManager.getAliasesForContainer(c)); + newAliases.add(c.getPath()); + ContainerManager.saveAliasesForContainer(c, newAliases, getUser()); + } + return true; + } + + @Override + public URLHelper getSuccessURL(ManageFoldersForm manageFoldersForm) + { + Container c = getContainer(); + c = ContainerManager.getForId(c.getId()); // Reload container to populate new location + return new AdminUrlsImpl().getManageFoldersURL(c); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Folder Management", getManageFoldersURL()); + root.addChild("Move Folder"); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ConfirmProjectMoveAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ManageFoldersForm form, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/confirmProjectMove.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm Project Move"); + } + } + + private static abstract class AbstractCreateFolderAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FORM target, Errors errors) + { + } + + @Override + public ModelAndView getView(FORM form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + if (!reshow) + { + FolderType folderType = FolderTypeManager.get().getDefaultFolderType(); + if (null != folderType) + { + // If a default folder type has been configured by a site admin set that as the default folder type choice + form.setFolderType(folderType.getName()); + } + form.setExportPhiLevel(ComplianceService.get().getMaxAllowedPhi(getContainer(), getUser())); + } + JspView statusView = new JspView<>("/org/labkey/core/admin/createFolder.jsp", form, errors); + vbox.addView(statusView); + + Container c = getViewContext().getContainerNoTab(); // Cannot create subfolder of tab folder + + setHelpTopic("createProject"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(null, c)); + getPageConfig().setTemplate(Template.Wizard); + + if (c.isRoot()) + getPageConfig().setTitle("Create Project"); + else + { + String title = "Create Folder"; + + title += " in /"; + if (c == ContainerManager.getHomeContainer()) + title += "Home"; + else + title += c.getName(); + + getPageConfig().setTitle(title); + } + + return vbox; + } + + @Override + public boolean handlePost(FORM form, BindException errors) throws Exception + { + Container parent = getViewContext().getContainerNoTab(); + String folderName = StringUtils.trimToNull(form.getName()); + String folderTitle = (form.isTitleSameAsName() || folderName.equals(form.getTitle())) ? null : form.getTitle(); + StringBuilder error = new StringBuilder(); + Consumer afterCreateHandler = getAfterCreateHandler(form); + + Container container; + + if (Container.isLegalName(folderName, parent.isRoot(), error)) + { + if (parent.hasChild(folderName)) + { + if (parent.isRoot()) + { + error.append("The server already has a project with this name."); + } + else + { + error.append("The ").append(parent.isProject() ? "project " : "folder ").append(parent.getPath()).append(" already has a folder with this name."); + } + } + else + { + String folderType = form.getFolderType(); + + if (null == folderType) + { + errors.reject(null, "Folder type must be specified"); + return false; + } + + if ("Template".equals(folderType)) // Create folder from selected template + { + Container sourceContainer = form.getTemplateSourceContainer(); + if (null == sourceContainer) + { + errors.reject(null, "Source template folder not selected"); + return false; + } + else if (!sourceContainer.hasPermission(getUser(), AdminPermission.class)) + { + errors.reject(null, "User does not have administrator permissions to the source container"); + return false; + } + else if (!sourceContainer.hasEnableRestrictedModules(getUser()) && sourceContainer.hasRestrictedActiveModule(sourceContainer.getActiveModules())) + { + errors.reject(null, "The source folder has a restricted module for which you do not have permission."); + return false; + } + + FolderExportContext exportCtx = new FolderExportContext(getUser(), sourceContainer, PageFlowUtil.set(form.getTemplateWriterTypes()), "new", + form.getTemplateIncludeSubfolders(), form.getExportPhiLevel(), false, false, false, + new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); + + container = ContainerManager.createContainerFromTemplate(parent, folderName, folderTitle, sourceContainer, getUser(), exportCtx, afterCreateHandler); + } + else + { + FolderType type = FolderTypeManager.get().getFolderType(folderType); + + if (type == null) + { + errors.reject(null, "Folder type not recognized"); + return false; + } + + String[] modules = form.getActiveModules(); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + if (null == modules || modules.length == 0) + { + errors.reject(null, "At least one module must be selected"); + return false; + } + } + + // Work done in this lambda will not fire container events. Only fireCreateContainer() will be called. + Consumer configureContainer = (newContainer) -> + { + afterCreateHandler.accept(newContainer); + newContainer.setFolderType(type, getUser(), errors); + + if (null == StringUtils.trimToNull(folderType) || FolderType.NONE.getName().equals(folderType)) + { + Set activeModules = new HashSet<>(); + for (String moduleName : modules) + { + Module module = ModuleLoader.getInstance().getModule(moduleName); + if (module != null) + activeModules.add(module); + } + + newContainer.setFolderType(FolderType.NONE, getUser(), errors, activeModules); + Module defaultModule = ModuleLoader.getInstance().getModule(form.getDefaultModule()); + newContainer.setDefaultModule(defaultModule); + } + }; + container = ContainerManager.createContainer(parent, folderName, folderTitle, null, NormalContainerType.NAME, getUser(), null, configureContainer); + } + + _successURL = new AdminUrlsImpl().getSetFolderPermissionsURL(container); + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + } + + errors.reject(ERROR_MSG, "Error: " + error + " Please enter a different name."); + return false; + } + + /** + * Return a Consumer that provides post-creation handling on the new Container + */ + abstract public Consumer getAfterCreateHandler(FORM form); + + @Override + protected String getCommandClassMethodName() + { + return "getAfterCreateHandler"; + } + + @Override + public ActionURL getSuccessURL(FORM form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminPermission.class) + public static class CreateFolderAction extends AbstractCreateFolderAction + { + @Override + public Consumer getAfterCreateHandler(ManageFoldersForm form) + { + // No special handling + return container -> {}; + } + } + + public static class CreateProjectForm extends ManageFoldersForm + { + private boolean _assignProjectAdmin = false; + + public boolean isAssignProjectAdmin() + { + return _assignProjectAdmin; + } + + @SuppressWarnings("unused") + public void setAssignProjectAdmin(boolean assignProjectAdmin) + { + _assignProjectAdmin = assignProjectAdmin; + } + } + + @RequiresPermission(CreateProjectPermission.class) + public static class CreateProjectAction extends AbstractCreateFolderAction + { + @Override + public void validateCommand(CreateProjectForm target, Errors errors) + { + super.validateCommand(target, errors); + if (!getContainer().isRoot()) + errors.reject(ERROR_MSG, "Must be invoked from the root"); + } + + @Override + public Consumer getAfterCreateHandler(CreateProjectForm form) + { + if (form.isAssignProjectAdmin()) + { + return c -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c.getPolicy()); + policy.addRoleAssignment(getUser(), ProjectAdminRole.class); + User savePolicyUser = getUser(); + if (c.isProject() && !c.hasPermission(savePolicyUser, AdminPermission.class) && ContainerManager.getRoot().hasPermission(savePolicyUser, CreateProjectPermission.class)) + { + // Special case for project creators who don't necessarily yet have permission to save the policy of + // the project they just created + savePolicyUser = User.getAdminServiceUser(); + } + + SecurityPolicyManager.savePolicy(policy, savePolicyUser); + }; + } + else + { + return c -> {}; + } + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetFolderPermissionsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(SetFolderPermissionsForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(SetFolderPermissionsForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setFolderPermissions.jsp", form, errors); + vbox.addView(statusView); + + Container c = getContainer(); + getPageConfig().setTitle("Users / Permissions"); + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + setHelpTopic("createProject"); + + return vbox; + } + + @Override + public boolean handlePost(SetFolderPermissionsForm form, BindException errors) + { + Container c = getContainer(); + String permissionType = form.getPermissionType(); + + if(c.isProject()){ + _successURL = new AdminUrlsImpl().getInitialFolderSettingsURL(c); + } + else + { + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + if (form.isAdvanced()) + { + _successURL = new SecurityController.SecurityUrlsImpl().getPermissionsURL(getContainer()); + } + else + { + _successURL = getContainer().getStartURL(getUser()); + } + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + } + + if(permissionType == null){ + errors.reject(ERROR_MSG, "You must select one of the options for permissions."); + return false; + } + + switch (permissionType) + { + case "CurrentUser" -> { + MutableSecurityPolicy policy = new MutableSecurityPolicy(c); + Role role = RoleManager.getRole(c.isProject() ? ProjectAdminRole.class : FolderAdminRole.class); + policy.addRoleAssignment(getUser(), role); + SecurityPolicyManager.savePolicy(policy, getUser()); + } + case "Inherit" -> SecurityManager.setInheritPermissions(c); + case "CopyExistingProject" -> { + String targetProject = form.getTargetProject(); + if (targetProject == null) + { + errors.reject(ERROR_MSG, "In order to copy permissions from an existing project, you must pick a project."); + return false; + } + Container source = ContainerManager.getForId(targetProject); + if (source == null) + { + source = ContainerManager.getForPath(targetProject); + } + if (source == null) + { + throw new NotFoundException("An unknown project was specified to copy permissions from: " + targetProject); + } + Map groupMap = GroupManager.copyGroupsToContainer(source, c, getUser()); + + //copy role assignments + SecurityPolicy op = SecurityPolicyManager.getPolicy(source); + MutableSecurityPolicy np = new MutableSecurityPolicy(c); + for (RoleAssignment assignment : op.getAssignments()) + { + int userId = assignment.getUserId(); + UserPrincipal p = SecurityManager.getPrincipal(userId); + Role r = assignment.getRole(); + + if (p instanceof Group g) + { + if (!g.isProjectGroup()) + { + np.addRoleAssignment(p, r, false); + } + else + { + np.addRoleAssignment(groupMap.get(p), r, false); + } + } + else + { + np.addRoleAssignment(p, r, false); + } + } + SecurityPolicyManager.savePolicy(np, getUser()); + } + default -> throw new UnsupportedOperationException("An Unknown permission type was supplied: " + permissionType); + } + _successURL.addParameter("wizard", Boolean.TRUE.toString()); + + return true; + } + + @Override + public ActionURL getSuccessURL(SetFolderPermissionsForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + } + } + + public static class SetFolderPermissionsForm + { + private String targetProject; + private String permissionType; + private boolean advanced; + + public String getPermissionType() + { + return permissionType; + } + + @SuppressWarnings("unused") + public void setPermissionType(String permissionType) + { + this.permissionType = permissionType; + } + + public String getTargetProject() + { + return targetProject; + } + + @SuppressWarnings("unused") + public void setTargetProject(String targetProject) + { + this.targetProject = targetProject; + } + + public boolean isAdvanced() + { + return advanced; + } + + @SuppressWarnings("unused") + public void setAdvanced(boolean advanced) + { + this.advanced = advanced; + } + } + + @RequiresPermission(AdminPermission.class) + public static class SetInitialFolderSettingsAction extends FormViewAction + { + private ActionURL _successURL; + + @Override + public void validateCommand(FilesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FilesForm form, boolean reshow, BindException errors) + { + VBox vbox = new VBox(); + Container c = getContainer(); + + JspView statusView = new JspView<>("/org/labkey/core/admin/setInitialFolderSettings.jsp", form, errors); + vbox.addView(statusView); + + getPageConfig().setNavTrail(ContainerManager.getCreateContainerWizardSteps(c, c.getParent())); + getPageConfig().setTemplate(Template.Wizard); + + String noun = c.isProject() ? "Project": "Folder"; + getPageConfig().setTitle(noun + " Settings"); + + return vbox; + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) + { + Container c = getContainer(); + String folderRootPath = StringUtils.trimToNull(form.getFolderRootPath()); + String fileRootOption = form.getFileRootOption() != null ? form.getFileRootOption() : "default"; + + if(folderRootPath == null && !fileRootOption.equals("default")) + { + errors.reject(ERROR_MSG, "Error: Must supply a default file location."); + return false; + } + + FileContentService service = FileContentService.get(); + if(fileRootOption.equals("default")) + { + service.setIsUseDefaultRoot(c, true); + } + // Requires AdminOperationsPermission to set file root + else if (c.hasPermission(getUser(), AdminOperationsPermission.class)) + { + if (!service.isValidProjectRoot(folderRootPath)) + { + errors.reject(ERROR_MSG, "File root '" + folderRootPath + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + return false; + } + + service.setIsUseDefaultRoot(c.getProject(), false); + service.setFileRootPath(c.getProject(), folderRootPath); + } + + List extraSteps = getContainer().getFolderType().getExtraSetupSteps(getContainer()); + if (extraSteps.isEmpty()) + { + _successURL = getContainer().getStartURL(getUser()); + } + else + { + _successURL = new ActionURL(extraSteps.get(0).getHref()); + } + + return true; + } + + @Override + public ActionURL getSuccessURL(FilesForm form) + { + return _successURL; + } + + @Override + public void addNavTrail(NavTree root) + { + getPageConfig().setFocusId("name"); + setHelpTopic("createProject"); + } + } + + @RequiresPermission(DeletePermission.class) + public static class DeleteWorkbooksAction extends SimpleRedirectAction + { + public void validateCommand(ReturnUrlForm target, Errors errors) + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + if (ids.isEmpty()) + { + errors.reject(ERROR_MSG, "No IDs provided"); + } + } + + @Override + public @Nullable URLHelper getRedirectURL(ReturnUrlForm form) throws Exception + { + Set ids = DataRegionSelection.getSelected(getViewContext(), true); + + ActionURL ret = new ActionURL(DeleteFolderAction.class, getContainer()); + ids.forEach(id -> ret.addParameter("targets", id)); + + ret.replaceParameter(ActionURL.Param.returnUrl, form.getReturnUrl()); + + return ret; + } + } + + //NOTE: some types of containers can be deleted by non-admin users, provided they have DeletePermission on the parent + @RequiresPermission(DeletePermission.class) + public static class DeleteFolderAction extends FormViewAction + { + private final List _deleted = new ArrayList<>(); + + @Override + public void validateCommand(ManageFoldersForm form, Errors errors) + { + try + { + List targets = form.getTargetContainers(getContainer()); + for (Container target : targets) + { + if (!ContainerManager.isDeletable(target)) + errors.reject(ERROR_MSG, "The path " + target.getPath() + " is not deletable."); + + if (target.isProject() && !getUser().hasRootAdminPermission()) + { + throw new UnauthorizedException(); + } + + Class permClass = target.getPermissionNeededToDelete(); + if (!target.hasPermission(getUser(), permClass)) + { + Permission perm = RoleManager.getPermission(permClass); + throw new UnauthorizedException("Cannot delete folder: " + target.getName() + ". " + perm.getName() + " permission required"); + } + + if (target.hasChildren() && !ContainerManager.hasTreePermission(target, getUser(), AdminPermission.class)) + { + throw new UnauthorizedException("Deleting the " + target.getContainerNoun() + " " + target.getName() + " requires admin permissions on that folder and all children. You do not have admin permission on all subfolders."); + } + + if (target.equals(ContainerManager.getSharedContainer()) || target.equals(ContainerManager.getHomeContainer())) + errors.reject(ERROR_MSG, "Deleting /Shared or /home is not possible."); + } + } + catch (IllegalArgumentException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + } + + @Override + public ModelAndView getView(ManageFoldersForm form, boolean reshow, BindException errors) + { + getPageConfig().setTemplate(Template.Dialog); + return new JspView<>("/org/labkey/core/admin/deleteFolder.jsp", form); + } + + @Override + public boolean handlePost(ManageFoldersForm form, BindException errors) + { + List targets = form.getTargetContainers(getContainer()); + + // Must be site/app admin to delete a project + for (Container c : targets) + { + ContainerManager.deleteAll(c, getUser()); + } + + _deleted.addAll(targets); + + return true; + } + + @Override + public ActionURL getSuccessURL(ManageFoldersForm form) + { + // Note: because in some scenarios we might be deleting children of the current contaner, in those cases we remain in this folder: + // If we just deleted a project then redirect to the home page, otherwise back to managing the project folders + if (_deleted.size() == 1 && _deleted.get(0).equals(getContainer())) + { + Container c = getContainer(); + if (c.isProject()) + return AppProps.getInstance().getHomePageActionURL(); + else + return new AdminUrlsImpl().getManageFoldersURL(c.getParent()); + } + else + { + if (form.getReturnUrl() != null) + { + return form.getReturnActionURL(); + } + else + { + return getContainer().getStartURL(getUser()); + } + } + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Confirm " + getContainer().getContainerNoun() + " deletion"); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersAction extends FormViewAction + { + @Override + public void validateCommand(FolderReorderForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(FolderReorderForm folderReorderForm, boolean reshow, BindException errors) + { + return new JspView("/org/labkey/core/admin/reorderFolders.jsp"); + } + + @Override + public boolean handlePost(FolderReorderForm form, BindException errors) + { + return ReorderFolders(form, errors); + } + + @Override + public ActionURL getSuccessURL(FolderReorderForm folderReorderForm) + { + if (getContainer().isRoot()) + return getShowAdminURL(); + else + return getManageFoldersURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + String title = "Reorder " + (getContainer().isRoot() || getContainer().getParent().isRoot() ? "Projects" : "Folders"); + addAdminNavTrail(root, title, this.getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public class ReorderFoldersApiAction extends MutatingApiAction + { + @Override + public ApiResponse execute(FolderReorderForm form, BindException errors) + { + return new ApiSimpleResponse("success", ReorderFolders(form, errors)); + } + } + + private boolean ReorderFolders(FolderReorderForm form, BindException errors) + { + Container parent = getContainer().isRoot() ? getContainer() : getContainer().getParent(); + if (form.isResetToAlphabetical()) + ContainerManager.setChildOrderToAlphabetical(parent); + else if (form.getOrder() != null) + { + List children = parent.getChildren(); + String[] order = form.getOrder().split(";"); + Map nameToContainer = new HashMap<>(); + for (Container child : children) + nameToContainer.put(child.getName(), child); + List sorted = new ArrayList<>(children.size()); + for (String childName : order) + { + Container child = nameToContainer.get(childName); + sorted.add(child); + } + + try + { + ContainerManager.setChildOrder(parent, sorted); + } + catch (ContainerException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + } + + return true; + } + + public static class FolderReorderForm + { + private String _order; + private boolean _resetToAlphabetical; + + public String getOrder() + { + return _order; + } + + @SuppressWarnings("unused") + public void setOrder(String order) + { + _order = order; + } + + public boolean isResetToAlphabetical() + { + return _resetToAlphabetical; + } + + @SuppressWarnings("unused") + public void setResetToAlphabetical(boolean resetToAlphabetical) + { + _resetToAlphabetical = resetToAlphabetical; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RevertFolderAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RevertFolderForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + + boolean success = false; + Container revertContainer = ContainerManager.getForPath(form.getContainerPath()); + if (null != revertContainer) + { + if (revertContainer.isContainerTab()) + { + FolderTab tab = revertContainer.getParent().getFolderType().findTab(revertContainer.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + revertContainer.setFolderType(origFolderType, getUser(), errors); + if (!errors.hasErrors()) + success = true; + } + } + } + else if (revertContainer.getFolderType().hasContainerTabs()) + { + try (DbScope.Transaction transaction = CoreSchema.getInstance().getSchema().getScope().ensureTransaction()) + { + List children = revertContainer.getChildren(); + for (Container container : children) + { + if (container.isContainerTab()) + { + FolderTab tab = revertContainer.getFolderType().findTab(container.getName()); + if (null != tab) + { + FolderType origFolderType = tab.getFolderType(); + if (null != origFolderType) + { + container.setFolderType(origFolderType, getUser(), errors); + } + } + } + } + if (!errors.hasErrors()) + { + transaction.commit(); + success = true; + } + } + } + } + return new ApiSimpleResponse("success", success); + } + } + + public static class RevertFolderForm + { + private String _containerPath; + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + } + + public static class EmailTestForm + { + private String _to; + private String _body; + private ConfigurationException _exception; + + public String getTo() + { + return _to; + } + + public void setTo(String to) + { + _to = to; + } + + public String getBody() + { + return _body; + } + + public void setBody(String body) + { + _body = body; + } + + public ConfigurationException getException() + { + return _exception; + } + + public void setException(ConfigurationException exception) + { + _exception = exception; + } + + public String getFrom(Container c) + { + LookAndFeelProperties props = LookAndFeelProperties.getInstance(c); + return props.getSystemEmailAddress(); + } + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class EmailTestAction extends FormViewAction + { + @Override + public void validateCommand(EmailTestForm form, Errors errors) + { + if(null == form.getTo() || form.getTo().isEmpty()) + { + errors.reject(ERROR_MSG, "To field cannot be blank."); + form.setException(new ConfigurationException("To field cannot be blank")); + return; + } + + try + { + ValidEmail email = new ValidEmail(form.getTo()); + } + catch(ValidEmail.InvalidEmailException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + form.setException(new ConfigurationException(e.getMessage())); + } + } + + @Override + public ModelAndView getView(EmailTestForm form, boolean reshow, BindException errors) + { + JspView testView = new JspView<>("/org/labkey/core/admin/emailTest.jsp", form); + testView.setTitle("Send a Test Email"); + + if(null != MailHelper.getSession() && null != MailHelper.getSession().getProperties()) + { + JspView emailPropsView = new JspView<>("/org/labkey/core/admin/emailProps.jsp"); + emailPropsView.setTitle("Current Email Settings"); + + return new VBox(emailPropsView, testView); + } + else + return testView; + } + + @Override + public boolean handlePost(EmailTestForm form, BindException errors) throws Exception + { + if (errors.hasErrors()) + { + return false; + } + + LookAndFeelProperties props = LookAndFeelProperties.getInstance(getContainer()); + try + { + MailHelper.ViewMessage msg = MailHelper.createMessage(props.getSystemEmailAddress(), new ValidEmail(form.getTo()).toString()); + msg.setSubject("Test email message sent from " + props.getShortName()); + msg.setText(PageFlowUtil.filter(form.getBody())); + + try + { + MailHelper.send(msg, getUser(), getContainer()); + } + catch (ConfigurationException e) + { + form.setException(e); + return false; + } + catch (Exception e) + { + form.setException(new ConfigurationException(e.getMessage())); + return false; + } + } + catch (MessagingException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return false; + } + return true; + } + + @Override + public URLHelper getSuccessURL(EmailTestForm emailTestForm) + { + return new ActionURL(EmailTestAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Test Email Configuration", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class RecreateViewsAction extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(Object o, BindException errors) + { + getPageConfig().setShowHeader(false); + getPageConfig().setTitle("Recreate Views?"); + return new HtmlView(HtmlString.of("Are you sure you want to drop and recreate all module views?")); + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + ModuleLoader.getInstance().recreateViews(); + return true; + } + + @Override + public void validateCommand(Object o, Errors errors) + { + } + + @Override + public @NotNull ActionURL getSuccessURL(Object o) + { + return AppProps.getInstance().getHomePageActionURL(); + } + } + + static public class LoggingForm + { + public boolean isLogging() + { + return logging; + } + + public void setLogging(boolean logging) + { + this.logging = logging; + } + + public boolean logging = false; + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class GetSessionLogEventsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + Integer eventId = null; + try + { + String s = getViewContext().getRequest().getParameter("eventId"); + if (null != s) + eventId = Integer.parseInt(s); + } + catch (NumberFormatException ignored) {} + ApiSimpleResponse res = new ApiSimpleResponse(); + res.put("success", true); + res.put("events", SessionAppender.getLoggingEvents(getViewContext().getRequest(), eventId)); + return res; + } + } + + @RequiresLogin + @AllowedBeforeInitialUserIsSet + @AllowedDuringUpgrade + @IgnoresAllocationTracking /* ignore so that we don't get an update in the UI for each time it requests the newest data */ + public static class GetTrackedAllocationsAction extends ReadOnlyApiAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ApiResponse execute(Object o, BindException errors) + { + long requestId = 0; + try + { + String s = getViewContext().getRequest().getParameter("requestId"); + if (null != s) + requestId = Long.parseLong(s); + } + catch (NumberFormatException ignored) {} + List requests = MemTracker.getInstance().getNewRequests(requestId); + List> jsonRequests = new ArrayList<>(requests.size()); + for (RequestInfo requestInfo : requests) + { + Map m = new HashMap<>(); + m.put("requestId", requestInfo.getId()); + m.put("url", requestInfo.getUrl()); + m.put("date", requestInfo.getDate()); + + + List> sortedObjects = sortByCounts(requestInfo); + + List> jsonObjects = new ArrayList<>(sortedObjects.size()); + for (Map.Entry entry : sortedObjects) + { + Map jsonObject = new HashMap<>(); + jsonObject.put("name", entry.getKey()); + jsonObject.put("count", entry.getValue()); + jsonObjects.add(jsonObject); + } + m.put("objects", jsonObjects); + jsonRequests.add(m); + } + return new ApiSimpleResponse("requests", jsonRequests); + } + + private List> sortByCounts(RequestInfo requestInfo) + { + List> objects = new ArrayList<>(requestInfo.getObjects().entrySet()); + objects.sort(Map.Entry.comparingByValue(Comparator.reverseOrder())); + return objects; + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class TrackedAllocationsViewerAction extends SimpleViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getUser().isPlatformDeveloper()) + throw new UnauthorizedException(); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + getPageConfig().setTemplate(Template.Print); + return new JspView<>("/org/labkey/core/admin/memTrackerViewer.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresLogin + @AllowedDuringUpgrade + @AllowedBeforeInitialUserIsSet + public static class SessionLoggingAction extends FormViewAction + { + @Override + public void checkPermissions() + { + super.checkPermissions(); + if (!getContainer().hasPermission(getUser(), PlatformDeveloperPermission.class)) + throw new UnauthorizedException(); + } + + @Override + public boolean handlePost(LoggingForm form, BindException errors) + { + boolean on = SessionAppender.isLogging(getViewContext().getRequest()); + if (form.logging != on) + { + if (!form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging OFF"); + SessionAppender.setLoggingForSession(getViewContext().getRequest(), form.logging); + if (form.logging) + LogManager.getLogger(AdminController.class).info("turn session logging ON"); + } + return true; + } + + @Override + public void validateCommand(LoggingForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(LoggingForm o, boolean reshow, BindException errors) + { + SessionAppender.setLoggingForSession(getViewContext().getRequest(), true); + getPageConfig().setTemplate(Template.Print); + return new LoggingView(); + } + + @Override + public ActionURL getSuccessURL(LoggingForm o) + { + return new ActionURL(SessionLoggingAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Admin Console", new ActionURL(ShowAdminAction.class, getContainer()).getLocalURIString()); + root.addChild("View Event Log"); + } + } + + static class LoggingView extends JspView + { + LoggingView() + { + super("/org/labkey/core/admin/logging.jsp", null); + } + } + + public static class LogForm + { + private String _message; + private String _level; + + public String getMessage() + { + return _message; + } + + public void setMessage(String message) + { + _message = message; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = level; + } + } + + + // Simple action that writes "message" parameter to the labkey log. Used by the test harness to indicate when + // each test begins and ends. Message parameter is output as sent, except that \n is translated to newline. + @RequiresLogin + public static class LogAction extends MutatingApiAction + { + @Override + public ApiResponse execute(LogForm logForm, BindException errors) + { + // Could use %A0 for newline in the middle of the message, however, parameter values get trimmed so translate + // \n to newlines to allow them at the beginning or end of the message as well. + StringBuilder message = new StringBuilder(); + message.append(StringUtils.replace(logForm.getMessage(), "\\n", "\n")); + + Level level = Level.toLevel(logForm.getLevel(), Level.INFO); + CLIENT_LOG.log(level, message); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class ValidateDomainsAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) throws Exception + { + // Find a valid pipeline root - we don't really care which one, we just need somewhere to write the log file + for (Container project : Arrays.asList(ContainerManager.getSharedContainer(), ContainerManager.getHomeContainer())) + { + PipeRoot root = PipelineService.get().findPipelineRoot(project); + if (root != null && root.isValid()) + { + ViewBackgroundInfo info = getViewBackgroundInfo(); + PipelineJob job = new ValidateDomainsPipelineJob(info, root); + PipelineService.get().queueJob(job); + return true; + } + } + return false; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return urlProvider(PipelineUrls.class).urlBegin(ContainerManager.getRoot()); + } + } + + public static class ModulesForm + { + private double[] _ignore = new double[0]; // Module versions to ignore (filter out of the results) + private boolean _managedOnly = false; + private boolean _unmanagedOnly = false; + + public double[] getIgnore() + { + return _ignore; + } + + public void setIgnore(double[] ignore) + { + _ignore = ignore; + } + + private Set getIgnoreSet() + { + return new LinkedHashSet<>(Arrays.asList(ArrayUtils.toObject(_ignore))); + } + + public boolean isManagedOnly() + { + return _managedOnly; + } + + @SuppressWarnings("unused") + public void setManagedOnly(boolean managedOnly) + { + _managedOnly = managedOnly; + } + + public boolean isUnmanagedOnly() + { + return _unmanagedOnly; + } + + @SuppressWarnings("unused") + public void setUnmanagedOnly(boolean unmanagedOnly) + { + _unmanagedOnly = unmanagedOnly; + } + } + + public enum ManageFilter + { + ManagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && module.shouldManageVersion(); + } + }, + UnmanagedOnly + { + @Override + public boolean accept(Module module) + { + return null != module && !module.shouldManageVersion(); + } + }, + All + { + @Override + public boolean accept(Module module) + { + return true; + } + }; + + public abstract boolean accept(Module module); + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public class ModulesAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModulesForm form, BindException errors) + { + ModuleLoader ml = ModuleLoader.getInstance(); + boolean hasAdminOpsPerm = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); + + Collection unknownModules = ml.getUnknownModuleContexts().values(); + Collection knownModules = ml.getAllModuleContexts(); + knownModules.removeAll(unknownModules); + + Set ignoreSet = form.getIgnoreSet(); + HtmlString managedLink = HtmlString.EMPTY_STRING; + HtmlString unmanagedLink = HtmlString.EMPTY_STRING; + + // Option to filter out all modules whose version shouldn't be managed, or whose version matches the previous release + // version or 0.00. This can be helpful during the end-of-release consolidation process. Show the link only in dev mode. + if (AppProps.getInstance().isDevMode()) + { + if (ignoreSet.isEmpty() && !form.isManagedOnly()) + { + String lowestSchemaVersion = ModuleContext.formatVersion(Constants.getLowestSchemaVersion()); + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("ignore", "0.00," + lowestSchemaVersion); + url.addParameter("managedOnly", true); + managedLink = LinkBuilder.labkeyLink("Click here to ignore null, " + lowestSchemaVersion + " and unmanaged modules", url).getHtmlString(); + } + else + { + List ignore = ignoreSet + .stream() + .map(ModuleContext::formatVersion) + .collect(Collectors.toCollection(LinkedList::new)); + + String ignoreString = ignore.isEmpty() ? null : ignore.toString(); + String unmanaged = form.isManagedOnly() ? "unmanaged" : null; + + managedLink = HtmlString.of("(Currently ignoring " + Joiner.on(" and ").skipNulls().join(new String[]{ignoreString, unmanaged}) + ") "); + } + + if (!form.isUnmanagedOnly()) + { + ActionURL url = new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + url.addParameter("unmanagedOnly", true); + unmanagedLink = LinkBuilder.labkeyLink("Click here to show unmanaged modules only", url).getHtmlString(); + } + else + { + unmanagedLink = HtmlString.of("(Currently showing unmanaged modules only)"); + } + } + + ManageFilter filter = form.isManagedOnly() ? ManageFilter.ManagedOnly : (form.isUnmanagedOnly() ? ManageFilter.UnmanagedOnly : ManageFilter.All); + + HtmlStringBuilder deleteInstructions = HtmlStringBuilder.of(); + if (hasAdminOpsPerm) + { + deleteInstructions.unsafeAppend("

    ").append( + "To delete a module that does not have a delete link, first delete its .module file and exploded module directory from your Labkey deployment directory, and restart the server. " + + "Module files are typically deployed in /modules and /externalModules.") + .unsafeAppend("

    ").append( + LinkBuilder.labkeyLink("Create new empty module", getCreateURL())); + } + + HtmlStringBuilder docLink = HtmlStringBuilder.of(); + docLink.unsafeAppend("

    ").append("Additional modules available, click ").append(new HelpTopic("defaultModules").getSimpleLinkHtml("here")).append(" to learn more."); + + HtmlStringBuilder knownDescription = HtmlStringBuilder.of() + .append("Each of these modules is installed and has a valid module file. ").append(managedLink).append(unmanagedLink).append(deleteInstructions).append(docLink); + HttpView known = new ModulesView(knownModules, "Known", knownDescription.getHtmlString(), null, ignoreSet, filter); + + HtmlStringBuilder unknownDescription = HtmlStringBuilder.of() + .append(1 == unknownModules.size() ? "This module" : "Each of these modules").append(" has been installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. Possible explanations: the " + + "module is no longer part of the deployed distribution, the module has been renamed, the server location where the module " + + "is stored is not accessible, or the module file is corrupted.") + .unsafeAppend("

    ").append("The delete links below will remove all record of a module from the database tables."); + HtmlString noModulesDescription = HtmlString.of("A module is considered \"unknown\" if it was installed on this server " + + "in the past but the corresponding module file is currently missing or invalid. This server has no unknown modules."); + HttpView unknown = new ModulesView(unknownModules, "Unknown", unknownDescription.getHtmlString(), noModulesDescription, Collections.emptySet(), filter); + + return new VBox(known, unknown); + } + + private class ModulesView extends WebPartView + { + private final Collection _contexts; + private final HtmlString _descriptionHtml; + private final HtmlString _noModulesDescriptionHtml; + private final Set _ignoreVersions; + private final ManageFilter _manageFilter; + + private ModulesView(Collection contexts, String type, HtmlString descriptionHtml, HtmlString noModulesDescriptionHtml, Set ignoreVersions, ManageFilter manageFilter) + { + super(FrameType.PORTAL); + List sorted = new ArrayList<>(contexts); + sorted.sort(Comparator.comparing(ModuleContext::getName, String.CASE_INSENSITIVE_ORDER)); + + _contexts = sorted; + _descriptionHtml = descriptionHtml; + _noModulesDescriptionHtml = noModulesDescriptionHtml; + _ignoreVersions = ignoreVersions; + _manageFilter = manageFilter; + setTitle(type + " Modules"); + } + + @Override + protected void renderView(Object model, HtmlWriter out) + { + boolean isDevMode = AppProps.getInstance().isDevMode(); + boolean hasAdminOpsPerm = getUser().hasRootPermission(AdminOperationsPermission.class); + boolean hasUploadModulePerm = getUser().hasRootPermission(UploadFileBasedModulePermission.class); + final AtomicInteger rowCount = new AtomicInteger(); + ExplodedModuleService moduleService = !hasUploadModulePerm ? null : ServiceRegistry.get().getService(ExplodedModuleService.class); + final File externalModulesDir = moduleService==null ? null : moduleService.getExternalModulesDirectory(); + final Path relativeRoot = ModuleLoader.getInstance().getCoreModule().getExplodedPath().getParentFile().getParentFile().toPath(); + + if (_contexts.isEmpty()) + { + out.write(_noModulesDescriptionHtml); + } + else + { + DIV( + DIV(_descriptionHtml), + TABLE(cl("labkey-data-region-legacy","labkey-show-borders","labkey-data-region-header-lock"), + TR( + TD(cl("labkey-column-header"),"Name"), + TD(cl("labkey-column-header"),"Release Version"), + TD(cl("labkey-column-header"),"Schema Version"), + TD(cl("labkey-column-header"),"Class"), + TD(cl("labkey-column-header"),"Location"), + TD(cl("labkey-column-header"),"Schemas"), + !AppProps.getInstance().isDevMode() ? null : TD(cl("labkey-column-header"),""), // edit actions + null == externalModulesDir ? null : TD(cl("labkey-column-header"),""), // upload actions + !hasAdminOpsPerm ? null : TD(cl("labkey-column-header"),"") // delete actions + ), + _contexts.stream() + .filter(moduleContext -> !_ignoreVersions.contains(moduleContext.getInstalledVersion())) + .map(moduleContext -> new Pair<>(moduleContext,ModuleLoader.getInstance().getModule(moduleContext.getName()))) + .filter(pair -> _manageFilter.accept(pair.getValue())) + .map(pair -> + { + ModuleContext moduleContext = pair.getKey(); + Module module = pair.getValue(); + List schemas = moduleContext.getSchemaList(); + Double schemaVersion = moduleContext.getSchemaVersion(); + boolean replaceableModule = false; + if (null != module && module.getClass() == SimpleModule.class && schemas.isEmpty()) + { + File zip = module.getZippedPath(); + if (null != zip && zip.getParentFile().equals(externalModulesDir)) + replaceableModule = true; + } + boolean deleteableModule = replaceableModule || null == module; + String className = StringUtils.trimToEmpty(moduleContext.getClassName()); + String fullPathToModule = ""; + String shortPathToModule = ""; + if (null != module) + { + Path p = module.getExplodedPath().toPath(); + if (null != module.getZippedPath()) + p = module.getZippedPath().toPath(); + if (isDevMode && ModuleEditorService.get().canEditSourceModule(module)) + if (!module.getExplodedPath().getPath().equals(module.getSourcePath())) + p = Paths.get(module.getSourcePath()); + fullPathToModule = p.toString(); + shortPathToModule = fullPathToModule; + Path rel = relativeRoot.relativize(p); + if (!rel.startsWith("..")) + shortPathToModule = rel.toString(); + } + ActionURL moduleEditorUrl = getModuleEditorURL(moduleContext.getName()); + + return TR(cl(rowCount.getAndIncrement()%2==0 ? "labkey-alternate-row" : "labkey-row").at(style,"vertical-align:top;"), + TD(moduleContext.getName()), + TD(at(style,"white-space:nowrap;"), null != module ? module.getReleaseVersion() : NBSP), + TD(null != schemaVersion ? ModuleContext.formatVersion(schemaVersion) : NBSP), + TD(SPAN(at(title,className), className.substring(className.lastIndexOf(".")+1))), + TD(SPAN(at(title,fullPathToModule),shortPathToModule)), + TD(schemas.stream().map(s -> createHtmlFragment(s, BR()))), + !AppProps.getInstance().isDevMode() ? null : TD((null == moduleEditorUrl) ? NBSP : LinkBuilder.labkeyLink("Edit module", moduleEditorUrl)), + null == externalModulesDir ? null : TD(!replaceableModule ? NBSP : LinkBuilder.labkeyLink("Upload Module", getUpdateURL(moduleContext.getName()))), + !hasAdminOpsPerm ? null : TD(!deleteableModule ? NBSP : LinkBuilder.labkeyLink("Delete Module" + (schemas.isEmpty() ? "" : (" and Schema" + (schemas.size() > 1 ? "s" : ""))), getDeleteURL(moduleContext.getName()))) + ); + }) + ) + ).appendTo(out); + } + } + } + + private ActionURL getDeleteURL(String name) + { + ActionURL url = ModuleEditorService.get().getDeleteModuleURL(name); + if (null != url) + return url; + url = new ActionURL(DeleteModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getUpdateURL(String name) + { + ActionURL url = ModuleEditorService.get().getUpdateModuleURL(name); + if (null != url) + return url; + url = new ActionURL(UpdateModuleAction.class, ContainerManager.getRoot()); + url.addParameter("name", name); + return url; + } + + private ActionURL getModuleEditorURL(String name) + { + return ModuleEditorService.get().getModuleEditorURL(name); + } + + private ActionURL getCreateURL() + { + ActionURL url = ModuleEditorService.get().getCreateModuleURL(); + if (null != url) + return url; + url = new ActionURL(CreateModuleAction.class, ContainerManager.getRoot()); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("defaultModules"); + addAdminNavTrail(root, "Modules", getClass()); + } + } + + public static class SchemaVersionTestCase extends Assert + { + @Test + public void verifyMinimumSchemaVersion() + { + List modulesTooLow = ModuleLoader.getInstance().getModules().stream() + .filter(ManageFilter.ManagedOnly::accept) + .filter(m -> null != m.getSchemaVersion()) + .filter(m -> m.getSchemaVersion() > 0.00 && m.getSchemaVersion() < Constants.getLowestSchemaVersion()) + .toList(); + + if (!modulesTooLow.isEmpty()) + fail("The following module" + (1 == modulesTooLow.size() ? " needs its schema version" : "s need their schema versions") + " increased to " + ModuleContext.formatVersion(Constants.getLowestSchemaVersion()) + ": " + modulesTooLow); + } + + @Test + public void modulesWithSchemaVersionButNoScripts() + { + // Flag all modules that have a schema version but don't have scripts. Their schema version should be null. + List moduleNames = ModuleLoader.getInstance().getModules().stream() + .filter(m -> m.getSchemaVersion() != null) + .filter(m -> m instanceof DefaultModule dm && !dm.hasScripts()) + .map(m -> m.getName() + ": " + m.getSchemaVersion()) + .toList(); + + if (!moduleNames.isEmpty()) + fail("The following module" + (1 == moduleNames.size() ? "" : "s") + " should have a null schema version: " + moduleNames); + } + } + + public static class ModuleForm + { + private String _name; + + public String getName() + { + return _name; + } + + @SuppressWarnings({"UnusedDeclaration"}) + public void setName(String name) + { + _name = name; + } + + @NotNull + private ModuleContext getModuleContext() + { + ModuleLoader ml = ModuleLoader.getInstance(); + ModuleContext ctx = ml.getModuleContextFromDatabase(getName()); + + if (null == ctx) + throw new NotFoundException("Module not found"); + + return ctx; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class DeleteModuleAction extends ConfirmAction + { + @Override + public void validateCommand(ModuleForm form, Errors errors) + { + } + + @Override + public ModelAndView getConfirmView(ModuleForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Module"); + + ModuleContext ctx = form.getModuleContext(); + Module module = ModuleLoader.getInstance().getModule(ctx.getName()); + boolean hasSchemas = !ctx.getSchemaList().isEmpty(); + boolean hasFiles = false; + if (null != module) + hasFiles = null!=module.getExplodedPath() && module.getExplodedPath().isDirectory() || null!=module.getZippedPath() && module.getZippedPath().isFile(); + + HtmlStringBuilder description = HtmlStringBuilder.of("\"" + ctx.getName() + "\" module"); + HtmlStringBuilder skippedSchemas = HtmlStringBuilder.of(); + if (hasSchemas) + { + SchemaActions schemaActions = ModuleLoader.getInstance().getSchemaActions(module, ctx); + List deleteList = schemaActions.deleteList(); + List skipList = schemaActions.skipList(); + + // List all the schemas that will be deleted + if (!deleteList.isEmpty()) + { + description.append(" and delete all data in "); + description.append(deleteList.size() > 1 ? "these schemas: " + StringUtils.join(deleteList, ", ") : "the \"" + deleteList.get(0) + "\" schema"); + } + + // For unknown modules, also list the schemas that won't be deleted + if (!skipList.isEmpty()) + { + skippedSchemas.append(HtmlString.BR); + skipList.forEach(sam -> skippedSchemas.append(HtmlString.BR) + .append("Note: Schema \"") + .append(sam.schema()) + .append("\" will not be deleted because it's in use by module \"") + .append(sam.module()) + .append("\"")); + } + } + + return new HtmlView(DIV( + !hasFiles ? null : DIV(cl("labkey-warning-messages"), + "This module still has files on disk. Consider, first stopping the server, deleting these files, and restarting the server before continuing.", + null==module.getExplodedPath()?null:UL(LI(module.getExplodedPath().getPath())), + null==module.getZippedPath()?null:UL(LI(module.getZippedPath().getPath())) + ), + BR(), + "Are you sure you want to remove the ", description, "? ", + "This operation cannot be undone!", + skippedSchemas, + BR(), + !hasFiles ? null : "Deleting modules on a running server could leave it in an unpredictable state; be sure to restart your server." + )); + } + + @Override + public boolean handlePost(ModuleForm form, BindException errors) + { + ModuleLoader.getInstance().removeModule(form.getModuleContext()); + + return true; + } + + @Override + public @NotNull URLHelper getSuccessURL(ModuleForm form) + { + return new ActionURL(ModulesAction.class, ContainerManager.getRoot()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class UpdateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class CreateModuleAction extends SimpleViewAction + { + @Override + public ModelAndView getView(ModuleForm moduleForm, BindException errors) throws Exception + { + return new HtmlView(HtmlString.of("This is a premium feature, please refer to our documentation on www.labkey.org")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + public static class OptionalFeatureForm + { + private String feature; + private boolean enabled; + + public String getFeature() + { + return feature; + } + + public void setFeature(String feature) + { + this.feature = feature; + } + + public boolean isEnabled() + { + return enabled; + } + + public void setEnabled(boolean enabled) + { + this.enabled = enabled; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + @ActionNames("OptionalFeature, ExperimentalFeature") + public static class OptionalFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(OptionalFeatureForm form, BindException errors) + { + String feature = StringUtils.trimToNull(form.getFeature()); + if (feature == null) + throw new ApiUsageException("feature is required"); + + OptionalFeatureService svc = OptionalFeatureService.get(); + if (svc == null) + throw new IllegalStateException(); + + Map ret = new HashMap<>(); + ret.put("feature", feature); + + if (isPost()) + { + ret.put("previouslyEnabled", svc.isFeatureEnabled(feature)); + svc.setFeatureEnabled(feature, form.isEnabled(), getUser()); + } + + ret.put("enabled", svc.isFeatureEnabled(feature)); + return new ApiSimpleResponse(ret); + } + } + + public static class OptionalFeaturesForm + { + private String _type; + private boolean _showHidden; + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + public @NotNull FeatureType getTypeEnum() + { + return EnumUtils.getEnum(FeatureType.class, getType(), FeatureType.Experimental); + } + + public boolean isShowHidden() + { + return _showHidden; + } + + @SuppressWarnings("unused") + public void setShowHidden(boolean showHidden) + { + _showHidden = showHidden; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class OptionalFeaturesAction extends SimpleViewAction + { + private FeatureType _type; + + @Override + public ModelAndView getView(OptionalFeaturesForm form, BindException errors) + { + _type = form.getTypeEnum(); + JspView view = new JspView<>("/org/labkey/core/admin/optionalFeatures.jsp", form); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("experimental"); + addAdminNavTrail(root, _type.name() + " Features", getClass()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ProductFeatureAction extends BaseApiAction + { + @Override + protected ModelAndView handleGet() throws Exception + { + return handlePost(); // 'execute' ensures that only POSTs are mutating + } + + @Override + public ApiResponse execute(ProductConfigForm form, BindException errors) + { + String productKey = StringUtils.trimToNull(form.getProductKey()); + + Map ret = new HashMap<>(); + + if (isPost()) + { + ProductConfiguration.setProductKey(productKey); + } + + ret.put("productKey", new ProductConfiguration().getCurrentProductKey()); + return new ApiSimpleResponse(ret); + } + } + + public static class ProductConfigForm + { + private String productKey; + + public String getProductKey() + { + return productKey; + } + + public void setProductKey(String productKey) + { + this.productKey = productKey; + } + + } + + @AdminConsoleAction + @RequiresPermission(AdminOperationsPermission.class) + public class ProductConfigurationAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Product Configuration", getClass()); + } + + @Override + public ModelAndView getView(Object o, BindException errors) throws Exception + { + JspView view = new JspView<>("/org/labkey/core/admin/productConfiguration.jsp"); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + } + + + public static class FolderTypesBean + { + private final Collection _allFolderTypes; + private final Collection _enabledFolderTypes; + private final FolderType _defaultFolderType; + + public FolderTypesBean(Collection allFolderTypes, Collection enabledFolderTypes, FolderType defaultFolderType) + { + _allFolderTypes = allFolderTypes; + _enabledFolderTypes = enabledFolderTypes; + _defaultFolderType = defaultFolderType; + } + + public Collection getAllFolderTypes() + { + return _allFolderTypes; + } + + public Collection getEnabledFolderTypes() + { + return _enabledFolderTypes; + } + + public FolderType getDefaultFolderType() + { + return _defaultFolderType; + } + } + + @AdminConsoleAction + @RequiresPermission(AdminPermission.class) + public class FolderTypesAction extends FormViewAction + { + @Override + public void validateCommand(Object form, Errors errors) + { + } + + @Override + public ModelAndView getView(Object form, boolean reshow, BindException errors) + { + FolderTypesBean bean; + if (reshow) + { + bean = getOptionsFromRequest(); + } + else + { + FolderTypeManager manager = FolderTypeManager.get(); + var defaultFolderType = manager.getDefaultFolderType(); + // If a default folder type has not yet been configuration use "Collaboration" folder type as the default + defaultFolderType = defaultFolderType != null ? defaultFolderType : manager.getFolderType(CollaborationFolderType.TYPE_NAME); + boolean userHasEnableRestrictedModulesPermission = getContainer().hasEnableRestrictedModules(getUser()); + bean = new FolderTypesBean(manager.getAllFolderTypes(), manager.getEnabledFolderTypes(userHasEnableRestrictedModulesPermission), defaultFolderType); + } + + return new JspView<>("/org/labkey/core/admin/enabledFolderTypes.jsp", bean, errors); + } + + @Override + public boolean handlePost(Object form, BindException errors) + { + FolderTypesBean bean = getOptionsFromRequest(); + var defaultFolderType = bean.getDefaultFolderType(); + if (defaultFolderType == null) + { + errors.reject(ERROR_MSG, "Please select a default folder type."); + return false; + } + var enabledFolderTypes = bean.getEnabledFolderTypes(); + if (!enabledFolderTypes.contains(defaultFolderType)) + { + errors.reject(ERROR_MSG, "Folder type selected as the default, '" + defaultFolderType.getName() + "', must be enabled."); + return false; + } + + FolderTypeManager.get().setEnabledFolderTypes(enabledFolderTypes, defaultFolderType); + return true; + } + + private FolderTypesBean getOptionsFromRequest() + { + var allFolderTypes = FolderTypeManager.get().getAllFolderTypes(); + List enabledFolderTypes = new ArrayList<>(); + FolderType defaultFolderType = null; + String defaultFolderTypeParam = getViewContext().getRequest().getParameter(FolderTypeManager.FOLDER_TYPE_DEFAULT); + + for (FolderType folderType : FolderTypeManager.get().getAllFolderTypes()) + { + boolean enabled = Boolean.TRUE.toString().equalsIgnoreCase(getViewContext().getRequest().getParameter(folderType.getName())); + if (enabled) + { + enabledFolderTypes.add(folderType); + } + if (folderType.getName().equals(defaultFolderTypeParam)) + { + defaultFolderType = folderType; + } + } + return new FolderTypesBean(allFolderTypes, enabledFolderTypes, defaultFolderType); + } + + @Override + public URLHelper getSuccessURL(Object form) + { + return getShowAdminURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Folder Types", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class CustomizeMenuAction extends MutatingApiAction + { + @Override + public ApiResponse execute(CustomizeMenuForm form, BindException errors) + { + if (null != form.getUrl()) + { + String errorMessage = StringExpressionFactory.validateURL(form.getUrl()); + if (null != errorMessage) + { + errors.reject(ERROR_MSG, errorMessage); + return new ApiSimpleResponse("success", false); + } + } + + setCustomizeMenuForm(form, getContainer(), getUser()); + return new ApiSimpleResponse("success", true); + } + } + + protected static final String CUSTOMMENU_SCHEMA = "customMenuSchemaName"; + protected static final String CUSTOMMENU_QUERY = "customMenuQueryName"; + protected static final String CUSTOMMENU_VIEW = "customMenuViewName"; + protected static final String CUSTOMMENU_COLUMN = "customMenuColumnName"; + protected static final String CUSTOMMENU_FOLDER = "customMenuFolderName"; + protected static final String CUSTOMMENU_TITLE = "customMenuTitle"; + protected static final String CUSTOMMENU_URL = "customMenuUrl"; + protected static final String CUSTOMMENU_ROOTFOLDER = "customMenuRootFolder"; + protected static final String CUSTOMMENU_FOLDERTYPES = "customMenuFolderTypes"; + protected static final String CUSTOMMENU_CHOICELISTQUERY = "customMenuChoiceListQuery"; + protected static final String CUSTOMMENU_INCLUDEALLDESCENDANTS = "customIncludeAllDescendants"; + protected static final String CUSTOMMENU_CURRENTPROJECTONLY = "customCurrentProjectOnly"; + + public static CustomizeMenuForm getCustomizeMenuForm(Portal.WebPart webPart) + { + CustomizeMenuForm form = new CustomizeMenuForm(); + Map menuProps = webPart.getPropertyMap(); + + String schemaName = menuProps.get(CUSTOMMENU_SCHEMA); + String queryName = menuProps.get(CUSTOMMENU_QUERY); + String columnName = menuProps.get(CUSTOMMENU_COLUMN); + String viewName = menuProps.get(CUSTOMMENU_VIEW); + String folderName = menuProps.get(CUSTOMMENU_FOLDER); + String title = menuProps.get(CUSTOMMENU_TITLE); if (null == title) title = "My Menu"; + String urlBottom = menuProps.get(CUSTOMMENU_URL); + String rootFolder = menuProps.get(CUSTOMMENU_ROOTFOLDER); + String folderTypes = menuProps.get(CUSTOMMENU_FOLDERTYPES); + String choiceListQueryString = menuProps.get(CUSTOMMENU_CHOICELISTQUERY); + boolean choiceListQuery = null == choiceListQueryString || choiceListQueryString.equalsIgnoreCase("true"); + String includeAllDescendantsString = menuProps.get(CUSTOMMENU_INCLUDEALLDESCENDANTS); + boolean includeAllDescendants = null == includeAllDescendantsString || includeAllDescendantsString.equalsIgnoreCase("true"); + String currentProjectOnlyString = menuProps.get(CUSTOMMENU_CURRENTPROJECTONLY); + boolean currentProjectOnly = null != currentProjectOnlyString && currentProjectOnlyString.equalsIgnoreCase("true"); + + form.setSchemaName(schemaName); + form.setQueryName(queryName); + form.setColumnName(columnName); + form.setViewName(viewName); + form.setFolderName(folderName); + form.setTitle(title); + form.setUrl(urlBottom); + form.setRootFolder(rootFolder); + form.setFolderTypes(folderTypes); + form.setChoiceListQuery(choiceListQuery); + form.setIncludeAllDescendants(includeAllDescendants); + form.setCurrentProjectOnly(currentProjectOnly); + + form.setWebPartIndex(webPart.getIndex()); + form.setPageId(webPart.getPageId()); + return form; + } + + private static void setCustomizeMenuForm(CustomizeMenuForm form, Container container, User user) + { + Portal.WebPart webPart = Portal.getPart(container, form.getPageId(), form.getWebPartIndex()); + if (null == webPart) + throw new NotFoundException(); + Map menuProps = webPart.getPropertyMap(); + + menuProps.put(CUSTOMMENU_SCHEMA, form.getSchemaName()); + menuProps.put(CUSTOMMENU_QUERY, form.getQueryName()); + menuProps.put(CUSTOMMENU_COLUMN, form.getColumnName()); + menuProps.put(CUSTOMMENU_VIEW, form.getViewName()); + menuProps.put(CUSTOMMENU_FOLDER, form.getFolderName()); + menuProps.put(CUSTOMMENU_TITLE, form.getTitle()); + menuProps.put(CUSTOMMENU_URL, form.getUrl()); + + // If root folder not specified, set as current container + menuProps.put(CUSTOMMENU_ROOTFOLDER, StringUtils.trimToNull(form.getRootFolder()) != null ? form.getRootFolder() : container.getPath()); + menuProps.put(CUSTOMMENU_FOLDERTYPES, form.getFolderTypes()); + menuProps.put(CUSTOMMENU_CHOICELISTQUERY, form.isChoiceListQuery() ? "true" : "false"); + menuProps.put(CUSTOMMENU_INCLUDEALLDESCENDANTS, form.isIncludeAllDescendants() ? "true" : "false"); + menuProps.put(CUSTOMMENU_CURRENTPROJECTONLY, form.isCurrentProjectOnly() ? "true" : "false"); + + Portal.updatePart(user, webPart); + } + + @RequiresPermission(AdminPermission.class) + public static class AddTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + if(tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot add tabs to custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + // Note: The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived + // from the name, and is editable, is allowed to be 64 characters, so we only error if passed something + // longer than 64 characters. + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + if (name.length() > 50) + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + CaseInsensitiveHashMap folderTabMap = new CaseInsensitiveHashMap<>(); + + for (FolderTab tab : tabContainer.getFolderType().getDefaultTabs()) + { + folderTabMap.put(tab.getName(), tab); + } + + if (pages.containsKey(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (page.getCaption() != null && page.getCaption().equals(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + else if (folderTabMap.containsKey(page.getPageId())) + { + if (folderTabMap.get(page.getPageId()).getCaption(getViewContext()).equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab of the same name already exists in this folder."); + return; + } + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + validateCommand(form, errors); + + if(errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + String name = form.getTabName(); + String caption = form.getTabName(); + + // The name, which shows up on the url, is trimmed to 50 characters. The caption, which is derived from the + // name, and is editable, is allowed to be 64 characters. + if (name.length() > 50) + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 50).trim(); + + Portal.saveParts(container, name); + Portal.addProperty(container, name, Portal.PROP_CUSTOMTAB); + + if (!name.equals(caption)) + { + // If we had to truncate the name then we want to set the caption to the un-truncated version of the name. + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(name); + // Get a mutable copy + page = page.copy(); + page.setCaption(caption); + Portal.updatePortalPage(container, page); + } + + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, container); + tabURL.addParameter("pageId", name); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ShowTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(getContainer().getContainerFor(ContainerType.DataType.tabParent), true)); + + if (form.getTabPageId() == null) + { + errors.reject(ERROR_MSG, "PageId cannot be blank."); + } + + if (!pages.containsKey(form.getTabPageId())) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + validateCommand(form, errors); + if (errors.hasErrors()) + return response; + + Portal.showPage(tabContainer, form.getTabPageId()); + ActionURL tabURL = new ActionURL(ProjectController.BeginAction.class, tabContainer); + tabURL.addParameter("pageId", form.getTabPageId()); + response.put("url", tabURL); + response.put("success", true); + return response; + } + } + + + public static class TabActionForm extends ReturnUrlForm + { + // This class is used for tab related actions (add, rename, show, etc.) + String _tabName; + String _tabPageId; + + public String getTabName() + { + return _tabName; + } + + public void setTabName(String name) + { + _tabName = name; + } + + public String getTabPageId() + { + return _tabPageId; + } + + public void setTabPageId(String tabPageId) + { + _tabPageId = tabPageId; + } + } + + @RequiresPermission(AdminPermission.class) + public class MoveTabAction extends MutatingApiAction + { + @Override + public ApiResponse execute(MoveTabForm form, BindException errors) + { + final Map properties = new HashMap<>(); + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage tab = pages.get(form.getPageId()); + + if (null != tab) + { + int oldIndex = tab.getIndex(); + Portal.PortalPage pageToSwap = handleMovePortalPage(tabContainer, getUser(), tab, form.getDirection()); + + if (null != pageToSwap) + { + properties.put("oldIndex", oldIndex); + properties.put("newIndex", tab.getIndex()); + properties.put("pageId", tab.getPageId()); + properties.put("pageIdToSwap", pageToSwap.getPageId()); + } + else + { + properties.put("error", "Unable to move tab."); + } + } + else + { + properties.put("error", "Requested tab does not exist."); + } + + return new ApiSimpleResponse(properties); + } + } + + public static class MoveTabForm implements HasViewContext + { + private int _direction; + private String _pageId; + private ViewContext _viewContext; + + public int getDirection() + { + // 0 moves left, 1 moves right. + return _direction; + } + + public void setDirection(int direction) + { + _direction = direction; + } + + public String getPageId() + { + return _pageId; + } + + public void setPageId(String pageId) + { + _pageId = pageId; + } + + @Override + public ViewContext getViewContext() + { + return _viewContext; + } + + @Override + public void setViewContext(ViewContext viewContext) + { + _viewContext = viewContext; + } + } + + private Portal.PortalPage handleMovePortalPage(Container c, User user, Portal.PortalPage page, int direction) + { + Map pageMap = new CaseInsensitiveHashMap<>(); + for (Portal.PortalPage pp : Portal.getTabPages(c, true)) + pageMap.put(pp.getPageId(), pp); + + for (FolderTab folderTab : c.getFolderType().getDefaultTabs()) + { + if (pageMap.containsKey(folderTab.getName())) + { + // Issue 46233 : folder tabs can conditionally hide/show themselves at render time, these need to + // be excluded when adjusting the relative indexes. + if (!folderTab.isVisible(c, user)) + pageMap.remove(folderTab.getName()); + } + } + List pagesList = new ArrayList<>(pageMap.values()); + pagesList.sort(Comparator.comparingInt(Portal.PortalPage::getIndex)); + + int visibleIndex; + for (visibleIndex = 0; visibleIndex < pagesList.size(); visibleIndex++) + { + if (pagesList.get(visibleIndex).getIndex() == page.getIndex()) + { + break; + } + } + + if (visibleIndex == pagesList.size()) + { + return null; + } + + if (direction == Portal.MOVE_DOWN) + { + if (visibleIndex == pagesList.size() - 1) + { + return page; + } + + Portal.PortalPage nextPage = pagesList.get(visibleIndex + 1); + + if (null == nextPage) + return null; + Portal.swapPageIndexes(c, page, nextPage); + return nextPage; + } + else + { + if (visibleIndex < 1) + { + return page; + } + + Portal.PortalPage prevPage = pagesList.get(visibleIndex - 1); + + if (null == prevPage) + return null; + Portal.swapPageIndexes(c, page, prevPage); + return prevPage; + } + } + + @RequiresPermission(AdminPermission.class) + public static class RenameTabAction extends MutatingApiAction + { + public void validateCommand(TabActionForm form, Errors errors) + { + Container tabContainer = getContainer().getContainerFor(ContainerType.DataType.tabParent); + + if (tabContainer.getFolderType() == FolderType.NONE) + { + errors.reject(ERROR_MSG, "Cannot change tab names in custom folder types."); + } + else + { + String name = form.getTabName(); + if (StringUtils.isEmpty(name)) + { + errors.reject(ERROR_MSG, "A tab name must be specified."); + return; + } + + if (name.length() > 64) + { + errors.reject(ERROR_MSG, "Tab name cannot be longer than 64 characters."); + return; + } + + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(tabContainer, true)); + Portal.PortalPage pageToChange = pages.get(form.getTabPageId()); + if (null == pageToChange) + { + errors.reject(ERROR_MSG, "Page cannot be found. Check with your system administrator."); + return; + } + + for (Portal.PortalPage page : pages.values()) + { + if (!page.equals(pageToChange)) + { + if (null != page.getCaption() && page.getCaption().equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + if (page.getPageId().equalsIgnoreCase(name)) + { + if (null != page.getCaption() || Portal.DEFAULT_PORTAL_PAGE_ID.equalsIgnoreCase(name)) + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + else + errors.reject(ERROR_MSG, "A tab with the same name already exists in this folder."); + return; + } + } + } + + List folderTabs = tabContainer.getFolderType().getDefaultTabs(); + for (FolderTab folderTab : folderTabs) + { + String folderTabCaption = folderTab.getCaption(getViewContext()); + if (!folderTab.getName().equalsIgnoreCase(pageToChange.getPageId()) && null != folderTabCaption && folderTabCaption.equalsIgnoreCase(name)) + { + errors.reject(ERROR_MSG, "You cannot change a tab's name to another tab's original name even if the original name is not visible."); + return; + } + } + } + } + + @Override + public ApiResponse execute(TabActionForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + validateCommand(form, errors); + + if (errors.hasErrors()) + { + return response; + } + + Container container = getContainer().getContainerFor(ContainerType.DataType.tabParent); + CaseInsensitiveHashMap pages = new CaseInsensitiveHashMap<>(Portal.getPages(container, true)); + Portal.PortalPage page = pages.get(form.getTabPageId()); + page = page.copy(); + page.setCaption(form.getTabName()); + // Update the page the caption is saved. + Portal.updatePortalPage(container, page); + + response.put("success", true); + return response; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ClearDeletedTabFoldersAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeletedFoldersForm form, BindException errors) + { + if (isBlank(form.getContainerPath())) + throw new NotFoundException(); + Container container = ContainerManager.getForPath(form.getContainerPath()); + for (String tabName : form.getResurrectFolders()) + { + ContainerManager.clearContainerTabDeleted(container, tabName, form.getNewFolderType()); + } + return new ApiSimpleResponse("success", true); + } + } + + @SuppressWarnings("unused") + public static class DeletedFoldersForm + { + private String _containerPath; + private String _newFolderType; + private List _resurrectFolders; + + public List getResurrectFolders() + { + return _resurrectFolders; + } + + public void setResurrectFolders(List resurrectFolders) + { + _resurrectFolders = resurrectFolders; + } + + public String getContainerPath() + { + return _containerPath; + } + + public void setContainerPath(String containerPath) + { + _containerPath = containerPath; + } + + public String getNewFolderType() + { + return _newFolderType; + } + + public void setNewFolderType(String newFolderType) + { + _newFolderType = newFolderType; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetFolderTabsAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object form, BindException errors) throws Exception + { + var data = getContainer() + .getFolderType() + .getAppBar(getViewContext(), getPageConfig()) + .getButtons() + .stream() + .map(this::getProperties) + .toList(); + + return success(data); + } + + private Map getProperties(NavTree navTree) + { + Map props = new HashMap<>(); + props.put("id", navTree.getId()); + props.put("text", navTree.getText()); + props.put("href", navTree.getHref()); + props.put("disabled", navTree.isDisabled()); + return props; + } + } + + @SuppressWarnings("unused") + public static class ShortURLForm + { + private String _shortURL; + private String _fullURL; + private boolean _delete; + + private List _savedShortURLs; + + public void setShortURL(String shortURL) + { + _shortURL = shortURL; + } + + public void setFullURL(String fullURL) + { + _fullURL = fullURL; + } + + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getShortURL() + { + return _shortURL; + } + + public String getFullURL() + { + return _fullURL; + } + + public boolean isDelete() + { + return _delete; + } + } + + public abstract static class AbstractShortURLAdminAction extends FormViewAction + { + @Override + public void validateCommand(ShortURLForm target, Errors errors) {} + + @Override + public boolean handlePost(ShortURLForm form, BindException errors) throws Exception + { + String shortURL = StringUtils.trimToEmpty(form.getShortURL()); + if (StringUtils.isEmpty(shortURL)) + { + errors.addError(new LabKeyError("Short URL must not be blank")); + } + if (shortURL.endsWith(".url")) + shortURL = shortURL.substring(0,shortURL.length()-".url".length()); + if (shortURL.contains("#") || shortURL.contains("/") || shortURL.contains(".")) + { + errors.addError(new LabKeyError("Short URLs may not contain '#' or '/' or '.'")); + } + URLHelper fullURL = null; + if (!form.isDelete()) + { + String trimmedFullURL = StringUtils.trimToNull(form.getFullURL()); + if (trimmedFullURL == null) + { + errors.addError(new LabKeyError("Target URL must not be blank")); + } + else + { + try + { + fullURL = new URLHelper(trimmedFullURL); + } + catch (URISyntaxException e) + { + errors.addError(new LabKeyError("Invalid Target URL. " + e.getMessage())); + } + } + } + if (errors.getErrorCount() > 0) + { + return false; + } + + ShortURLService service = ShortURLService.get(); + if (form.isDelete()) + { + ShortURLRecord shortURLRecord = service.resolveShortURL(shortURL); + if (shortURLRecord == null) + { + throw new NotFoundException("No such short URL: " + shortURL); + } + try + { + service.deleteShortURL(shortURLRecord, getUser()); + } + catch (ValidationException e) + { + errors.addError(new LabKeyError("Error deleting short URL:")); + for(ValidationError error: e.getErrors()) + { + errors.addError(new LabKeyError(error.getMessage())); + } + } + + if (errors.getErrorCount() > 0) + { + return false; + } + } + else + { + ShortURLRecord shortURLRecord = service.saveShortURL(shortURL, fullURL, getUser()); + MutableSecurityPolicy policy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(shortURLRecord)); + // Add a role assignment to let another group manage the URL. This grants permission to the journal + // to change where the URL redirects you to after they copy the data + SecurityPolicyManager.savePolicy(policy, getUser()); + } + return true; + } + } + + @AdminConsoleAction + public class ShortURLAdminAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + JspView newView = new JspView<>("/org/labkey/core/admin/createNewShortURL.jsp", form, errors); + boolean isAppAdmin = getUser().hasRootPermission(ApplicationAdminPermission.class); + newView.setTitle(isAppAdmin ? "Create New Short URL" : "Short URLs"); + newView.setFrame(WebPartView.FrameType.PORTAL); + + QuerySettings qSettings = new QuerySettings(getViewContext(), "ShortURL", CoreQuerySchema.SHORT_URL_TABLE_NAME); + qSettings.setBaseSort(new Sort("-Created")); + QueryView existingView = new QueryView(new CoreQuerySchema(getUser(), getContainer()), qSettings, null); + if (!isAppAdmin) + { + existingView.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + } + existingView.setTitle("Existing Short URLs"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Short URL Admin", getClass()); + } + } + + @RequiresPermission(ApplicationAdminPermission.class) + public class UpdateShortURLAction extends AbstractShortURLAdminAction + { + @Override + public ModelAndView getView(ShortURLForm form, boolean reshow, BindException errors) + { + var shortUrlRecord = ShortURLService.get().resolveShortURL(form.getShortURL()); + if (shortUrlRecord == null) + { + errors.addError(new LabKeyError("Short URL does not exist: " + form.getShortURL())); + return new SimpleErrorView(errors); + } + form.setFullURL(shortUrlRecord.getFullURL()); + + JspView view = new JspView<>("/org/labkey/core/admin/updateShortURL.jsp", form, errors); + view.setTitle("Update Short URL"); + view.setFrame(WebPartView.FrameType.PORTAL); + return view; + } + + @Override + public URLHelper getSuccessURL(ShortURLForm form) + { + return new ActionURL(ShortURLAdminAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("shortURL"); + addAdminNavTrail(root, "Update Short URL", getClass()); + } + } + + // API for reporting client-side exceptions. + // UNDONE: Throttle by IP to avoid DOS from buggy clients. + @Marshal(Marshaller.Jackson) + @SuppressWarnings("UnusedDeclaration") + @RequiresLogin // Issue 52520: Prevent bots from submitting reports + @IgnoresForbiddenProjectCheck // Skip the "forbidden project" check since it disallows root + public static class LogClientExceptionAction extends MutatingApiAction + { + @Override + public Object execute(ExceptionForm form, BindException errors) + { + String errorCode = ExceptionUtil.logClientExceptionToMothership( + form.getStackTrace(), + form.getExceptionMessage(), + form.getBrowser(), + null, + form.getRequestURL(), + form.getReferrerURL(), + form.getUsername() + ); + + Map results = new HashMap<>(); + results.put("errorCode", errorCode); + results.put("loggedToMothership", errorCode != null); + + return success(results); + } + } + + @SuppressWarnings("unused") + public static class ExceptionForm + { + private String _exceptionMessage; + private String _stackTrace; + private String _requestURL; + private String _browser; + private String _username; + private String _referrerURL; + private String _file; + private String _line; + private String _platform; + + public String getExceptionMessage() + { + return _exceptionMessage; + } + + public void setExceptionMessage(String exceptionMessage) + { + _exceptionMessage = exceptionMessage; + } + + public String getUsername() + { + return _username; + } + + public void setUsername(String username) + { + _username = username; + } + + public String getStackTrace() + { + return _stackTrace; + } + + public void setStackTrace(String stackTrace) + { + _stackTrace = stackTrace; + } + + public String getRequestURL() + { + return _requestURL; + } + + public void setRequestURL(String requestURL) + { + _requestURL = requestURL; + } + + public String getBrowser() + { + return _browser; + } + + public void setBrowser(String browser) + { + _browser = browser; + } + + public String getReferrerURL() + { + return _referrerURL; + } + + public void setReferrerURL(String referrerURL) + { + _referrerURL = referrerURL; + } + + public String getFile() + { + return _file; + } + + public void setFile(String file) + { + _file = file; + } + + public String getLine() + { + return _line; + } + + public void setLine(String line) + { + _line = line; + } + + public String getPlatform() + { + return _platform; + } + + public void setPlatform(String platform) + { + _platform = platform; + } + } + + + /** generate URLS to seed web-site scanner */ + @SuppressWarnings("UnusedDeclaration") + @RequiresSiteAdmin + public static class SpiderAction extends SimpleViewAction + { + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Spider Initialization"); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + List urls = new ArrayList<>(1000); + + if (getContainer().equals(ContainerManager.getRoot())) + { + for (Container c : ContainerManager.getAllChildren(ContainerManager.getRoot())) + { + urls.add(c.getStartURL(getUser()).toString()); + urls.add(new ActionURL(SpiderAction.class, c).toString()); + } + + Container home = ContainerManager.getHomeContainer(); + for (ActionDescriptor d : SpringActionController.getRegisteredActionDescriptors()) + { + ActionURL url = new ActionURL(d.getControllerName(), d.getPrimaryName(), home); + urls.add(url.toString()); + } + } + else + { + DefaultSchema def = DefaultSchema.get(getUser(), getContainer()); + def.getSchemaNames().forEach(name -> + { + QuerySchema q = def.getSchema(name); + if (null == q) + return; + var tableNames = q.getTableNames(); + if (null == tableNames) + return; + tableNames.forEach(table -> + { + try + { + var t = q.getTable(table); + if (null != t) + { + ActionURL grid = t.getGridURL(getContainer()); + if (null != grid) + urls.add(grid.toString()); + else + urls.add(new ActionURL("query", "executeQuery.view", getContainer()) + .addParameter("schemaName", q.getSchemaName()) + .addParameter("query.queryName", t.getName()) + .toString()); + } + } + catch (Exception x) + { + // pass + } + }); + }); + + ModuleLoader.getInstance().getModules().forEach(m -> + { + ActionURL url = m.getTabURL(getContainer(), getUser()); + if (null != url) + urls.add(url.toString()); + }); + } + + return new HtmlView(DIV(urls.stream().map(url -> createHtmlFragment(A(at(href,url),url),BR())))); + } + } + + @SuppressWarnings("UnusedDeclaration") + @RequiresPermission(TroubleshooterPermission.class) + public static class TestMothershipReportAction extends ReadOnlyApiAction + { + @Override + public Object execute(MothershipReportSelectionForm form, BindException errors) throws Exception + { + MothershipReport report; + MothershipReport.Target target = form.isTestMode() ? MothershipReport.Target.test : MothershipReport.Target.local; + if (MothershipReport.Type.CheckForUpdates.toString().equals(form.getType())) + { + report = UsageReportingLevel.generateReport(UsageReportingLevel.valueOf(form.getLevel()), target); + } + else + { + report = ExceptionUtil.createReportFromThrowable(getViewContext().getRequest(), + new SQLException("Intentional exception for testing purposes", "400"), + (String)getViewContext().getRequest().getAttribute(ViewServlet.ORIGINAL_URL_STRING), + target, + ExceptionReportingLevel.valueOf(form.getLevel()), null, null, null); + } + + final Map params; + if (report == null) + { + params = new LinkedHashMap<>(); + } + else + { + params = report.getJsonFriendlyParams(); + if (form.isSubmit()) + { + report.setForwardedFor(form.getForwardedFor()); + report.run(); + if (null != report.getUpgradeMessage()) + params.put("upgradeMessage", report.getUpgradeMessage()); + } + } + if (form.isDownload()) + { + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, "metrics.json"); + } + return new ApiSimpleResponse(params); + } + } + + + static class MothershipReportSelectionForm + { + private String _type = MothershipReport.Type.CheckForUpdates.toString(); + private String _level = UsageReportingLevel.ON.toString(); + private boolean _submit = false; + private boolean _download = false; + private String _forwardedFor = null; + // indicates action is being invoked for dev/test + private boolean _testMode = false; + + public String getType() + { + return _type; + } + + public void setType(String type) + { + _type = type; + } + + public String getLevel() + { + return _level; + } + + public void setLevel(String level) + { + _level = StringUtils.upperCase(level); + } + + public boolean isSubmit() + { + return _submit; + } + + public void setSubmit(boolean submit) + { + _submit = submit; + } + + public String getForwardedFor() + { + return _forwardedFor; + } + + public void setForwardedFor(String forwardedFor) + { + _forwardedFor = forwardedFor; + } + + public boolean isTestMode() + { + return _testMode; + } + + public void setTestMode(boolean testMode) + { + _testMode = testMode; + } + + public boolean isDownload() + { + return _download; + } + + public void setDownload(boolean download) + { + _download = download; + } + } + + + @RequiresPermission(TroubleshooterPermission.class) + public class SuspiciousAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + Collection list = BlockListFilter.reportSuspicious(); + HtmlStringBuilder html = HtmlStringBuilder.of(); + if (list.isEmpty()) + { + html.append("No suspicious activity.\n"); + } + else + { + html.unsafeAppend("") + .unsafeAppend("\n"); + for (BlockListFilter.Suspicious s : list) + { + html.unsafeAppend("\n"); + } + html.unsafeAppend("
    host (user)user-agentcount
    ") + .append(s.host); + if (!isBlank(s.user)) + html.append(HtmlString.NBSP).append("(" + s.user + ")"); + html.unsafeAppend("") + .append(s.userAgent) + .unsafeAppend("") + .append(s.count) + .unsafeAppend("
    "); + } + return new HtmlView(html); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Suspicious activity", SuspiciousAction.class); + } + } + + /** This is a very crude API right now, mostly using default serialization of pre-existing objects + * NOTE: callers should expect that the return shape of this method may and will change in non-backward-compatible ways + */ + @Marshal(Marshaller.Jackson) + @RequiresNoPermission + @AllowedBeforeInitialUserIsSet + public static class ConfigurationSummaryAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) + { + if (!getContainer().isRoot()) + throw new NotFoundException("Must be invoked in the root"); + + // requires site-admin, unless there are no users + if (!UserManager.hasNoRealUsers() && !getContainer().hasPermission(getUser(), AdminOperationsPermission.class)) + throw new UnauthorizedException(); + + return getConfigurationJson(); + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + ObjectMapper result = JsonUtil.createDefaultMapper(); + result.addMixIn(ExternalScriptEngineDefinitionImpl.class, IgnorePasswordMixIn.class); + return result; + } + + /* returns a jackson serializable object that reports superset of information returned in admin console */ + private JSONObject getConfigurationJson() + { + JSONObject res = new JSONObject(); + + res.put("server", AdminBean.getPropertyMap()); + + final Map> sets = new TreeMap<>(); + new SqlSelector(CoreSchema.getInstance().getScope(), + new SQLFragment("SELECT category, name, value FROM prop.propertysets PS inner join prop.properties P on PS.\"set\" = P.\"set\"\n" + + "WHERE objectid = ? AND category IN ('SiteConfig') AND encryption='None' AND LOWER(name) NOT LIKE '%password%'", ContainerManager.getRoot())).forEachMap(m -> + { + String category = (String)m.get("category"); + String name = (String)m.get("name"); + Object value = m.get("value"); + if (!sets.containsKey(category)) + sets.put(category, new TreeMap<>()); + sets.get(category).put(name,value); + } + ); + res.put("siteSettings", sets); + + HealthCheck.Result result = HealthCheckRegistry.get().checkHealth(Arrays.asList("all")); + res.put("health", result); + + LabKeyScriptEngineManager mgr = LabKeyScriptEngineManager.get(); + res.put("scriptEngines", mgr.getEngineDefinitions()); + + return res; + } + } + + @JsonIgnoreProperties(value = { "password", "changePassword", "configuration" }) + private static class IgnorePasswordMixIn + { + } + + @AdminConsoleAction() + public class AllowListAction extends FormViewAction + { + private AllowListType _type; + + @Override + public void validateCommand(AllowListForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AllowListForm form, boolean reshow, BindException errors) + { + _type = form.getTypeEnum(); + + form.setExistingValuesList(form.getTypeEnum().getValues()); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewListValue.jsp", form, errors); + newView.setTitle("Register New " + form.getTypeEnum().getTitle()); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingListValues.jsp", form, errors); + existingView.setTitle("Existing " + form.getTypeEnum().getTitle() + "s"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + AllowListType allowListType = form.getTypeEnum(); + //handle delete of existing value + if (form.isDelete()) + { + String urlToDelete = form.getExistingValue(); + List values = new ArrayList<>(allowListType.getValues()); + for (String value : values) + { + if (null != urlToDelete && urlToDelete.trim().equalsIgnoreCase(value.trim())) + { + values.remove(value); + allowListType.setValues(values, getUser()); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated urls + else if (form.isSaveAll()) + { + Set validatedValues = form.validateValues(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(validatedValues.stream().toList(), getUser()); + } + //save new external value + else if (form.isSaveNew()) + { + Set valueSet = form.validateNewValue(errors); + if (errors.hasErrors()) + return false; + + allowListType.setValues(valueSet, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic(_type.getHelpTopic()); + addAdminNavTrail(root, String.format("%1$s Admin", _type.getTitle()), getClass()); + } + } + + public static class AllowListForm + { + private String _newValue; + private String _existingValue; + private boolean _delete; + private String _existingValues; + private boolean _saveAll; + private boolean _saveNew; + private String _type; + + private List _existingValuesList; + + public String getNewValue() + { + return _newValue; + } + + @SuppressWarnings("unused") + public void setNewValue(String newValue) + { + _newValue = newValue; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public String getExistingValues() + { + return _existingValues; + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public List getExistingValuesList() + { + //for updated urls that comes in as String values from the jsp/html form + if (null != getExistingValues()) + { + // The JavaScript delimits with "\n". Not sure where these "\r"s are coming from, but we need to strip them. + return new ArrayList<>(Arrays.asList(getExistingValues().replace("\r", "").split("\n"))); + } + return _existingValuesList; + } + + public void setExistingValuesList(List valuesList) + { + _existingValuesList = valuesList; + } + + public String getType() + { + return _type; + } + + @SuppressWarnings("unused") + public void setType(String type) + { + _type = type; + } + + @NotNull + public AllowListType getTypeEnum() + { + return EnumUtils.getEnum(AllowListType.class, getType(), AllowListType.Redirect); + } + + @JsonIgnore + public Set validateNewValue(BindException errors) + { + String value = StringUtils.trimToEmpty(getNewValue()); + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + return null; + + Set valueSet = new CaseInsensitiveHashSet(getTypeEnum().getValues()); + checkDuplicatesByAddition(value, valueSet, errors); + return valueSet; + } + + @JsonIgnore + public Set validateValues(BindException errors) + { + List values = getExistingValuesList(); //get values from the form, this includes updated values + Set valueSet = new CaseInsensitiveHashSet(); + + if (null != values && !values.isEmpty()) + { + for (String value : values) + { + getTypeEnum().validateValueFormat(value, errors); + if (errors.hasErrors()) + continue; + + checkDuplicatesByAddition(value, valueSet, errors); + } + } + + return valueSet; + } + + /** + * Adds value to value set unless it is a duplicate, in which case it adds an error + * @param value to check + * @param valueSet of existing values + * @param errors collections of errors observed + */ + @JsonIgnore + private void checkDuplicatesByAddition(String value, Set valueSet, BindException errors) + { + String trimValue = StringUtils.trimToEmpty(value); + if (!valueSet.add(trimValue)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values not allowed.", trimValue))); + } + } + + @AdminConsoleAction + public static class DeleteAllValuesAction extends FormHandlerAction + { + @Override + public void validateCommand(AllowListForm form, Errors errors) + { + } + + @Override + public boolean handlePost(AllowListForm form, BindException errors) throws Exception + { + form.getTypeEnum().setValues(Collections.emptyList(), getUser()); + return true; + } + + @Override + public URLHelper getSuccessURL(AllowListForm form) + { + return form.getTypeEnum().getSuccessURL(getContainer()); + } + } + + public static class ExternalSourcesForm + { + private boolean _delete; + private boolean _saveNew; + private boolean _saveAll; + + private String _newDirective; + private String _newHost; + private String _existingValue; + private String _existingValues; + + public boolean isDelete() + { + return _delete; + } + + @SuppressWarnings("unused") + public void setDelete(boolean delete) + { + _delete = delete; + } + + public boolean isSaveNew() + { + return _saveNew; + } + + @SuppressWarnings("unused") + public void setSaveNew(boolean saveNew) + { + _saveNew = saveNew; + } + + public boolean isSaveAll() + { + return _saveAll; + } + + @SuppressWarnings("unused") + public void setSaveAll(boolean saveAll) + { + _saveAll = saveAll; + } + + public String getNewDirective() + { + return _newDirective; + } + + @SuppressWarnings("unused") + public void setNewDirective(String newDirective) + { + _newDirective = newDirective; + } + + public String getNewHost() + { + return _newHost; + } + + @SuppressWarnings("unused") + public void setNewHost(String newHost) + { + _newHost = newHost; + } + + public String getExistingValue() + { + return _existingValue; + } + + @SuppressWarnings("unused") + public void setExistingValue(String existingValue) + { + _existingValue = existingValue; + } + + public List getExistingValues() + { + return Arrays.stream(StringUtils.trimToEmpty(_existingValues).split("\n")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .toList(); + } + + @SuppressWarnings("unused") + public void setExistingValues(String existingValues) + { + _existingValues = existingValues; + } + + private AllowedHost getExistingAllowedHost(BindException errors) + { + return getAllowedHost(getExistingValue(), errors); + } + + private AllowedHost getAllowedHost(String value, BindException errors) + { + String[] parts = value.split("\\|", 2); // Stop after the first bar to produce two parts + if (parts.length != 2) + { + errors.addError(new LabKeyError("Can't parse allowed host.")); + return null; + } + return validateHost(parts[0], parts[1], errors); + } + + private List getExistingAllowedHosts(BindException errors) + { + List existing = getExistingValues().stream() + .map(value-> getAllowedHost(value, errors)) + .toList(); + + if (errors.hasErrors()) + return null; + + return checkDuplicates(existing, errors); + } + + private List validateNewAllowedHost(BindException errors) throws JsonProcessingException + { + AllowedHost newAllowedHost = validateHost(getNewDirective(), getNewHost(), errors); + + if (errors.hasErrors()) + return null; + + List hosts = getSavedAllowedHosts(); + hosts.add(newAllowedHost); + + return checkDuplicates(hosts, errors); + } + + // Lenient for now: no unknown directives, no blank hosts or hosts with semicolons + public static AllowedHost validateHost(String directiveString, String host, BindException errors) + { + AllowedHost ret = null; + + if (StringUtils.isEmpty(directiveString)) + { + errors.addError(new LabKeyError("Directive must not be blank")); + } + else if (StringUtils.isEmpty(host)) + { + errors.addError(new LabKeyError("Host must not be blank")); + } + else if (host.contains(";")) + { + errors.addError(new LabKeyError("Semicolons are not allowed in host names")); + } + else + { + Directive directive = EnumUtils.getEnum(Directive.class, directiveString); + + if (null == directive) + { + errors.addError(new LabKeyError("Unknown directive: " + directiveString)); + } + else + { + ret = new AllowedHost(directive, host.trim()); + } + } + + return ret; + } + + /** + * Check for duplicates in hosts: within each Directive, hosts are checked using case-insensitive comparisons + + * @param hosts a list of AllowedHost objects to check for duplicates + * @param errors errors to populate + * @return hosts if there are no duplicates, otherwise {@code null} + */ + public static @Nullable List checkDuplicates(List hosts, BindException errors) + { + // Not a simple Set check since we want host check to be case-insensitive + MultiValuedMap map = new CaseInsensitiveHashSetValuedMap<>(); + + hosts.forEach(allowedHost -> { + String host = allowedHost.host().trim(); + if (!map.put(allowedHost.directive(), host)) + errors.addError(new LabKeyError(String.format("'%1$s' already exists. Duplicate values are not allowed.", allowedHost))); + }); + + return errors.hasErrors() ? null : hosts; + } + + // Returns a mutable list + public List getSavedAllowedHosts() throws JsonProcessingException + { + return AllowedExternalResourceHosts.readAllowedHosts(); + } + } + + @AdminConsoleAction() + public class ExternalSourcesAction extends FormViewAction + { + @Override + public void validateCommand(ExternalSourcesForm form, Errors errors) + { + } + + @Override + public ModelAndView getView(ExternalSourcesForm form, boolean reshow, BindException errors) + { + boolean isTroubleshooter = !getContainer().hasPermission(getUser(), ApplicationAdminPermission.class); + + JspView newView = new JspView<>("/org/labkey/core/admin/addNewExternalSource.jsp", form, errors); + newView.setTitle(isTroubleshooter ? "Overview" : "Register New External Resource Host"); + newView.setFrame(WebPartView.FrameType.PORTAL); + JspView existingView = new JspView<>("/org/labkey/core/admin/existingExternalSources.jsp", form, errors); + existingView.setTitle("Existing External Resource Hosts"); + existingView.setFrame(WebPartView.FrameType.PORTAL); + + return new VBox(newView, existingView); + } + + private static final Object HOST_LOCK = new Object(); + + @Override + public boolean handlePost(ExternalSourcesForm form, BindException errors) throws Exception + { + List allowedHosts = null; + + // Multiple requests could access this in parallel, so synchronize access, Issue 53457 + synchronized (HOST_LOCK) + { + //handle delete of an existing value + if (form.isDelete()) + { + AllowedHost subToDelete = form.getExistingAllowedHost(errors); + if (errors.hasErrors()) + return false; + allowedHosts = form.getSavedAllowedHosts(); + var iter = allowedHosts.listIterator(); + while (iter.hasNext()) + { + AllowedHost sub = iter.next(); + if (sub.equals(subToDelete)) + { + iter.remove(); + break; + } + } + } + //handle updates - clicking on Save button under Existing will save the updated hosts + else if (form.isSaveAll()) + { + allowedHosts = form.getExistingAllowedHosts(errors); + if (errors.hasErrors()) + return false; + } + //save new external value + else if (form.isSaveNew()) + { + allowedHosts = form.validateNewAllowedHost(errors); + } + + if (errors.hasErrors()) + return false; + + AllowedExternalResourceHosts.saveAllowedHosts(allowedHosts, getUser()); + } + + return true; + } + + @Override + public URLHelper getSuccessURL(ExternalSourcesForm form) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("externalHosts"); + addAdminNavTrail(root, "Allowed External Resource Hosts", getClass()); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ProjectSettingsAction extends ProjectSettingsViewPostAction + { + @Override + protected LookAndFeelView getTabView(ProjectSettingsForm form, boolean reshow, BindException errors) + { + return new LookAndFeelView(errors); + } + + @Override + public void validateCommand(ProjectSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ProjectSettingsForm form, BindException errors) throws Exception + { + return saveProjectSettings(getContainer(), getUser(), form, errors); + } + } + + private static boolean saveProjectSettings(Container c, User user, ProjectSettingsForm form, BindException errors) + { + WriteableLookAndFeelProperties props = LookAndFeelProperties.getWriteableInstance(c); + boolean hasAdminOpsPerm = c.hasPermission(user, AdminOperationsPermission.class); + + // Site-only properties + + if (c.isRoot()) + { + DateParsingMode dateParsingMode = DateParsingMode.fromString(form.getDateParsingMode()); + props.setDateParsingMode(dateParsingMode); + + if (hasAdminOpsPerm) + { + String customWelcome = form.getCustomWelcome(); + String welcomeUrl = StringUtils.trimToNull(customWelcome); + if ("/".equals(welcomeUrl) || AppProps.getInstance().getContextPath().equalsIgnoreCase(welcomeUrl)) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid welcome URL. The url cannot equal '/' or the contextPath (" + AppProps.getInstance().getContextPath() + ")"); + } + else + { + props.setCustomWelcome(welcomeUrl); + } + } + } + + // Site & project properties + + boolean shouldInherit = form.getShouldInherit(); + if (shouldInherit != SecurityManager.shouldNewSubfoldersInheritPermissions(c)) + { + SecurityManager.setNewSubfoldersInheritPermissions(c, user, shouldInherit); + } + + setProperty(form.isSystemDescriptionInherited(), props::clearSystemDescription, () -> props.setSystemDescription(form.getSystemDescription())); + setProperty(form.isSystemShortNameInherited(), props::clearSystemShortName, () -> props.setSystemShortName(form.getSystemShortName())); + setProperty(form.isThemeNameInherited(), props::clearThemeName, () -> props.setThemeName(form.getThemeName())); + setProperty(form.isFolderDisplayModeInherited(), props::clearFolderDisplayMode, () -> props.setFolderDisplayMode(FolderDisplayMode.fromString(form.getFolderDisplayMode()))); + setProperty(form.isApplicationMenuDisplayModeInherited(), props::clearApplicationMenuDisplayMode, () -> props.setApplicationMenuDisplayMode(FolderDisplayMode.fromString(form.getApplicationMenuDisplayMode()))); + setProperty(form.isHelpMenuEnabledInherited(), props::clearHelpMenuEnabled, () -> props.setHelpMenuEnabled(form.isHelpMenuEnabled())); + + // a few properties on this page should be restricted to operational permissions (i.e. site admin) + if (hasAdminOpsPerm) + { + setProperty(form.isSystemEmailAddressInherited(), props::clearSystemEmailAddress, () -> { + String systemEmailAddress = form.getSystemEmailAddress(); + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(systemEmailAddress); + props.setSystemEmailAddress(email); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid System Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + }); + + setProperty(form.isCustomLoginInherited(), props::clearCustomLogin, () -> { + String customLogin = form.getCustomLogin(); + if (props.isValidUrl(customLogin)) + { + props.setCustomLogin(customLogin); + } + else + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid login URL. Should be in the form -."); + } + }); + } + + setProperty(form.isCompanyNameInherited(), props::clearCompanyName, () -> props.setCompanyName(form.getCompanyName())); + setProperty(form.isLogoHrefInherited(), props::clearLogoHref, () -> props.setLogoHref(form.getLogoHref())); + setProperty(form.isReportAProblemPathInherited(), props::clearReportAProblemPath, () -> props.setReportAProblemPath(form.getReportAProblemPath())); + setProperty(form.isSupportEmailInherited(), props::clearSupportEmail, () -> { + String supportEmail = form.getSupportEmail(); + + if (!isBlank(supportEmail)) + { + try + { + // this will throw an InvalidEmailException for invalid email addresses + ValidEmail email = new ValidEmail(supportEmail); + props.setSupportEmail(email.toString()); + } + catch (ValidEmail.InvalidEmailException e) + { + errors.reject(SpringActionController.ERROR_MSG, "Invalid Support Email Address: [" + + e.getBadEmail() + "]. Please enter a valid email address."); + } + } + else + { + // This stores a blank value, not null (which would mean inherit) + props.setSupportEmail(null); + } + }); + + boolean noErrors = !saveFolderSettings(c, user, props, form, errors); + + if (noErrors) + { + // Bump the look & feel revision so browsers retrieve the new theme stylesheet + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + } + + return noErrors; + } + + private static void setProperty(boolean inherited, Runnable clear, Runnable set) + { + if (inherited) + clear.run(); + else + set.run(); + } + + // Same as ProjectSettingsAction, but provides special admin console permissions handling + @AdminConsoleAction(ApplicationAdminPermission.class) + public static class LookAndFeelSettingsAction extends ProjectSettingsAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class UpdateContainerSettingsAction extends MutatingApiAction + { + @Override + public Object execute(FolderSettingsForm form, BindException errors) + { + boolean saved = saveFolderSettings(getContainer(), getUser(), LookAndFeelProperties.getWriteableFolderInstance(getContainer()), form, errors); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("success", saved && !errors.hasErrors()); + return response; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ResourcesAction extends ProjectSettingsViewPostAction + { + @Override + protected JspView getTabView(Object o, boolean reshow, BindException errors) + { + LookAndFeelBean bean = new LookAndFeelBean(); + return new JspView<>("/org/labkey/core/admin/lookAndFeelResources.jsp", bean, errors); + } + + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + Container c = getContainer(); + Map fileMap = getFileMap(); + + for (ResourceType type : ResourceType.values()) + { + MultipartFile file = fileMap.get(type.name()); + + if (file != null && !file.isEmpty()) + { + try + { + type.save(file, c, getUser()); + } + catch (Exception e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + return false; + } + } + } + + // Note that audit logging happens via the attachment code, so we don't log separately here + + // Bump the look & feel revision so browsers retrieve the new logo, custom stylesheet, etc. + WriteableAppProps.incrementLookAndFeelRevisionAndSave(); + + return true; + } + } + + // Same as ResourcesAction, but provides special admin console permissions handling + @AdminConsoleAction + public static class AdminConsoleResourcesAction extends ResourcesAction + { + @Override + protected TYPE getType() + { + return TYPE.LookAndFeelSettings; + } + } + + @RequiresPermission(AdminPermission.class) + public static class MenuBarAction extends ProjectSettingsViewAction + { + @Override + protected HttpView getTabView() + { + if (getContainer().isRoot()) + return HtmlView.err("Menu bar must be configured for each project separately."); + + WebPartView v = new JspView<>("/org/labkey/core/admin/editMenuBar.jsp", null); + v.setView("menubar", new VBox()); + Portal.populatePortalView(getViewContext(), Portal.DEFAULT_PORTAL_PAGE_ID, v, false, true, true, false); + + return v; + } + } + + @RequiresPermission(AdminPermission.class) + public static class FilesAction extends ProjectSettingsViewPostAction + { + @Override + protected HttpView getTabView(FilesForm form, boolean reshow, BindException errors) + { + Container c = getContainer(); + + if (c.isRoot()) + return HtmlView.err("Files must be configured for each project separately."); + + if (!reshow || form.isPipelineRootForm()) + { + try + { + AdminController.setFormAndConfirmMessage(getViewContext(), form); + } + catch (IllegalArgumentException e) + { + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + } + VBox box = new VBox(); + JspView view = new JspView<>("/org/labkey/core/admin/view/filesProjectSettings.jsp", form, errors); + String title = "Configure File Root"; + if (CloudStoreService.get() != null) + title += " And Enable Cloud Stores"; + view.setTitle(title); + box.addView(view); + + // only site admins (i.e. AdminOperationsPermission) can configure the pipeline root + if (c.hasPermission(getViewContext().getUser(), AdminOperationsPermission.class)) + { + SetupForm setupForm = SetupForm.init(c); + setupForm.setShowAdditionalOptionsLink(true); + setupForm.setErrors(errors); + PipeRoot pipeRoot = SetupForm.getPipelineRoot(c); + + if (pipeRoot != null) + { + for (String errorMessage : pipeRoot.validate()) + errors.addError(new LabKeyError(errorMessage)); + } + JspView pipelineView = (JspView) PipelineService.get().getSetupView(setupForm); + pipelineView.setTitle("Configure Data Processing Pipeline"); + box.addView(pipelineView); + } + + return box; + } + + @Override + public void validateCommand(FilesForm form, Errors errors) + { + if (!form.isPipelineRootForm() && !form.isDisableFileSharing() && !form.hasSiteDefaultRoot() && !form.isCloudFileRoot()) + { + String root = StringUtils.trimToNull(form.getFolderRootPath()); + if (root != null) + { + File f = new File(root); + if (!f.exists() || !f.isDirectory()) + { + errors.reject(SpringActionController.ERROR_MSG, "File root '" + root + "' does not appear to be a valid directory accessible to the server at " + getViewContext().getRequest().getServerName() + "."); + } + } + else + errors.reject(SpringActionController.ERROR_MSG, "A Project specified file root cannot be blank, to disable file sharing for this project, select the disable option."); + } + else if (form.isCloudFileRoot()) + { + AdminController.validateCloudFileRoot(form, getContainer(), errors); + } + } + + @Override + public boolean handlePost(FilesForm form, BindException errors) throws Exception + { + FileContentService service = FileContentService.get(); + if (service != null) + { + if (form.isPipelineRootForm()) + return PipelineService.get().savePipelineSetup(getViewContext(), form, errors); + else + { + AdminController.setFileRootFromForm(getViewContext(), form, errors); + } + } + + // Cloud settings + AdminController.setEnabledCloudStores(getViewContext(), form, errors); + + return !errors.hasErrors(); + } + + @Override + public URLHelper getSuccessURL(FilesForm form) + { + ActionURL url = new AdminController.AdminUrlsImpl().getProjectSettingsFileURL(getContainer()); + if (form.isPipelineRootForm()) + { + url.addParameter("piperootSet", true); + } + else + { + if (form.isFileRootChanged()) + url.addParameter("rootSet", form.getMigrateFilesOption()); + if (form.isEnabledCloudStoresChanged()) + url.addParameter("cloudChanged", true); + } + return url; + } + } + + public static class LookAndFeelView extends JspView + { + LookAndFeelView(BindException errors) + { + super("/org/labkey/core/admin/lookAndFeelProperties.jsp", new LookAndFeelBean(), errors); + } + } + + + public static class LookAndFeelBean + { + public final HtmlString helpLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString welcomeLink = new HelpTopic("customizeLook").getSimpleLinkHtml("more info..."); + public final HtmlString customColumnRestrictionHelpLink = new HelpTopic("chartTrouble").getSimpleLinkHtml("more info..."); + } + + @RequiresPermission(AdminPermission.class) + public static class AdjustSystemTimestampsAction extends FormViewAction + { + @Override + public void addNavTrail(NavTree root) + { + } + + @Override + public void validateCommand(AdjustTimestampsForm form, Errors errors) + { + if (form.getHourDelta() == null || form.getHourDelta() == 0) + errors.reject(ERROR_MSG, "You must specify a non-zero value for 'Hour Delta'"); + } + + @Override + public ModelAndView getView(AdjustTimestampsForm form, boolean reshow, BindException errors) throws Exception + { + return new JspView<>("/org/labkey/core/admin/adjustTimestamps.jsp", form, errors); + } + + private void updateFields(TableInfo tInfo, Collection fieldNames, int delta) + { + SQLFragment sql = new SQLFragment(); + DbSchema schema = tInfo.getSchema(); + String comma = ""; + List updating = new ArrayList<>(); + + for (String fieldName: fieldNames) + { + ColumnInfo col = tInfo.getColumn(FieldKey.fromParts(fieldName)); + if (col != null && col.getJdbcType() == JdbcType.TIMESTAMP) + { + updating.add(fieldName); + if (sql.isEmpty()) + sql.append("UPDATE ").append(tInfo, "").append(" SET "); + sql.append(comma) + .append(String.format(" %s = {fn timestampadd(SQL_TSI_HOUR, %d, %s)}", col.getSelectIdentifier(), delta, col.getSelectIdentifier())); + comma = ", "; + } + } + + if (!sql.isEmpty()) + { + logger.info(String.format("Updating %s in table %s.%s", updating, schema.getName(), tInfo.getName())); + logger.debug(sql.toDebugString()); + int numRows = new SqlExecutor(schema).execute(sql); + logger.info(String.format("Updated %d rows for table %s.%s", numRows, schema.getName(), tInfo.getName())); + } + } + + @Override + public boolean handlePost(AdjustTimestampsForm form, BindException errors) throws Exception + { + List toUpdate = Arrays.asList("Created", "Modified", "lastIndexed", "diCreated", "diModified"); + logger.info("Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + DbScope scope = DbScope.getLabKeyScope(); + try (DbScope.Transaction t = scope.ensureTransaction()) + { + ModuleLoader.getInstance().getModules().forEach(module -> { + logger.info("==> Beginning update of timestamps for module: " + module.getName()); + module.getSchemaNames().stream().sorted().forEach(schemaName -> { + DbSchema schema = DbSchema.get(schemaName, DbSchemaType.Module); + scope.invalidateSchema(schema); // Issue 44452: assure we have a fresh set of tables to work from + schema.getTableNames().forEach(tableName -> { + TableInfo tInfo = schema.getTable(tableName); + if (tInfo.getTableType() == DatabaseTableType.TABLE) + { + updateFields(tInfo, toUpdate, form.getHourDelta()); + } + }); + }); + logger.info("<== DONE updating timestamps for module: " + module.getName()); + }); + t.commit(); + } + logger.info("DONE Adjusting all " + toUpdate + " timestamp fields in all tables by " + form.getHourDelta() + " hours."); + return true; + } + + @Override + public URLHelper getSuccessURL(AdjustTimestampsForm adjustTimestampsForm) + { + return PageFlowUtil.urlProvider(AdminUrls.class).getAdminConsoleURL(); + } + } + + public static class AdjustTimestampsForm + { + private Integer hourDelta; + + public Integer getHourDelta() + { + return hourDelta; + } + + public void setHourDelta(Integer hourDelta) + { + this.hourDelta = hourDelta; + } + } + + @RequiresPermission(TroubleshooterPermission.class) + public class ViewUsageStatistics extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("ViewUsageStatistics")); + } + + @Override + public void addNavTrail(NavTree root) + { + addAdminNavTrail(root, "Usage Statistics", this.getClass()); + } + } + + private static final URI LABKEY_ORG_REPORT_ACTION; + + static + { + LABKEY_ORG_REPORT_ACTION = URI.create("https://www.labkey.org/admin-contentSecurityPolicyReport.api"); + } + + @RequiresNoPermission + @CSRF(CSRF.Method.NONE) + public static class ContentSecurityPolicyReportAction extends ReadOnlyApiAction + { + private static final Logger _log = LogHelper.getLogger(ContentSecurityPolicyReportAction.class, "CSP warnings"); + + // recent reports, to help avoid log spam + private static final Map reports = Collections.synchronizedMap(new LRUMap<>(20)); + + @Override + public Object execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + var ret = new JSONObject().put("success", true); + + // fail fast + if (!_log.isWarnEnabled()) + return ret; + + var request = getViewContext().getRequest(); + assert null != request; + + var userAgent = request.getHeader("User-Agent"); + if (PageFlowUtil.isRobotUserAgent(userAgent) && !_log.isDebugEnabled()) + return ret; + + // NOTE User may be "guest", and will always be guest if being relayed to labkey.org + var jsonObj = form.getJsonObject(); + if (null != jsonObj) + { + JSONObject cspReport = jsonObj.optJSONObject("csp-report"); + if (cspReport != null) + { + String blockedUri = cspReport.optString("blocked-uri", null); + + // Issue 52933 - suppress base-uri problems from a crawler or bot on labkey.org + if (blockedUri != null && + blockedUri.startsWith("https://labkey.org%2C") && + blockedUri.endsWith("undefined") && + !_log.isDebugEnabled()) + { + return ret; + } + + String urlString = cspReport.optString("document-uri", null); + if (urlString != null) + { + String path = new URLHelper(urlString).deleteParameters().getURIString(); + if (null == reports.put(path, Boolean.TRUE) || _log.isDebugEnabled()) + { + // Don't modify forwarded reports; they already have user, ip, user-agent, etc. from the forwarding server. + boolean forwarded = jsonObj.optBoolean("forwarded", false); + if (!forwarded) + { + User user = getUser(); + String email = null; + // If the user is not logged in, we may still be able to snag the email address from our cookie + if (user.isGuest()) + email = LoginController.getEmailFromCookie(getViewContext().getRequest()); + if (null == email) + email = user.getEmail(); + jsonObj.put("user", email); + String ipAddress = request.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null) + ipAddress = request.getRemoteAddr(); + jsonObj.put("ip", ipAddress); + if (isNotBlank(userAgent)) + jsonObj.put("user-agent", userAgent); + String labkeyVersion = request.getParameter("labkeyVersion"); + if (null != labkeyVersion) + jsonObj.put("labkeyVersion", labkeyVersion); + String cspVersion = request.getParameter("cspVersion"); + if (null != cspVersion) + jsonObj.put("cspVersion", cspVersion); + } + + var jsonStr = jsonObj.toString(2); + _log.warn("ContentSecurityPolicy warning on page: {}\n{}", urlString, jsonStr); + + if (!forwarded && OptionalFeatureService.get().isFeatureEnabled(ContentSecurityPolicyFilter.FEATURE_FLAG_FORWARD_CSP_REPORTS)) + { + jsonObj.put("forwarded", true); + + // Create an HttpClient + HttpClient client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + // Create the POST request + HttpRequest remoteRequest = HttpRequest.newBuilder() + .uri(LABKEY_ORG_REPORT_ACTION) + .header("Content-Type", request.getContentType()) // Use whatever the browser set + .POST(HttpRequest.BodyPublishers.ofString(jsonObj.toString(2))) + .build(); + + // Send the request and get the response + HttpResponse response = client.send(remoteRequest, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != 200) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}\n{}", response.statusCode(), response.body()); + } + else + { + JSONObject jsonResponse = new JSONObject(response.body()); + boolean success = jsonResponse.optBoolean("success", false); + if (!success) + { + _log.error("ContentSecurityPolicy report forwarding to https://www.labkey.org failed: {}", jsonResponse); + } + } + } + } + } + } + } + return ret; + } + } + + + public static class TestCase extends AbstractActionPermissionTest + { + @Override + @Test + public void testActionPermissions() + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + AdminController controller = new AdminController(); + + // @RequiresPermission(ReadPermission.class) + assertForReadPermission(user, false, + new GetModulesAction(), + new GetFolderTabsAction(), + new ClearDeletedTabFoldersAction() + ); + + // @RequiresPermission(DeletePermission.class) + assertForUpdateOrDeletePermission(user, + new DeleteFolderAction() + ); + + // @RequiresPermission(AdminPermission.class) + assertForAdminPermission(user, + controller.new CustomizeEmailAction(), + controller.new FolderAliasesAction(), + controller.new MoveFolderAction(), + controller.new MoveTabAction(), + controller.new RenameFolderAction(), + controller.new ReorderFoldersAction(), + controller.new ReorderFoldersApiAction(), + controller.new SiteValidationAction(), + new AddTabAction(), + new ConfirmProjectMoveAction(), + new CreateFolderAction(), + new CustomizeMenuAction(), + new DeleteCustomEmailAction(), + new FilesAction(), + new MenuBarAction(), + new ProjectSettingsAction(), + new RenameContainerAction(), + new RenameTabAction(), + new ResetPropertiesAction(), + new ResetQueryStatisticsAction(), + new ResetResourceAction(), + new ResourcesAction(), + new RevertFolderAction(), + new SetFolderPermissionsAction(), + new SetInitialFolderSettingsAction(), + new ShowTabAction() + ); + + //TODO @RequiresPermission(AdminReadPermission.class) + //controller.new TestMothershipReportAction() + + // @RequiresPermission(AdminOperationsPermission.class) + assertForAdminOperationsPermission(ContainerManager.getRoot(), user, + controller.new DbCheckerAction(), + controller.new DeleteModuleAction(), + controller.new DoCheckAction(), + controller.new EmailTestAction(), + controller.new ShowNetworkDriveTestAction(), + controller.new ValidateDomainsAction(), + new OptionalFeatureAction(), + new GetSchemaXmlDocAction(), + new RecreateViewsAction() + ); + + // @AdminConsoleAction + assertForAdminPermission(ContainerManager.getRoot(), user, + controller.new ActionsAction(), + controller.new CachesAction(), + controller.new ConfigureSystemMaintenanceAction(), + controller.new CustomizeSiteAction(), + controller.new DumpHeapAction(), + controller.new ExecutionPlanAction(), + controller.new FolderTypesAction(), + controller.new MemTrackerAction(), + controller.new ModulesAction(), + controller.new QueriesAction(), + controller.new QueryStackTracesAction(), + controller.new ResetErrorMarkAction(), + controller.new ShortURLAdminAction(), + controller.new ShowAllErrorsAction(), + controller.new ShowErrorsSinceMarkAction(), + controller.new ShowPrimaryLogAction(), + controller.new ShowCspReportLogAction(), + controller.new ShowThreadsAction(), + new ExportActionsAction(), + new ExportQueriesAction(), + new MemoryChartAction(), + new ShowAdminAction() + ); + + // @RequiresSiteAdmin + assertForRequiresSiteAdmin(user, + controller.new EnvironmentVariablesAction(), + controller.new SystemMaintenanceAction(), + controller.new SystemPropertiesAction(), + new GetPendingRequestCountAction(), + new InstallCompleteAction(), + new NewInstallSiteSettingsAction() + ); + + assertForTroubleshooterPermission(ContainerManager.getRoot(), user, + controller.new OptionalFeaturesAction(), + controller.new ShowModuleErrorsAction(), + new ModuleStatusAction() + ); + } + } + + public static class SerializationTest extends PipelineJob.TestSerialization + { + static class TestJob extends PipelineJob + { + ImpersonationContext _impersonationContext; + ImpersonationContext _impersonationContext1; + ImpersonationContext _impersonationContext2; + + @Override + public URLHelper getStatusHref() + { + return null; + } + + @Override + public String getDescription() + { + return "Test Job"; + } + } + + @Test + public void testSerialization() + { + TestJob job = new TestJob(); + TestContext ctx = TestContext.get(); + ViewContext viewContext = new ViewContext(); + viewContext.setContainer(ContainerManager.getSharedContainer()); + viewContext.setUser(ctx.getUser()); + RoleImpersonationContextFactory factory = new RoleImpersonationContextFactory( + viewContext.getContainer(), viewContext.getUser(), + Collections.singleton(RoleManager.getRole(SharedViewEditorRole.class)), Collections.emptySet(), null); + job._impersonationContext = factory.getImpersonationContext(); + + try + { + UserImpersonationContextFactory factory1 = new UserImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + UserManager.getGuestUser(), null); + job._impersonationContext1 = factory1.getImpersonationContext(); + } + catch (Exception e) + { + LOG.error("Invalid user email for impersonating."); + } + + GroupImpersonationContextFactory factory2 = new GroupImpersonationContextFactory(viewContext.getContainer(), viewContext.getUser(), + GroupManager.getGroup(ContainerManager.getRoot(), "Users", GroupEnumType.SITE), null); + job._impersonationContext2 = factory2.getImpersonationContext(); + testSerialize(job, LOG); + } + } + + public static class WorkbookDeleteTestCase extends Assert + { + private static final String FOLDER_NAME = "WorkbookDeleteTestCaseFolder"; + private static final String TEST_EMAIL = "testDelete@myDomain.com"; + + @Test + public void testWorkbookDelete() throws Exception + { + doCleanup(); + + Container project = ContainerManager.createContainer(ContainerManager.getRoot(), FOLDER_NAME, TestContext.get().getUser()); + Container workbook = ContainerManager.createContainer(project, null, "Title1", null, WorkbookContainerType.NAME, TestContext.get().getUser()); + + ValidEmail email = new ValidEmail(TEST_EMAIL); + SecurityManager.NewUserStatus newUserStatus = SecurityManager.addUser(email, null); + User nonAdminUser = newUserStatus.getUser(); + MutableSecurityPolicy policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, ReaderRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + // User lacks any permission, throw unauthorized for parent and workbook: + HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + MockHttpServletResponse response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Grant permission, should be able to delete the workbook but not parent: + policy = new MutableSecurityPolicy(project.getPolicy()); + policy.addRoleAssignment(nonAdminUser, EditorRole.class); + SecurityPolicyManager.savePolicy(policy, TestContext.get().getUser()); + + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, project), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FORBIDDEN, response.getStatus()); + + // Hitting delete action results in a redirect: + request = ViewServlet.mockRequest(RequestMethod.POST.name(), new ActionURL(DeleteFolderAction.class, workbook), nonAdminUser, Map.of("Content-Type", "application/json"), null); + response = ViewServlet.mockDispatch(request, null); + Assert.assertEquals("Incorrect response code", HttpServletResponse.SC_FOUND, response.getStatus()); + + doCleanup(); + } + + protected static void doCleanup() throws Exception + { + Container project = ContainerManager.getForPath(FOLDER_NAME); + if (project != null) + { + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + if (UserManager.userExists(new ValidEmail(TEST_EMAIL))) + { + User u = UserManager.getUser(new ValidEmail(TEST_EMAIL)); + UserManager.deleteUser(u.getUserId()); + } + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/AbstractRunInput.java b/experiment/src/org/labkey/experiment/api/AbstractRunInput.java index 43349a47470..7982c76963e 100644 --- a/experiment/src/org/labkey/experiment/api/AbstractRunInput.java +++ b/experiment/src/org/labkey/experiment/api/AbstractRunInput.java @@ -1,125 +1,125 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment.api; - -import org.jetbrains.annotations.Nullable; -import org.labkey.api.exp.IdentifiableBase; -import org.labkey.api.exp.Lsid; -import org.labkey.api.util.StringUtilsLabKey; - -import java.util.Objects; - -/** - * Base class for the beans that wire up material and data objects to be inputs to a protocol application. - * User: jeckels - * Date: Oct 31, 2008 - */ -public abstract class AbstractRunInput extends IdentifiableBase -{ - private final String _defaultRole; - - private long _targetApplicationId; - protected String _role; - protected Long _protocolInputId; - - protected AbstractRunInput(String defaultRole) - { - _defaultRole = defaultRole; - } - - public long getTargetApplicationId() - { - return _targetApplicationId; - } - - public void setTargetApplicationId(long targetApplicationId) - { - _targetApplicationId = targetApplicationId; - } - - public String getRole() - { - return _role; - } - - public void setRole(@Nullable String role) - { - if (role == null) - { - role = _defaultRole; - } - // Issue 17590. For a while, we've had a bug with exporting material role names in XARs. We exported the - // material name instead of the role name itself. The material names can be longer than the role names, - // so truncate here if needed to prevent a SQLException later - if (role.length() > 50) - { - role = StringUtilsLabKey.leftSurrogatePairFriendly(role, 49); - } - _role = role; - } - - public Long getProtocolInputId() - { - return _protocolInputId; - } - - public void setProtocolInputId(Long protocolInputId) - { - _protocolInputId = protocolInputId; - } - - protected abstract long getInputKey(); - - - protected static String lsid(String namespace, long inputKey, long targetApplicationId) - { - if (targetApplicationId == 0 || inputKey == 0) - throw new IllegalStateException("LSID requires targetApplicationId and input id"); - Lsid lsid = new Lsid(namespace, inputKey + "." + targetApplicationId); - return lsid.toString(); - } - - - @Override - public void setLSID(String lsid) - { - throw new UnsupportedOperationException(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AbstractRunInput input = (AbstractRunInput) o; - - return getInputKey() == input.getInputKey() && - Objects.equals(_role, input._role) && - _targetApplicationId == input._targetApplicationId && - Objects.equals(_protocolInputId, input._protocolInputId); - } - - @Override - public int hashCode() - { - int result = (int)getInputKey(); - result = 31 * result + (int)_targetApplicationId; - result = 31 * result + (_role == null ? 0 : _role.hashCode()); - result = 31 * result + (_protocolInputId == null ? 0 : _protocolInputId.hashCode()); - return result; - } -} +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment.api; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.exp.IdentifiableBase; +import org.labkey.api.exp.Lsid; +import org.labkey.api.util.StringUtilsLabKey; + +import java.util.Objects; + +/** + * Base class for the beans that wire up material and data objects to be inputs to a protocol application. + * User: jeckels + * Date: Oct 31, 2008 + */ +public abstract class AbstractRunInput extends IdentifiableBase +{ + private final String _defaultRole; + + private long _targetApplicationId; + protected String _role; + protected Long _protocolInputId; + + protected AbstractRunInput(String defaultRole) + { + _defaultRole = defaultRole; + } + + public long getTargetApplicationId() + { + return _targetApplicationId; + } + + public void setTargetApplicationId(long targetApplicationId) + { + _targetApplicationId = targetApplicationId; + } + + public String getRole() + { + return _role; + } + + public void setRole(@Nullable String role) + { + if (role == null) + { + role = _defaultRole; + } + // Issue 17590. For a while, we've had a bug with exporting material role names in XARs. We exported the + // material name instead of the role name itself. The material names can be longer than the role names, + // so truncate here if needed to prevent a SQLException later + if (role.length() > 50) + { + role = StringUtilsLabKey.leftSurrogatePairFriendly(role, 49); + } + _role = role; + } + + public Long getProtocolInputId() + { + return _protocolInputId; + } + + public void setProtocolInputId(Long protocolInputId) + { + _protocolInputId = protocolInputId; + } + + protected abstract long getInputKey(); + + + protected static String lsid(String namespace, long inputKey, long targetApplicationId) + { + if (targetApplicationId == 0 || inputKey == 0) + throw new IllegalStateException("LSID requires targetApplicationId and input id"); + Lsid lsid = new Lsid(namespace, inputKey + "." + targetApplicationId); + return lsid.toString(); + } + + + @Override + public void setLSID(String lsid) + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AbstractRunInput input = (AbstractRunInput) o; + + return getInputKey() == input.getInputKey() && + Objects.equals(_role, input._role) && + _targetApplicationId == input._targetApplicationId && + Objects.equals(_protocolInputId, input._protocolInputId); + } + + @Override + public int hashCode() + { + int result = (int)getInputKey(); + result = 31 * result + (int)_targetApplicationId; + result = 31 * result + (_role == null ? 0 : _role.hashCode()); + result = 31 * result + (_protocolInputId == null ? 0 : _protocolInputId.hashCode()); + return result; + } +} diff --git a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java index a59e474df7c..b76ee431b56 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataImpl.java @@ -1,978 +1,978 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.ExperimentDataHandler; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Handler; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.XarFormatException; -import org.labkey.api.exp.XarSource; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.query.ExpDataClassDataTable; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineJob; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryRowReference; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchResultTemplate; -import org.labkey.api.search.SearchScope; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DataClassReadPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.MediaReadPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.MimeMap; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.InputBuilder; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.ViewContext; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.labkey.api.exp.query.ExpSchema.SCHEMA_EXP_DATA; - -public class ExpDataImpl extends AbstractRunItemImpl implements ExpData -{ - public enum DataOperations - { - Edit("editing", UpdatePermission.class), - EditLineage("editing lineage", UpdatePermission.class), - Delete("deleting", DeletePermission.class), - Move("moving", MoveEntitiesPermission.class); - - private final String _description; // used as a suffix in messaging users about what is not allowed - private final Class _permissionClass; - - DataOperations(String description, Class permissionClass) - { - _description = description; - _permissionClass = permissionClass; - } - - public String getDescription() - { - return _description; - } - - public Class getPermissionClass() - { - return _permissionClass; - } - } - - public static final SearchService.SearchCategory expDataCategory = new SearchService.SearchCategory("data", "ExpData", false) { - @Override - public Set getPermittedContainerIds(User user, Map containers) - { - return getPermittedContainerIds(user, containers, DataClassReadPermission.class); - } - }; - public static final SearchService.SearchCategory expMediaDataCategory = new SearchService.SearchCategory("mediaData", "ExpData for media objects", false) { - @Override - public Set getPermittedContainerIds(User user, Map containers) - { - return getPermittedContainerIds(user, containers, MediaReadPermission.class); - } - }; - - /** Cache this because it can be expensive to recompute */ - private Boolean _finalRunOutput; - - /** - * Temporary mapping until experiment.xml contains the mime type - */ - private static final MimeMap MIME_MAP = new MimeMap(); - - static public List fromDatas(List datas) - { - List ret = new ArrayList<>(datas.size()); - for (Data data : datas) - { - ret.add(new ExpDataImpl(data)); - } - return ret; - } - - // For serialization - protected ExpDataImpl() {} - - public ExpDataImpl(Data data) - { - super(data); - } - - @Override - public void setComment(User user, String comment) throws ValidationException - { - setComment(user, comment, true); - } - - @Override - public void setComment(User user, String comment, boolean index) throws ValidationException - { - super.setComment(user, comment); - - if (index) - index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); - } - - @Override - @Nullable - public ActionURL detailsURL() - { - DataType dataType = getDataType(); - if (dataType != null) - { - ActionURL url = dataType.getDetailsURL(this); - if (url != null) - return url; - } - - return _object.detailsURL(); - } - - @Override - public @Nullable QueryRowReference getQueryRowReference() - { - return getQueryRowReference(null); - } - - @Override - public @Nullable QueryRowReference getQueryRowReference(@Nullable User user) - { - ExpDataClassImpl dc = getDataClass(user); - if (dc != null) - return new QueryRowReference(getContainer(), SCHEMA_EXP_DATA, dc.getName(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); - - // Issue 40123: see MedImmuneDataHandler MEDIMMUNE_DATA_TYPE, this claims the "Data" namespace - DataType type = getDataType(); - if (type != null) - { - QueryRowReference queryRowReference = type.getQueryRowReference(this); - if (queryRowReference != null) - return queryRowReference; - } - - return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Data.name(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); - } - - @Override - public List getTargetApplications() - { - return getTargetApplications(new SimpleFilter(FieldKey.fromParts("DataId"), getRowId()), ExperimentServiceImpl.get().getTinfoDataInput()); - } - - @Override - public List getTargetRuns() - { - return getTargetRuns(ExperimentServiceImpl.get().getTinfoDataInput(), "DataId"); - } - - @Override - public DataType getDataType() - { - return ExperimentService.get().getDataType(getLSIDNamespacePrefix()); - } - - @Override - public void setDataFileURI(URI uri) - { - ensureUnlocked(); - _object.setDataFileUrl(ExpData.normalizeDataFileURI(uri)); - } - - @Override - public void save(User user) - { - // Replace the default "Data" cpastype if the Data belongs to a DataClass - ExpDataClassImpl dataClass = getDataClass(); - if (dataClass != null && ExpData.DEFAULT_CPAS_TYPE.equals(getCpasType())) - setCpasType(dataClass.getLSID()); - - boolean isNew = getRowId() == 0; - save(user, ExperimentServiceImpl.get().getTinfoData(), true); - - if (isNew) - { - if (dataClass != null) - { - Map map = new HashMap<>(); - map.put("lsid", getLSID()); - Table.insert(user, dataClass.getTinfo(), map); - } - } - index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); - } - - @Override - protected void save(User user, TableInfo table, boolean ensureObject) - { - assert ensureObject; - super.save(user, table, true); - } - - @Override - public URI getDataFileURI() - { - String url = _object.getDataFileUrl(); - if (url == null) - return null; - try - { - return new URI(_object.getDataFileUrl()); - } - catch (URISyntaxException use) - { - return null; - } - } - - @Override - public ExperimentDataHandler findDataHandler() - { - return Handler.Priority.findBestHandler(ExperimentServiceImpl.get().getExperimentDataHandlers(), this); - } - - @Override - public String getDataFileUrl() - { - return _object.getDataFileUrl(); - } - - @Override - public boolean hasFileScheme() - { - return !FileUtil.hasCloudScheme(getDataFileUrl()); - } - - @Override - @Nullable - public File getFile() - { - return _object.getFile(); - } - - @Override - public @Nullable FileLike getFileLike() - { - return _object.getFileLike(); - } - - @Override - @Nullable - public java.nio.file.Path getFilePath() - { - return _object.getFilePath(); - } - - @Override - public boolean isInlineImage() - { - return null != getFile() && MIME_MAP.isInlineImageFor(getFile()); - } - - @Override - public void delete(User user) - { - delete(user, true); - } - - @Override - public void delete(User user, boolean deleteRunsUsingData) - { - ExperimentServiceImpl.get().deleteDataByRowIds(user, getContainer(), Collections.singleton(getRowId()), deleteRunsUsingData); - } - - public String getMimeType() - { - if (null != getDataFileUrl()) - return MIME_MAP.getContentTypeFor(getDataFileUrl()); - else - return null; - } - - @Override - public boolean isFileOnDisk() - { - java.nio.file.Path f = getFilePath(); - if (f != null) - if (!FileUtil.hasCloudScheme(f)) - return NetworkDrive.exists(f.toFile()) && !Files.isDirectory(f); - else - return Files.exists(f); - else - return false; - } - - public boolean isPathAccessible() - { - java.nio.file.Path path = getFilePath(); - return (null != path && Files.exists(path)); - } - - @Override - public String getCpasType() - { - String result = _object.getCpasType(); - if (result != null) - return result; - - ExpDataClass dataClass = getDataClass(); - if (dataClass != null) - return dataClass.getLSID(); - - return ExpData.DEFAULT_CPAS_TYPE; - } - - public void setGenerated(boolean generated) - { - ensureUnlocked(); - _object.setGenerated(generated); - } - - @Override - public boolean isGenerated() - { - return _object.isGenerated(); - } - - @Override - public boolean isFinalRunOutput() - { - if (_finalRunOutput == null) - { - ExpRun run = getRun(); - _finalRunOutput = run != null && run.isFinalOutput(this); - } - return _finalRunOutput.booleanValue(); - } - - @Override - @Nullable - public ExpDataClassImpl getDataClass() - { - return getDataClass(null); - } - - @Override - @Nullable - public ExpDataClassImpl getDataClass(@Nullable User user) - { - if (_object.getClassId() != null && getContainer() != null) - { - if (user == null) - return ExperimentServiceImpl.get().getDataClass(getContainer(), _object.getClassId()); - else - return ExperimentServiceImpl.get().getDataClass(getContainer(), user, _object.getClassId()); - } - - return null; - } - - @Override - public void importDataFile(PipelineJob job, XarSource xarSource) throws ExperimentException - { - String dataFileURL = getDataFileUrl(); - if (dataFileURL == null) - return; - - if (xarSource.shouldIgnoreDataFiles()) - { - job.debug("Skipping load of data file " + dataFileURL + " based on the XAR source"); - return; - } - - job.debug("Trying to load data file " + dataFileURL + " into the system"); - - java.nio.file.Path path = FileUtil.stringToPath(getContainer(), dataFileURL); - - if (!Files.exists(path)) - { - job.debug("Unable to find the data file " + FileUtil.getAbsolutePath(getContainer(), path) + " on disk."); - return; - } - - // Check that the file is under the pipeline root to prevent users from referencing a file that they - // don't have permission to import - PipeRoot pr = PipelineService.get().findPipelineRoot(job.getContainer()); - if (!xarSource.allowImport(pr, job.getContainer(), path)) - { - if (pr == null) - { - job.warn("No pipeline root was set, skipping load of file " + FileUtil.getAbsolutePath(getContainer(), path)); - return; - } - job.debug("The data file " + FileUtil.getAbsolutePath(getContainer(), path) + " is not under the folder's pipeline root: " + pr + ". It will not be loaded directly, but may be loaded if referenced from other files that are under the pipeline root."); - return; - } - - ExperimentDataHandler handler = findDataHandler(); - try - { - handler.importFile(this, FileSystemLike.wrapFile(path), job.getInfo(), job.getLogger(), xarSource.getXarContext()); - } - catch (ExperimentException e) - { - throw new XarFormatException(e); - } - - job.debug("Finished trying to load data file " + dataFileURL + " into the system"); - } - - // Get all text and int strings from the data class for indexing - private void getIndexValues( - Map props, - @NotNull ExpDataClassDataTableImpl table, - Set identifiersHi, - Set identifiersMed, - Set identifiersLo, - Set keywordHi, - Set keywordMed, - Set keywordsLo, - JSONObject jsonData - ) - { - CaseInsensitiveHashSet skipColumns = new CaseInsensitiveHashSet(); - for (ExpDataClassDataTable.Column column : ExpDataClassDataTable.Column.values()) - skipColumns.add(column.name()); - skipColumns.add("Ancestors"); - skipColumns.add("Container"); - - processIndexValues(props, table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); - } - - @Override - @NotNull - public Collection getAliases() - { - TableInfo mapTi = ExperimentService.get().getTinfoDataAliasMap(); - TableInfo ti = ExperimentService.get().getTinfoAlias(); - SQLFragment sql = new SQLFragment() - .append("SELECT a.name FROM ").append(mapTi, "m") - .append(" JOIN ").append(ti, "a") - .append(" ON m.alias = a.RowId WHERE m.lsid = ? "); - sql.add(getLSID()); - ArrayList aliases = new SqlSelector(mapTi.getSchema(), sql).getArrayList(String.class); - return Collections.unmodifiableList(aliases); - } - - @Override - public String getDocumentId() - { - String dataClassName = "-"; - ExpDataClass dc = getDataClass(); - if (dc != null) - dataClassName = dc.getName(); - // why not just data:rowId? - return "data:" + new Path(getContainer().getId(), dataClassName, Long.toString(getRowId())).encode(); - } - - @Override - public Map getObjectProperties() - { - return getObjectProperties(getDataClass()); - } - - @Override - public Map getObjectProperties(@Nullable User user) - { - return getObjectProperties(getDataClass(user)); - } - - private Map getObjectProperties(ExpDataClassImpl dataClass) - { - HashMap ret = new HashMap<>(super.getObjectProperties()); - var ti = null == dataClass ? null : dataClass.getTinfo(); - if (null != ti) - { - ret.putAll(getObjectProperties(ti)); - } - return ret; - } - - private static Pair getRowIdClassNameContainerFromDocumentId(String resourceIdentifier, Map dcCache) - { - if (resourceIdentifier.startsWith("data:")) - resourceIdentifier = resourceIdentifier.substring("data:".length()); - - Path path = Path.parse(resourceIdentifier); - if (path.size() != 3) - return null; - String containerId = path.get(0); - String dataClassName = path.get(1); - String rowIdString = path.get(2); - - long rowId; - try - { - rowId = Long.parseLong(rowIdString); - if (rowId == 0) - return null; - } - catch (NumberFormatException ex) - { - return null; - } - - Container c = ContainerManager.getForId(containerId); - if (c == null) - return null; - - ExpDataClass dc = null; - if (!StringUtils.isEmpty(dataClassName) && !dataClassName.equals("-")) - { - String dcKey = containerId + '-' + dataClassName; - dc = dcCache.computeIfAbsent(dcKey, (x) -> ExperimentServiceImpl.get().getDataClass(c, dataClassName)); - } - - return new Pair<>(rowId, dc); - } - - @Nullable - public static ExpDataImpl fromDocumentId(String resourceIdentifier) - { - Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, new HashMap<>()); - if (rowIdDataClass == null) - return null; - - Long rowId = rowIdDataClass.first; - ExpDataClass dc = rowIdDataClass.second; - - if (dc != null) - return ExperimentServiceImpl.get().getExpData(dc, rowId); - else - return ExperimentServiceImpl.get().getExpData(rowId); - } - - @Nullable - public static Map fromDocumentIds(Collection resourceIdentifiers) - { - Map rowIdIdentifierMap = new LongHashMap<>(); - Map dcCache = new HashMap<>(); - Map dcMap = new LongHashMap<>(); - Map> dcRowIdMap = new LongHashMap<>(); // data rowIds with dataClass - List rowIds = new ArrayList<>(); // data rowIds without dataClass - for (String resourceIdentifier : resourceIdentifiers) - { - Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, dcCache); - if (rowIdDataClass == null) - continue; - - Long rowId = rowIdDataClass.first; - ExpDataClass dc = rowIdDataClass.second; - - rowIdIdentifierMap.put(rowId, resourceIdentifier); - - if (dc != null) - { - dcMap.put(dc.getRowId(), dc); - dcRowIdMap - .computeIfAbsent(dc.getRowId(), (k) -> new ArrayList<>()) - .add(rowId); - } - else - rowIds.add(rowId); - } - - List expDatas = new ArrayList<>(); - if (!rowIds.isEmpty()) - expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(rowIds)); - - if (!dcRowIdMap.isEmpty()) - { - for (Long dataClassId : dcRowIdMap.keySet()) - { - ExpDataClass dc = dcMap.get(dataClassId); - if (dc != null) - expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(dc, dcRowIdMap.get(dataClassId))); - } - } - - Map identifierDatas = new HashMap<>(); - for (ExpData data : expDatas) - { - identifierDatas.put(rowIdIdentifierMap.get(data.getRowId()), data); - } - - return identifierDatas; - } - - @Override - public @Nullable URI getWebDavURL(@NotNull FileContentService.PathType type) - { - java.nio.file.Path path = getFilePath(); - if (path == null) - { - return null; - } - - Container c = getContainer(); - if (c == null) - { - return null; - } - - return FileContentService.get().getWebDavUrl(path, c, type); - } - - @Override - public @Nullable WebdavResource createIndexDocument(@Nullable TableInfo tableInfo) - { - Container container = getContainer(); - if (container == null) - return null; - - Map props = new HashMap<>(); - JSONObject jsonData = new JSONObject(); - Set keywordsHi = new HashSet<>(); - Set keywordsMed = new HashSet<>(); - Set keywordsLo = new HashSet<>(); - - Set identifiersHi = new HashSet<>(); - Set identifiersMed = new HashSet<>(); - Set identifiersLo = new HashSet<>(); - - StringBuilder body = new StringBuilder(); - - // Name is an identifier with the highest weight - identifiersHi.add(getName()); - keywordsMed.add(getName()); // also add to keywords since those are stemmed - - // Description is added as a keywordsLo -- in Biologics it is common for the description to - // contain names of other DataClasses, e.g., "Mature desK of PS-10", which would be tokenized as - // [mature, desk, ps, 10] if added it as a keyword so we lower its priority to avoid useless results. - // CONSIDER: tokenize the description and extract identifiers - if (null != getDescription()) - keywordsLo.add(getDescription()); - - String comment = getComment(); - if (comment != null) - keywordsMed.add(comment); - - // Add aliases in parentheses in the title - StringBuilder title = new StringBuilder(getName()); - Collection aliases = getAliases(); - if (!aliases.isEmpty()) - { - title.append(" (").append(StringUtils.join(aliases, ", ")).append(")"); - identifiersHi.addAll(aliases); - } - - ExpDataClassImpl dc = getDataClass(User.getSearchUser()); - if (dc != null) - { - ActionURL show = new ActionURL(ExperimentController.ShowDataClassAction.class, container).addParameter("rowId", dc.getRowId()); - NavTree t = new NavTree(dc.getName(), show); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - props.put(SearchService.PROPERTY.navtrail.toString(), nav); - - props.put(DataSearchResultTemplate.PROPERTY, dc.getName()); - body.append(dc.getName()); - - if (tableInfo == null) - tableInfo = QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_EXP_DATA).getTable(dc.getName()); - - if (!(tableInfo instanceof ExpDataClassDataTableImpl expDataClassDataTable)) - throw new IllegalArgumentException(String.format("Unable to index data class item in %s. Table must be an instance of %s", dc.getName(), ExpDataClassDataTableImpl.class.getName())); - - if (!expDataClassDataTable.getDataClass().equals(dc)) - throw new IllegalArgumentException(String.format("Data class table mismatch for %s", dc.getName())); - - // Collect other text columns and lookup display columns - getIndexValues(props, expDataClassDataTable, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); - } - - // === Stored, not indexed - if (dc != null && dc.isMedia()) - props.put(SearchService.PROPERTY.categories.toString(), expMediaDataCategory.toString()); - else - props.put(SearchService.PROPERTY.categories.toString(), expDataCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), title.toString()); - props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); - - ActionURL view = ExperimentController.ExperimentUrlsImpl.get().getDataDetailsURL(this); - view.setExtraPath(container.getId()); - String docId = getDocumentId(); - - // Generate a summary explicitly instead of relying on a summary to be extracted - // from the document body. Placing lookup values and the description in the body - // would tokenize using the English analyzer and index "PS-12" as ["ps", "12"] which leads to poor results. - StringBuilder summary = new StringBuilder(); - if (StringUtils.isNotEmpty(getDescription())) - summary.append(getDescription()).append("\n"); - - appendTokens(summary, keywordsMed); - appendTokens(summary, identifiersMed); - appendTokens(summary, identifiersLo); - - props.put(SearchService.PROPERTY.summary.toString(), summary); - - return new ExpDataResource( - getRowId(), - new Path(docId), - docId, - container.getEntityId(), - "text/plain", - body.toString(), - view, - props, - getCreatedBy(), - getCreated(), - getModifiedBy(), - getModified() - ); - } - - private static void appendTokens(StringBuilder sb, Collection toks) - { - if (toks.isEmpty()) - return; - - sb.append(toks.stream().map(s -> s.length() > 30 ? StringUtilsLabKey.leftSurrogatePairFriendly(s, 30) + "\u2026" : s).collect(Collectors.joining(", "))).append("\n"); - } - - private static class ExpDataResource extends SimpleDocumentResource - { - final long _rowId; - - public ExpDataResource(long rowId, Path path, String documentId, GUID containerId, String contentType, String body, URLHelper executeUrl, Map properties, User createdBy, Date created, User modifiedBy, Date modified) - { - super(path, documentId, containerId, contentType, body, executeUrl, createdBy, created, modifiedBy, modified, properties); - _rowId = rowId; - } - - @Override - public void setLastIndexed(long ms, long modified) - { - ExperimentServiceImpl.get().setDataLastIndexed(_rowId, ms); - } - } - - public static class DataSearchResultTemplate implements SearchResultTemplate - { - public static final String NAME = "data"; - public static final String PROPERTY = "dataclass"; - - @Nullable - @Override - public String getName() - { - return NAME; - } - - private ExpDataClass getDataClass() - { - if (HttpView.hasCurrentView()) - { - ViewContext ctx = HttpView.currentContext(); - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - return ExperimentService.get().getDataClass(ctx.getContainer(), ctx.getUser(), dataclass); - } - return null; - } - - @Nullable - @Override - public String getCategories() - { - ExpDataClass dataClass = getDataClass(); - - if (dataClass != null && dataClass.isMedia()) - return expMediaDataCategory.getName(); - - return expDataCategory.getName(); - } - - @Nullable - @Override - public SearchScope getSearchScope() - { - return SearchScope.FolderAndSubfolders; - } - - @NotNull - @Override - public String getResultNameSingular() - { - ExpDataClass dc = getDataClass(); - if (dc != null) - return dc.getName(); - return "data"; - } - - @NotNull - @Override - public String getResultNamePlural() - { - return getResultNameSingular(); - } - - @Override - public boolean includeNavigationLinks() - { - return true; - } - - @Override - public boolean includeAdvanceUI() - { - return false; - } - - @Nullable - @Override - public HtmlString getExtraHtml(ViewContext ctx) - { - String q = ctx.getActionURL().getParameter("q"); - - if (StringUtils.isNotBlank(q)) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - ActionURL url = ctx.cloneActionURL().deleteParameter(PROPERTY); - url.replaceParameter(ActionURL.Param._dc, (int)Math.round(1000 * Math.random())); - - StringBuilder html = new StringBuilder(); - html.append("
    "); - - appendParam(html, null, dataclass, "All", false, url); - for (ExpDataClass dc : ExperimentService.get().getDataClasses(ctx.getContainer(), ctx.getUser(), true)) - { - appendParam(html, dc.getName(), dataclass, dc.getName(), true, url); - } - - html.append("
    "); - return HtmlString.unsafe(html.toString()); - } - else - { - return null; - } - } - - private void appendParam(StringBuilder sb, @Nullable String dataclass, @Nullable String current, @NotNull String label, boolean addParam, ActionURL url) - { - sb.append(""); - - if (!Objects.equals(dataclass, current)) - { - if (addParam) - url = url.clone().addParameter(PROPERTY, dataclass); - - sb.append(LinkBuilder.simpleLink(label, url)); - } - else - { - sb.append(label); - } - - sb.append(" "); - } - - @Override - public HtmlString getHiddenInputsHtml(ViewContext ctx) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - { - return InputBuilder.hidden().id("search-type").name(PROPERTY).value(dataclass).getHtmlString(); - } - - return null; - } - - - @Override - public String reviseQuery(ViewContext ctx, String q) - { - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - - if (null != dataclass) - return "+(" + q + ") +" + PROPERTY + ":" + dataclass; - else - return q; - } - - @Override - public void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) - { - SearchResultTemplate.super.addNavTrail(root, ctx, scope, category); - - String dataclass = ctx.getActionURL().getParameter(PROPERTY); - if (dataclass != null) - { - String text = root.getText(); - root.setText(text + " - " + dataclass); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.ExperimentDataHandler; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Handler; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.XarFormatException; +import org.labkey.api.exp.XarSource; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataClassDataTable; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryRowReference; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchResultTemplate; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DataClassReadPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MediaReadPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.MimeMap; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.InputBuilder; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.ViewContext; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.labkey.api.exp.query.ExpSchema.SCHEMA_EXP_DATA; + +public class ExpDataImpl extends AbstractRunItemImpl implements ExpData +{ + public enum DataOperations + { + Edit("editing", UpdatePermission.class), + EditLineage("editing lineage", UpdatePermission.class), + Delete("deleting", DeletePermission.class), + Move("moving", MoveEntitiesPermission.class); + + private final String _description; // used as a suffix in messaging users about what is not allowed + private final Class _permissionClass; + + DataOperations(String description, Class permissionClass) + { + _description = description; + _permissionClass = permissionClass; + } + + public String getDescription() + { + return _description; + } + + public Class getPermissionClass() + { + return _permissionClass; + } + } + + public static final SearchService.SearchCategory expDataCategory = new SearchService.SearchCategory("data", "ExpData", false) { + @Override + public Set getPermittedContainerIds(User user, Map containers) + { + return getPermittedContainerIds(user, containers, DataClassReadPermission.class); + } + }; + public static final SearchService.SearchCategory expMediaDataCategory = new SearchService.SearchCategory("mediaData", "ExpData for media objects", false) { + @Override + public Set getPermittedContainerIds(User user, Map containers) + { + return getPermittedContainerIds(user, containers, MediaReadPermission.class); + } + }; + + /** Cache this because it can be expensive to recompute */ + private Boolean _finalRunOutput; + + /** + * Temporary mapping until experiment.xml contains the mime type + */ + private static final MimeMap MIME_MAP = new MimeMap(); + + static public List fromDatas(List datas) + { + List ret = new ArrayList<>(datas.size()); + for (Data data : datas) + { + ret.add(new ExpDataImpl(data)); + } + return ret; + } + + // For serialization + protected ExpDataImpl() {} + + public ExpDataImpl(Data data) + { + super(data); + } + + @Override + public void setComment(User user, String comment) throws ValidationException + { + setComment(user, comment, true); + } + + @Override + public void setComment(User user, String comment, boolean index) throws ValidationException + { + super.setComment(user, comment); + + if (index) + index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); + } + + @Override + @Nullable + public ActionURL detailsURL() + { + DataType dataType = getDataType(); + if (dataType != null) + { + ActionURL url = dataType.getDetailsURL(this); + if (url != null) + return url; + } + + return _object.detailsURL(); + } + + @Override + public @Nullable QueryRowReference getQueryRowReference() + { + return getQueryRowReference(null); + } + + @Override + public @Nullable QueryRowReference getQueryRowReference(@Nullable User user) + { + ExpDataClassImpl dc = getDataClass(user); + if (dc != null) + return new QueryRowReference(getContainer(), SCHEMA_EXP_DATA, dc.getName(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + + // Issue 40123: see MedImmuneDataHandler MEDIMMUNE_DATA_TYPE, this claims the "Data" namespace + DataType type = getDataType(); + if (type != null) + { + QueryRowReference queryRowReference = type.getQueryRowReference(this); + if (queryRowReference != null) + return queryRowReference; + } + + return new QueryRowReference(getContainer(), ExpSchema.SCHEMA_EXP, ExpSchema.TableType.Data.name(), FieldKey.fromParts(ExpDataTable.Column.RowId), getRowId()); + } + + @Override + public List getTargetApplications() + { + return getTargetApplications(new SimpleFilter(FieldKey.fromParts("DataId"), getRowId()), ExperimentServiceImpl.get().getTinfoDataInput()); + } + + @Override + public List getTargetRuns() + { + return getTargetRuns(ExperimentServiceImpl.get().getTinfoDataInput(), "DataId"); + } + + @Override + public DataType getDataType() + { + return ExperimentService.get().getDataType(getLSIDNamespacePrefix()); + } + + @Override + public void setDataFileURI(URI uri) + { + ensureUnlocked(); + _object.setDataFileUrl(ExpData.normalizeDataFileURI(uri)); + } + + @Override + public void save(User user) + { + // Replace the default "Data" cpastype if the Data belongs to a DataClass + ExpDataClassImpl dataClass = getDataClass(); + if (dataClass != null && ExpData.DEFAULT_CPAS_TYPE.equals(getCpasType())) + setCpasType(dataClass.getLSID()); + + boolean isNew = getRowId() == 0; + save(user, ExperimentServiceImpl.get().getTinfoData(), true); + + if (isNew) + { + if (dataClass != null) + { + Map map = new HashMap<>(); + map.put("lsid", getLSID()); + Table.insert(user, dataClass.getTinfo(), map); + } + } + index(SearchService.get().defaultTask().getQueue(getContainer(), SearchService.PRIORITY.modified), null); + } + + @Override + protected void save(User user, TableInfo table, boolean ensureObject) + { + assert ensureObject; + super.save(user, table, true); + } + + @Override + public URI getDataFileURI() + { + String url = _object.getDataFileUrl(); + if (url == null) + return null; + try + { + return new URI(_object.getDataFileUrl()); + } + catch (URISyntaxException use) + { + return null; + } + } + + @Override + public ExperimentDataHandler findDataHandler() + { + return Handler.Priority.findBestHandler(ExperimentServiceImpl.get().getExperimentDataHandlers(), this); + } + + @Override + public String getDataFileUrl() + { + return _object.getDataFileUrl(); + } + + @Override + public boolean hasFileScheme() + { + return !FileUtil.hasCloudScheme(getDataFileUrl()); + } + + @Override + @Nullable + public File getFile() + { + return _object.getFile(); + } + + @Override + public @Nullable FileLike getFileLike() + { + return _object.getFileLike(); + } + + @Override + @Nullable + public java.nio.file.Path getFilePath() + { + return _object.getFilePath(); + } + + @Override + public boolean isInlineImage() + { + return null != getFile() && MIME_MAP.isInlineImageFor(getFile()); + } + + @Override + public void delete(User user) + { + delete(user, true); + } + + @Override + public void delete(User user, boolean deleteRunsUsingData) + { + ExperimentServiceImpl.get().deleteDataByRowIds(user, getContainer(), Collections.singleton(getRowId()), deleteRunsUsingData); + } + + public String getMimeType() + { + if (null != getDataFileUrl()) + return MIME_MAP.getContentTypeFor(getDataFileUrl()); + else + return null; + } + + @Override + public boolean isFileOnDisk() + { + java.nio.file.Path f = getFilePath(); + if (f != null) + if (!FileUtil.hasCloudScheme(f)) + return NetworkDrive.exists(f.toFile()) && !Files.isDirectory(f); + else + return Files.exists(f); + else + return false; + } + + public boolean isPathAccessible() + { + java.nio.file.Path path = getFilePath(); + return (null != path && Files.exists(path)); + } + + @Override + public String getCpasType() + { + String result = _object.getCpasType(); + if (result != null) + return result; + + ExpDataClass dataClass = getDataClass(); + if (dataClass != null) + return dataClass.getLSID(); + + return ExpData.DEFAULT_CPAS_TYPE; + } + + public void setGenerated(boolean generated) + { + ensureUnlocked(); + _object.setGenerated(generated); + } + + @Override + public boolean isGenerated() + { + return _object.isGenerated(); + } + + @Override + public boolean isFinalRunOutput() + { + if (_finalRunOutput == null) + { + ExpRun run = getRun(); + _finalRunOutput = run != null && run.isFinalOutput(this); + } + return _finalRunOutput.booleanValue(); + } + + @Override + @Nullable + public ExpDataClassImpl getDataClass() + { + return getDataClass(null); + } + + @Override + @Nullable + public ExpDataClassImpl getDataClass(@Nullable User user) + { + if (_object.getClassId() != null && getContainer() != null) + { + if (user == null) + return ExperimentServiceImpl.get().getDataClass(getContainer(), _object.getClassId()); + else + return ExperimentServiceImpl.get().getDataClass(getContainer(), user, _object.getClassId()); + } + + return null; + } + + @Override + public void importDataFile(PipelineJob job, XarSource xarSource) throws ExperimentException + { + String dataFileURL = getDataFileUrl(); + if (dataFileURL == null) + return; + + if (xarSource.shouldIgnoreDataFiles()) + { + job.debug("Skipping load of data file " + dataFileURL + " based on the XAR source"); + return; + } + + job.debug("Trying to load data file " + dataFileURL + " into the system"); + + java.nio.file.Path path = FileUtil.stringToPath(getContainer(), dataFileURL); + + if (!Files.exists(path)) + { + job.debug("Unable to find the data file " + FileUtil.getAbsolutePath(getContainer(), path) + " on disk."); + return; + } + + // Check that the file is under the pipeline root to prevent users from referencing a file that they + // don't have permission to import + PipeRoot pr = PipelineService.get().findPipelineRoot(job.getContainer()); + if (!xarSource.allowImport(pr, job.getContainer(), path)) + { + if (pr == null) + { + job.warn("No pipeline root was set, skipping load of file " + FileUtil.getAbsolutePath(getContainer(), path)); + return; + } + job.debug("The data file " + FileUtil.getAbsolutePath(getContainer(), path) + " is not under the folder's pipeline root: " + pr + ". It will not be loaded directly, but may be loaded if referenced from other files that are under the pipeline root."); + return; + } + + ExperimentDataHandler handler = findDataHandler(); + try + { + handler.importFile(this, FileSystemLike.wrapFile(path), job.getInfo(), job.getLogger(), xarSource.getXarContext()); + } + catch (ExperimentException e) + { + throw new XarFormatException(e); + } + + job.debug("Finished trying to load data file " + dataFileURL + " into the system"); + } + + // Get all text and int strings from the data class for indexing + private void getIndexValues( + Map props, + @NotNull ExpDataClassDataTableImpl table, + Set identifiersHi, + Set identifiersMed, + Set identifiersLo, + Set keywordHi, + Set keywordMed, + Set keywordsLo, + JSONObject jsonData + ) + { + CaseInsensitiveHashSet skipColumns = new CaseInsensitiveHashSet(); + for (ExpDataClassDataTable.Column column : ExpDataClassDataTable.Column.values()) + skipColumns.add(column.name()); + skipColumns.add("Ancestors"); + skipColumns.add("Container"); + + processIndexValues(props, table, skipColumns, identifiersHi, identifiersMed, identifiersLo, keywordHi, keywordMed, keywordsLo, jsonData); + } + + @Override + @NotNull + public Collection getAliases() + { + TableInfo mapTi = ExperimentService.get().getTinfoDataAliasMap(); + TableInfo ti = ExperimentService.get().getTinfoAlias(); + SQLFragment sql = new SQLFragment() + .append("SELECT a.name FROM ").append(mapTi, "m") + .append(" JOIN ").append(ti, "a") + .append(" ON m.alias = a.RowId WHERE m.lsid = ? "); + sql.add(getLSID()); + ArrayList aliases = new SqlSelector(mapTi.getSchema(), sql).getArrayList(String.class); + return Collections.unmodifiableList(aliases); + } + + @Override + public String getDocumentId() + { + String dataClassName = "-"; + ExpDataClass dc = getDataClass(); + if (dc != null) + dataClassName = dc.getName(); + // why not just data:rowId? + return "data:" + new Path(getContainer().getId(), dataClassName, Long.toString(getRowId())).encode(); + } + + @Override + public Map getObjectProperties() + { + return getObjectProperties(getDataClass()); + } + + @Override + public Map getObjectProperties(@Nullable User user) + { + return getObjectProperties(getDataClass(user)); + } + + private Map getObjectProperties(ExpDataClassImpl dataClass) + { + HashMap ret = new HashMap<>(super.getObjectProperties()); + var ti = null == dataClass ? null : dataClass.getTinfo(); + if (null != ti) + { + ret.putAll(getObjectProperties(ti)); + } + return ret; + } + + private static Pair getRowIdClassNameContainerFromDocumentId(String resourceIdentifier, Map dcCache) + { + if (resourceIdentifier.startsWith("data:")) + resourceIdentifier = resourceIdentifier.substring("data:".length()); + + Path path = Path.parse(resourceIdentifier); + if (path.size() != 3) + return null; + String containerId = path.get(0); + String dataClassName = path.get(1); + String rowIdString = path.get(2); + + long rowId; + try + { + rowId = Long.parseLong(rowIdString); + if (rowId == 0) + return null; + } + catch (NumberFormatException ex) + { + return null; + } + + Container c = ContainerManager.getForId(containerId); + if (c == null) + return null; + + ExpDataClass dc = null; + if (!StringUtils.isEmpty(dataClassName) && !dataClassName.equals("-")) + { + String dcKey = containerId + '-' + dataClassName; + dc = dcCache.computeIfAbsent(dcKey, (x) -> ExperimentServiceImpl.get().getDataClass(c, dataClassName)); + } + + return new Pair<>(rowId, dc); + } + + @Nullable + public static ExpDataImpl fromDocumentId(String resourceIdentifier) + { + Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, new HashMap<>()); + if (rowIdDataClass == null) + return null; + + Long rowId = rowIdDataClass.first; + ExpDataClass dc = rowIdDataClass.second; + + if (dc != null) + return ExperimentServiceImpl.get().getExpData(dc, rowId); + else + return ExperimentServiceImpl.get().getExpData(rowId); + } + + @Nullable + public static Map fromDocumentIds(Collection resourceIdentifiers) + { + Map rowIdIdentifierMap = new LongHashMap<>(); + Map dcCache = new HashMap<>(); + Map dcMap = new LongHashMap<>(); + Map> dcRowIdMap = new LongHashMap<>(); // data rowIds with dataClass + List rowIds = new ArrayList<>(); // data rowIds without dataClass + for (String resourceIdentifier : resourceIdentifiers) + { + Pair rowIdDataClass = getRowIdClassNameContainerFromDocumentId(resourceIdentifier, dcCache); + if (rowIdDataClass == null) + continue; + + Long rowId = rowIdDataClass.first; + ExpDataClass dc = rowIdDataClass.second; + + rowIdIdentifierMap.put(rowId, resourceIdentifier); + + if (dc != null) + { + dcMap.put(dc.getRowId(), dc); + dcRowIdMap + .computeIfAbsent(dc.getRowId(), (k) -> new ArrayList<>()) + .add(rowId); + } + else + rowIds.add(rowId); + } + + List expDatas = new ArrayList<>(); + if (!rowIds.isEmpty()) + expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(rowIds)); + + if (!dcRowIdMap.isEmpty()) + { + for (Long dataClassId : dcRowIdMap.keySet()) + { + ExpDataClass dc = dcMap.get(dataClassId); + if (dc != null) + expDatas.addAll(ExperimentServiceImpl.get().getExpDatas(dc, dcRowIdMap.get(dataClassId))); + } + } + + Map identifierDatas = new HashMap<>(); + for (ExpData data : expDatas) + { + identifierDatas.put(rowIdIdentifierMap.get(data.getRowId()), data); + } + + return identifierDatas; + } + + @Override + public @Nullable URI getWebDavURL(@NotNull FileContentService.PathType type) + { + java.nio.file.Path path = getFilePath(); + if (path == null) + { + return null; + } + + Container c = getContainer(); + if (c == null) + { + return null; + } + + return FileContentService.get().getWebDavUrl(path, c, type); + } + + @Override + public @Nullable WebdavResource createIndexDocument(@Nullable TableInfo tableInfo) + { + Container container = getContainer(); + if (container == null) + return null; + + Map props = new HashMap<>(); + JSONObject jsonData = new JSONObject(); + Set keywordsHi = new HashSet<>(); + Set keywordsMed = new HashSet<>(); + Set keywordsLo = new HashSet<>(); + + Set identifiersHi = new HashSet<>(); + Set identifiersMed = new HashSet<>(); + Set identifiersLo = new HashSet<>(); + + StringBuilder body = new StringBuilder(); + + // Name is an identifier with the highest weight + identifiersHi.add(getName()); + keywordsMed.add(getName()); // also add to keywords since those are stemmed + + // Description is added as a keywordsLo -- in Biologics it is common for the description to + // contain names of other DataClasses, e.g., "Mature desK of PS-10", which would be tokenized as + // [mature, desk, ps, 10] if added it as a keyword so we lower its priority to avoid useless results. + // CONSIDER: tokenize the description and extract identifiers + if (null != getDescription()) + keywordsLo.add(getDescription()); + + String comment = getComment(); + if (comment != null) + keywordsMed.add(comment); + + // Add aliases in parentheses in the title + StringBuilder title = new StringBuilder(getName()); + Collection aliases = getAliases(); + if (!aliases.isEmpty()) + { + title.append(" (").append(StringUtils.join(aliases, ", ")).append(")"); + identifiersHi.addAll(aliases); + } + + ExpDataClassImpl dc = getDataClass(User.getSearchUser()); + if (dc != null) + { + ActionURL show = new ActionURL(ExperimentController.ShowDataClassAction.class, container).addParameter("rowId", dc.getRowId()); + NavTree t = new NavTree(dc.getName(), show); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + props.put(SearchService.PROPERTY.navtrail.toString(), nav); + + props.put(DataSearchResultTemplate.PROPERTY, dc.getName()); + body.append(dc.getName()); + + if (tableInfo == null) + tableInfo = QueryService.get().getUserSchema(User.getSearchUser(), container, SCHEMA_EXP_DATA).getTable(dc.getName()); + + if (!(tableInfo instanceof ExpDataClassDataTableImpl expDataClassDataTable)) + throw new IllegalArgumentException(String.format("Unable to index data class item in %s. Table must be an instance of %s", dc.getName(), ExpDataClassDataTableImpl.class.getName())); + + if (!expDataClassDataTable.getDataClass().equals(dc)) + throw new IllegalArgumentException(String.format("Data class table mismatch for %s", dc.getName())); + + // Collect other text columns and lookup display columns + getIndexValues(props, expDataClassDataTable, identifiersHi, identifiersMed, identifiersLo, keywordsHi, keywordsMed, keywordsLo, jsonData); + } + + // === Stored, not indexed + if (dc != null && dc.isMedia()) + props.put(SearchService.PROPERTY.categories.toString(), expMediaDataCategory.toString()); + else + props.put(SearchService.PROPERTY.categories.toString(), expDataCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), title.toString()); + props.put(SearchService.PROPERTY.jsonData.toString(), jsonData); + + ActionURL view = ExperimentController.ExperimentUrlsImpl.get().getDataDetailsURL(this); + view.setExtraPath(container.getId()); + String docId = getDocumentId(); + + // Generate a summary explicitly instead of relying on a summary to be extracted + // from the document body. Placing lookup values and the description in the body + // would tokenize using the English analyzer and index "PS-12" as ["ps", "12"] which leads to poor results. + StringBuilder summary = new StringBuilder(); + if (StringUtils.isNotEmpty(getDescription())) + summary.append(getDescription()).append("\n"); + + appendTokens(summary, keywordsMed); + appendTokens(summary, identifiersMed); + appendTokens(summary, identifiersLo); + + props.put(SearchService.PROPERTY.summary.toString(), summary); + + return new ExpDataResource( + getRowId(), + new Path(docId), + docId, + container.getEntityId(), + "text/plain", + body.toString(), + view, + props, + getCreatedBy(), + getCreated(), + getModifiedBy(), + getModified() + ); + } + + private static void appendTokens(StringBuilder sb, Collection toks) + { + if (toks.isEmpty()) + return; + + sb.append(toks.stream().map(s -> s.length() > 30 ? StringUtilsLabKey.leftSurrogatePairFriendly(s, 30) + "\u2026" : s).collect(Collectors.joining(", "))).append("\n"); + } + + private static class ExpDataResource extends SimpleDocumentResource + { + final long _rowId; + + public ExpDataResource(long rowId, Path path, String documentId, GUID containerId, String contentType, String body, URLHelper executeUrl, Map properties, User createdBy, Date created, User modifiedBy, Date modified) + { + super(path, documentId, containerId, contentType, body, executeUrl, createdBy, created, modifiedBy, modified, properties); + _rowId = rowId; + } + + @Override + public void setLastIndexed(long ms, long modified) + { + ExperimentServiceImpl.get().setDataLastIndexed(_rowId, ms); + } + } + + public static class DataSearchResultTemplate implements SearchResultTemplate + { + public static final String NAME = "data"; + public static final String PROPERTY = "dataclass"; + + @Nullable + @Override + public String getName() + { + return NAME; + } + + private ExpDataClass getDataClass() + { + if (HttpView.hasCurrentView()) + { + ViewContext ctx = HttpView.currentContext(); + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + return ExperimentService.get().getDataClass(ctx.getContainer(), ctx.getUser(), dataclass); + } + return null; + } + + @Nullable + @Override + public String getCategories() + { + ExpDataClass dataClass = getDataClass(); + + if (dataClass != null && dataClass.isMedia()) + return expMediaDataCategory.getName(); + + return expDataCategory.getName(); + } + + @Nullable + @Override + public SearchScope getSearchScope() + { + return SearchScope.FolderAndSubfolders; + } + + @NotNull + @Override + public String getResultNameSingular() + { + ExpDataClass dc = getDataClass(); + if (dc != null) + return dc.getName(); + return "data"; + } + + @NotNull + @Override + public String getResultNamePlural() + { + return getResultNameSingular(); + } + + @Override + public boolean includeNavigationLinks() + { + return true; + } + + @Override + public boolean includeAdvanceUI() + { + return false; + } + + @Nullable + @Override + public HtmlString getExtraHtml(ViewContext ctx) + { + String q = ctx.getActionURL().getParameter("q"); + + if (StringUtils.isNotBlank(q)) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + ActionURL url = ctx.cloneActionURL().deleteParameter(PROPERTY); + url.replaceParameter(ActionURL.Param._dc, (int)Math.round(1000 * Math.random())); + + StringBuilder html = new StringBuilder(); + html.append("
    "); + + appendParam(html, null, dataclass, "All", false, url); + for (ExpDataClass dc : ExperimentService.get().getDataClasses(ctx.getContainer(), ctx.getUser(), true)) + { + appendParam(html, dc.getName(), dataclass, dc.getName(), true, url); + } + + html.append("
    "); + return HtmlString.unsafe(html.toString()); + } + else + { + return null; + } + } + + private void appendParam(StringBuilder sb, @Nullable String dataclass, @Nullable String current, @NotNull String label, boolean addParam, ActionURL url) + { + sb.append(""); + + if (!Objects.equals(dataclass, current)) + { + if (addParam) + url = url.clone().addParameter(PROPERTY, dataclass); + + sb.append(LinkBuilder.simpleLink(label, url)); + } + else + { + sb.append(label); + } + + sb.append(" "); + } + + @Override + public HtmlString getHiddenInputsHtml(ViewContext ctx) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + { + return InputBuilder.hidden().id("search-type").name(PROPERTY).value(dataclass).getHtmlString(); + } + + return null; + } + + + @Override + public String reviseQuery(ViewContext ctx, String q) + { + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + + if (null != dataclass) + return "+(" + q + ") +" + PROPERTY + ":" + dataclass; + else + return q; + } + + @Override + public void addNavTrail(NavTree root, ViewContext ctx, @NotNull SearchScope scope, @Nullable String category) + { + SearchResultTemplate.super.addNavTrail(root, ctx, scope, category); + + String dataclass = ctx.getActionURL().getParameter(PROPERTY); + if (dataclass != null) + { + String text = root.getText(); + root.setText(text + " - " + dataclass); + } + } + } +} diff --git a/mothership/src/org/labkey/mothership/MothershipManager.java b/mothership/src/org/labkey/mothership/MothershipManager.java index 7d60c0a1979..efc3cf368ee 100644 --- a/mothership/src/org/labkey/mothership/MothershipManager.java +++ b/mothership/src/org/labkey/mothership/MothershipManager.java @@ -1,627 +1,627 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.mothership; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.query.FieldKey; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.MothershipReport; -import org.labkey.api.util.ReentrantLockWithName; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; - -import static org.labkey.api.security.UserManager.USER_DISPLAY_NAME_COMPARATOR; - -public class MothershipManager -{ - private static final MothershipManager INSTANCE = new MothershipManager(); - private static final String SCHEMA_NAME = "mothership"; - private static final String UPGRADE_MESSAGE_PROPERTY_CATEGORY = "upgradeMessage"; - private static final String MOTHERSHIP_SECURE_CATEGORY = "mothershipSecure"; - private static final String CURRENT_BUILD_DATE_PROP = "currentBuildDate"; - private static final String UPGRADE_MESSAGE_PROP = "upgradeMessage"; - private static final String CREATE_ISSUE_URL_PROP = "createIssueURL"; - private static final String ISSUES_CONTAINER_PROP = "issuesContainer"; - private static final String MARKETING_MESSAGE_PROP = "marketingMessage"; - private static final String UPTIME_CONTAINER_PROP = "uptimeContainer"; - private static final String STATUS_CAKE_API_KEY_PROP = "statusCakeApiKey"; - private static final ReentrantLock INSERT_EXCEPTION_LOCK = new ReentrantLockWithName(MothershipManager.class, "INSERT_EXCEPTION_LOCK"); - - private static final Logger log = LogHelper.getLogger(MothershipManager.class, "Persists mothership records like sessions and installs"); - - public static MothershipManager get() - { - return INSTANCE; - } - - private MothershipManager() {} - - String getSchemaName() - { - return SCHEMA_NAME; - } - - /* package */ - public DbSchema getSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); - } - - public void insertException(ExceptionStackTrace stackTrace, ExceptionReport report) - { - // Synchronize to prevent two different threads from creating duplicate rows in the ExceptionStackTrace table - try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction(INSERT_EXCEPTION_LOCK)) - { - boolean isNew = false; - ExceptionStackTrace existingStackTrace = getExceptionStackTrace(stackTrace.getStackTraceHash(), stackTrace.getContainer()); - if (existingStackTrace != null) - { - stackTrace = existingStackTrace; - } - else - { - stackTrace = Table.insert(null, getTableInfoExceptionStackTrace(), stackTrace); - isNew = true; - } - - report.setExceptionStackTraceId(stackTrace.getExceptionStackTraceId()); - - String url = report.getUrl(); - if (null != url && url.length() > 512) - report.setURL(StringUtilsLabKey.leftSurrogatePairFriendly(url, 506) + "..."); - - String referrerURL = report.getReferrerURL(); - if (null != referrerURL && referrerURL.length() > 512) - report.setReferrerURL(StringUtilsLabKey.leftSurrogatePairFriendly(referrerURL, 506) + "..."); - - String browser = report.getBrowser(); - if (null != browser && browser.length() > 100) - report.setBrowser(browser.substring(0,90) + "..."); - - String exceptionMessage = report.getExceptionMessage(); - if (null != exceptionMessage && exceptionMessage.length() > 1000) - report.setExceptionMessage(StringUtilsLabKey.leftSurrogatePairFriendly(exceptionMessage, 990) + "..."); - - String actionName = report.getPageflowAction(); - if (null != actionName && actionName.length() > 40) - { - report.setPageflowAction(actionName.substring(0, 39)); - } - - String controllerName = report.getPageflowName(); - if (null != controllerName && controllerName.length() > 30) - { - report.setPageflowName(controllerName.substring(0, 29)); - } - - String errorCode = report.getErrorCode(); - if (null != errorCode && errorCode.length() > MothershipReport.ERROR_CODE_LENGTH) - { - report.setErrorCode(errorCode.substring(0, MothershipReport.ERROR_CODE_LENGTH - 1)); - } - - report = Table.insert(null, getTableInfoExceptionReport(), report); - stackTrace.setInstances(stackTrace.getInstances() + 1); - stackTrace.setLastReport(report.getCreated()); - if (isNew) - { - stackTrace.setFirstReport(report.getCreated()); - } - Table.update(null, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); - - transaction.commit(); - } - } - - private static final Object ENSURE_SOFTWARE_RELEASE_LOCK = new Object(); - - private void addFilter(SimpleFilter filter, String fieldKey, Object value) - { - if (value == null) - { - filter.addCondition(FieldKey.fromString(fieldKey), null, CompareType.ISBLANK); - } - else - { - filter.addCondition(FieldKey.fromString(fieldKey), value); - } - - } - - public SoftwareRelease ensureSoftwareRelease(Container container, String revision, String url, String branch, String tag, Date buildTime, String buildNumber) - { - synchronized (ENSURE_SOFTWARE_RELEASE_LOCK) - { - // Issue 48270 - transform empty strings to nulls before querying for existing row - revision = StringUtils.trimToNull(revision); - url = StringUtils.trimToNull(url); - branch = StringUtils.trimToNull(branch); - tag = StringUtils.trimToNull(tag); - revision = StringUtils.trimToNull(revision); - buildNumber = StringUtils.trimToNull(buildNumber); - - // Filter on the columns that are part of the unique constraint - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - addFilter(filter, "VcsRevision", revision); - addFilter(filter, "VcsUrl", url); - addFilter(filter, "VcsBranch", branch); - addFilter(filter, "VcsTag", tag); - addFilter(filter, "BuildTime", buildTime); - - SoftwareRelease result = new TableSelector(getTableInfoSoftwareRelease(), filter, null).getObject(SoftwareRelease.class); - if (result == null) - { - if (buildNumber == null) - { - buildNumber = "Unknown VCS"; - } - - result = new SoftwareRelease(); - result.setVcsUrl(url); - result.setVcsRevision(revision); - result.setVcsBranch(branch); - result.setVcsTag(tag); - result.setBuildTime(buildTime); - result.setBuildNumber(buildNumber); - result.setContainer(container.getId()); - result = Table.insert(null, getTableInfoSoftwareRelease(), result); - } - return result; - } - } - - public ServerInstallation getServerInstallation(@NotNull String serverGUID, @NotNull String serverHostName, @NotNull Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerInstallationGUID"), serverGUID); - filter.addCondition(FieldKey.fromString("ServerHostName"), serverHostName); - return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); - } - - public ServerSession getServerSession(String serverSessionGUID, Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerSessionGUID"), serverSessionGUID); - return new TableSelector(getTableInfoServerSession(), filter, null).getObject(ServerSession.class); - } - - public ExceptionStackTrace getExceptionStackTrace(String stackTraceHash, String containerId) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), containerId); - filter.addCondition(FieldKey.fromString("StackTraceHash"), stackTraceHash); - return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); - } - - public ExceptionStackTrace getExceptionStackTrace(int exceptionStackTraceId, Container container) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromString("ExceptionStackTraceId"), exceptionStackTraceId); - return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); - } - - public void deleteForContainer(Container c) - { - SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); - sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionReport() + " WHERE ExceptionStackTraceId IN (SELECT ExceptionStackTraceId FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?)", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoServerSession() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoServerInstallation() + " WHERE Container = ?", c); - sqlExecutor.execute("DELETE FROM " + getTableInfoSoftwareRelease() + " WHERE Container = ?", c); - } - - public void deleteForUser(User u) - { - SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); - sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET AssignedTo = NULL WHERE AssignedTo = ?", u.getUserId()); - sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET ModifiedBy = NULL WHERE ModifiedBy = ?", u.getUserId()); - } - - public synchronized ServerSession updateServerSession(MothershipController.ServerInfoForm form, String serverIP, ServerSession session, ServerInstallation installation, Container container) - { - try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction()) - { - String hostName = form.getBestServerHostName(serverIP); - ServerInstallation existingInstallation = getServerInstallation(installation.getServerInstallationGUID(), hostName, container); - - if (existingInstallation == null) - { - installation.setContainer(container.getId()); - installation.setServerHostName(hostName); - installation = Table.insert(null, getTableInfoServerInstallation(), installation); - } - else - { - existingInstallation.setServerHostName(hostName); - installation = Table.update(null, getTableInfoServerInstallation(), existingInstallation, existingInstallation.getServerInstallationId()); - } - - Date now = new Date(); - ServerSession existingSession = getServerSession(session.getServerSessionGUID(), container); - if (existingSession != null) - { - // Issue 50876: Reparent mothership server session when the base URL changes mid-session - existingSession.setServerInstallationId(installation.getServerInstallationId()); - - Calendar existingCal = Calendar.getInstance(); - existingCal.setTime(existingSession.getLastKnownTime()); - Calendar nowCal = Calendar.getInstance(); - - // Check if this session is straddling months. If so, break it into two so that we - // retain metrics with month level granularity - if (existingCal.get(Calendar.MONTH) != nowCal.get(Calendar.MONTH)) - { - // Chain the two sessions together - session.setOriginalServerSessionId(existingSession.getServerSessionId()); - - // Update the GUID for the old one as we're capturing in a new record going forward - existingSession.setServerSessionGUID(GUID.makeGUID()); - Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); - existingSession = null; - } - } - - if (existingSession == null) - { - session.setEarliestKnownTime(now); - session.setServerInstallationId(installation.getServerInstallationId()); - - configureSession(session, now, serverIP, form); - session = Table.insert(null, getTableInfoServerSession(), session); - } - else - { - configureSession(existingSession, now, serverIP, form); - session = Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); - } - - transaction.commit(); - return session; - } - } - - private void configureSession(@NotNull ServerSession session, Date now, String serverIP, MothershipController.ServerInfoForm form) - { - session.setLastKnownTime(now); - session.setServerIP(serverIP); - session.setServerHostName(form.getBestServerHostName(serverIP)); - - session.setLogoLink(getBestString(session.getLogoLink(), form.getLogoLink())); - session.setOrganizationName(getBestString(session.getOrganizationName(), form.getOrganizationName())); - session.setSystemDescription(getBestString(session.getSystemDescription(), form.getSystemDescription())); - session.setSystemShortName(getBestString(session.getSystemShortName(), form.getSystemShortName())); - - session.setContainerCount(getBestInteger(session.getContainerCount(), form.getContainerCount())); - session.setProjectCount(getBestInteger(session.getProjectCount(), form.getProjectCount())); - session.setRecentUserCount(getBestInteger(session.getRecentUserCount(), form.getRecentUserCount())); - session.setUserCount(getBestInteger(session.getUserCount(), form.getUserCount())); - session.setAdministratorEmail(getBestString(session.getAdministratorEmail(), form.getAdministratorEmail())); - session.setDistribution(getBestString(session.getDistribution(), form.getDistribution())); - session.setUsageReportingLevel(getBestString(session.getUsageReportingLevel(), form.getUsageReportingLevel())); - session.setExceptionReportingLevel(getBestString(session.getExceptionReportingLevel(), form.getExceptionReportingLevel())); - session.setJsonMetrics(getBestJson(session.getJsonMetrics(), form.getJsonMetrics(), session.getServerSessionGUID())); - - } - - private String getBestString(String currentValue, String newValue) - { - if (StringUtils.isEmpty(newValue)) - { - return currentValue; - } - return newValue; - } - - private Integer getBestInteger(Integer currentValue, Integer newValue) - { - if (newValue == null) - { - return currentValue; - } - return newValue; - } - - private Boolean getBestBoolean(Boolean currentValue, Boolean newValue) - { - if (newValue == null) - { - return currentValue; - } - return newValue; - } - - private String getBestJson(String currentValue, String newValue, String serverSessionGUID) - { - if (StringUtils.isEmpty(newValue)) - { - return currentValue; - } - if (StringUtils.isEmpty(currentValue)) - { - // Verify the newValue as valid json; if it is, return it. Otherwise, return null. - try - { - JsonUtil.DEFAULT_MAPPER.readTree(newValue); - return newValue; - } - catch (IOException e) - { - logJsonError(newValue, serverSessionGUID, e); - return null; - } - } - - // Rather than overwrite the current json map, merge the new with the current. - ObjectMapper mapper = JsonUtil.createDefaultMapper(); - try - { - log.debug("Merging JSON. Old is " + currentValue.length() + " characters, new is " + newValue.length()); - Map currentMap = mapper.readValue(currentValue, Map.class); - Map newMap = mapper.readValue(newValue, Map.class); - merge(currentMap, newMap); - return mapper.writeValueAsString(currentMap); - } - catch (IOException e) - { - logJsonError(newValue, serverSessionGUID, e); - return currentValue; - } - } - - /** Merges the values from newMap into currentMap, recursing through child maps. See issue 50665 */ - private void merge(Map currentMap, Map newMap) - { - for (Map.Entry entry : newMap.entrySet()) - { - String key = entry.getKey(); - Object currentChild = currentMap.get(key); - if (currentChild instanceof Map currentChildMap && entry.getValue() instanceof Map newChildMap) - { - merge(currentChildMap, newChildMap); - } - else - { - currentMap.put(entry.getKey(), entry.getValue()); - } - } - } - - private void logJsonError(String newValue, String serverSessionGUID, Exception e) - { - log.error("Malformed json in mothership report from server session '"+serverSessionGUID + "': " + newValue, e); - } - - public TableInfo getTableInfoExceptionStackTrace() - { - return getSchema().getTable("ExceptionStackTrace"); - } - - public TableInfo getTableInfoExceptionReport() - { - return getSchema().getTable("ExceptionReport"); - } - - public TableInfo getTableInfoSoftwareRelease() - { - return getSchema().getTable("SoftwareRelease"); - } - - public TableInfo getTableInfoServerSession() - { - return getSchema().getTable("ServerSession"); - } - - public TableInfo getTableInfoServerInstallation() - { - return getSchema().getTable("ServerInstallation"); - } - - public SqlDialect getDialect() - { - return getSchema().getSqlDialect(); - } - - private WritablePropertyMap getWritableProperties(Container c, boolean secure) - { - if (secure) - { - return PropertyManager.getEncryptedStore().getWritableProperties(c, MOTHERSHIP_SECURE_CATEGORY, true); - } - else - { - return PropertyManager.getWritableProperties(c, UPGRADE_MESSAGE_PROPERTY_CATEGORY, true); - } - } - - private @NotNull Map getProperties(boolean secure) - { - if (secure) - { - return PropertyManager.getEncryptedStore().getProperties(getContainer(), MOTHERSHIP_SECURE_CATEGORY); - } - else - { - return PropertyManager.getProperties(getContainer(), UPGRADE_MESSAGE_PROPERTY_CATEGORY); - } - } - - private static Container getContainer() - { - return ContainerManager.getForPath(MothershipReport.CONTAINER_PATH); - } - - public Date getCurrentBuildDate() - { - Map props = getProperties(false); - String buildDate = props.get(CURRENT_BUILD_DATE_PROP); - return null == buildDate ? null : new Date(DateUtil.parseISODateTime(buildDate)); - } - - private String getStringProperty(String name) - { - return getStringProperty(name, false); - } - - private String getStringProperty(String name, boolean secure) - { - Map props = getProperties(secure); - String message = props.get(name); - if (message == null) - { - return ""; - } - return message; - } - - public String getUpgradeMessage() - { - return getStringProperty(UPGRADE_MESSAGE_PROP); - } - - public String getMarketingMessage() - { - return getStringProperty(MARKETING_MESSAGE_PROP); - } - - private void saveProperty(String name, String value) - { - saveProperty(name, value, false); - } - - private void saveProperty(String name, String value, boolean secure) - { - WritablePropertyMap props = getWritableProperties(getContainer(), secure); - props.put(name, value); - props.save(); - } - - public void setCurrentBuildDate(Date buildDate) - { - saveProperty(CURRENT_BUILD_DATE_PROP, DateUtil.formatIsoDateShortTime(buildDate)); - } - - public void setUpgradeMessage(String message) - { - saveProperty(UPGRADE_MESSAGE_PROP, message); - } - - public void setMarketingMessage(String message) - { - saveProperty(MARKETING_MESSAGE_PROP, message); - } - - public String getCreateIssueURL() - { - return getStringProperty(CREATE_ISSUE_URL_PROP); - } - - public void setCreateIssueURL(String url) - { - saveProperty(CREATE_ISSUE_URL_PROP, url); - } - - public void updateExceptionStackTrace(ExceptionStackTrace stackTrace, User user) - { - Table.update(user, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); - } - - public String getIssuesContainer() - { - return getStringProperty(ISSUES_CONTAINER_PROP); - } - - public void setIssuesContainer(String container) - { - saveProperty(ISSUES_CONTAINER_PROP, container); - } - - public String getUptimeContainer() - { - return getStringProperty(UPTIME_CONTAINER_PROP); - } - - public void setUptimeContainer(String uptimeContainer) - { - saveProperty(UPTIME_CONTAINER_PROP, uptimeContainer); - } - - public String getStatusCakeApiKey() - { - return getStringProperty(STATUS_CAKE_API_KEY_PROP, true); - } - - public void setStatusCakeApiKey(String statusCakeApiKey) - { - saveProperty(STATUS_CAKE_API_KEY_PROP, statusCakeApiKey, true); - } - - public void updateSoftwareRelease(Container container, User user, SoftwareRelease bean) - { - bean.setContainer(container.getId()); - Table.update(user, getTableInfoSoftwareRelease(), bean, bean.getSoftwareReleaseId()); - } - - public ServerInstallation getServerInstallation(int id, Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromString("ServerInstallationId"), id); - return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); - } - - public List getAssignedToList(Container container) - { - List projectUsers = org.labkey.api.security.SecurityManager.getProjectUsers(container.getProject()); - List list = new ArrayList<>(); - // Filter list to only show active users - for (User user : projectUsers) - { - if (user.isActive()) - { - list.add(user); - } - } - list.sort(USER_DISPLAY_NAME_COMPARATOR); - return list; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.mothership; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.query.FieldKey; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.MothershipReport; +import org.labkey.api.util.ReentrantLockWithName; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; + +import static org.labkey.api.security.UserManager.USER_DISPLAY_NAME_COMPARATOR; + +public class MothershipManager +{ + private static final MothershipManager INSTANCE = new MothershipManager(); + private static final String SCHEMA_NAME = "mothership"; + private static final String UPGRADE_MESSAGE_PROPERTY_CATEGORY = "upgradeMessage"; + private static final String MOTHERSHIP_SECURE_CATEGORY = "mothershipSecure"; + private static final String CURRENT_BUILD_DATE_PROP = "currentBuildDate"; + private static final String UPGRADE_MESSAGE_PROP = "upgradeMessage"; + private static final String CREATE_ISSUE_URL_PROP = "createIssueURL"; + private static final String ISSUES_CONTAINER_PROP = "issuesContainer"; + private static final String MARKETING_MESSAGE_PROP = "marketingMessage"; + private static final String UPTIME_CONTAINER_PROP = "uptimeContainer"; + private static final String STATUS_CAKE_API_KEY_PROP = "statusCakeApiKey"; + private static final ReentrantLock INSERT_EXCEPTION_LOCK = new ReentrantLockWithName(MothershipManager.class, "INSERT_EXCEPTION_LOCK"); + + private static final Logger log = LogHelper.getLogger(MothershipManager.class, "Persists mothership records like sessions and installs"); + + public static MothershipManager get() + { + return INSTANCE; + } + + private MothershipManager() {} + + String getSchemaName() + { + return SCHEMA_NAME; + } + + /* package */ + public DbSchema getSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Module); + } + + public void insertException(ExceptionStackTrace stackTrace, ExceptionReport report) + { + // Synchronize to prevent two different threads from creating duplicate rows in the ExceptionStackTrace table + try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction(INSERT_EXCEPTION_LOCK)) + { + boolean isNew = false; + ExceptionStackTrace existingStackTrace = getExceptionStackTrace(stackTrace.getStackTraceHash(), stackTrace.getContainer()); + if (existingStackTrace != null) + { + stackTrace = existingStackTrace; + } + else + { + stackTrace = Table.insert(null, getTableInfoExceptionStackTrace(), stackTrace); + isNew = true; + } + + report.setExceptionStackTraceId(stackTrace.getExceptionStackTraceId()); + + String url = report.getUrl(); + if (null != url && url.length() > 512) + report.setURL(StringUtilsLabKey.leftSurrogatePairFriendly(url, 506) + "..."); + + String referrerURL = report.getReferrerURL(); + if (null != referrerURL && referrerURL.length() > 512) + report.setReferrerURL(StringUtilsLabKey.leftSurrogatePairFriendly(referrerURL, 506) + "..."); + + String browser = report.getBrowser(); + if (null != browser && browser.length() > 100) + report.setBrowser(browser.substring(0,90) + "..."); + + String exceptionMessage = report.getExceptionMessage(); + if (null != exceptionMessage && exceptionMessage.length() > 1000) + report.setExceptionMessage(StringUtilsLabKey.leftSurrogatePairFriendly(exceptionMessage, 990) + "..."); + + String actionName = report.getPageflowAction(); + if (null != actionName && actionName.length() > 40) + { + report.setPageflowAction(actionName.substring(0, 39)); + } + + String controllerName = report.getPageflowName(); + if (null != controllerName && controllerName.length() > 30) + { + report.setPageflowName(controllerName.substring(0, 29)); + } + + String errorCode = report.getErrorCode(); + if (null != errorCode && errorCode.length() > MothershipReport.ERROR_CODE_LENGTH) + { + report.setErrorCode(errorCode.substring(0, MothershipReport.ERROR_CODE_LENGTH - 1)); + } + + report = Table.insert(null, getTableInfoExceptionReport(), report); + stackTrace.setInstances(stackTrace.getInstances() + 1); + stackTrace.setLastReport(report.getCreated()); + if (isNew) + { + stackTrace.setFirstReport(report.getCreated()); + } + Table.update(null, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); + + transaction.commit(); + } + } + + private static final Object ENSURE_SOFTWARE_RELEASE_LOCK = new Object(); + + private void addFilter(SimpleFilter filter, String fieldKey, Object value) + { + if (value == null) + { + filter.addCondition(FieldKey.fromString(fieldKey), null, CompareType.ISBLANK); + } + else + { + filter.addCondition(FieldKey.fromString(fieldKey), value); + } + + } + + public SoftwareRelease ensureSoftwareRelease(Container container, String revision, String url, String branch, String tag, Date buildTime, String buildNumber) + { + synchronized (ENSURE_SOFTWARE_RELEASE_LOCK) + { + // Issue 48270 - transform empty strings to nulls before querying for existing row + revision = StringUtils.trimToNull(revision); + url = StringUtils.trimToNull(url); + branch = StringUtils.trimToNull(branch); + tag = StringUtils.trimToNull(tag); + revision = StringUtils.trimToNull(revision); + buildNumber = StringUtils.trimToNull(buildNumber); + + // Filter on the columns that are part of the unique constraint + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + addFilter(filter, "VcsRevision", revision); + addFilter(filter, "VcsUrl", url); + addFilter(filter, "VcsBranch", branch); + addFilter(filter, "VcsTag", tag); + addFilter(filter, "BuildTime", buildTime); + + SoftwareRelease result = new TableSelector(getTableInfoSoftwareRelease(), filter, null).getObject(SoftwareRelease.class); + if (result == null) + { + if (buildNumber == null) + { + buildNumber = "Unknown VCS"; + } + + result = new SoftwareRelease(); + result.setVcsUrl(url); + result.setVcsRevision(revision); + result.setVcsBranch(branch); + result.setVcsTag(tag); + result.setBuildTime(buildTime); + result.setBuildNumber(buildNumber); + result.setContainer(container.getId()); + result = Table.insert(null, getTableInfoSoftwareRelease(), result); + } + return result; + } + } + + public ServerInstallation getServerInstallation(@NotNull String serverGUID, @NotNull String serverHostName, @NotNull Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerInstallationGUID"), serverGUID); + filter.addCondition(FieldKey.fromString("ServerHostName"), serverHostName); + return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); + } + + public ServerSession getServerSession(String serverSessionGUID, Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerSessionGUID"), serverSessionGUID); + return new TableSelector(getTableInfoServerSession(), filter, null).getObject(ServerSession.class); + } + + public ExceptionStackTrace getExceptionStackTrace(String stackTraceHash, String containerId) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), containerId); + filter.addCondition(FieldKey.fromString("StackTraceHash"), stackTraceHash); + return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); + } + + public ExceptionStackTrace getExceptionStackTrace(int exceptionStackTraceId, Container container) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromString("ExceptionStackTraceId"), exceptionStackTraceId); + return new TableSelector(getTableInfoExceptionStackTrace(), filter, null).getObject(ExceptionStackTrace.class); + } + + public void deleteForContainer(Container c) + { + SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); + sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionReport() + " WHERE ExceptionStackTraceId IN (SELECT ExceptionStackTraceId FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?)", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoExceptionStackTrace() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoServerSession() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoServerInstallation() + " WHERE Container = ?", c); + sqlExecutor.execute("DELETE FROM " + getTableInfoSoftwareRelease() + " WHERE Container = ?", c); + } + + public void deleteForUser(User u) + { + SqlExecutor sqlExecutor = new SqlExecutor(getSchema()); + sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET AssignedTo = NULL WHERE AssignedTo = ?", u.getUserId()); + sqlExecutor.execute("UPDATE " + getTableInfoExceptionStackTrace() + " SET ModifiedBy = NULL WHERE ModifiedBy = ?", u.getUserId()); + } + + public synchronized ServerSession updateServerSession(MothershipController.ServerInfoForm form, String serverIP, ServerSession session, ServerInstallation installation, Container container) + { + try (DbScope.Transaction transaction = getSchema().getScope().ensureTransaction()) + { + String hostName = form.getBestServerHostName(serverIP); + ServerInstallation existingInstallation = getServerInstallation(installation.getServerInstallationGUID(), hostName, container); + + if (existingInstallation == null) + { + installation.setContainer(container.getId()); + installation.setServerHostName(hostName); + installation = Table.insert(null, getTableInfoServerInstallation(), installation); + } + else + { + existingInstallation.setServerHostName(hostName); + installation = Table.update(null, getTableInfoServerInstallation(), existingInstallation, existingInstallation.getServerInstallationId()); + } + + Date now = new Date(); + ServerSession existingSession = getServerSession(session.getServerSessionGUID(), container); + if (existingSession != null) + { + // Issue 50876: Reparent mothership server session when the base URL changes mid-session + existingSession.setServerInstallationId(installation.getServerInstallationId()); + + Calendar existingCal = Calendar.getInstance(); + existingCal.setTime(existingSession.getLastKnownTime()); + Calendar nowCal = Calendar.getInstance(); + + // Check if this session is straddling months. If so, break it into two so that we + // retain metrics with month level granularity + if (existingCal.get(Calendar.MONTH) != nowCal.get(Calendar.MONTH)) + { + // Chain the two sessions together + session.setOriginalServerSessionId(existingSession.getServerSessionId()); + + // Update the GUID for the old one as we're capturing in a new record going forward + existingSession.setServerSessionGUID(GUID.makeGUID()); + Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); + existingSession = null; + } + } + + if (existingSession == null) + { + session.setEarliestKnownTime(now); + session.setServerInstallationId(installation.getServerInstallationId()); + + configureSession(session, now, serverIP, form); + session = Table.insert(null, getTableInfoServerSession(), session); + } + else + { + configureSession(existingSession, now, serverIP, form); + session = Table.update(null, getTableInfoServerSession(), existingSession, existingSession.getServerSessionId()); + } + + transaction.commit(); + return session; + } + } + + private void configureSession(@NotNull ServerSession session, Date now, String serverIP, MothershipController.ServerInfoForm form) + { + session.setLastKnownTime(now); + session.setServerIP(serverIP); + session.setServerHostName(form.getBestServerHostName(serverIP)); + + session.setLogoLink(getBestString(session.getLogoLink(), form.getLogoLink())); + session.setOrganizationName(getBestString(session.getOrganizationName(), form.getOrganizationName())); + session.setSystemDescription(getBestString(session.getSystemDescription(), form.getSystemDescription())); + session.setSystemShortName(getBestString(session.getSystemShortName(), form.getSystemShortName())); + + session.setContainerCount(getBestInteger(session.getContainerCount(), form.getContainerCount())); + session.setProjectCount(getBestInteger(session.getProjectCount(), form.getProjectCount())); + session.setRecentUserCount(getBestInteger(session.getRecentUserCount(), form.getRecentUserCount())); + session.setUserCount(getBestInteger(session.getUserCount(), form.getUserCount())); + session.setAdministratorEmail(getBestString(session.getAdministratorEmail(), form.getAdministratorEmail())); + session.setDistribution(getBestString(session.getDistribution(), form.getDistribution())); + session.setUsageReportingLevel(getBestString(session.getUsageReportingLevel(), form.getUsageReportingLevel())); + session.setExceptionReportingLevel(getBestString(session.getExceptionReportingLevel(), form.getExceptionReportingLevel())); + session.setJsonMetrics(getBestJson(session.getJsonMetrics(), form.getJsonMetrics(), session.getServerSessionGUID())); + + } + + private String getBestString(String currentValue, String newValue) + { + if (StringUtils.isEmpty(newValue)) + { + return currentValue; + } + return newValue; + } + + private Integer getBestInteger(Integer currentValue, Integer newValue) + { + if (newValue == null) + { + return currentValue; + } + return newValue; + } + + private Boolean getBestBoolean(Boolean currentValue, Boolean newValue) + { + if (newValue == null) + { + return currentValue; + } + return newValue; + } + + private String getBestJson(String currentValue, String newValue, String serverSessionGUID) + { + if (StringUtils.isEmpty(newValue)) + { + return currentValue; + } + if (StringUtils.isEmpty(currentValue)) + { + // Verify the newValue as valid json; if it is, return it. Otherwise, return null. + try + { + JsonUtil.DEFAULT_MAPPER.readTree(newValue); + return newValue; + } + catch (IOException e) + { + logJsonError(newValue, serverSessionGUID, e); + return null; + } + } + + // Rather than overwrite the current json map, merge the new with the current. + ObjectMapper mapper = JsonUtil.createDefaultMapper(); + try + { + log.debug("Merging JSON. Old is " + currentValue.length() + " characters, new is " + newValue.length()); + Map currentMap = mapper.readValue(currentValue, Map.class); + Map newMap = mapper.readValue(newValue, Map.class); + merge(currentMap, newMap); + return mapper.writeValueAsString(currentMap); + } + catch (IOException e) + { + logJsonError(newValue, serverSessionGUID, e); + return currentValue; + } + } + + /** Merges the values from newMap into currentMap, recursing through child maps. See issue 50665 */ + private void merge(Map currentMap, Map newMap) + { + for (Map.Entry entry : newMap.entrySet()) + { + String key = entry.getKey(); + Object currentChild = currentMap.get(key); + if (currentChild instanceof Map currentChildMap && entry.getValue() instanceof Map newChildMap) + { + merge(currentChildMap, newChildMap); + } + else + { + currentMap.put(entry.getKey(), entry.getValue()); + } + } + } + + private void logJsonError(String newValue, String serverSessionGUID, Exception e) + { + log.error("Malformed json in mothership report from server session '"+serverSessionGUID + "': " + newValue, e); + } + + public TableInfo getTableInfoExceptionStackTrace() + { + return getSchema().getTable("ExceptionStackTrace"); + } + + public TableInfo getTableInfoExceptionReport() + { + return getSchema().getTable("ExceptionReport"); + } + + public TableInfo getTableInfoSoftwareRelease() + { + return getSchema().getTable("SoftwareRelease"); + } + + public TableInfo getTableInfoServerSession() + { + return getSchema().getTable("ServerSession"); + } + + public TableInfo getTableInfoServerInstallation() + { + return getSchema().getTable("ServerInstallation"); + } + + public SqlDialect getDialect() + { + return getSchema().getSqlDialect(); + } + + private WritablePropertyMap getWritableProperties(Container c, boolean secure) + { + if (secure) + { + return PropertyManager.getEncryptedStore().getWritableProperties(c, MOTHERSHIP_SECURE_CATEGORY, true); + } + else + { + return PropertyManager.getWritableProperties(c, UPGRADE_MESSAGE_PROPERTY_CATEGORY, true); + } + } + + private @NotNull Map getProperties(boolean secure) + { + if (secure) + { + return PropertyManager.getEncryptedStore().getProperties(getContainer(), MOTHERSHIP_SECURE_CATEGORY); + } + else + { + return PropertyManager.getProperties(getContainer(), UPGRADE_MESSAGE_PROPERTY_CATEGORY); + } + } + + private static Container getContainer() + { + return ContainerManager.getForPath(MothershipReport.CONTAINER_PATH); + } + + public Date getCurrentBuildDate() + { + Map props = getProperties(false); + String buildDate = props.get(CURRENT_BUILD_DATE_PROP); + return null == buildDate ? null : new Date(DateUtil.parseISODateTime(buildDate)); + } + + private String getStringProperty(String name) + { + return getStringProperty(name, false); + } + + private String getStringProperty(String name, boolean secure) + { + Map props = getProperties(secure); + String message = props.get(name); + if (message == null) + { + return ""; + } + return message; + } + + public String getUpgradeMessage() + { + return getStringProperty(UPGRADE_MESSAGE_PROP); + } + + public String getMarketingMessage() + { + return getStringProperty(MARKETING_MESSAGE_PROP); + } + + private void saveProperty(String name, String value) + { + saveProperty(name, value, false); + } + + private void saveProperty(String name, String value, boolean secure) + { + WritablePropertyMap props = getWritableProperties(getContainer(), secure); + props.put(name, value); + props.save(); + } + + public void setCurrentBuildDate(Date buildDate) + { + saveProperty(CURRENT_BUILD_DATE_PROP, DateUtil.formatIsoDateShortTime(buildDate)); + } + + public void setUpgradeMessage(String message) + { + saveProperty(UPGRADE_MESSAGE_PROP, message); + } + + public void setMarketingMessage(String message) + { + saveProperty(MARKETING_MESSAGE_PROP, message); + } + + public String getCreateIssueURL() + { + return getStringProperty(CREATE_ISSUE_URL_PROP); + } + + public void setCreateIssueURL(String url) + { + saveProperty(CREATE_ISSUE_URL_PROP, url); + } + + public void updateExceptionStackTrace(ExceptionStackTrace stackTrace, User user) + { + Table.update(user, getTableInfoExceptionStackTrace(), stackTrace, stackTrace.getExceptionStackTraceId()); + } + + public String getIssuesContainer() + { + return getStringProperty(ISSUES_CONTAINER_PROP); + } + + public void setIssuesContainer(String container) + { + saveProperty(ISSUES_CONTAINER_PROP, container); + } + + public String getUptimeContainer() + { + return getStringProperty(UPTIME_CONTAINER_PROP); + } + + public void setUptimeContainer(String uptimeContainer) + { + saveProperty(UPTIME_CONTAINER_PROP, uptimeContainer); + } + + public String getStatusCakeApiKey() + { + return getStringProperty(STATUS_CAKE_API_KEY_PROP, true); + } + + public void setStatusCakeApiKey(String statusCakeApiKey) + { + saveProperty(STATUS_CAKE_API_KEY_PROP, statusCakeApiKey, true); + } + + public void updateSoftwareRelease(Container container, User user, SoftwareRelease bean) + { + bean.setContainer(container.getId()); + Table.update(user, getTableInfoSoftwareRelease(), bean, bean.getSoftwareReleaseId()); + } + + public ServerInstallation getServerInstallation(int id, Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromString("ServerInstallationId"), id); + return new TableSelector(getTableInfoServerInstallation(), filter, null).getObject(ServerInstallation.class); + } + + public List getAssignedToList(Container container) + { + List projectUsers = org.labkey.api.security.SecurityManager.getProjectUsers(container.getProject()); + List list = new ArrayList<>(); + // Filter list to only show active users + for (User user : projectUsers) + { + if (user.isActive()) + { + list.add(user); + } + } + list.sort(USER_DISPLAY_NAME_COMPARATOR); + return list; + } +} diff --git a/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java b/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java index 0b16a0b4eb8..f6ab4a307d9 100644 --- a/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java +++ b/pipeline/src/org/labkey/pipeline/api/WorkDirectoryRemote.java @@ -1,594 +1,594 @@ -/* - * Copyright (c) 2008-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.pipeline.api; - -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.labkey.api.pipeline.WorkDirFactory; -import org.labkey.api.pipeline.WorkDirectory; -import org.labkey.api.pipeline.file.FileAnalysisJobSupport; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.URIUtil; -import org.springframework.beans.factory.InitializingBean; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * Used to copy files from (and back to) a remote file system so that they can be used directly on the local file system, - * improving performance on high-latency and/or low-bandwidth network file systems - * - * @author jeckels - */ -public class WorkDirectoryRemote extends AbstractWorkDirectory -{ - private static final Logger _systemLog = LogManager.getLogger(WorkDirectoryRemote.class); - - private static final int FILE_LOCKS_DEFAULT = 5; - - private final File _lockDirectory; - private final File _folderToClean; - - private static final Map _locks = new HashMap<>(); - - @Override - public File inputFile(File fileInput, boolean forceCopy) throws IOException - { - return inputFile(fileInput, newFile(fileInput.getName()), forceCopy); - } - - @Override - public File inputFile(File fileInput, File fileWork, boolean forceCopy) throws IOException - { - //can be used to prevent duplicate copy attempts - if (fileWork.exists() && !forceCopy) - { - _copiedInputs.put(fileInput, fileWork); - } - - return copyInputFile(fileInput, fileWork); - } - - public static class Factory extends AbstractFactory implements InitializingBean - { - private String _lockDirectory; - private String _tempDirectory; - private boolean _sharedTempDirectory; - private boolean _allowReuseExistingTempDirectory; - private boolean _deterministicWorkingDirName; - private boolean _cleanupOnStartup; - private String _transferToDirOnFailure = null; - - @Override - public void afterPropertiesSet() - { - if (_tempDirectory == null) - { - throw new IllegalStateException("tempDirectory not set - set it directly using the tempDirectory property or use the tempDirectoryEnv property to point to an environment variable"); - } - if (_cleanupOnStartup) - { - FileUtil.deleteDirectoryContents(new File(_tempDirectory)); - } - } - - @Override - public WorkDirectory createWorkDirectory(String jobId, FileAnalysisJobSupport support, boolean useDeterministicFolderPath, Logger log) throws IOException - { - if (useDeterministicFolderPath) - { - _sharedTempDirectory = true; - _allowReuseExistingTempDirectory = true; - _deterministicWorkingDirName = true; - } - - File tempDir; - File tempDirBase = null; - int attempt = 0; - do - { - // We've seen very intermittent problems failing to create temp files in the past during the DRTs, - // so try a few times before failing - File dirParent = (_tempDirectory == null ? null : new File(_tempDirectory)); - - // If the temp directory is shared, then create a jobId directory to be sure the - // work directory path is unique. - try - { - if (_sharedTempDirectory) - { - if (_deterministicWorkingDirName) - { - dirParent = new File(dirParent, jobId); - tempDirBase = dirParent; - } - else - { - dirParent = FileUtil.createTempFile(jobId, "", dirParent); - tempDirBase = dirParent; - } - - if (_allowReuseExistingTempDirectory && dirParent.exists()) - { - log.info("parent directory exists, reusing: " + dirParent.getPath()); - } - else - { - dirParent.delete(); - FileUtil.mkdirs(dirParent); - } - } - - String name = support.getBaseName(); - if (name.length() > 10) - { - // Don't let the total path get too long - Windows doesn't like paths longer than 255 characters - // so if there's a ridiculously long file name, we don't want to duplicate its name in the - // directory too - name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 9); - } - else if (name.length() < 3) - { - //File.createTempFile() does not allow prefixes <3 chars - name = "wd_" + name; - } - - if (_deterministicWorkingDirName) - { - tempDir = new File(dirParent, name + WORK_DIR_SUFFIX); - } - else - { - tempDir = FileUtil.createTempFile(name, WORK_DIR_SUFFIX, dirParent); - } - - if (_allowReuseExistingTempDirectory && tempDir.exists()) - { - log.info("working directory exists, reusing: " + dirParent.getPath()); - } - else - { - tempDir.delete(); - FileUtil.mkdirs(tempDir); - } - } - catch (IOException e) - { - IOException ioException = new IOException("Failed to create local working directory in the tempDirectory " - + dirParent + ", specified in the tempDirectory property in the pipeline configuration"); - ioException.initCause(e); - _systemLog.error(ioException.getMessage(), e); - throw ioException; - } - attempt++; - } - while (attempt < 5 && !tempDir.isDirectory()); - if (!tempDir.isDirectory()) - { - throw new IOException("Failed to create local working directory " + tempDir); - } - - File lockDir = (_lockDirectory == null ? null : new File(_lockDirectory)); - File transferToDirOnFailure = (_transferToDirOnFailure == null ? null : new File(_transferToDirOnFailure)); - return new WorkDirectoryRemote(support, this, log, lockDir, tempDir, transferToDirOnFailure, _allowReuseExistingTempDirectory, tempDirBase); - } - - public String getLockDirectory() - { - if (_lockDirectory!= null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File lockDir = new File(_lockDirectory); - if (!NetworkDrive.exists(lockDir) || !lockDir.isDirectory()) - throw new IllegalArgumentException("The lock directory " + _lockDirectory + " does not exist."); - } - return _lockDirectory; - } - - public void setLockDirectory(String directoryString) - { - _lockDirectory = directoryString; - } - - public String getTempDirectory() - { - if (_tempDirectory != null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File tempDir = new File(_tempDirectory); - if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) - throw new IllegalArgumentException("The temporary directory " + _tempDirectory + " does not exist."); - } - return _tempDirectory; - } - - /** @param directoryString path of the directory to be used as scratch space */ - public void setTempDirectory(String directoryString) - { - _tempDirectory = directoryString; - } - - public void setCleanupOnStartup(boolean cleanupOnStartup) - { - _cleanupOnStartup = cleanupOnStartup; - } - - /** - * Set to an environment variable set to the path to use for the temporary directory. - * (e.g. some cluster schedulers initialize TMPDIR to a job specific temporary directory - * which will be removed, if the job is cancelled) - * - * @param tempDirectoryVar environment variable name - */ - public void setTempDirectoryEnv(String tempDirectoryVar) - { - String tempDirectory = System.getenv(tempDirectoryVar); - if (tempDirectory == null || tempDirectory.isEmpty()) - throw new IllegalArgumentException("The environment variable " + tempDirectoryVar + " does not exist:\n" + System.getenv()); - setTempDirectory(tempDirectory); - } - - /** - * @return true if the root temporary directory will be shared by multiple tasks - */ - public boolean isSharedTempDirectory() - { - return _sharedTempDirectory; - } - - /** - * Set to true, if the root temporary directory will be shared by multiple tasks. - * This is usually not necessary on a scheduled computational cluster, where each - * task is given a separate working environment. - * - * @param sharedTempDirectory true if the root temporary directory will be shared by multiple tasks - */ - public void setSharedTempDirectory(boolean sharedTempDirectory) - { - _sharedTempDirectory = sharedTempDirectory; - } - - public String getTransferToDirOnFailure() - { - if (_transferToDirOnFailure != null) - { - // Do the validation on get instead of set because we may not have the NetworkDrive - // configuration loaded in time at startup - File tempDir = new File(_transferToDirOnFailure); - if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) - throw new IllegalArgumentException("The directory " + _transferToDirOnFailure + " does not exist."); - } - - return _transferToDirOnFailure; - } - - /** - * If a directory is provided, when a remote job fails, the working directory will - * be moved from the working location to a directory under this folder - */ - public void setTransferToDirOnFailure(String transferToDirOnFailure) - { - _transferToDirOnFailure = transferToDirOnFailure; - } - - public boolean isAllowReuseExistingTempDirectory() - { - return _allowReuseExistingTempDirectory; - } - - /** - * If true, instead of deleting an existing working directory on job startup, an existing directory will be reused. - * This is mostly used to allow job resume, and should only be used - */ - public void setAllowReuseExistingTempDirectory(boolean allowReuseExistingTempDirectory) - { - _allowReuseExistingTempDirectory = allowReuseExistingTempDirectory; - } - - public boolean isDeterministicWorkingDirName() - { - return _deterministicWorkingDirName; - } - - /** - * If true, the working directory for each job will be named using the job' name alone (as opposed to a random temp file based on jobName) - * This is intended to support job resume, and should be used with sharedTempDirectory=true to avoid conflicts. - */ - public void setDeterministicWorkingDirName(boolean deterministicWorkingDirName) - { - _deterministicWorkingDirName = deterministicWorkingDirName; - } - } - - public WorkDirectoryRemote(FileAnalysisJobSupport support, WorkDirFactory factory, Logger log, File lockDir, File tempDir, File transferToDirOnFailure, boolean reuseExistingDirectory, File folderToClean) throws IOException - { - super(support, factory, tempDir, reuseExistingDirectory, log); - - _lockDirectory = lockDir; - _transferToDirOnFailure = transferToDirOnFailure; - _folderToClean = folderToClean; - } - - /** - * @return a pair, where the first value is the total number of locks, and the second value is the lock index that - * should be used next - */ - private MasterLockInfo parseMasterLock(RandomAccessFile masterIn, File masterLockFile) throws IOException - { - ByteArrayOutputStream bOut = new ByteArrayOutputStream(); - byte[] b = new byte[128]; - int i; - while ((i = masterIn.read(b)) != -1) - { - bOut.write(b, 0, i); - } - String line = new String(bOut.toByteArray(), StringUtilsLabKey.DEFAULT_CHARSET).trim(); - int totalLocks = FILE_LOCKS_DEFAULT; - int currentIndex = 0; - if (!line.isEmpty()) - { - String[] parts = line.split(" "); - try - { - currentIndex = Integer.parseInt(parts[0]); - } - catch (NumberFormatException e) - { - throw new IOException("Could not parse the current lock index from the master lock file " + masterLockFile + ", the value was: " + parts[0]); - } - - if (parts.length > 1) - { - try - { - totalLocks = Integer.parseInt(parts[1]); - } - catch (NumberFormatException e) - { - throw new IOException("Could not parse the total number of locks from the master lock file " + masterLockFile + ", the value was: " + parts[1]); - } - } - - if (totalLocks < 1) - totalLocks = FILE_LOCKS_DEFAULT; - } - - if (currentIndex >= totalLocks) - { - currentIndex = 0; - } - return new MasterLockInfo(totalLocks, currentIndex); - } - - /** - * File system locks are fine to communicate locking between two different processes, but they don't work for - * multiple threads inside the same VM. We need to do Java-level locking as well. - */ - private static synchronized Lock getInMemoryLockObject(File f) - { - Lock result = _locks.get(f); - if (result == null) - { - result = new ReentrantLock(); - _locks.put(f, result); - } - return result; - } - - @Override - public void remove(boolean success) throws IOException - { - super.remove(success); - - // Issue 25166: this was a pre-existing potential bug. If _sharedTempDirectory is true, we create a second level - // of temp directory above the primary working dir. this is added to make sure we clean this up. - _jobLog.debug("inspecting remote work dir: " + (_folderToClean == null ? _dir.getPath() : _folderToClean.getPath())); - if (success && _folderToClean != null && !_dir.equals(_folderToClean)) - { - _jobLog.debug("removing entire work dir through: " + _folderToClean.getPath()); - _jobLog.debug("starting with: " + _dir.getPath()); - File toCheck = _dir; - - //debugging only: - if (!URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) - { - _jobLog.warn("not a descendant!"); - } - - while (toCheck != null && URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) - { - if (!toCheck.exists()) - { - _jobLog.debug("directory does not exist: " + toCheck.getPath()); - toCheck = toCheck.getParentFile(); - continue; - } - - String[] children = toCheck.list(); - if (children != null && children.length == 0) - { - _jobLog.debug("removing directory: " + toCheck.getPath()); - FileUtils.deleteDirectory(toCheck); - toCheck = toCheck.getParentFile(); - } - else if (children == null) - { - _jobLog.debug("unable to list children, will not delete: " + toCheck.getPath()); - continue; - } - else - { - _jobLog.debug("work directory has children, will not delete: " + toCheck.getPath()); - _jobLog.debug("files:"); - for (String fn : children) - { - _jobLog.debug(fn); - } - break; - } - } - } - } - - @Override - protected CopyingResource createCopyingLock() throws IOException - { - if (_lockDirectory == null) - { - return new SimpleCopyingResource(); - } - - _jobLog.debug("Starting to acquire lock for copying files"); - - MasterLockInfo lockInfo; - - // Synchronize to prevent multiple threads from trying to lock the master file from within the same VM - synchronized (WorkDirectoryRemote.class) - { - RandomAccessFile randomAccessFile = null; - FileLock masterLock = null; - - try - { - File masterLockFile = new File(_lockDirectory, "counter"); - randomAccessFile = new RandomAccessFile(masterLockFile, "rw"); - FileChannel masterChannel = randomAccessFile.getChannel(); - masterLock = masterChannel.lock(); - - lockInfo = parseMasterLock(randomAccessFile, masterLockFile); - int nextIndex = (lockInfo.getCurrentLock() + 1) % lockInfo.getTotalLocks(); - rewriteMasterLock(randomAccessFile, new MasterLockInfo(lockInfo.getTotalLocks(), nextIndex)); - } - finally - { - if (randomAccessFile != null) { try { randomAccessFile.close(); } catch (IOException e) {} } - if (masterLock != null) { try { masterLock.release(); } catch (IOException e) {} } - } - } - - _jobLog.debug("Acquiring lock #" + lockInfo.getCurrentLock()); - File f = new File(_lockDirectory, "lock" + lockInfo.getCurrentLock()); - FileChannel lockChannel = new FileOutputStream(f, true).getChannel(); - FileLockCopyingResource result = new FileLockCopyingResource(lockChannel, lockInfo.getCurrentLock(), f); - _jobLog.debug("Lock #" + lockInfo.getCurrentLock() + " acquired"); - - return result; - } - - private void rewriteMasterLock(RandomAccessFile masterFile, MasterLockInfo lockInfo) - throws IOException - { - masterFile.seek(0); - - String output = Integer.toString(lockInfo.getCurrentLock()); - if (lockInfo.getTotalLocks() != FILE_LOCKS_DEFAULT) - output += " " + Integer.toString(lockInfo.getTotalLocks()); - byte[] outputBytes = output.getBytes(StringUtilsLabKey.DEFAULT_CHARSET); - masterFile.write(outputBytes); - masterFile.setLength(outputBytes.length); - } - - private static class MasterLockInfo - { - private final int _totalLocks; - private final int _currentLock; - - private MasterLockInfo(int totalLocks, int currentLock) - { - assert totalLocks > 0 : "Total locks must be greater than 0."; - - _totalLocks = totalLocks; - _currentLock = currentLock; - } - - public int getTotalLocks() - { - return _totalLocks; - } - - public int getCurrentLock() - { - return _currentLock; - } - } - - public class FileLockCopyingResource extends SimpleCopyingResource - { - private FileChannel _channel; - private final int _lockNumber; - private FileLock _lock; - private final Throwable _creationStack; - private Lock _memoryLock; - - public FileLockCopyingResource(FileChannel channel, int lockNumber, File f) throws IOException - { - _channel = channel; - _lockNumber = lockNumber; - _creationStack = new Throwable(); - - // Lock the memory part first to eliminate multi-threaded access to the same file - _memoryLock = getInMemoryLockObject(f); - _memoryLock.lock(); - - // Lock the file part second - _lock = _channel.lock(); - } - - @Override - protected void finalize() throws Throwable - { - super.finalize(); - if (_lock != null) - { - _systemLog.error("FileLockCopyingResource was not released before it was garbage collected. Creation stack is: ", _creationStack); - } - close(); - } - - @Override - public void close() - { - if (_lock != null) - { - // Unlock the file part first - try { _lock.release(); } catch (IOException e) {} - try { _channel.close(); } catch (IOException e) {} - _jobLog.debug("Lock #" + _lockNumber + " released"); - _lock = null; - _channel = null; - super.close(); - - // Unlock the memory part last - _memoryLock.unlock(); - _memoryLock = null; - } - } - } +/* + * Copyright (c) 2008-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.pipeline.api; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.labkey.api.pipeline.WorkDirFactory; +import org.labkey.api.pipeline.WorkDirectory; +import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URIUtil; +import org.springframework.beans.factory.InitializingBean; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Used to copy files from (and back to) a remote file system so that they can be used directly on the local file system, + * improving performance on high-latency and/or low-bandwidth network file systems + * + * @author jeckels + */ +public class WorkDirectoryRemote extends AbstractWorkDirectory +{ + private static final Logger _systemLog = LogManager.getLogger(WorkDirectoryRemote.class); + + private static final int FILE_LOCKS_DEFAULT = 5; + + private final File _lockDirectory; + private final File _folderToClean; + + private static final Map _locks = new HashMap<>(); + + @Override + public File inputFile(File fileInput, boolean forceCopy) throws IOException + { + return inputFile(fileInput, newFile(fileInput.getName()), forceCopy); + } + + @Override + public File inputFile(File fileInput, File fileWork, boolean forceCopy) throws IOException + { + //can be used to prevent duplicate copy attempts + if (fileWork.exists() && !forceCopy) + { + _copiedInputs.put(fileInput, fileWork); + } + + return copyInputFile(fileInput, fileWork); + } + + public static class Factory extends AbstractFactory implements InitializingBean + { + private String _lockDirectory; + private String _tempDirectory; + private boolean _sharedTempDirectory; + private boolean _allowReuseExistingTempDirectory; + private boolean _deterministicWorkingDirName; + private boolean _cleanupOnStartup; + private String _transferToDirOnFailure = null; + + @Override + public void afterPropertiesSet() + { + if (_tempDirectory == null) + { + throw new IllegalStateException("tempDirectory not set - set it directly using the tempDirectory property or use the tempDirectoryEnv property to point to an environment variable"); + } + if (_cleanupOnStartup) + { + FileUtil.deleteDirectoryContents(new File(_tempDirectory)); + } + } + + @Override + public WorkDirectory createWorkDirectory(String jobId, FileAnalysisJobSupport support, boolean useDeterministicFolderPath, Logger log) throws IOException + { + if (useDeterministicFolderPath) + { + _sharedTempDirectory = true; + _allowReuseExistingTempDirectory = true; + _deterministicWorkingDirName = true; + } + + File tempDir; + File tempDirBase = null; + int attempt = 0; + do + { + // We've seen very intermittent problems failing to create temp files in the past during the DRTs, + // so try a few times before failing + File dirParent = (_tempDirectory == null ? null : new File(_tempDirectory)); + + // If the temp directory is shared, then create a jobId directory to be sure the + // work directory path is unique. + try + { + if (_sharedTempDirectory) + { + if (_deterministicWorkingDirName) + { + dirParent = new File(dirParent, jobId); + tempDirBase = dirParent; + } + else + { + dirParent = FileUtil.createTempFile(jobId, "", dirParent); + tempDirBase = dirParent; + } + + if (_allowReuseExistingTempDirectory && dirParent.exists()) + { + log.info("parent directory exists, reusing: " + dirParent.getPath()); + } + else + { + dirParent.delete(); + FileUtil.mkdirs(dirParent); + } + } + + String name = support.getBaseName(); + if (name.length() > 10) + { + // Don't let the total path get too long - Windows doesn't like paths longer than 255 characters + // so if there's a ridiculously long file name, we don't want to duplicate its name in the + // directory too + name = StringUtilsLabKey.leftSurrogatePairFriendly(name, 9); + } + else if (name.length() < 3) + { + //File.createTempFile() does not allow prefixes <3 chars + name = "wd_" + name; + } + + if (_deterministicWorkingDirName) + { + tempDir = new File(dirParent, name + WORK_DIR_SUFFIX); + } + else + { + tempDir = FileUtil.createTempFile(name, WORK_DIR_SUFFIX, dirParent); + } + + if (_allowReuseExistingTempDirectory && tempDir.exists()) + { + log.info("working directory exists, reusing: " + dirParent.getPath()); + } + else + { + tempDir.delete(); + FileUtil.mkdirs(tempDir); + } + } + catch (IOException e) + { + IOException ioException = new IOException("Failed to create local working directory in the tempDirectory " + + dirParent + ", specified in the tempDirectory property in the pipeline configuration"); + ioException.initCause(e); + _systemLog.error(ioException.getMessage(), e); + throw ioException; + } + attempt++; + } + while (attempt < 5 && !tempDir.isDirectory()); + if (!tempDir.isDirectory()) + { + throw new IOException("Failed to create local working directory " + tempDir); + } + + File lockDir = (_lockDirectory == null ? null : new File(_lockDirectory)); + File transferToDirOnFailure = (_transferToDirOnFailure == null ? null : new File(_transferToDirOnFailure)); + return new WorkDirectoryRemote(support, this, log, lockDir, tempDir, transferToDirOnFailure, _allowReuseExistingTempDirectory, tempDirBase); + } + + public String getLockDirectory() + { + if (_lockDirectory!= null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File lockDir = new File(_lockDirectory); + if (!NetworkDrive.exists(lockDir) || !lockDir.isDirectory()) + throw new IllegalArgumentException("The lock directory " + _lockDirectory + " does not exist."); + } + return _lockDirectory; + } + + public void setLockDirectory(String directoryString) + { + _lockDirectory = directoryString; + } + + public String getTempDirectory() + { + if (_tempDirectory != null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File tempDir = new File(_tempDirectory); + if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) + throw new IllegalArgumentException("The temporary directory " + _tempDirectory + " does not exist."); + } + return _tempDirectory; + } + + /** @param directoryString path of the directory to be used as scratch space */ + public void setTempDirectory(String directoryString) + { + _tempDirectory = directoryString; + } + + public void setCleanupOnStartup(boolean cleanupOnStartup) + { + _cleanupOnStartup = cleanupOnStartup; + } + + /** + * Set to an environment variable set to the path to use for the temporary directory. + * (e.g. some cluster schedulers initialize TMPDIR to a job specific temporary directory + * which will be removed, if the job is cancelled) + * + * @param tempDirectoryVar environment variable name + */ + public void setTempDirectoryEnv(String tempDirectoryVar) + { + String tempDirectory = System.getenv(tempDirectoryVar); + if (tempDirectory == null || tempDirectory.isEmpty()) + throw new IllegalArgumentException("The environment variable " + tempDirectoryVar + " does not exist:\n" + System.getenv()); + setTempDirectory(tempDirectory); + } + + /** + * @return true if the root temporary directory will be shared by multiple tasks + */ + public boolean isSharedTempDirectory() + { + return _sharedTempDirectory; + } + + /** + * Set to true, if the root temporary directory will be shared by multiple tasks. + * This is usually not necessary on a scheduled computational cluster, where each + * task is given a separate working environment. + * + * @param sharedTempDirectory true if the root temporary directory will be shared by multiple tasks + */ + public void setSharedTempDirectory(boolean sharedTempDirectory) + { + _sharedTempDirectory = sharedTempDirectory; + } + + public String getTransferToDirOnFailure() + { + if (_transferToDirOnFailure != null) + { + // Do the validation on get instead of set because we may not have the NetworkDrive + // configuration loaded in time at startup + File tempDir = new File(_transferToDirOnFailure); + if (!NetworkDrive.exists(tempDir) || !tempDir.isDirectory()) + throw new IllegalArgumentException("The directory " + _transferToDirOnFailure + " does not exist."); + } + + return _transferToDirOnFailure; + } + + /** + * If a directory is provided, when a remote job fails, the working directory will + * be moved from the working location to a directory under this folder + */ + public void setTransferToDirOnFailure(String transferToDirOnFailure) + { + _transferToDirOnFailure = transferToDirOnFailure; + } + + public boolean isAllowReuseExistingTempDirectory() + { + return _allowReuseExistingTempDirectory; + } + + /** + * If true, instead of deleting an existing working directory on job startup, an existing directory will be reused. + * This is mostly used to allow job resume, and should only be used + */ + public void setAllowReuseExistingTempDirectory(boolean allowReuseExistingTempDirectory) + { + _allowReuseExistingTempDirectory = allowReuseExistingTempDirectory; + } + + public boolean isDeterministicWorkingDirName() + { + return _deterministicWorkingDirName; + } + + /** + * If true, the working directory for each job will be named using the job' name alone (as opposed to a random temp file based on jobName) + * This is intended to support job resume, and should be used with sharedTempDirectory=true to avoid conflicts. + */ + public void setDeterministicWorkingDirName(boolean deterministicWorkingDirName) + { + _deterministicWorkingDirName = deterministicWorkingDirName; + } + } + + public WorkDirectoryRemote(FileAnalysisJobSupport support, WorkDirFactory factory, Logger log, File lockDir, File tempDir, File transferToDirOnFailure, boolean reuseExistingDirectory, File folderToClean) throws IOException + { + super(support, factory, tempDir, reuseExistingDirectory, log); + + _lockDirectory = lockDir; + _transferToDirOnFailure = transferToDirOnFailure; + _folderToClean = folderToClean; + } + + /** + * @return a pair, where the first value is the total number of locks, and the second value is the lock index that + * should be used next + */ + private MasterLockInfo parseMasterLock(RandomAccessFile masterIn, File masterLockFile) throws IOException + { + ByteArrayOutputStream bOut = new ByteArrayOutputStream(); + byte[] b = new byte[128]; + int i; + while ((i = masterIn.read(b)) != -1) + { + bOut.write(b, 0, i); + } + String line = new String(bOut.toByteArray(), StringUtilsLabKey.DEFAULT_CHARSET).trim(); + int totalLocks = FILE_LOCKS_DEFAULT; + int currentIndex = 0; + if (!line.isEmpty()) + { + String[] parts = line.split(" "); + try + { + currentIndex = Integer.parseInt(parts[0]); + } + catch (NumberFormatException e) + { + throw new IOException("Could not parse the current lock index from the master lock file " + masterLockFile + ", the value was: " + parts[0]); + } + + if (parts.length > 1) + { + try + { + totalLocks = Integer.parseInt(parts[1]); + } + catch (NumberFormatException e) + { + throw new IOException("Could not parse the total number of locks from the master lock file " + masterLockFile + ", the value was: " + parts[1]); + } + } + + if (totalLocks < 1) + totalLocks = FILE_LOCKS_DEFAULT; + } + + if (currentIndex >= totalLocks) + { + currentIndex = 0; + } + return new MasterLockInfo(totalLocks, currentIndex); + } + + /** + * File system locks are fine to communicate locking between two different processes, but they don't work for + * multiple threads inside the same VM. We need to do Java-level locking as well. + */ + private static synchronized Lock getInMemoryLockObject(File f) + { + Lock result = _locks.get(f); + if (result == null) + { + result = new ReentrantLock(); + _locks.put(f, result); + } + return result; + } + + @Override + public void remove(boolean success) throws IOException + { + super.remove(success); + + // Issue 25166: this was a pre-existing potential bug. If _sharedTempDirectory is true, we create a second level + // of temp directory above the primary working dir. this is added to make sure we clean this up. + _jobLog.debug("inspecting remote work dir: " + (_folderToClean == null ? _dir.getPath() : _folderToClean.getPath())); + if (success && _folderToClean != null && !_dir.equals(_folderToClean)) + { + _jobLog.debug("removing entire work dir through: " + _folderToClean.getPath()); + _jobLog.debug("starting with: " + _dir.getPath()); + File toCheck = _dir; + + //debugging only: + if (!URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) + { + _jobLog.warn("not a descendant!"); + } + + while (toCheck != null && URIUtil.isDescendant(_folderToClean.toURI(), toCheck.toURI())) + { + if (!toCheck.exists()) + { + _jobLog.debug("directory does not exist: " + toCheck.getPath()); + toCheck = toCheck.getParentFile(); + continue; + } + + String[] children = toCheck.list(); + if (children != null && children.length == 0) + { + _jobLog.debug("removing directory: " + toCheck.getPath()); + FileUtils.deleteDirectory(toCheck); + toCheck = toCheck.getParentFile(); + } + else if (children == null) + { + _jobLog.debug("unable to list children, will not delete: " + toCheck.getPath()); + continue; + } + else + { + _jobLog.debug("work directory has children, will not delete: " + toCheck.getPath()); + _jobLog.debug("files:"); + for (String fn : children) + { + _jobLog.debug(fn); + } + break; + } + } + } + } + + @Override + protected CopyingResource createCopyingLock() throws IOException + { + if (_lockDirectory == null) + { + return new SimpleCopyingResource(); + } + + _jobLog.debug("Starting to acquire lock for copying files"); + + MasterLockInfo lockInfo; + + // Synchronize to prevent multiple threads from trying to lock the master file from within the same VM + synchronized (WorkDirectoryRemote.class) + { + RandomAccessFile randomAccessFile = null; + FileLock masterLock = null; + + try + { + File masterLockFile = new File(_lockDirectory, "counter"); + randomAccessFile = new RandomAccessFile(masterLockFile, "rw"); + FileChannel masterChannel = randomAccessFile.getChannel(); + masterLock = masterChannel.lock(); + + lockInfo = parseMasterLock(randomAccessFile, masterLockFile); + int nextIndex = (lockInfo.getCurrentLock() + 1) % lockInfo.getTotalLocks(); + rewriteMasterLock(randomAccessFile, new MasterLockInfo(lockInfo.getTotalLocks(), nextIndex)); + } + finally + { + if (randomAccessFile != null) { try { randomAccessFile.close(); } catch (IOException e) {} } + if (masterLock != null) { try { masterLock.release(); } catch (IOException e) {} } + } + } + + _jobLog.debug("Acquiring lock #" + lockInfo.getCurrentLock()); + File f = new File(_lockDirectory, "lock" + lockInfo.getCurrentLock()); + FileChannel lockChannel = new FileOutputStream(f, true).getChannel(); + FileLockCopyingResource result = new FileLockCopyingResource(lockChannel, lockInfo.getCurrentLock(), f); + _jobLog.debug("Lock #" + lockInfo.getCurrentLock() + " acquired"); + + return result; + } + + private void rewriteMasterLock(RandomAccessFile masterFile, MasterLockInfo lockInfo) + throws IOException + { + masterFile.seek(0); + + String output = Integer.toString(lockInfo.getCurrentLock()); + if (lockInfo.getTotalLocks() != FILE_LOCKS_DEFAULT) + output += " " + Integer.toString(lockInfo.getTotalLocks()); + byte[] outputBytes = output.getBytes(StringUtilsLabKey.DEFAULT_CHARSET); + masterFile.write(outputBytes); + masterFile.setLength(outputBytes.length); + } + + private static class MasterLockInfo + { + private final int _totalLocks; + private final int _currentLock; + + private MasterLockInfo(int totalLocks, int currentLock) + { + assert totalLocks > 0 : "Total locks must be greater than 0."; + + _totalLocks = totalLocks; + _currentLock = currentLock; + } + + public int getTotalLocks() + { + return _totalLocks; + } + + public int getCurrentLock() + { + return _currentLock; + } + } + + public class FileLockCopyingResource extends SimpleCopyingResource + { + private FileChannel _channel; + private final int _lockNumber; + private FileLock _lock; + private final Throwable _creationStack; + private Lock _memoryLock; + + public FileLockCopyingResource(FileChannel channel, int lockNumber, File f) throws IOException + { + _channel = channel; + _lockNumber = lockNumber; + _creationStack = new Throwable(); + + // Lock the memory part first to eliminate multi-threaded access to the same file + _memoryLock = getInMemoryLockObject(f); + _memoryLock.lock(); + + // Lock the file part second + _lock = _channel.lock(); + } + + @Override + protected void finalize() throws Throwable + { + super.finalize(); + if (_lock != null) + { + _systemLog.error("FileLockCopyingResource was not released before it was garbage collected. Creation stack is: ", _creationStack); + } + close(); + } + + @Override + public void close() + { + if (_lock != null) + { + // Unlock the file part first + try { _lock.release(); } catch (IOException e) {} + try { _channel.close(); } catch (IOException e) {} + _jobLog.debug("Lock #" + _lockNumber + " released"); + _lock = null; + _channel = null; + super.close(); + + // Unlock the memory part last + _memoryLock.unlock(); + _memoryLock = null; + } + } + } } \ No newline at end of file diff --git a/search/src/org/labkey/search/SearchController.java b/search/src/org/labkey/search/SearchController.java index fe1ddb5291a..c43f5c5d9b9 100644 --- a/search/src/org/labkey/search/SearchController.java +++ b/search/src/org/labkey/search/SearchController.java @@ -1,1198 +1,1198 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.search; - -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hc.core5.http.HttpStatus; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONObject; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasBindParameters; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReturnUrlForm; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.search.SearchResultTemplate; -import org.labkey.api.search.SearchScope; -import org.labkey.api.search.SearchService; -import org.labkey.api.search.SearchService.SearchResult; -import org.labkey.api.search.SearchUrls; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.RequiresSiteAdmin; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.ApplicationAdminPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.HtmlStringBuilder; -import org.labkey.api.util.Path; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.webdav.WebdavService; -import org.labkey.search.audit.SearchAuditProvider; -import org.labkey.search.model.AbstractSearchService; -import org.labkey.search.model.CrawlerRunningState; -import org.labkey.search.model.IndexInspector; -import org.labkey.search.model.LuceneDirectoryType; -import org.labkey.search.model.SearchPropertyManager; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.PropertyValues; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.servlet.ModelAndView; - -import java.io.IOException; -import java.sql.Date; -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -import static org.labkey.api.action.BaseViewAction.springBindParameters; - -public class SearchController extends SpringActionController -{ - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); - - private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); - - public SearchController() - { - setActionResolver(_actionResolver); - } - - - @SuppressWarnings("unused") - public static class SearchUrlsImpl implements SearchUrls - { - @Override - public ActionURL getSearchURL(Container c, @Nullable String query) - { - return SearchController.getSearchURL(c, query); - } - - @Override - public ActionURL getSearchURL(String query, String category) - { - return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); - } - - @Override - public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) - { - return SearchController.getSearchURL(c, query, null, template); - } - } - - - @RequiresPermission(ReadPermission.class) - public class BeginAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(Object o) - { - return getSearchURL(); - } - } - - - public static class AdminForm - { - public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; - private int msg = 0; - private boolean pause; - private boolean start; - private boolean delete; - private String indexPath; - - private boolean limit; - private int fileLimitMB; - - private boolean _path; - - private boolean _directory; - private String _directoryType; - - public String getMessage() - { - return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; - } - - public void setMsg(int m) - { - msg = m; - } - - public boolean isDelete() - { - return delete; - } - - public void setDelete(boolean delete) - { - this.delete = delete; - } - - public boolean isStart() - { - return start; - } - - public void setStart(boolean start) - { - this.start = start; - } - - public boolean isPause() - { - return pause; - } - - public void setPause(boolean pause) - { - this.pause = pause; - } - - public String getIndexPath() - { - return indexPath; - } - - public void setIndexPath(String indexPath) - { - this.indexPath = indexPath; - } - - public boolean isPath() - { - return _path; - } - - public void setPath(boolean path) - { - _path = path; - } - - public boolean isDirectory() - { - return _directory; - } - - public void setDirectory(boolean directory) - { - _directory = directory; - } - - public String getDirectoryType() - { - return _directoryType; - } - - public void setDirectoryType(String directoryType) - { - _directoryType = directoryType; - } - - public boolean isLimit() - { - return limit; - } - - public void setLimit(boolean limit) - { - this.limit = limit; - } - - public int getFileLimitMB() - { - return fileLimitMB; - } - - public int setFileLimitMB(int fileLimitMB) - { - return this.fileLimitMB = fileLimitMB; - } - } - - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class AdminAction extends FormViewAction - { - @SuppressWarnings("UnusedDeclaration") - public AdminAction() - { - } - - public AdminAction(ViewContext ctx, PageConfig pageConfig) - { - setViewContext(ctx); - setPageConfig(pageConfig); - } - - private int _msgid = 0; - - @Override - public void validateCommand(AdminForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) - { - @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) - Throwable t = SearchService.get().getConfigurationError(); - - VBox vbox = new VBox(); - - if (null != t) - { - HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); - builder.append(ExceptionUtil.renderException(t)); - WebPartView configErrorView = new HtmlView(builder); - configErrorView.setTitle("Search Configuration Error"); - configErrorView.setFrame(WebPartView.FrameType.PORTAL); - vbox.addView(configErrorView); - } - - // Spring errors get displayed in the "Index Configuration" pane - WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); - indexerView.setTitle("Index Configuration"); - vbox.addView(indexerView); - - // Won't be able to gather statistics if the search index is misconfigured - if (null == t) - { - WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); - indexerStatsView.setTitle("Index Statistics"); - vbox.addView(indexerStatsView); - } - - WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); - searchStatsView.setTitle("Search Statistics"); - vbox.addView(searchStatsView); - - return vbox; - } - - @Override - public boolean handlePost(AdminForm form, BindException errors) - { - SearchService ss = SearchService.get(); - - if (form.isStart()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); - ss.startCrawler(); - } - else if (form.isPause()) - { - SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); - ss.pauseCrawler(); - } - else if (form.isDelete()) - { - ss.deleteIndex("a site admin requested it"); - ss.start(); - SearchPropertyManager.audit(getUser(), "Index Deleted"); - _msgid = 1; - } - else if (form.isPath()) - { - SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); - ss.updateIndex(); - _msgid = 2; - } - else if (form.isDirectory()) - { - LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); - if (null == type) - { - errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); - return false; - } - SearchPropertyManager.setDirectoryType(getUser(), type); - ss.resetIndex(); - _msgid = 3; - } - else if (form.isLimit()) - { - SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); - ss.resetIndex(); - _msgid = 4; - } - - return true; - } - - @Override - public URLHelper getSuccessURL(AdminForm o) - { - ActionURL success = new ActionURL(AdminAction.class, getContainer()); - if (0 != _msgid) - success.addParameter("msg", String.valueOf(_msgid)); - return success; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("searchAdmin"); - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); - } - } - - - @AdminConsoleAction - public static class IndexContentsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/exportContents.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); - root.addChild("Index Contents"); - } - } - - - public static class ExportForm - { - private String _format = "Text"; - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - - @AdminConsoleAction - public static class ExportIndexContentsAction extends ExportAction - { - @Override - public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception - { - new IndexInspector().export(response, form.getFormat()); - } - } - - - /** for selenium testing */ - @RequiresSiteAdmin - public static class WaitForIdleAction extends SimpleRedirectAction - { - @Override - public URLHelper getRedirectURL(Object o) throws Exception - { - SearchService ss = AbstractSearchService.get(); - ss.waitForIdle(); - return new ActionURL(AdminAction.class, getContainer()); - } - } - - // UNDONE: remove; for testing only - @RequiresSiteAdmin - public class CancelAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - SearchService.IndexTask defaultTask = ss.defaultTask(); - for (SearchService.IndexTask task : ss.getTasks()) - { - if (task != defaultTask && !task.isCancelled()) - task.cancel(true); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - - // UNDONE: remove; for testing only - // cause the current directory to be crawled soon - @RequiresSiteAdmin - public class CrawlAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(ReturnUrlForm form) - { - SearchService ss = SearchService.get(); - - ss.addPathToCrawl( - WebdavService.getPath().append(getContainer().getParsedPath()), - new Date(System.currentTimeMillis())); - - return form.getReturnActionURL(getSearchURL()); - } - } - - public static class IndexForm extends ReturnUrlForm - { - boolean _full = false; - boolean _wait = false; - boolean _since = false; - - public boolean isFull() - { - return _full; - } - - @SuppressWarnings("unused") - public void setFull(boolean full) - { - _full = full; - } - - public boolean isWait() - { - return _wait; - } - - @SuppressWarnings("unused") - public void setWait(boolean wait) - { - _wait = wait; - } - - public boolean isSince() - { - return _since; - } - - @SuppressWarnings("unused") - public void setSince(boolean since) - { - _since = since; - } - } - - // for testing only - @RequiresSiteAdmin - public class IndexAction extends SimpleRedirectAction - { - @Override - public ActionURL getRedirectURL(IndexForm form) throws Exception - { - SearchService ss = SearchService.get(); - - SearchService.IndexTask task = null; - - try (var ignored = SpringActionController.ignoreSqlUpdates()) - { - if (form.isFull()) - { - ss.indexFull(true, "a site admin requested it"); - } - else if (form.isSince()) - { - task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); - } - else - { - task = ss.indexContainer(null, getContainer(), null); - } - } - - if (form.isWait() && null != task) - { - task.get(); // wait for completion - if (ss instanceof AbstractSearchService) - ((AbstractSearchService)ss).commit(); - } - - return form.getReturnActionURL(getSearchURL()); - } - } - - @RequiresPermission(ReadPermission.class) - public class JsonAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SearchForm form, BindException errors) - { - SearchService ss = SearchService.get(); - - audit(form); - - final Path contextPath = Path.parse(getViewContext().getContextPath()); - - final String query = form.getQ() - .replaceAll("(? hits = result.hits; - totalHits = result.totalHits; - - arr = new Object[hits.size()]; - - int i = 0; - int batchSize = 1000; - Map> docDataMap = new HashMap<>(); - for (int ind = 0; ind < hits.size(); ind++) - { - SearchService.SearchHit hit = hits.get(ind); - JSONObject o = new JSONObject(); - String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; - - o.put("id", id); - o.put("title", hit.title); - o.put("container", hit.container); - o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); - o.put("summary", StringUtils.trimToEmpty(hit.summary)); - o.put("score", hit.score); - o.put("identifiers", hit.identifiers); - o.put("category", StringUtils.trimToEmpty(hit.category)); - - if (form.isExperimentalCustomJson()) - { - o.put("jsonData", hit.jsonData); - - if (ind % batchSize == 0) - { - int batchEnd = Math.min(hits.size(), ind + batchSize); - List docIds = new ArrayList<>(); - for (int j = ind; j < batchEnd; j++) - docIds.add(hits.get(j).docid); - - docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); - } - - Map custom = docDataMap.get(hit.docid); - if (custom != null) - o.put("data", custom); - } - - arr[i++] = o; - } - } - - JSONObject metaData = new JSONObject(); - metaData.put("idProperty","id"); - metaData.put("root", "hits"); - metaData.put("successProperty", "success"); - - response.put("metaData", metaData); - response.put("success",true); - response.put("hits", arr); - response.put("totalHits", totalHits); - response.put("q", query); - - return new ApiSimpleResponse(response); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class TestJson extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - public static ActionURL getSearchURL(Container c) - { - return new ActionURL(SearchAction.class, c); - } - - private ActionURL getSearchURL() - { - return getSearchURL(getContainer()); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString) - { - return getSearchURL(c, queryString, null, null); - } - - private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) - { - ActionURL url = getSearchURL(c); - - if (null != queryString) - url.addParameter("q", queryString); - - if (null != category) - url.addParameter("category", category); - - if (null != template) - url.addParameter("template", template); - - return url; - } - - // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. - public interface SearchConfiguration - { - ActionURL getPostURL(Container c); // Search does not actually post - String getDescription(Container c); - SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; - boolean includeAdvancedUI(); - boolean includeNavigationLinks(); - } - - - public static class InternalSearchConfiguration implements SearchConfiguration - { - private final SearchService _ss = SearchService.get(); - - private InternalSearchConfiguration() - { - } - - @Override - public ActionURL getPostURL(Container c) - { - return getSearchURL(c); - } - - @Override - public String getDescription(Container c) - { - return LookAndFeelProperties.getInstance(c).getShortName(); - } - - @Override - public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException - { - SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); - options.categories = _ss.getCategories(category); - options.invertResults = invertSort; - options.limit = limit; - options.offset = offset; - options.scope = scope; - options.sortField = sortField; - - return _ss.search(options.build()); - } - - @Override - public boolean includeAdvancedUI() - { - return true; - } - - @Override - public boolean includeNavigationLinks() - { - return true; - } - } - - - @RequiresPermission(ReadPermission.class) - public class SearchAction extends SimpleViewAction - { - private String _category = null; - private SearchScope _scope = null; - private SearchForm _form = null; - - @Override - public ModelAndView getView(SearchForm form, BindException errors) - { - _category = form.getCategory(); - _scope = form.getSearchScope(); - _form = form; - - if (null == _scope || null == _scope.getRoot(getContainer())) - { - throw new NotFoundException(); - } - - form.setPrint(isPrint()); - - audit(form); - - // reenable caching for search results page (fast browser back button) - HttpServletResponse response = getViewContext().getResponse(); - ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); - getPageConfig().setNoIndex(); - setHelpTopic("luceneSearch"); - - return new JspView<>("/org/labkey/search/view/search.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); - } - } - - public static class PriorityForm - { - SearchService.PRIORITY priority = SearchService.PRIORITY.modified; - - public SearchService.PRIORITY getPriority() - { - return priority; - } - - public void setPriority(SearchService.PRIORITY priority) - { - this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); - } - } - - // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue - // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). - // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It - // does not guarantee that all indexed content has been committed... but that may not be required in practice. - - @RequiresPermission(ApplicationAdminPermission.class) - public static class WaitForIndexerAction extends ExportAction - { - @Override - public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception - { - SearchService ss = SearchService.get(); - long startTime = System.currentTimeMillis(); - boolean success = ss.drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); - - LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); - - // Return an error if we time out - if (!success) - response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); - } - } - - @RequiresPermission(ReadPermission.class) - public class CommentAction extends FormHandlerAction - { - @Override - public void validateCommand(SearchForm target, Errors errors) - { - } - - @Override - public boolean handlePost(SearchForm searchForm, BindException errors) - { - audit(searchForm); - return true; - } - - @Override - public URLHelper getSuccessURL(SearchForm searchForm) - { - return getSearchURL(); - } - } - - - @SuppressWarnings("unused") - public static class SearchForm implements HasBindParameters - { - private String _query = ""; - private String _sortField; - private boolean _print = false; - private int _offset = 0; - private int _limit = 1000; - private String _category = null; - private String _comment = null; - private int _textBoxWidth = 50; // default size - private List _fields; - private boolean _includeHelpLink = true; - private boolean _webpart = false; - private boolean _showAdvanced = false; - private boolean _invertSort = false; - private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) - private String _template = null; - private SearchScope _scope = SearchScope.All; - private boolean _normalizeUrls = false; - private boolean _experimentalCustomJson = false; - - public void setConfiguration(SearchConfiguration config) - { - _config = config; - } - - public SearchConfiguration getConfig() - { - return _config; - } - - public String getQ() - { - return _query; - } - - public void setQ(String query) - { - _query = StringUtils.trimToEmpty(query); - } - - public String getSortField() - { - return _sortField; - } - - public void setSortField(String sortField) - { - _sortField = sortField; - } - - public boolean isPrint() - { - return _print; - } - - public void setPrint(boolean print) - { - _print = print; - } - - public int getOffset() - { - return _offset; - } - - public void setOffset(int o) - { - _offset = o; - } - - public int getLimit() - { - return _limit; - } - - public void setLimit(int o) - { - _limit = o; - } - - public String getScope() - { - return _scope.name(); - } - - public void setScope(String scope) - { - try - { - _scope = SearchScope.valueOf(scope); - } - catch (IllegalArgumentException e) - { - _scope = SearchScope.All; - } - } - - public SearchScope getSearchScope() - { - return _scope; - } - - public String getCategory() - { - return _category; - } - - public void setCategory(String category) - { - _category = category; - } - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public int getTextBoxWidth() - { - return _textBoxWidth; - } - - public void setTextBoxWidth(int textBoxWidth) - { - _textBoxWidth = textBoxWidth; - } - - public boolean getIncludeHelpLink() - { - return _includeHelpLink; - } - - public void setIncludeHelpLink(boolean includeHelpLink) - { - _includeHelpLink = includeHelpLink; - } - - public boolean isWebPart() - { - return _webpart; - } - - public void setWebPart(boolean webpart) - { - _webpart = webpart; - } - - public boolean isShowAdvanced() - { - return _showAdvanced; - } - - public void setShowAdvanced(boolean showAdvanced) - { - _showAdvanced = showAdvanced; - } - - public boolean isInvertSort() - { - return _invertSort; - } - - public void setInvertSort(boolean invertSort) - { - _invertSort = invertSort; - } - - public String getTemplate() - { - return _template; - } - - public void setTemplate(String template) - { - _template = template; - } - - public SearchResultTemplate getSearchResultTemplate() - { - SearchService ss = AbstractSearchService.get(); - return ss.getSearchResultTemplate(getTemplate()); - } - - public boolean isNormalizeUrls() - { - return _normalizeUrls; - } - - public void setNormalizeUrls(boolean normalizeUrls) - { - _normalizeUrls = normalizeUrls; - } - - public boolean isExperimentalCustomJson() - { - return _experimentalCustomJson; - } - - public void setExperimentalCustomJson(boolean experimentalCustomJson) - { - _experimentalCustomJson = experimentalCustomJson; - } - - public List getFields() - { - return _fields; - } - - public void setFields(List fields) - { - _fields = fields; - } - - @Override - public @NotNull BindException bindParameters(PropertyValues m) - { - MutablePropertyValues mpvs = new MutablePropertyValues(m); - var q = mpvs.getPropertyValue("q"); - if (null != q && q.getValue() instanceof String[] arr) - { - mpvs.removePropertyValue("q"); - mpvs.addPropertyValue("q", StringUtils.join(arr," ")); - } - return springBindParameters(this, "form", mpvs); - } - } - - - protected void audit(SearchForm form) - { - ViewContext ctx = getViewContext(); - String comment = form.getComment(); - - audit(ctx.getUser(), ctx.getContainer(), form.getQ(), comment); - } - - - public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) - { - if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) - return; - - AuditLogService audit = AuditLogService.get(); - if (null == audit) - return; - - if (null == c) - c = ContainerManager.getRoot(); - - if (query.length() > 200) - query = StringUtilsLabKey.leftSurrogatePairFriendly(query, 197) + "..."; - - SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); - event.setQuery(query); - - AuditLogService.get().addEvent(user, event); - } - - - public static class SearchSettingsForm - { - private boolean _searchable; - - public boolean isSearchable() - { - return _searchable; - } - - @SuppressWarnings("unused") - public void setSearchable(boolean searchable) - { - _searchable = searchable; - } - } - - - @RequiresPermission(AdminPermission.class) - public static class SearchSettingsAction extends FolderManagementViewPostAction - { - @Override - protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); - } - - @Override - public void validateCommand(SearchSettingsForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SearchSettingsForm form, BindException errors) - { - Container container = getContainer(); - if (container.isRoot()) - { - throw new NotFoundException(); - } - - ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); - - return true; - } - - @Override - public URLHelper getSuccessURL(SearchSettingsForm searchForm) - { - // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) - return getViewContext().getActionURL(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - setHelpTopic("searchAdmin"); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.search; + +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasBindParameters; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReturnUrlForm; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.search.SearchResultTemplate; +import org.labkey.api.search.SearchScope; +import org.labkey.api.search.SearchService; +import org.labkey.api.search.SearchService.SearchResult; +import org.labkey.api.search.SearchUrls; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.RequiresSiteAdmin; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.ApplicationAdminPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.HtmlStringBuilder; +import org.labkey.api.util.Path; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.FolderManagement.FolderManagementViewPostAction; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.webdav.WebdavService; +import org.labkey.search.audit.SearchAuditProvider; +import org.labkey.search.model.AbstractSearchService; +import org.labkey.search.model.CrawlerRunningState; +import org.labkey.search.model.IndexInspector; +import org.labkey.search.model.LuceneDirectoryType; +import org.labkey.search.model.SearchPropertyManager; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValues; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.servlet.ModelAndView; + +import java.io.IOException; +import java.sql.Date; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.labkey.api.action.BaseViewAction.springBindParameters; + +public class SearchController extends SpringActionController +{ + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SearchController.class); + + private static final Logger LOG = LogHelper.getLogger(SearchController.class, "Search UI and admin"); + + public SearchController() + { + setActionResolver(_actionResolver); + } + + + @SuppressWarnings("unused") + public static class SearchUrlsImpl implements SearchUrls + { + @Override + public ActionURL getSearchURL(Container c, @Nullable String query) + { + return SearchController.getSearchURL(c, query); + } + + @Override + public ActionURL getSearchURL(String query, String category) + { + return SearchController.getSearchURL(ContainerManager.getRoot(), query, category, null); + } + + @Override + public ActionURL getSearchURL(Container c, @Nullable String query, @NotNull String template) + { + return SearchController.getSearchURL(c, query, null, template); + } + } + + + @RequiresPermission(ReadPermission.class) + public class BeginAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(Object o) + { + return getSearchURL(); + } + } + + + public static class AdminForm + { + public String[] _messages = {"", "Index deleted", "Index path changed", "Directory type changed", "File size limit changed"}; + private int msg = 0; + private boolean pause; + private boolean start; + private boolean delete; + private String indexPath; + + private boolean limit; + private int fileLimitMB; + + private boolean _path; + + private boolean _directory; + private String _directoryType; + + public String getMessage() + { + return msg >= 0 && msg < _messages.length ? _messages[msg] : ""; + } + + public void setMsg(int m) + { + msg = m; + } + + public boolean isDelete() + { + return delete; + } + + public void setDelete(boolean delete) + { + this.delete = delete; + } + + public boolean isStart() + { + return start; + } + + public void setStart(boolean start) + { + this.start = start; + } + + public boolean isPause() + { + return pause; + } + + public void setPause(boolean pause) + { + this.pause = pause; + } + + public String getIndexPath() + { + return indexPath; + } + + public void setIndexPath(String indexPath) + { + this.indexPath = indexPath; + } + + public boolean isPath() + { + return _path; + } + + public void setPath(boolean path) + { + _path = path; + } + + public boolean isDirectory() + { + return _directory; + } + + public void setDirectory(boolean directory) + { + _directory = directory; + } + + public String getDirectoryType() + { + return _directoryType; + } + + public void setDirectoryType(String directoryType) + { + _directoryType = directoryType; + } + + public boolean isLimit() + { + return limit; + } + + public void setLimit(boolean limit) + { + this.limit = limit; + } + + public int getFileLimitMB() + { + return fileLimitMB; + } + + public int setFileLimitMB(int fileLimitMB) + { + return this.fileLimitMB = fileLimitMB; + } + } + + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class AdminAction extends FormViewAction + { + @SuppressWarnings("UnusedDeclaration") + public AdminAction() + { + } + + public AdminAction(ViewContext ctx, PageConfig pageConfig) + { + setViewContext(ctx); + setPageConfig(pageConfig); + } + + private int _msgid = 0; + + @Override + public void validateCommand(AdminForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(AdminForm form, boolean reshow, BindException errors) + { + @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"}) + Throwable t = SearchService.get().getConfigurationError(); + + VBox vbox = new VBox(); + + if (null != t) + { + HtmlStringBuilder builder = HtmlStringBuilder.of(HtmlString.unsafe("Your search index is misconfigured. Search is disabled and documents are not being indexed, pending resolution of this issue. See below for details about the cause of the problem.

    ")); + builder.append(ExceptionUtil.renderException(t)); + WebPartView configErrorView = new HtmlView(builder); + configErrorView.setTitle("Search Configuration Error"); + configErrorView.setFrame(WebPartView.FrameType.PORTAL); + vbox.addView(configErrorView); + } + + // Spring errors get displayed in the "Index Configuration" pane + WebPartView indexerView = new JspView<>("/org/labkey/search/view/indexerAdmin.jsp", form, errors); + indexerView.setTitle("Index Configuration"); + vbox.addView(indexerView); + + // Won't be able to gather statistics if the search index is misconfigured + if (null == t) + { + WebPartView indexerStatsView = new JspView<>("/org/labkey/search/view/indexerStats.jsp", form); + indexerStatsView.setTitle("Index Statistics"); + vbox.addView(indexerStatsView); + } + + WebPartView searchStatsView = new JspView<>("/org/labkey/search/view/searchStats.jsp", form); + searchStatsView.setTitle("Search Statistics"); + vbox.addView(searchStatsView); + + return vbox; + } + + @Override + public boolean handlePost(AdminForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + if (form.isStart()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Start); + ss.startCrawler(); + } + else if (form.isPause()) + { + SearchPropertyManager.setCrawlerRunningState(getUser(), CrawlerRunningState.Pause); + ss.pauseCrawler(); + } + else if (form.isDelete()) + { + ss.deleteIndex("a site admin requested it"); + ss.start(); + SearchPropertyManager.audit(getUser(), "Index Deleted"); + _msgid = 1; + } + else if (form.isPath()) + { + SearchPropertyManager.setIndexPath(getUser(), form.getIndexPath()); + ss.updateIndex(); + _msgid = 2; + } + else if (form.isDirectory()) + { + LuceneDirectoryType type = EnumUtils.getEnum(LuceneDirectoryType.class, form.getDirectoryType()); + if (null == type) + { + errors.reject(ERROR_MSG, "Unrecognized value for \"directoryType\": \"" + form.getDirectoryType() + "\""); + return false; + } + SearchPropertyManager.setDirectoryType(getUser(), type); + ss.resetIndex(); + _msgid = 3; + } + else if (form.isLimit()) + { + SearchPropertyManager.setFileSizeLimitMB(getUser(), form.getFileLimitMB()); + ss.resetIndex(); + _msgid = 4; + } + + return true; + } + + @Override + public URLHelper getSuccessURL(AdminForm o) + { + ActionURL success = new ActionURL(AdminAction.class, getContainer()); + if (0 != _msgid) + success.addParameter("msg", String.valueOf(_msgid)); + return success; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("searchAdmin"); + urlProvider(AdminUrls.class).addAdminNavTrail(root, "Full-Text Search Configuration", getClass(), getContainer()); + } + } + + + @AdminConsoleAction + public static class IndexContentsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/exportContents.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext(), getPageConfig()).addNavTrail(root); + root.addChild("Index Contents"); + } + } + + + public static class ExportForm + { + private String _format = "Text"; + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + + @AdminConsoleAction + public static class ExportIndexContentsAction extends ExportAction + { + @Override + public void export(ExportForm form, HttpServletResponse response, BindException errors) throws Exception + { + new IndexInspector().export(response, form.getFormat()); + } + } + + + /** for selenium testing */ + @RequiresSiteAdmin + public static class WaitForIdleAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(Object o) throws Exception + { + SearchService ss = AbstractSearchService.get(); + ss.waitForIdle(); + return new ActionURL(AdminAction.class, getContainer()); + } + } + + // UNDONE: remove; for testing only + @RequiresSiteAdmin + public class CancelAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + SearchService.IndexTask defaultTask = ss.defaultTask(); + for (SearchService.IndexTask task : ss.getTasks()) + { + if (task != defaultTask && !task.isCancelled()) + task.cancel(true); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + + // UNDONE: remove; for testing only + // cause the current directory to be crawled soon + @RequiresSiteAdmin + public class CrawlAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(ReturnUrlForm form) + { + SearchService ss = SearchService.get(); + + ss.addPathToCrawl( + WebdavService.getPath().append(getContainer().getParsedPath()), + new Date(System.currentTimeMillis())); + + return form.getReturnActionURL(getSearchURL()); + } + } + + public static class IndexForm extends ReturnUrlForm + { + boolean _full = false; + boolean _wait = false; + boolean _since = false; + + public boolean isFull() + { + return _full; + } + + @SuppressWarnings("unused") + public void setFull(boolean full) + { + _full = full; + } + + public boolean isWait() + { + return _wait; + } + + @SuppressWarnings("unused") + public void setWait(boolean wait) + { + _wait = wait; + } + + public boolean isSince() + { + return _since; + } + + @SuppressWarnings("unused") + public void setSince(boolean since) + { + _since = since; + } + } + + // for testing only + @RequiresSiteAdmin + public class IndexAction extends SimpleRedirectAction + { + @Override + public ActionURL getRedirectURL(IndexForm form) throws Exception + { + SearchService ss = SearchService.get(); + + SearchService.IndexTask task = null; + + try (var ignored = SpringActionController.ignoreSqlUpdates()) + { + if (form.isFull()) + { + ss.indexFull(true, "a site admin requested it"); + } + else if (form.isSince()) + { + task = ss.indexContainer(null, getContainer(), new Date(System.currentTimeMillis()- TimeUnit.DAYS.toMillis(1))); + } + else + { + task = ss.indexContainer(null, getContainer(), null); + } + } + + if (form.isWait() && null != task) + { + task.get(); // wait for completion + if (ss instanceof AbstractSearchService) + ((AbstractSearchService)ss).commit(); + } + + return form.getReturnActionURL(getSearchURL()); + } + } + + @RequiresPermission(ReadPermission.class) + public class JsonAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SearchForm form, BindException errors) + { + SearchService ss = SearchService.get(); + + audit(form); + + final Path contextPath = Path.parse(getViewContext().getContextPath()); + + final String query = form.getQ() + .replaceAll("(? hits = result.hits; + totalHits = result.totalHits; + + arr = new Object[hits.size()]; + + int i = 0; + int batchSize = 1000; + Map> docDataMap = new HashMap<>(); + for (int ind = 0; ind < hits.size(); ind++) + { + SearchService.SearchHit hit = hits.get(ind); + JSONObject o = new JSONObject(); + String id = StringUtils.isEmpty(hit.docid) ? String.valueOf(i) : hit.docid; + + o.put("id", id); + o.put("title", hit.title); + o.put("container", hit.container); + o.put("url", form.isNormalizeUrls() ? hit.normalizeHref(contextPath) : hit.url); + o.put("summary", StringUtils.trimToEmpty(hit.summary)); + o.put("score", hit.score); + o.put("identifiers", hit.identifiers); + o.put("category", StringUtils.trimToEmpty(hit.category)); + + if (form.isExperimentalCustomJson()) + { + o.put("jsonData", hit.jsonData); + + if (ind % batchSize == 0) + { + int batchEnd = Math.min(hits.size(), ind + batchSize); + List docIds = new ArrayList<>(); + for (int j = ind; j < batchEnd; j++) + docIds.add(hits.get(j).docid); + + docDataMap = ss.getCustomSearchJsonMap(getUser(), docIds); + } + + Map custom = docDataMap.get(hit.docid); + if (custom != null) + o.put("data", custom); + } + + arr[i++] = o; + } + } + + JSONObject metaData = new JSONObject(); + metaData.put("idProperty","id"); + metaData.put("root", "hits"); + metaData.put("successProperty", "success"); + + response.put("metaData", metaData); + response.put("success",true); + response.put("hits", arr); + response.put("totalHits", totalHits); + response.put("q", query); + + return new ApiSimpleResponse(response); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class TestJson extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/search/view/testJson.jsp", null, null); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + public static ActionURL getSearchURL(Container c) + { + return new ActionURL(SearchAction.class, c); + } + + private ActionURL getSearchURL() + { + return getSearchURL(getContainer()); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString) + { + return getSearchURL(c, queryString, null, null); + } + + private static ActionURL getSearchURL(Container c, @Nullable String queryString, @Nullable String category, @Nullable String template) + { + ActionURL url = getSearchURL(c); + + if (null != queryString) + url.addParameter("q", queryString); + + if (null != category) + url.addParameter("category", category); + + if (null != template) + url.addParameter("template", template); + + return url; + } + + // This interface used to be used to hide all the specifics of internal vs. external index search, but we no longer support external indexes. This interface could be removed. + public interface SearchConfiguration + { + ActionURL getPostURL(Container c); // Search does not actually post + String getDescription(Container c); + SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, @Nullable String sortField, int offset, int limit, boolean invertSort) throws IOException; + boolean includeAdvancedUI(); + boolean includeNavigationLinks(); + } + + + public static class InternalSearchConfiguration implements SearchConfiguration + { + private final SearchService _ss = SearchService.get(); + + private InternalSearchConfiguration() + { + } + + @Override + public ActionURL getPostURL(Container c) + { + return getSearchURL(c); + } + + @Override + public String getDescription(Container c) + { + return LookAndFeelProperties.getInstance(c).getShortName(); + } + + @Override + public SearchResult getSearchResult(String queryString, @Nullable String category, User user, Container currentContainer, SearchScope scope, String sortField, int offset, int limit, boolean invertSort) throws IOException + { + SearchService.SearchOptions.Builder options = new SearchService.SearchOptions.Builder(queryString, user, currentContainer); + options.categories = _ss.getCategories(category); + options.invertResults = invertSort; + options.limit = limit; + options.offset = offset; + options.scope = scope; + options.sortField = sortField; + + return _ss.search(options.build()); + } + + @Override + public boolean includeAdvancedUI() + { + return true; + } + + @Override + public boolean includeNavigationLinks() + { + return true; + } + } + + + @RequiresPermission(ReadPermission.class) + public class SearchAction extends SimpleViewAction + { + private String _category = null; + private SearchScope _scope = null; + private SearchForm _form = null; + + @Override + public ModelAndView getView(SearchForm form, BindException errors) + { + _category = form.getCategory(); + _scope = form.getSearchScope(); + _form = form; + + if (null == _scope || null == _scope.getRoot(getContainer())) + { + throw new NotFoundException(); + } + + form.setPrint(isPrint()); + + audit(form); + + // reenable caching for search results page (fast browser back button) + HttpServletResponse response = getViewContext().getResponse(); + ResponseHelper.setPrivate(response, Duration.ofMinutes(5)); + getPageConfig().setNoIndex(); + setHelpTopic("luceneSearch"); + + return new JspView<>("/org/labkey/search/view/search.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + _form.getSearchResultTemplate().addNavTrail(root, getViewContext(), _scope, _category); + } + } + + public static class PriorityForm + { + SearchService.PRIORITY priority = SearchService.PRIORITY.modified; + + public SearchService.PRIORITY getPriority() + { + return priority; + } + + public void setPriority(SearchService.PRIORITY priority) + { + this.priority = Objects.requireNonNullElse(priority, SearchService.PRIORITY.modified); + } + } + + // This is intended to help test search indexing. This action sticks a special runnable in the indexer queue + // and then returns when that runnable is executed (or if five minutes goes by without the runnable executing). + // The tests can invoke this action to ensure that the indexer has executed all previous indexing tasks. It + // does not guarantee that all indexed content has been committed... but that may not be required in practice. + + @RequiresPermission(ApplicationAdminPermission.class) + public static class WaitForIndexerAction extends ExportAction + { + @Override + public void export(PriorityForm form, HttpServletResponse response, BindException errors) throws Exception + { + SearchService ss = SearchService.get(); + long startTime = System.currentTimeMillis(); + boolean success = ss.drainQueue(form.getPriority(), 5, TimeUnit.MINUTES); + + LOG.info("Spent {}ms draining the search indexer queue at priority {}. Success: {}", System.currentTimeMillis() - startTime, form.getPriority(), success); + + // Return an error if we time out + if (!success) + response.setStatus(HttpStatus.SC_INTERNAL_SERVER_ERROR); + } + } + + @RequiresPermission(ReadPermission.class) + public class CommentAction extends FormHandlerAction + { + @Override + public void validateCommand(SearchForm target, Errors errors) + { + } + + @Override + public boolean handlePost(SearchForm searchForm, BindException errors) + { + audit(searchForm); + return true; + } + + @Override + public URLHelper getSuccessURL(SearchForm searchForm) + { + return getSearchURL(); + } + } + + + @SuppressWarnings("unused") + public static class SearchForm implements HasBindParameters + { + private String _query = ""; + private String _sortField; + private boolean _print = false; + private int _offset = 0; + private int _limit = 1000; + private String _category = null; + private String _comment = null; + private int _textBoxWidth = 50; // default size + private List _fields; + private boolean _includeHelpLink = true; + private boolean _webpart = false; + private boolean _showAdvanced = false; + private boolean _invertSort = false; + private SearchConfiguration _config = new InternalSearchConfiguration(); // Assume internal search (for webparts, etc.) + private String _template = null; + private SearchScope _scope = SearchScope.All; + private boolean _normalizeUrls = false; + private boolean _experimentalCustomJson = false; + + public void setConfiguration(SearchConfiguration config) + { + _config = config; + } + + public SearchConfiguration getConfig() + { + return _config; + } + + public String getQ() + { + return _query; + } + + public void setQ(String query) + { + _query = StringUtils.trimToEmpty(query); + } + + public String getSortField() + { + return _sortField; + } + + public void setSortField(String sortField) + { + _sortField = sortField; + } + + public boolean isPrint() + { + return _print; + } + + public void setPrint(boolean print) + { + _print = print; + } + + public int getOffset() + { + return _offset; + } + + public void setOffset(int o) + { + _offset = o; + } + + public int getLimit() + { + return _limit; + } + + public void setLimit(int o) + { + _limit = o; + } + + public String getScope() + { + return _scope.name(); + } + + public void setScope(String scope) + { + try + { + _scope = SearchScope.valueOf(scope); + } + catch (IllegalArgumentException e) + { + _scope = SearchScope.All; + } + } + + public SearchScope getSearchScope() + { + return _scope; + } + + public String getCategory() + { + return _category; + } + + public void setCategory(String category) + { + _category = category; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public int getTextBoxWidth() + { + return _textBoxWidth; + } + + public void setTextBoxWidth(int textBoxWidth) + { + _textBoxWidth = textBoxWidth; + } + + public boolean getIncludeHelpLink() + { + return _includeHelpLink; + } + + public void setIncludeHelpLink(boolean includeHelpLink) + { + _includeHelpLink = includeHelpLink; + } + + public boolean isWebPart() + { + return _webpart; + } + + public void setWebPart(boolean webpart) + { + _webpart = webpart; + } + + public boolean isShowAdvanced() + { + return _showAdvanced; + } + + public void setShowAdvanced(boolean showAdvanced) + { + _showAdvanced = showAdvanced; + } + + public boolean isInvertSort() + { + return _invertSort; + } + + public void setInvertSort(boolean invertSort) + { + _invertSort = invertSort; + } + + public String getTemplate() + { + return _template; + } + + public void setTemplate(String template) + { + _template = template; + } + + public SearchResultTemplate getSearchResultTemplate() + { + SearchService ss = AbstractSearchService.get(); + return ss.getSearchResultTemplate(getTemplate()); + } + + public boolean isNormalizeUrls() + { + return _normalizeUrls; + } + + public void setNormalizeUrls(boolean normalizeUrls) + { + _normalizeUrls = normalizeUrls; + } + + public boolean isExperimentalCustomJson() + { + return _experimentalCustomJson; + } + + public void setExperimentalCustomJson(boolean experimentalCustomJson) + { + _experimentalCustomJson = experimentalCustomJson; + } + + public List getFields() + { + return _fields; + } + + public void setFields(List fields) + { + _fields = fields; + } + + @Override + public @NotNull BindException bindParameters(PropertyValues m) + { + MutablePropertyValues mpvs = new MutablePropertyValues(m); + var q = mpvs.getPropertyValue("q"); + if (null != q && q.getValue() instanceof String[] arr) + { + mpvs.removePropertyValue("q"); + mpvs.addPropertyValue("q", StringUtils.join(arr," ")); + } + return springBindParameters(this, "form", mpvs); + } + } + + + protected void audit(SearchForm form) + { + ViewContext ctx = getViewContext(); + String comment = form.getComment(); + + audit(ctx.getUser(), ctx.getContainer(), form.getQ(), comment); + } + + + public static void audit(@Nullable User user, @Nullable Container c, String query, String comment) + { + if ((null != user && user.isSearchUser()) || StringUtils.isEmpty(query)) + return; + + AuditLogService audit = AuditLogService.get(); + if (null == audit) + return; + + if (null == c) + c = ContainerManager.getRoot(); + + if (query.length() > 200) + query = StringUtilsLabKey.leftSurrogatePairFriendly(query, 197) + "..."; + + SearchAuditProvider.SearchAuditEvent event = new SearchAuditProvider.SearchAuditEvent(c, comment); + event.setQuery(query); + + AuditLogService.get().addEvent(user, event); + } + + + public static class SearchSettingsForm + { + private boolean _searchable; + + public boolean isSearchable() + { + return _searchable; + } + + @SuppressWarnings("unused") + public void setSearchable(boolean searchable) + { + _searchable = searchable; + } + } + + + @RequiresPermission(AdminPermission.class) + public static class SearchSettingsAction extends FolderManagementViewPostAction + { + @Override + protected JspView getTabView(SearchSettingsForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/search/view/fullTextSearch.jsp", form, errors); + } + + @Override + public void validateCommand(SearchSettingsForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SearchSettingsForm form, BindException errors) + { + Container container = getContainer(); + if (container.isRoot()) + { + throw new NotFoundException(); + } + + ContainerManager.updateSearchable(container, form.isSearchable(), getUser()); + + return true; + } + + @Override + public URLHelper getSuccessURL(SearchSettingsForm searchForm) + { + // In this case, must redirect back to view so Container is reloaded (simple reshow will continue to show the old value) + return getViewContext().getActionURL(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + setHelpTopic("searchAdmin"); + } + } +} From 42735591490112ef8c838dcd1849bf468e08d9b0 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 13 Nov 2025 15:50:04 -0800 Subject: [PATCH 3/4] fix --- api/src/org/labkey/api/data/Container.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/data/Container.java b/api/src/org/labkey/api/data/Container.java index b4d69f94270..b2ca3dd25f2 100644 --- a/api/src/org/labkey/api/data/Container.java +++ b/api/src/org/labkey/api/data/Container.java @@ -1595,7 +1595,7 @@ public String getContainerNoun(boolean titleCase) String noun = _containerType.getContainerNoun(this); if (titleCase) { - return StringUtilsLabKey.leftSurrogatePairFriendly(noun, 1).toUpperCase() + StringUtilsLabKey.rightSurrogatePairFriendly(noun,1); + return StringUtilsLabKey.leftSurrogatePairFriendly(noun, 1).toUpperCase() + StringUtilsLabKey.rightSurrogatePairFriendly(noun, noun.length() - 1); } return noun; From e5cc3ef465423a7f1978f3afe5549fc0bc421585 Mon Sep 17 00:00:00 2001 From: labkey-tchad Date: Mon, 17 Nov 2025 16:52:54 -0800 Subject: [PATCH 4/4] Fix overlapping sheet names --- query/src/org/labkey/query/controllers/QueryController.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 35ba9a5d411..da1f6703863 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -235,6 +235,7 @@ import org.labkey.api.util.ResponseHelper; import org.labkey.api.util.ReturnURLString; import org.labkey.api.util.StringExpression; +import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.TestContext; import org.labkey.api.util.URLHelper; import org.labkey.api.util.UnexpectedException; @@ -2134,7 +2135,7 @@ public Object execute(ExportQueriesForm form, BindException errors) throws Excep throw new IllegalArgumentException("Cannot create sheet names from overlapping query names."); for (int i = 0; i < queryForms.size(); i++) { - sheetNames.put(entry.getValue().get(i), name.substring(0, name.length() - countLength) + "(" + i + ")"); + sheetNames.put(entry.getValue().get(i), StringUtilsLabKey.leftSurrogatePairFriendly(name, name.length() - countLength) + "(" + i + ")"); } } else