From c5d28af172b2a589b376d2fadc01836ce6da9787 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 22 Oct 2025 19:51:44 -0700 Subject: [PATCH 01/13] Add auditing of what method was used for data imports, updates, deletes --- .../org/labkey/api/action/BaseViewAction.java | 1531 +- .../api/assay/AssayFilePropertyWriter.java | 5 +- .../org/labkey/api/assay/AssayRunCreator.java | 14 +- .../api/assay/DefaultAssayRunCreator.java | 2547 ++- .../api/assay/actions/UploadWizardAction.java | 2 +- .../pipeline/AssayUploadPipelineJob.java | 7 +- .../org/labkey/api/audit/AuditTypeEvent.java | 525 +- .../api/audit/ExperimentAuditEvent.java | 3 +- .../api/audit/SampleTimelineAuditEvent.java | 3 +- .../api/audit/TransactionAuditProvider.java | 96 + .../provider/FileSystemAuditProvider.java | 469 +- .../api/query/AbstractQueryImportAction.java | 33 +- .../api/query/AbstractQueryUpdateService.java | 3032 +-- .../api/query/QueryImportPipelineJob.java | 15 +- .../api/query/SimpleQueryUpdateService.java | 10 + .../labkey/api/query/UserSchemaAction.java | 670 +- api/src/org/labkey/api/util/HttpUtil.java | 27 +- .../api/assay/DefaultAssaySaveHandler.java | 8 +- .../assay/actions/ImportRunApiAction.java | 15 +- .../assay/pipeline/AssayImportRunTask.java | 7 +- .../org/labkey/assay/plate/PlateManager.java | 10 +- .../assay/plate/audit/PlateAuditEvent.java | 5 +- .../assay/plate/audit/PlateAuditProvider.java | 13 +- .../assay/plate/audit/PlateSetAuditEvent.java | 5 +- .../plate/audit/PlateSetAuditProvider.java | 9 +- .../experiment/SampleTypeAuditProvider.java | 441 +- .../api/ExpDataClassDataTableImpl.java | 5 + .../experiment/api/ExperimentServiceImpl.java | 12 +- .../experiment/api/SampleTypeServiceImpl.java | 4740 ++--- .../api/SampleTypeUpdateServiceDI.java | 4 + .../labkey/list/model/ListAuditProvider.java | 547 +- .../list/model/ListQueryUpdateService.java | 1774 +- .../query/audit/QueryUpdateAuditProvider.java | 609 +- .../query/controllers/QueryController.java | 17548 ++++++++-------- .../study/query/DatasetUpdateService.java | 2280 +- 35 files changed, 18636 insertions(+), 18385 deletions(-) diff --git a/api/src/org/labkey/api/action/BaseViewAction.java b/api/src/org/labkey/api/action/BaseViewAction.java index d31dda91433..0ea6e665da5 100644 --- a/api/src/org/labkey/api/action/BaseViewAction.java +++ b/api/src/org/labkey/api/action/BaseViewAction.java @@ -1,755 +1,776 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.action; - -import jakarta.servlet.ServletRequest; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.beanutils.DynaBean; -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.data.Container; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.security.User; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.writer.ContainerUser; -import org.springframework.beans.AbstractPropertyAccessor; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.BeanWrapper; -import org.springframework.beans.BeansException; -import org.springframework.beans.InvalidPropertyException; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.NotReadablePropertyException; -import org.springframework.beans.NotWritablePropertyException; -import org.springframework.beans.PropertyAccessException; -import org.springframework.beans.PropertyValue; -import org.springframework.beans.PropertyValues; -import org.springframework.beans.TypeMismatchException; -import org.springframework.core.MethodParameter; -import org.springframework.core.convert.TypeDescriptor; -import org.springframework.lang.Nullable; -import org.springframework.validation.BeanPropertyBindingResult; -import org.springframework.validation.BindException; -import org.springframework.validation.BindingErrorProcessor; -import org.springframework.validation.BindingResult; -import org.springframework.validation.FieldError; -import org.springframework.validation.ObjectError; -import org.springframework.validation.Validator; -import org.springframework.web.bind.ServletRequestDataBinder; -import org.springframework.web.bind.ServletRequestParameterPropertyValues; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.MultipartHttpServletRequest; -import org.springframework.web.servlet.ModelAndView; - -import java.beans.PropertyDescriptor; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; - -public abstract class BaseViewAction
extends PermissionCheckableAction implements Validator, HasPageConfig, ContainerUser -{ - protected static final Logger logger = LogHelper.getLogger(BaseViewAction.class, "BaseViewAction"); - - private PageConfig _pageConfig = null; - private PropertyValues _pvs; - private boolean _robot = false; // Is this request from GoogleBot or some other crawler? - private boolean _debug = false; - - protected boolean _print = false; - protected Class _commandClass; - protected String _commandName = "form"; - - protected BaseViewAction() - { - String methodName = getCommandClassMethodName(); - - if (null == methodName) - return; - - // inspect the action's *public* methods to determine form class - Class typeBest = null; - for (Method m : this.getClass().getMethods()) - { - if (methodName.equals(m.getName())) - { - Class[] types = m.getParameterTypes(); - if (types.length < 1) - continue; - Class typeCurrent = types[0]; - assert null == _commandClass || typeCurrent.equals(_commandClass); - - // Using templated classes to extend a base action can lead to multiple - // versions of a method with acceptable types, so take the most extended - // type we can find. - if (typeBest == null || typeBest.isAssignableFrom(typeCurrent)) - typeBest = typeCurrent; - } - } - if (typeBest != null) - setCommandClass(typeBest); - } - - - protected abstract String getCommandClassMethodName(); - - - protected BaseViewAction(@NotNull Class commandClass) - { - setCommandClass(commandClass); - } - - - public void setProperties(PropertyValues pvs) - { - _pvs = pvs; - } - - - public void setProperties(Map m) - { - _pvs = new MutablePropertyValues(m); - } - - - /* Doesn't guarantee non-null, non-empty */ - public Object getProperty(String key, String d) - { - PropertyValue pv = _pvs.getPropertyValue(key); - return pv == null ? d : pv.getValue(); - } - - - public Object getProperty(Enum key) - { - PropertyValue pv = _pvs.getPropertyValue(key.name()); - return pv == null ? null : pv.getValue(); - } - - - public Object getProperty(String key) - { - PropertyValue pv = _pvs.getPropertyValue(key); - return pv == null ? null : pv.getValue(); - } - - public PropertyValues getPropertyValues() - { - return _pvs; - } - - - public static PropertyValues getPropertyValuesForFormBinding(PropertyValues pvs, @NotNull Predicate allowBind) - { - if (null == pvs) - return null; - MutablePropertyValues ret = new MutablePropertyValues(); - for (PropertyValue pv : pvs.getPropertyValues()) - { - if (allowBind.test(pv.getName())) - ret.addPropertyValue(pv); - } - return ret; - } - - static final String FORM_DATE_ENCODED_PARAM = "formDataEncoded"; - - /** - * When a double quote is encountered in a multipart/form-data context, it is encoded as %22 using URL-encoding by browsers. - * This process replaces the double quote with its hexadecimal equivalent in a URL-safe format, preventing it from being misinterpreted as the end of a value or a boundary. - * The consequence of such encoding is we can't distinguish '"' from the actual '%22' in parameter name. - * As a workaround, a client-side util `encodeFormDataQuote` is used to convert %22 to %2522 and " to %22 explicitly, while passing in an additional param formDataEncoded=true. - * This class converts those encoded param names back to its decoded form during PropertyValues binding. - * See Issue 52827, 52925 and 52119 for more information. - */ - static public class ViewActionParameterPropertyValues extends ServletRequestParameterPropertyValues - { - - public ViewActionParameterPropertyValues(ServletRequest request) { - this(request, null, null); - } - - public ViewActionParameterPropertyValues(ServletRequest request, @Nullable String prefix, @Nullable String prefixSeparator) - { - super(request, prefix, prefixSeparator); - if (isFormDataEncoded()) - { - for (int i = 0; i < getPropertyValues().length; i++) - { - PropertyValue formDataPropValue = getPropertyValues()[i]; - String propValueName = formDataPropValue.getName(); - String decoded = PageFlowUtil.decodeQuoteEncodedFormDataKey(propValueName); - if (!propValueName.equals(decoded)) - setPropertyValueAt(new PropertyValue(decoded, formDataPropValue.getValue()), i); - } - } - } - - private boolean isFormDataEncoded() - { - PropertyValue formDataPropValue = getPropertyValue(FORM_DATE_ENCODED_PARAM); - if (formDataPropValue != null) - { - Object v = formDataPropValue.getValue(); - String formDataPropValueStr = v == null ? null : String.valueOf(v); - if (StringUtils.isNotBlank(formDataPropValueStr)) - return (Boolean) ConvertUtils.convert(formDataPropValueStr, Boolean.class); - } - - return false; - } - } - - @Override - public ModelAndView handleRequest(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) throws Exception - { - if (null == getPropertyValues()) - setProperties(new ViewActionParameterPropertyValues(request)); - getViewContext().setBindPropertyValues(getPropertyValues()); - handleSpecialProperties(); - - return handleRequest(); - } - - - private void handleSpecialProperties() - { - _robot = PageFlowUtil.isRobotUserAgent(getViewContext().getRequest().getHeader("User-Agent")); - - // Special flag puts actions in "debug" mode, during which they should log extra information that would be - // helpful for testing or debugging problems - if (!_robot && hasStringValue("_debug")) - { - _debug = true; - } - - // SpringActionController.defaultPageConfig() has logic for isPrint, don't need to duplicate here - _print = PageConfig.Template.Print == HttpView.currentPageConfig().getTemplate(); - } - - private boolean hasStringValue(String propertyName) - { - Object o = getProperty(propertyName); - if (o == null) - { - return false; - } - if (o instanceof String s) - { - return null != StringUtils.trimToNull(s); - } - if (o instanceof String[] strings) - { - for (String s : strings) - { - if (null != StringUtils.trimToNull(s)) - { - return true; - } - } - } - return false; - } - - public abstract ModelAndView handleRequest() throws Exception; - - - @Override - public void setPageConfig(PageConfig page) - { - _pageConfig = page; - } - - - @Override - public Container getContainer() - { - return getViewContext().getContainer(); - } - - - @Override - public User getUser() - { - return getViewContext().getUser(); - } - - - @Override - public PageConfig getPageConfig() - { - return _pageConfig; - } - - - public void setTitle(String title) - { - assert null != getPageConfig() : "action not initialized property"; - getPageConfig().setTitle(title); - } - - - public void setHelpTopic(String topicName) - { - setHelpTopic(new HelpTopic(topicName)); - } - - - public void setHelpTopic(HelpTopic topic) - { - assert null != getPageConfig() : "action not initialized properly"; - getPageConfig().setHelpTopic(topic); - } - - - protected Object newInstance(Class c) - { - try - { - return c == null ? null : c.getConstructor().newInstance(); - } - catch (Exception x) - { - if (x instanceof RuntimeException) - throw ((RuntimeException)x); - else - throw new RuntimeException(x); - } - } - - - protected @NotNull FORM getCommand(HttpServletRequest request) throws Exception - { - FORM command = (FORM) createCommand(); - - if (command instanceof HasViewContext) - ((HasViewContext)command).setViewContext(getViewContext()); - - return command; - } - - - protected @NotNull FORM getCommand() throws Exception - { - return getCommand(getViewContext().getRequest()); - } - - - // - // PARAMETER BINDING - // - // don't assume parameters always come from a request, use PropertyValues interface - // - - public @NotNull BindException defaultBindParameters(FORM form, PropertyValues params) - { - return defaultBindParameters(form, getCommandName(), params); - } - - - public static @NotNull BindException defaultBindParameters(Object form, String commandName, PropertyValues params) - { - /* check for do-it-myself forms */ - if (form instanceof HasBindParameters) - { - return ((HasBindParameters)form).bindParameters(params); - } - - if (form instanceof DynaBean) - { - return simpleBindParameters(form, commandName, params); - } - else - { - return springBindParameters(form, commandName, params); - } - } - - public static @NotNull BindException springBindParameters(Object command, String commandName, PropertyValues params) - { - Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); - ServletRequestDataBinder binder = new ServletRequestDataBinder(command, commandName); - - String[] fields = binder.getDisallowedFields(); - List fieldList = new ArrayList<>(fields != null ? Arrays.asList(fields) : Collections.emptyList()); - fieldList.addAll(Arrays.asList("class.*", "Class.*", "*.class.*", "*.Class.*")); - binder.setDisallowedFields(fieldList.toArray(new String[] {})); - - ConvertHelper.getPropertyEditorRegistrar().registerCustomEditors(binder); - BindingErrorProcessor defaultBEP = binder.getBindingErrorProcessor(); - binder.setBindingErrorProcessor(getBindingErrorProcessor(defaultBEP)); - binder.setFieldMarkerPrefix(SpringActionController.FIELD_MARKER); - try - { - // most paths probably called getPropertyValuesForFormBinding() already, but this is a public static method, so call it again - binder.bind(getPropertyValuesForFormBinding(params, allow)); - BindException errors = new NullSafeBindException(binder.getBindingResult()); - return errors; - } - catch (InvalidPropertyException x) - { - // Maybe we should propagate exception and return SC_BAD_REQUEST (in ExceptionUtil.handleException()) - // most POST handlers check errors.hasErrors(), but not all GET handlers do - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding property: " + x.getPropertyName()); - return errors; - } - catch (NumberFormatException x) - { - // Malformed array parameter throws this exception, unfortunately. Just reject the request. #21931 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; invalid array index (" + x.getMessage() + ")"); - return errors; - } - catch (NegativeArraySizeException x) - { - // Another malformed array parameter throws this exception. #23929 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; negative array size (" + x.getMessage() + ")"); - return errors; - } - catch (IllegalArgumentException x) - { - // General bean binding problem. #23929 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding property; (" + x.getMessage() + ")"); - return errors; - } - } - - - static BindingErrorProcessor getBindingErrorProcessor(final BindingErrorProcessor defaultBEP) - { - return new BindingErrorProcessor() - { - @Override - public void processMissingFieldError(String missingField, BindingResult bindingResult) - { - defaultBEP.processMissingFieldError(missingField, bindingResult); - } - - @Override - public void processPropertyAccessException(PropertyAccessException ex, BindingResult bindingResult) - { - Object newValue = ex.getPropertyChangeEvent().getNewValue(); - if (newValue instanceof String) - newValue = StringUtils.trimToNull((String)newValue); - - // convert NULL conversion errors to required errors - if (null == newValue) - defaultBEP.processMissingFieldError(ex.getPropertyChangeEvent().getPropertyName(), bindingResult); - else - defaultBEP.processPropertyAccessException(ex, bindingResult); - } - }; - } - - - /* - * This binder doesn't have much to offer over the standard spring data binding except that it will - * handle DynaBeans. - */ - public static @NotNull BindException simpleBindParameters(Object command, String commandName, PropertyValues params) - { - Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); - - BindException errors = new NullSafeBindException(command, "Form"); - - // unfortunately ObjectFactory and BeanObjectFactory are not good about reporting errors - // do this by hand - for (PropertyValue pv : params.getPropertyValues()) - { - String propertyName = pv.getName(); - Object value = pv.getValue(); - if (!allow.test(propertyName)) - continue; - - try - { - Object converted = value; - Class propClass = PropertyUtils.getPropertyType(command, propertyName); - if (null == propClass) - continue; - if (value == null) - { - /* */ - } - else if (propClass.isPrimitive()) - { - converted = ConvertUtils.convert(String.valueOf(value), propClass); - } - else if (propClass.isArray()) - { - if (value instanceof Collection) - value = ((Collection) value).toArray(new String[0]); - else if (!value.getClass().isArray()) - value = new String[] {String.valueOf(value)}; - converted = ConvertUtils.convert((String[])value, propClass); - } - PropertyUtils.setProperty(command, propertyName, converted); - } - catch (ConversionException x) - { - errors.addError(new FieldError(commandName, propertyName, value, true, new String[] {"ConversionError", "typeMismatch"}, null, "Could not convert to value: " + value)); - } - catch (Exception x) - { - errors.addError(new ObjectError(commandName, new String[]{"Error"}, new Object[] {value}, x.getMessage())); - logger.error("unexpected error", x); - } - } - return errors; - } - - @Override - public boolean supports(Class clazz) - { - return getCommandClass().isAssignableFrom(clazz); - } - - - /* for TableViewForm, uses BeanUtils to work with DynaBeans */ - static public class BeanUtilsPropertyBindingResult extends BeanPropertyBindingResult - { - public BeanUtilsPropertyBindingResult(Object target, String objectName) - { - super(target, objectName); - } - - @Override - protected BeanWrapper createBeanWrapper() - { - return new BeanUtilsWrapperImpl((DynaBean)getTarget()); - } - } - - static public class BeanUtilsWrapperImpl extends AbstractPropertyAccessor implements BeanWrapper - { - private Object object; - private boolean autoGrowNestedPaths = false; - private int autoGrowCollectionLimit = 0; - - public BeanUtilsWrapperImpl() - { - // registerDefaultEditors(); - } - - public BeanUtilsWrapperImpl(DynaBean target) - { - this(); - object = target; - } - - @Override - public Object getPropertyValue(String propertyName) throws BeansException - { - try - { - return PropertyUtils.getProperty(object, propertyName); - } - catch (Exception e) - { - throw new NotReadablePropertyException(object.getClass(), propertyName); - } - } - - @Override - public void setPropertyValue(String propertyName, Object value) throws BeansException - { - try - { - PropertyUtils.setProperty(object, propertyName, value); - } - catch (Exception e) - { - throw new NotWritablePropertyException(object.getClass(), propertyName); - } - } - - @Override - public boolean isReadableProperty(String propertyName) - { - return true; - } - - @Override - public boolean isWritableProperty(String propertyName) - { - return true; - } - - @Override - public TypeDescriptor getPropertyTypeDescriptor(String s) throws BeansException - { - return null; - } - - public void setWrappedInstance(Object obj) - { - object = obj; - } - - @Override - public Object getWrappedInstance() - { - return object; - } - - @Override - public Class getWrappedClass() - { - return object.getClass(); - } - - @Override - public PropertyDescriptor[] getPropertyDescriptors() - { - throw new UnsupportedOperationException(); - } - - @Override - public PropertyDescriptor getPropertyDescriptor(String propertyName) throws BeansException - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAutoGrowNestedPaths(boolean b) - { - this.autoGrowNestedPaths = b; - } - - @Override - public boolean isAutoGrowNestedPaths() - { - return this.autoGrowNestedPaths; - } - - @Override - public void setAutoGrowCollectionLimit(int i) - { - this.autoGrowCollectionLimit = i; - } - - @Override - public int getAutoGrowCollectionLimit() - { - return this.autoGrowCollectionLimit; - } - - @Override - public T convertIfNecessary(Object value, Class requiredType) throws TypeMismatchException - { - if (value == null) - return null; - return (T)ConvertUtils.convert(String.valueOf(value), requiredType); - } - - @Override - public T convertIfNecessary(Object value, Class requiredType, MethodParameter methodParam) throws TypeMismatchException - { - return convertIfNecessary(value, requiredType); - } - } - - /** - * @return a map from form element name to uploaded files - */ - protected Map getFileMap() - { - if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) - return ((MultipartHttpServletRequest)getViewContext().getRequest()).getFileMap(); - return Collections.emptyMap(); - } - - protected List getAttachmentFileList() - { - return SpringAttachmentFile.createList(getFileMap()); - } - - public boolean isRobot() - { - return _robot; - } - - public boolean isPrint() - { - return _print; - } - - public boolean isDebug() - { - return _debug; - } - - public @NotNull Class getCommandClass() - { - if (null == _commandClass) - throw new IllegalStateException("NULL _commandClass in " + getClass().getName()); - return _commandClass; - } - - public void setCommandClass(@NotNull Class commandClass) - { - _commandClass = commandClass; - } - - protected final @NotNull Object createCommand() - { - return BeanUtils.instantiateClass(getCommandClass()); - } - - public void setCommandName(String commandName) - { - _commandName = commandName; - } - - public String getCommandName() - { - return _commandName; - } - - /** - * Cacheable resources can calculate a last modified timestamp to send to the browser. - */ - protected long getLastModified(FORM form) - { - return Long.MIN_VALUE; - } - - /** - * Cacheable resources can calculate an ETag header to send to the browser. - */ - protected String getETag(FORM form) - { - return null; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.action; + +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.beanutils.DynaBean; +import org.apache.commons.beanutils.PropertyUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.data.Container; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.security.User; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HttpUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.writer.ContainerUser; +import org.springframework.beans.AbstractPropertyAccessor; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.BeansException; +import org.springframework.beans.InvalidPropertyException; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.NotReadablePropertyException; +import org.springframework.beans.NotWritablePropertyException; +import org.springframework.beans.PropertyAccessException; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.TypeMismatchException; +import org.springframework.core.MethodParameter; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.lang.Nullable; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.BindException; +import org.springframework.validation.BindingErrorProcessor; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.ObjectError; +import org.springframework.validation.Validator; +import org.springframework.web.bind.ServletRequestDataBinder; +import org.springframework.web.bind.ServletRequestParameterPropertyValues; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.servlet.ModelAndView; + +import java.beans.PropertyDescriptor; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +public abstract class BaseViewAction extends PermissionCheckableAction implements Validator, HasPageConfig, ContainerUser +{ + protected static final Logger logger = LogHelper.getLogger(BaseViewAction.class, "BaseViewAction"); + + private PageConfig _pageConfig = null; + private PropertyValues _pvs; + private boolean _robot = false; // Is this request from GoogleBot or some other crawler? + private boolean _debug = false; + + protected boolean _print = false; + protected Class _commandClass; + protected String _commandName = "form"; + + protected BaseViewAction() + { + String methodName = getCommandClassMethodName(); + + if (null == methodName) + return; + + // inspect the action's *public* methods to determine form class + Class typeBest = null; + for (Method m : this.getClass().getMethods()) + { + if (methodName.equals(m.getName())) + { + Class[] types = m.getParameterTypes(); + if (types.length < 1) + continue; + Class typeCurrent = types[0]; + assert null == _commandClass || typeCurrent.equals(_commandClass); + + // Using templated classes to extend a base action can lead to multiple + // versions of a method with acceptable types, so take the most extended + // type we can find. + if (typeBest == null || typeBest.isAssignableFrom(typeCurrent)) + typeBest = typeCurrent; + } + } + if (typeBest != null) + setCommandClass(typeBest); + } + + + protected abstract String getCommandClassMethodName(); + + + protected BaseViewAction(@NotNull Class commandClass) + { + setCommandClass(commandClass); + } + + + public void setProperties(PropertyValues pvs) + { + _pvs = pvs; + } + + + public void setProperties(Map m) + { + _pvs = new MutablePropertyValues(m); + } + + + /* Doesn't guarantee non-null, non-empty */ + public Object getProperty(String key, String d) + { + PropertyValue pv = _pvs.getPropertyValue(key); + return pv == null ? d : pv.getValue(); + } + + + public Object getProperty(Enum key) + { + PropertyValue pv = _pvs.getPropertyValue(key.name()); + return pv == null ? null : pv.getValue(); + } + + + public Object getProperty(String key) + { + PropertyValue pv = _pvs.getPropertyValue(key); + return pv == null ? null : pv.getValue(); + } + + public PropertyValues getPropertyValues() + { + return _pvs; + } + + + public static PropertyValues getPropertyValuesForFormBinding(PropertyValues pvs, @NotNull Predicate allowBind) + { + if (null == pvs) + return null; + MutablePropertyValues ret = new MutablePropertyValues(); + for (PropertyValue pv : pvs.getPropertyValues()) + { + if (allowBind.test(pv.getName())) + ret.addPropertyValue(pv); + } + return ret; + } + + static final String FORM_DATE_ENCODED_PARAM = "formDataEncoded"; + + /** + * When a double quote is encountered in a multipart/form-data context, it is encoded as %22 using URL-encoding by browsers. + * This process replaces the double quote with its hexadecimal equivalent in a URL-safe format, preventing it from being misinterpreted as the end of a value or a boundary. + * The consequence of such encoding is we can't distinguish '"' from the actual '%22' in parameter name. + * As a workaround, a client-side util `encodeFormDataQuote` is used to convert %22 to %2522 and " to %22 explicitly, while passing in an additional param formDataEncoded=true. + * This class converts those encoded param names back to its decoded form during PropertyValues binding. + * See Issue 52827, 52925 and 52119 for more information. + */ + static public class ViewActionParameterPropertyValues extends ServletRequestParameterPropertyValues + { + + public ViewActionParameterPropertyValues(ServletRequest request) { + this(request, null, null); + } + + public ViewActionParameterPropertyValues(ServletRequest request, @Nullable String prefix, @Nullable String prefixSeparator) + { + super(request, prefix, prefixSeparator); + if (isFormDataEncoded()) + { + for (int i = 0; i < getPropertyValues().length; i++) + { + PropertyValue formDataPropValue = getPropertyValues()[i]; + String propValueName = formDataPropValue.getName(); + String decoded = PageFlowUtil.decodeQuoteEncodedFormDataKey(propValueName); + if (!propValueName.equals(decoded)) + setPropertyValueAt(new PropertyValue(decoded, formDataPropValue.getValue()), i); + } + } + } + + private boolean isFormDataEncoded() + { + PropertyValue formDataPropValue = getPropertyValue(FORM_DATE_ENCODED_PARAM); + if (formDataPropValue != null) + { + Object v = formDataPropValue.getValue(); + String formDataPropValueStr = v == null ? null : String.valueOf(v); + if (StringUtils.isNotBlank(formDataPropValueStr)) + return (Boolean) ConvertUtils.convert(formDataPropValueStr, Boolean.class); + } + + return false; + } + } + + @Override + public ModelAndView handleRequest(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) throws Exception + { + if (null == getPropertyValues()) + setProperties(new ViewActionParameterPropertyValues(request)); + getViewContext().setBindPropertyValues(getPropertyValues()); + handleSpecialProperties(); + + return handleRequest(); + } + + + private void handleSpecialProperties() + { + _robot = PageFlowUtil.isRobotUserAgent(getViewContext().getRequest().getHeader("User-Agent")); + + // Special flag puts actions in "debug" mode, during which they should log extra information that would be + // helpful for testing or debugging problems + if (!_robot && hasStringValue("_debug")) + { + _debug = true; + } + + // SpringActionController.defaultPageConfig() has logic for isPrint, don't need to duplicate here + _print = PageConfig.Template.Print == HttpView.currentPageConfig().getTemplate(); + } + + private boolean hasStringValue(String propertyName) + { + Object o = getProperty(propertyName); + if (o == null) + { + return false; + } + if (o instanceof String s) + { + return null != StringUtils.trimToNull(s); + } + if (o instanceof String[] strings) + { + for (String s : strings) + { + if (null != StringUtils.trimToNull(s)) + { + return true; + } + } + } + return false; + } + + public abstract ModelAndView handleRequest() throws Exception; + + + @Override + public void setPageConfig(PageConfig page) + { + _pageConfig = page; + } + + + @Override + public Container getContainer() + { + return getViewContext().getContainer(); + } + + + @Override + public User getUser() + { + return getViewContext().getUser(); + } + + + @Override + public PageConfig getPageConfig() + { + return _pageConfig; + } + + + public void setTitle(String title) + { + assert null != getPageConfig() : "action not initialized property"; + getPageConfig().setTitle(title); + } + + + public void setHelpTopic(String topicName) + { + setHelpTopic(new HelpTopic(topicName)); + } + + + public void setHelpTopic(HelpTopic topic) + { + assert null != getPageConfig() : "action not initialized properly"; + getPageConfig().setHelpTopic(topic); + } + + + protected Object newInstance(Class c) + { + try + { + return c == null ? null : c.getConstructor().newInstance(); + } + catch (Exception x) + { + if (x instanceof RuntimeException) + throw ((RuntimeException)x); + else + throw new RuntimeException(x); + } + } + + + protected @NotNull FORM getCommand(HttpServletRequest request) throws Exception + { + FORM command = (FORM) createCommand(); + + if (command instanceof HasViewContext) + ((HasViewContext)command).setViewContext(getViewContext()); + + return command; + } + + + protected @NotNull FORM getCommand() throws Exception + { + return getCommand(getViewContext().getRequest()); + } + + + // + // PARAMETER BINDING + // + // don't assume parameters always come from a request, use PropertyValues interface + // + + public @NotNull BindException defaultBindParameters(FORM form, PropertyValues params) + { + return defaultBindParameters(form, getCommandName(), params); + } + + + public static @NotNull BindException defaultBindParameters(Object form, String commandName, PropertyValues params) + { + /* check for do-it-myself forms */ + if (form instanceof HasBindParameters) + { + return ((HasBindParameters)form).bindParameters(params); + } + + if (form instanceof DynaBean) + { + return simpleBindParameters(form, commandName, params); + } + else + { + return springBindParameters(form, commandName, params); + } + } + + public static @NotNull BindException springBindParameters(Object command, String commandName, PropertyValues params) + { + Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); + ServletRequestDataBinder binder = new ServletRequestDataBinder(command, commandName); + + String[] fields = binder.getDisallowedFields(); + List fieldList = new ArrayList<>(fields != null ? Arrays.asList(fields) : Collections.emptyList()); + fieldList.addAll(Arrays.asList("class.*", "Class.*", "*.class.*", "*.Class.*")); + binder.setDisallowedFields(fieldList.toArray(new String[] {})); + + ConvertHelper.getPropertyEditorRegistrar().registerCustomEditors(binder); + BindingErrorProcessor defaultBEP = binder.getBindingErrorProcessor(); + binder.setBindingErrorProcessor(getBindingErrorProcessor(defaultBEP)); + binder.setFieldMarkerPrefix(SpringActionController.FIELD_MARKER); + try + { + // most paths probably called getPropertyValuesForFormBinding() already, but this is a public static method, so call it again + binder.bind(getPropertyValuesForFormBinding(params, allow)); + BindException errors = new NullSafeBindException(binder.getBindingResult()); + return errors; + } + catch (InvalidPropertyException x) + { + // Maybe we should propagate exception and return SC_BAD_REQUEST (in ExceptionUtil.handleException()) + // most POST handlers check errors.hasErrors(), but not all GET handlers do + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding property: " + x.getPropertyName()); + return errors; + } + catch (NumberFormatException x) + { + // Malformed array parameter throws this exception, unfortunately. Just reject the request. #21931 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; invalid array index (" + x.getMessage() + ")"); + return errors; + } + catch (NegativeArraySizeException x) + { + // Another malformed array parameter throws this exception. #23929 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; negative array size (" + x.getMessage() + ")"); + return errors; + } + catch (IllegalArgumentException x) + { + // General bean binding problem. #23929 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding property; (" + x.getMessage() + ")"); + return errors; + } + } + + + static BindingErrorProcessor getBindingErrorProcessor(final BindingErrorProcessor defaultBEP) + { + return new BindingErrorProcessor() + { + @Override + public void processMissingFieldError(String missingField, BindingResult bindingResult) + { + defaultBEP.processMissingFieldError(missingField, bindingResult); + } + + @Override + public void processPropertyAccessException(PropertyAccessException ex, BindingResult bindingResult) + { + Object newValue = ex.getPropertyChangeEvent().getNewValue(); + if (newValue instanceof String) + newValue = StringUtils.trimToNull((String)newValue); + + // convert NULL conversion errors to required errors + if (null == newValue) + defaultBEP.processMissingFieldError(ex.getPropertyChangeEvent().getPropertyName(), bindingResult); + else + defaultBEP.processPropertyAccessException(ex, bindingResult); + } + }; + } + + + /* + * This binder doesn't have much to offer over the standard spring data binding except that it will + * handle DynaBeans. + */ + public static @NotNull BindException simpleBindParameters(Object command, String commandName, PropertyValues params) + { + Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); + + BindException errors = new NullSafeBindException(command, "Form"); + + // unfortunately ObjectFactory and BeanObjectFactory are not good about reporting errors + // do this by hand + for (PropertyValue pv : params.getPropertyValues()) + { + String propertyName = pv.getName(); + Object value = pv.getValue(); + if (!allow.test(propertyName)) + continue; + + try + { + Object converted = value; + Class propClass = PropertyUtils.getPropertyType(command, propertyName); + if (null == propClass) + continue; + if (value == null) + { + /* */ + } + else if (propClass.isPrimitive()) + { + converted = ConvertUtils.convert(String.valueOf(value), propClass); + } + else if (propClass.isArray()) + { + if (value instanceof Collection) + value = ((Collection) value).toArray(new String[0]); + else if (!value.getClass().isArray()) + value = new String[] {String.valueOf(value)}; + converted = ConvertUtils.convert((String[])value, propClass); + } + PropertyUtils.setProperty(command, propertyName, converted); + } + catch (ConversionException x) + { + errors.addError(new FieldError(commandName, propertyName, value, true, new String[] {"ConversionError", "typeMismatch"}, null, "Could not convert to value: " + value)); + } + catch (Exception x) + { + errors.addError(new ObjectError(commandName, new String[]{"Error"}, new Object[] {value}, x.getMessage())); + logger.error("unexpected error", x); + } + } + return errors; + } + + @Override + public boolean supports(Class clazz) + { + return getCommandClass().isAssignableFrom(clazz); + } + + public Map getTransactionAuditDetails() + { + return getTransactionAuditDetails(getViewContext()); + } + + public static Map getTransactionAuditDetails(ViewContext viewContext) + { + Map map = new HashMap<>(); + map.put(TransactionAuditProvider.TransactionDetail.APIAction, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); + String productName = HttpUtil.getProductName(viewContext.getRequest()); + if (null != productName) + map.put(TransactionAuditProvider.TransactionDetail.Product, productName); + String clientLibrary = HttpUtil.getClientLibrary(viewContext.getRequest()); + if (null != clientLibrary) + map.put(TransactionAuditProvider.TransactionDetail.ClientLibrary, clientLibrary); + return map; + } + + /* for TableViewForm, uses BeanUtils to work with DynaBeans */ + static public class BeanUtilsPropertyBindingResult extends BeanPropertyBindingResult + { + public BeanUtilsPropertyBindingResult(Object target, String objectName) + { + super(target, objectName); + } + + @Override + protected BeanWrapper createBeanWrapper() + { + return new BeanUtilsWrapperImpl((DynaBean)getTarget()); + } + } + + static public class BeanUtilsWrapperImpl extends AbstractPropertyAccessor implements BeanWrapper + { + private Object object; + private boolean autoGrowNestedPaths = false; + private int autoGrowCollectionLimit = 0; + + public BeanUtilsWrapperImpl() + { + // registerDefaultEditors(); + } + + public BeanUtilsWrapperImpl(DynaBean target) + { + this(); + object = target; + } + + @Override + public Object getPropertyValue(String propertyName) throws BeansException + { + try + { + return PropertyUtils.getProperty(object, propertyName); + } + catch (Exception e) + { + throw new NotReadablePropertyException(object.getClass(), propertyName); + } + } + + @Override + public void setPropertyValue(String propertyName, Object value) throws BeansException + { + try + { + PropertyUtils.setProperty(object, propertyName, value); + } + catch (Exception e) + { + throw new NotWritablePropertyException(object.getClass(), propertyName); + } + } + + @Override + public boolean isReadableProperty(String propertyName) + { + return true; + } + + @Override + public boolean isWritableProperty(String propertyName) + { + return true; + } + + @Override + public TypeDescriptor getPropertyTypeDescriptor(String s) throws BeansException + { + return null; + } + + public void setWrappedInstance(Object obj) + { + object = obj; + } + + @Override + public Object getWrappedInstance() + { + return object; + } + + @Override + public Class getWrappedClass() + { + return object.getClass(); + } + + @Override + public PropertyDescriptor[] getPropertyDescriptors() + { + throw new UnsupportedOperationException(); + } + + @Override + public PropertyDescriptor getPropertyDescriptor(String propertyName) throws BeansException + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAutoGrowNestedPaths(boolean b) + { + this.autoGrowNestedPaths = b; + } + + @Override + public boolean isAutoGrowNestedPaths() + { + return this.autoGrowNestedPaths; + } + + @Override + public void setAutoGrowCollectionLimit(int i) + { + this.autoGrowCollectionLimit = i; + } + + @Override + public int getAutoGrowCollectionLimit() + { + return this.autoGrowCollectionLimit; + } + + @Override + public T convertIfNecessary(Object value, Class requiredType) throws TypeMismatchException + { + if (value == null) + return null; + return (T)ConvertUtils.convert(String.valueOf(value), requiredType); + } + + @Override + public T convertIfNecessary(Object value, Class requiredType, MethodParameter methodParam) throws TypeMismatchException + { + return convertIfNecessary(value, requiredType); + } + } + + /** + * @return a map from form element name to uploaded files + */ + protected Map getFileMap() + { + if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) + return ((MultipartHttpServletRequest)getViewContext().getRequest()).getFileMap(); + return Collections.emptyMap(); + } + + protected List getAttachmentFileList() + { + return SpringAttachmentFile.createList(getFileMap()); + } + + public boolean isRobot() + { + return _robot; + } + + public boolean isPrint() + { + return _print; + } + + public boolean isDebug() + { + return _debug; + } + + public @NotNull Class getCommandClass() + { + if (null == _commandClass) + throw new IllegalStateException("NULL _commandClass in " + getClass().getName()); + return _commandClass; + } + + public void setCommandClass(@NotNull Class commandClass) + { + _commandClass = commandClass; + } + + protected final @NotNull Object createCommand() + { + return BeanUtils.instantiateClass(getCommandClass()); + } + + public void setCommandName(String commandName) + { + _commandName = commandName; + } + + public String getCommandName() + { + return _commandName; + } + + /** + * Cacheable resources can calculate a last modified timestamp to send to the browser. + */ + protected long getLastModified(FORM form) + { + return Long.MIN_VALUE; + } + + /** + * Cacheable resources can calculate an ETag header to send to the browser. + */ + protected String getETag(FORM form) + { + return null; + } +} diff --git a/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java b/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java index 4d7c60f0458..3c489ba02a1 100644 --- a/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java +++ b/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.Logger; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.provider.FileSystemAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; @@ -59,7 +60,7 @@ public static void cleanupPostedFiles(Container container, Set files, } } - public CaseInsensitiveHashMap savePostedFiles(Container container, User user, Map filePropertyMap, Long auditTransactionId, String auditComment) throws ExperimentException + public CaseInsensitiveHashMap savePostedFiles(Container container, User user, Map filePropertyMap, TransactionAuditProvider.TransactionAuditEvent auditTransactionEvent, String auditComment) throws ExperimentException { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(container); CaseInsensitiveHashMap properties = new CaseInsensitiveHashMap<>(); @@ -87,7 +88,7 @@ public CaseInsensitiveHashMap savePostedFiles(Container container, Use event.setProvidedFileName(originalName); event.setFile(file.getName()); event.setDirectory(file.getParent()); - event.setTransactionId(auditTransactionId); + event.setTransactionEvent(auditTransactionEvent, FileSystemAuditProvider.EVENT_TYPE); event.setFieldName(entry.getKey()); AuditLogService.get().addEvent(user, event); diff --git a/api/src/org/labkey/api/assay/AssayRunCreator.java b/api/src/org/labkey/api/assay/AssayRunCreator.java index 86766eff41f..15bbbd5c12f 100644 --- a/api/src/org/labkey/api/assay/AssayRunCreator.java +++ b/api/src/org/labkey/api/assay/AssayRunCreator.java @@ -16,12 +16,15 @@ package org.labkey.api.assay; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.query.ValidationException; import org.labkey.api.util.Pair; +import java.util.Map; + /** * An AssayRunCreator does the actual work of constructing an assay run and saving it to the database. It gets * data about the run to be created from a AssayRunUploadContext and translates it into objects in the Experiment @@ -40,14 +43,17 @@ public interface AssayRunCreator * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. */ - Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) - throws ExperimentException, ValidationException; + default Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) + throws ExperimentException, ValidationException + { + return saveExperimentRun(context, batchId, false, null); + } - Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId, boolean forceAsync) + Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId, boolean forceAsync, Map transactionDetails) throws ExperimentException, ValidationException; /** * @return the batch to which the run has been assigned */ - ExpExperiment saveExperimentRun(AssayRunUploadContext context, @Nullable ExpExperiment batch, ExpRun run, boolean forceSaveBatchProps) + ExpExperiment saveExperimentRun(AssayRunUploadContext context, @Nullable ExpExperiment batch, ExpRun run, boolean forceSaveBatchProps, @Nullable Map transactionDetails) throws ExperimentException, ValidationException; } diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index f48283bf482..8f273f1aba9 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -1,1276 +1,1271 @@ -/* - * Copyright (c) 2011-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.assay; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.assay.actions.AssayRunUploadForm; -import org.labkey.api.assay.pipeline.AssayRunAsyncContext; -import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; -import org.labkey.api.assay.sample.AssaySampleLookupContext; -import org.labkey.api.assay.transform.DataTransformService; -import org.labkey.api.assay.transform.TransformDataHandler; -import org.labkey.api.assay.transform.TransformResult; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.RemapCache; -import org.labkey.api.data.validator.ColumnValidator; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.ExperimentDataHandler; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.XarContext; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataRunInput; -import org.labkey.api.exp.api.ExpExperiment; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpObject; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpRunItem; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ProvenanceService; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.study.assay.ParticipantVisitResolver; -import org.labkey.api.study.assay.ParticipantVisitResolverType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.writer.ContainerUser; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Collections.unmodifiableCollection; -import static org.labkey.api.assay.AssayFileWriter.TEMP_DIR_NAME; - -public class DefaultAssayRunCreator implements AssayRunCreator -{ - private static final Logger LOG = LogManager.getLogger(DefaultAssayRunCreator.class); - public static final String CROSS_RUN_DATA_INPUT_ROLE = "cross run input"; - - private final ProviderType _provider; - - public DefaultAssayRunCreator(ProviderType provider) - { - _provider = provider; - } - - public TransformResult transform(AssayRunUploadContext context, ExpRun run) throws ValidationException - { - return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); - } - - @Override - public Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) throws ExperimentException, ValidationException - { - return saveExperimentRun(context, batchId, false); - } - - /** - * Create and save an experiment run synchronously or asynchronously in a background job depending upon the assay design. - * - * @param context The context used to create and save the batch and run. - * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. - * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. - */ - @Override - public Pair saveExperimentRun( - AssayRunUploadContext context, - @Nullable Long batchId, - boolean forceAsync - ) throws ExperimentException, ValidationException - { - ExpExperiment exp = null; - if (batchId != null) - { - exp = ExperimentService.get().getExpExperiment(batchId); - } - - AssayProvider provider = context.getProvider(); - ExpProtocol protocol = context.getProtocol(); - ExpRun run = null; - - try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) - { - if (transaction.getAuditId() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); - } - context.init(); - // Check if assay protocol is configured to import in the background. - // Issue 26811: If we don't have a view, assume that we are on a background job thread already. - boolean importInBackground = forceAsync || (provider.isBackgroundUpload(protocol) && HttpView.hasCurrentView()); - if (!importInBackground) - { - if ((Object) context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE) instanceof File errFile) - { - throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); - } - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); - run.setComments(context.getComments()); - run.setWorkflowTaskId(context.getWorkflowTask()); - - exp = saveExperimentRun(context, exp, run, false); - - // re-fetch the run after it has been fully constructed - run = ExperimentService.get().getExpRun(run.getRowId()); - - context.uploadComplete(run); - } - else - { - context.uploadComplete(null); - context.setTransactionAuditId(transaction.getAuditId()); - exp = saveExperimentRunAsync(context, exp); - } - transaction.commit(); - } - - return Pair.of(exp, run); - } - - private ExpExperiment saveExperimentRunAsync(AssayRunUploadContext context, @Nullable ExpExperiment batch) throws ExperimentException - { - try - { - // Whether we need to save batch properties - boolean forceSaveBatchProps = false; - if (batch == null) - { - // No batch yet, so make one - batch = AssayService.get().createStandardBatch(context.getContainer(), null, context.getProtocol()); - batch.save(context.getUser()); - // It's brand new, so we need to eventually set its properties - forceSaveBatchProps = true; - } - - // Queue up a pipeline job to do the actual import in the background - ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); - - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - // Check if the primary file from the previous import is no longer present for a re-run - if (primaryFile == null && !context.getUploadedData().isEmpty()) - { - // Choose another file as the primary - primaryFile = context.getUploadedData().entrySet().iterator().next().getValue(); - } - primaryFile = Objects.requireNonNull(primaryFile); - AssayRunAsyncContext asyncContext = context.getProvider().createRunAsyncContext(context); - final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob( - asyncContext, - info, - batch, - forceSaveBatchProps, - PipelineService.get().getPipelineRootSetting(context.getContainer()), - primaryFile.toNioPathForRead().toFile() - ); - - context.setPipelineJobGUID(pipelineJob.getJobGUID()); - - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), null, pipelineJob.getJobGUID()); - resultsFileWriter.savePostedFiles(context); - - // Don't queue the job until the transaction is committed, since otherwise the thread - // that's running the job might start before it can access the job's row in the database. - ExperimentService.get().getSchema().getScope().addCommitTask(() -> { - try - { - PipelineService.get().queueJob(pipelineJob, asyncContext.getJobNotificationProvider()); - } - catch (PipelineValidationException e) - { - throw UnexpectedException.wrap(e); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - - return batch; - } - - /** - * @param batch if not null, the run group that's already created for this batch. If null, a new one needs to be created - * @param run The run to save - * @return the run and batch that were inserted - */ - @Override - public ExpExperiment saveExperimentRun( - final AssayRunUploadContext context, - @Nullable ExpExperiment batch, - @NotNull ExpRun run, - boolean forceSaveBatchProps - ) throws ExperimentException, ValidationException - { - context.setAutoFillDefaultResultColumns(run.getRowId() > 0); // need to setAutoFillDefaultResultColumns before run is saved - - final Container container = context.getContainer(); - - Map inputMaterials = new HashMap<>(); - Map inputDatas = new HashMap<>(); - Map outputMaterials = new HashMap<>(); - Map outputDatas = new HashMap<>(); - Map transformedDatas = new HashMap<>(); - - Map runProperties = context.getRunProperties(); - Map unresolvedRunProperties = context.getUnresolvedRunProperties(); - Map batchProperties = context.getBatchProperties(); - - Map allProperties = new HashMap<>(); - allProperties.putAll(runProperties); - allProperties.putAll(batchProperties); - - ParticipantVisitResolverType resolverType = null; - for (Map.Entry entry : allProperties.entrySet()) - { - if (entry.getKey().getName().equals(AbstractAssayProvider.PARTICIPANT_VISIT_RESOLVER_PROPERTY_NAME)) - { - resolverType = AbstractAssayProvider.findType(entry.getValue(), getProvider().getParticipantVisitResolverTypes()); - if (resolverType != null) - { - resolverType.configureRun(context, run, inputDatas); - } - break; - } - } - - // TODO: Share these RemapCache and materialCache instances with AbstractAssayTsvDataHandler.checkData and ExpressionMatrixDataHandler.importFile - // Cache of resolved alternate lookup keys -> rowId - final RemapCache cache = new RemapCache(true); - // Cache of rowId -> ExpMaterial - final Map materialCache = new LongHashMap<>(); - - addInputMaterials(context, inputMaterials, cache, materialCache); - addInputDatas(context, inputDatas); - addOutputMaterials(context, outputMaterials, cache, materialCache); - addOutputDatas(context, inputDatas, outputDatas); - - boolean success = false; - DbScope scope = ExperimentService.get().getSchema().getScope(); - try (DbScope.Transaction transaction = scope.ensureTransaction(ExperimentService.get().getProtocolImportLock())) - { - if (transaction.getAuditId() == null) - { - var auditAction = context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT; - if (context.getTransactionAuditId() != null) - { - var auditEvent = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, context.getTransactionAuditId()); - transaction.setAuditEvent(auditEvent); - } - else - { - var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); - } - } - boolean saveBatchProps = forceSaveBatchProps; - - // Add any material/data inputs related to the specimen IDs, etc in the incoming data. - // Some subclasses may actually create ExpMaterials or do other database changes, so do this inside the - // overall transaction - resolveParticipantVisits(context, inputMaterials, inputDatas, outputMaterials, outputDatas, allProperties, resolverType); - - // Check for circular inputs/outputs - checkForCycles(inputMaterials, outputMaterials); - checkForCycles(inputDatas, outputDatas); - - // Create the batch, if needed - if (batch == null) - { - // Make sure that we have a batch to associate with this run - batch = AssayService.get().createStandardBatch(run.getContainer(), null, context.getProtocol()); - batch.save(context.getUser()); - saveBatchProps = true; - } - run.save(context.getUser()); - // Add the run to the batch so that we can find it when we're loading the data files - batch.addRuns(context.getUser(), run); - assert batch.equals(run.getBatch()) : "Run's batch should be the current batch"; - - ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); - XarContext xarContext = new AssayUploadXarContext("Simple Run Creation", context); - - run = ExperimentService.get().saveSimpleExperimentRun( - run, - inputMaterials, - inputDatas, - outputMaterials, - outputDatas, - transformedDatas, - info, - context.getLogger() != null ? context.getLogger() : LOG, - false - ); - - // handle data transformation - TransformResult transformResult = transform(context, run); - - if (transformResult.getWarnings() != null && context instanceof AssayRunUploadForm uploadForm) - { - context.setTransformResult(transformResult); - uploadForm.setName(run.getName()); - uploadForm.setComments(run.getComments()); - throw new ValidationException(" "); - } - - if (saveBatchProps) - saveProperties(context, batch, transformResult.getBatchProperties(), batchProperties); - if (null != transformResult.getAssayId()) - run.setName(transformResult.getAssayId()); - if (null != transformResult.getComments()) - run.setComments(transformResult.getComments()); - saveProperties(context, run, transformResult.getRunProperties(), runProperties); - - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); - resultsFileWriter.savePostedFiles(context); - - Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run); - if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir)) - { - FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir); - if (assayResultFileRoot != null) - QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot); - } - - importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult); - - var reRunId = context.getReRunId(); - if (reRunId != null && getProvider().getReRunSupport() == AssayProvider.ReRunSupport.ReRunAndReplace) - { - final ExpRun replacedRun = ExperimentService.get().getExpRun(reRunId); - if (replacedRun == null) - throw new ExperimentException(String.format("Unable to find run to be replaced (RowId %d)", reRunId)); - - if (replacedRun.getContainer().hasPermission(context.getUser(), UpdatePermission.class)) - { - replacedRun.setReplacedByRun(run); - replacedRun.save(context.getUser()); - } - - String auditMessage = String.format("Run id %d was replaced by run id %d", replacedRun.getRowId(), run.getRowId()); - ExperimentService.get().auditRunEvent(context.getUser(), context.getProtocol(), replacedRun, null, auditMessage, context.getAuditUserComment()); - - transaction.addCommitTask(() -> replacedRun.archiveDataFiles(context.getUser()), DbScope.CommitTaskOption.POSTCOMMIT); - // Issue 51710: Remove replaced assay runs from the search index - transaction.addCommitTask(() -> AssayService.get().deindexAssayRuns(List.of(replacedRun)), DbScope.CommitTaskOption.POSTCOMMIT); - } - - AssayService.get().ensureUniqueBatchName(batch, context.getProtocol(), context.getUser()); - - ExperimentService.get().onRunDataCreated(context.getProtocol(), run, container, context.getUser()); - - transaction.commit(); - success = true; - - // Inspect the run properties for a “prov:objectInputs” property that is a list of LSID strings. - // Attach run's starting protocol application with starting input LSIDs. - Object provInputsProperty = unresolvedRunProperties.get(ProvenanceService.PROVENANCE_INPUT_PROPERTY); - if (provInputsProperty != null) - { - ProvenanceService pvs = ProvenanceService.get(); - Set runInputLSIDs = null; - if (provInputsProperty instanceof String provInputs) - { - // parse as a JSONArray of values or a comma-separated list of values - if (provInputs.startsWith("[") && provInputs.endsWith("]")) - provInputsProperty = new JSONArray(provInputs); - else - runInputLSIDs = Set.of(provInputs.split(",")); - } - - if (provInputsProperty instanceof JSONArray jsonArray) - { - runInputLSIDs = jsonArray.toList().stream() - .map(String::valueOf) - .collect(Collectors.toSet()); - } - - if (runInputLSIDs != null && !runInputLSIDs.isEmpty()) - { - ExpProtocolApplication inputProtocolApp = run.getInputProtocolApplication(); - pvs.addProvenanceInputs(container, inputProtocolApp, runInputLSIDs); - } - } - - ExperimentService.get().queueSyncRunEdges(run); - - return batch; - } - catch (IOException | ConvertHelper.FileConversionException | BatchValidationException e) - { - // HACK: Rethrowing these as ApiUsageException avoids any upstream consequences of wrapping them in ExperimentException. - // Namely, that they are logged to the server/mothership. There has to be a better way. - if (e instanceof ConvertHelper.FileConversionException fce) - throw new ApiUsageException(fce.getMessage(), fce); - else if (e instanceof BatchValidationException bve) - throw new ApiUsageException(bve.getMessage(), bve); - - throw new ExperimentException(e); - } - finally - { - if (!success) - { - // clean up the run results file dir here if it was created, for non-async imports - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); - resultsFileWriter.cleanupPostedFiles(context.getContainer(), false); - - cleanPrimaryFile(context); - } - } - } - - private void cleanPrimaryFile(AssayRunUploadContext context) throws ExperimentException - { - // Do not clear the primary file for run re-imports - if (context.getReRunId() != null) - return; - - try - { - // Issue 51300: don't keep the primary file if the new run failed to save - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - - // If the uploaded file is in the temp directory, then do not delete it as it may be reused in the next import attempt. - if (primaryFile != null && primaryFile.exists() && !primaryFile.getPath().contains(TEMP_DIR_NAME)) - primaryFile.delete(); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - } - - private void resolveParticipantVisits( - AssayRunUploadContext context, - Map inputMaterials, - Map inputDatas, - Map outputMaterials, - Map outputDatas, - Map allProperties, - @Nullable ParticipantVisitResolverType resolverType - ) throws ExperimentException - { - try - { - ParticipantVisitResolver resolver = null; - if (resolverType != null) - { - String targetStudyId = null; - for (Map.Entry property : allProperties.entrySet()) - { - if (AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME.equals(property.getKey().getName())) - { - targetStudyId = property.getValue(); - break; - } - } - Container targetStudy = null; - if (targetStudyId != null && !targetStudyId.isEmpty()) - targetStudy = ContainerManager.getForId(targetStudyId); - - resolver = resolverType.createResolver( - unmodifiableCollection(inputMaterials.keySet()), - unmodifiableCollection(inputDatas.keySet()), - unmodifiableCollection(outputMaterials.keySet()), - unmodifiableCollection(outputDatas.keySet()), - context.getContainer(), - targetStudy, context.getUser()); - } - - resolveExtraRunData(resolver, context, inputMaterials, inputDatas, outputMaterials, outputDatas); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - } - - protected void importStandardResultData( - AssayRunUploadContext context, - ExpRun run, - Map inputDatas, - Map outputDatas, - ViewBackgroundInfo info, - XarContext xarContext - ) throws ExperimentException, BatchValidationException - { - DataIteratorBuilder rawData = context.getRawData(); - List insertedDatas = new ArrayList<>(); - - if (rawData != null) - { - insertedDatas.addAll(outputDatas.keySet()); - - ExpData primaryData = null; - // Decide which file to treat as the primary, to which the data rows will be attached - for (Map.Entry entry : outputDatas.entrySet()) - { - if (ExpDataRunInput.DEFAULT_ROLE.equalsIgnoreCase(entry.getValue())) - { - primaryData = entry.getKey(); - } - } - if (primaryData == null && !insertedDatas.isEmpty()) - primaryData = insertedDatas.get(0); - - if (primaryData != null) - { - TsvDataHandler dataHandler = new TsvDataHandler(); - dataHandler.setAllowEmptyData(true); - dataHandler.importRows(primaryData, context.getUser(), run, context.getProtocol(), getProvider(), rawData, null, context.shouldAutoFillDefaultResultColumns(), context); - } - } - else - { - for (Map.Entry entry : inputDatas.entrySet()) - { - // skip any of the cross run inputData that are already in the outputData - if (CROSS_RUN_DATA_INPUT_ROLE.equals(entry.getValue())) - continue; - - insertedDatas.add(entry.getKey()); - } - - insertedDatas.addAll(outputDatas.keySet()); - - Logger logger = context.getLogger() != null ? context.getLogger() : LOG; - for (ExpData insertedData : insertedDatas) - { - ExperimentDataHandler dataHandler = insertedData.findDataHandler(); - - FileLike fileLike = FileSystemLike.wrapFile(insertedData.getFile()); - if (dataHandler instanceof AbstractAssayTsvDataHandler tsvHandler) - { - tsvHandler.importFile(insertedData, fileLike, info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); - } - else - { - dataHandler.importFile(insertedData, fileLike, info, logger, xarContext); - } - } - } - } - - private void importResultData( - AssayRunUploadContext context, - ExpRun run, - Map inputDatas, - Map outputDatas, - ViewBackgroundInfo info, - XarContext xarContext, - TransformResult transformResult - ) throws ExperimentException, BatchValidationException - { - if (transformResult.getTransformedData().isEmpty()) - { - importStandardResultData(context, run, inputDatas, outputDatas, info, xarContext); - return; - } - - DataType dataType = context.getProvider().getDataType(); - if (dataType == null) - { - // we know that we are importing transformed data at this point - dataType = TsvDataHandler.RELATED_TRANSFORM_FILE_DATA_TYPE; - } - - ExpData data = ExperimentService.get().createData(context.getContainer(), dataType); - ExperimentDataHandler handler = data.findDataHandler(); - - // this should assert to always be true - if (handler instanceof TransformDataHandler transformDataHandler) - { - for (Map.Entry entry : transformResult.getTransformedData().entrySet()) - { - ExpData expData = entry.getKey(); - // The object may have already been claimed by - if (expData.getSourceApplication() == null) - { - expData.setSourceApplication(run.getOutputProtocolApplication()); - } - expData.save(context.getUser()); - - run.getOutputProtocolApplication().addDataInput(context.getUser(), expData, ExpDataRunInput.IMPORTED_DATA_ROLE); - // Add to the cached list of outputs - run.getDataOutputs().add(expData); - - transformDataHandler.importTransformDataMap(expData, context, run, entry.getValue()); - } - } - } - - protected void addInputMaterials( - AssayRunUploadContext context, - Map inputMaterials, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - addMaterials(context, inputMaterials, context.getInputMaterials(), null, cache, materialCache); - - // Find lookups to a SampleType and add the resolved material as an input sample - for (Map.Entry entry : context.getRunProperties().entrySet()) - { - String value = StringUtils.trimToNull(entry.getValue()); - if (value == null) - continue; - - // Lookup must point at "Samples.*", "exp.materials.*", or "exp.Materials" - DomainProperty dp = entry.getKey(); - var sampleLookup = AssaySampleLookupContext.checkSampleLookup(context.getContainer(), context.getUser(), dp); - if (!sampleLookup.isLookup()) - continue; - - String role = AssayService.get().getPropertyInputLineageRole(dp); - addMaterials(context, inputMaterials, Map.of(value, role), sampleLookup.expSampleType(), cache, materialCache); - } - } - - protected void addInputDatas( - AssayRunUploadContext context, - @NotNull Map inputDatas - ) throws ExperimentException, ValidationException - { - Logger log = context.getLogger() != null ? context.getLogger() : LOG; - - Map inputs = context.getInputDatas(); - addDatas(context.getContainer(), inputDatas, inputs, log); - - // Inspect the uploaded files which will be added as outputs of the run - if (context.isAllowCrossRunFileInputs()) - { - Map files = context.getUploadedData(); - for (Map.Entry entry : files.entrySet()) - { - String key = entry.getKey(); - if (AssayDataCollector.PRIMARY_FILE.equals(key)) - { - FileLike file = entry.getValue(); - - // Check if the file is created by a run - // Don't use getExpDataByURL(String). That method expects string in a very particular format. - ExpData existingData = ExperimentService.get().getExpDataByURL(file.toNioPathForRead(), context.getContainer()); - if (existingData != null && existingData.getRunId() != null && !inputDatas.containsKey(existingData)) - { - // Add this file as an input to the run. When we add the outputs to the run, we will detect - // that this file was already added as an input and create a new exp.data for the same file - // path and attach it as an output. - log.debug("found existing cross run file input: name={}, rowId={}, dataFileUrl={}", existingData.getName(), existingData.getRowId(), existingData.getDataFileUrl()); - inputDatas.put(existingData, CROSS_RUN_DATA_INPUT_ROLE); - } - } - } - } - } - - // CONSIDER: Move this to ExperimentService - // Resolve submitted values into ExpData objects - protected void addDatas(Container c, @NotNull Map resolved, @NotNull Map unresolved, @Nullable Logger log) throws ValidationException - { - for (Map.Entry entry : unresolved.entrySet()) - { - Object o = entry.getKey(); - String role = entry.getValue(); - - if (o instanceof ExpData expData) - { - resolved.put(expData, role); - } - else - { - File file = ExpDataFileConverter.convert(o); - if (file != null) - { - ExpData data = ExperimentService.get().getExpDataByURL(file, c); - if (data == null) - { - DataType dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - data = createData(c, file, file.getName(), dataType, false, true, log); - } - - resolved.put(data, role); - } - } - } - } - - public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData) throws ValidationException - { - return generateResultData(user, container, provider, dataArray, outputData, null); - } - - public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData, @Nullable Logger log) throws ValidationException - { - if (log == null) - log = LOG; - - ExpData newData = null; - - // Don't create an empty result data file if there are other outputs from this run, or if the user didn't - // include any data rows - if (!dataArray.isEmpty() && outputData.isEmpty()) - { - DataType dataType = provider.getDataType(); - if (dataType == null) - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - - newData = createData(container, "Analysis Results", dataType, log); - newData.save(user); - outputData.put(newData, ExpDataRunInput.DEFAULT_ROLE); - } - - return newData; - } - - // Find an existing ExpData for the File or null. - public static @Nullable ExpData findExistingData(Container c, @Nullable File file, @Nullable Logger log) - { - if (file == null) - return null; - - if (log == null) - log = LOG; - - List existing = ExperimentService.get().getAllExpDataByURL(file, c); - if (!existing.isEmpty()) - { - for (ExpData d : existing) - { - log.debug("found existing exp.data for file, rowId={}, runId={}, dataFileUrl={}", d.getRowId(), d.getRunId(), d.getDataFileUrl()); - } - - // pick the most recently created one - return existing.get(0); - } - - return null; - } - - public static @NotNull ExpData createData(Container c, String name, @NotNull DataType dataType, @Nullable Logger log) throws ValidationException - { - // NOTE: reuseExistingData and errorOnDataOwned flags are irrelevant when we aren't providing a File - return createData(c, null, name, dataType, false, false, log); - } - - public static @NotNull ExpData createData( - Container c, - File file, - String name, - @Nullable DataType dataType, - boolean reuseExistingData, - boolean errorIfDataOwned, - @Nullable Logger log - ) throws ValidationException - { - if (log == null) - log = LOG; - - ExpData data = findExistingData(c, file, log); - - ExpRun previousRun; - if (data != null && null != (previousRun = data.getRun())) - { - // There's an existing data, but it's already marked as being created by another run - String msg = "File '" + data.getName() + "' has been previously imported in run '" + previousRun.getName() + "' (" + previousRun.getRowId() + ")"; - if (reuseExistingData && errorIfDataOwned) - throw new ValidationException(msg); - - log.debug(msg); - - // Create a new one for the same path so the new run can claim it as its own - if (!reuseExistingData) - { - log.debug("ignoring existing exp.data, will create a new one"); - data = null; - } - } - - if (data == null) - { - if (dataType == null) - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - - log.debug("creating assay exp.data for file. dataType={}, file={}", dataType.getNamespacePrefix(), file); - data = ExperimentService.get().createData(c, dataType, name); - data.setLSID(ExperimentService.get().generateGuidLSID(c, dataType)); - if (file != null) - { - data.setDataFileURI(FileUtil.getAbsoluteCaseSensitiveFile(file).toURI()); - } - } - else - { - if (dataType != null && !dataType.matches(new Lsid(data.getLSID()))) - { - // Reset its LSID so that it's the correct type // CONSIDER: creating a new ExpData with the correct type instead - String newLsid = ExperimentService.get().generateGuidLSID(c, dataType); - log.debug("LSID doesn't match desired type. Changed the LSID from '{}' to '{}'", data.getLSID(), newLsid); - data.setLSID(newLsid); - } - } - return data; - } - - protected void addOutputMaterials( - AssayRunUploadContext context, - Map outputMaterials, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - addMaterials(context, outputMaterials, context.getOutputMaterials(), null, cache, materialCache); - } - - protected void addMaterials( - AssayRunUploadContext context, - @NotNull Map resolved, - @NotNull Map unresolved, - @Nullable ExpSampleType sampleType, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - for (Map.Entry entry : unresolved.entrySet()) - { - Object sampleIdentifier = entry.getKey(); - ExpMaterial material = ExperimentService.get().findExpMaterial(context.getContainer(), context.getUser(), sampleIdentifier, sampleType, cache, materialCache); - if (material == null) - throw new ExperimentException("Unable to resolve sample: " + sampleIdentifier); - - if (!resolved.containsKey(material)) - { - if (!material.isOperationPermitted(SampleTypeService.SampleOperations.AddAssayData)) - throw new ExperimentException(SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddAssayData)); - if (sampleType == null || sampleType.getLSID().equals(material.getCpasType())) - resolved.put(material, entry.getValue()); - } - } - } - - protected void addOutputDatas( - AssayRunUploadContext context, - Map inputDatas, - Map outputDatas - ) throws ExperimentException, ValidationException - { - Logger log = context.getLogger() != null ? context.getLogger() : LOG; - - // Create set of existing input files - Set inputFiles = new HashSet<>(); - for (ExpData inputData : inputDatas.keySet()) - { - FileLike f = inputData.getFileLike(); - if (f != null) - inputFiles.add(f); - } - - Map files = context.getUploadedData(); - - AssayDataType dataType; - for (Map.Entry entry : files.entrySet()) - { - FileLike file = entry.getValue(); - dataType = context.getProvider().getDataType(); - - // Reuse existing exp.data as the assay output file unless: - // - we are re-importing the run - // - or the output file is already one of the input files and if we are allowing cross-run file inputs - boolean reuseExistingData = true; - if (context.getReRunId() != null) - reuseExistingData = false; - if (context.isAllowCrossRunFileInputs() && inputFiles.contains(file)) - reuseExistingData = false; - - // For Luminex re-import, we want to reuse the existing exp.data but not - // throw an error when we discover that the exp.data is already owned. The - // original run will be duplicated for re-import and then will be deleted. - boolean errorIfDataOwned = getProvider().getReRunSupport() != AssayProvider.ReRunSupport.ReRunAndDelete; - - log.debug("adding output data: file={}", file.toNioPathForRead()); - log.debug(" context.getReRunId()={}", context.getReRunId()); - log.debug(" provider.getReRunSupport()={}", getProvider().getReRunSupport()); - log.debug(" context.allowCrossRunFileInputs={}", context.isAllowCrossRunFileInputs()); - log.debug(" inputFiles.contains(file)={}", inputFiles.contains(file)); - log.debug("==> reuseExistingData = {}", reuseExistingData); - log.debug("==> errorIfDataOwned = {}", errorIfDataOwned); - - ExpData data = DefaultAssayRunCreator.createData(context.getContainer(), file.toNioPathForRead().toFile(), file.getName(), dataType, reuseExistingData, errorIfDataOwned, log); - String role = ExpDataRunInput.DEFAULT_ROLE; - if (dataType != null && dataType.getFileType().isType(file)) - { - if (dataType.getRole() != null) - { - role = dataType.getRole(); - } - } - outputDatas.put(data, role); - } - - FileLike primaryFile = files.get(AssayDataCollector.PRIMARY_FILE); - if (primaryFile != null) - { - addRelatedOutputDatas(context, inputFiles, outputDatas, primaryFile); - } - - Map outputs = context.getOutputDatas(); - addDatas(context.getContainer(), outputDatas, outputs, log); - } - - /** - * Add files that follow the general naming convention (same basename) as the primary file - */ - public void addRelatedOutputDatas( - AssayRunUploadContext context, - Set inputFiles, - Map outputDatas, - final FileLike primaryFile - ) throws ValidationException - { - AssayDataType dataType = getProvider().getDataType(); - final String baseName = dataType == null ? null : dataType.getFileType().getBaseName(primaryFile.toNioPathForRead()); - if (baseName != null) - { - // Grab all the files that are related based on naming convention - File primary = primaryFile.toNioPathForRead().toFile(); - File parent = primary.getParentFile(); - // converting to File land to reuse the FileFilter - File[] relatedFiles = parent.listFiles(getRelatedOutputDataFileFilter(primary, baseName)); - if (relatedFiles != null) - { - for (File f : relatedFiles) - { - FileLike relatedFile = primaryFile.getParent().resolveChild(f.getName()); - // Ignore files already considered inputs to the run - if (inputFiles.contains(relatedFile)) - continue; - - Pair dataOutput = createdRelatedOutputData(context, baseName, f); - if (dataOutput != null) - { - outputDatas.put(dataOutput.getKey(), dataOutput.getValue()); - } - } - } - } - } - - protected void resolveExtraRunData( - ParticipantVisitResolver resolver, - AssayRunUploadContext context, - Map inputMaterials, - Map inputDatas, - Map outputMaterials, - Map outputDatas - ) throws ExperimentException - { - } - - /** - * Create an ExpData object for the file, and figure out what its role name should be - * @return null if the file is already linked to another run - */ - @Nullable - public static Pair createdRelatedOutputData(AssayRunUploadContext context, String baseName, File relatedFile) throws ValidationException - { - String roleName = null; - DataType dataType = null; - for (AssayDataType inputType : context.getProvider().getRelatedDataTypes()) - { - // Check if we recognize it as a specially handled file type - if (inputType.getFileType().isMatch(relatedFile.getName(), baseName)) - { - roleName = inputType.getRole(); - dataType = inputType; - break; - } - } - // If not, make up a new type and role for it - if (roleName == null) - { - roleName = relatedFile.getName().substring(baseName.length()); - while (!roleName.isEmpty() && (roleName.startsWith(".") || roleName.startsWith("-") || roleName.startsWith("_") || roleName.startsWith(" "))) - { - roleName = roleName.substring(1); - } - if (roleName.isEmpty()) - { - roleName = null; - } - } - if (dataType == null) - { - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - } - - // Find an existing data that isn't owned by another run or create a new own - ExpData data = findExistingData(context.getContainer(), relatedFile, context.getLogger()); - if (data != null) - { - if (data.getSourceApplication() == null) - return new Pair<>(data, roleName); - - // The file is already linked to another run, so this one must have not created it - return null; - } - - data = createData(context.getContainer(), relatedFile, relatedFile.getName(), dataType, true, true, context.getLogger()); - assert data.getSourceApplication() == null; - return Pair.of(data, roleName); - } - - // Disallow creating a run with inputs which are also outputs - protected void checkForCycles( - Map inputs, - Map outputs - ) throws ExperimentException - { - for (ExpRunItem input : inputs.keySet()) - { - if (outputs.containsKey(input)) - { - String role = outputs.get(input); - throw new ExperimentException("Circular input/output '" + input.getName() + "' with role '" + role + "'"); - } - } - } - - private void saveProperties( - final AssayRunUploadContext context, - ExpObject expObject, - Map transformResultProperties, - Map properties - ) throws ValidationException - { - Map propsToSave = transformResultProperties.isEmpty() ? properties : transformResultProperties; - List errors = validateProperties(context, propsToSave); - if (!errors.isEmpty()) - throw new ValidationException(errors); - - savePropertyObject(expObject, propsToSave, context.getUser()); - } - - protected void savePropertyObject(ExpObject object, Map properties, User user) throws ValidationException - { - for (Map.Entry entry : properties.entrySet()) - { - DomainProperty pd = entry.getKey(); - String value = entry.getValue(); - - // resolve any file links for batch or run properties - if (PropertyType.FILE_LINK.getTypeUri().equals(pd.getType().getTypeURI())) - { - File resolvedFile = ExpDataFileConverter.convert(value); - if (resolvedFile != null) - value = resolvedFile.getAbsolutePath(); - } - - // Treat the empty string as a null in the database, which is our normal behavior when receiving data - // from HTML forms. - if (StringUtils.trimToNull(value) == null) - { - value = null; - } - if (value != null) - { - object.setProperty(user, pd.getPropertyDescriptor(), value); - } - else - { - // We still need to validate blanks - List errors = new ArrayList<>(); - OntologyManager.validateProperty(pd.getValidators(), pd.getPropertyDescriptor(), new ObjectProperty(object.getLSID(), object.getContainer(), pd.getPropertyDescriptor(), value), errors, new ValidatorContext(pd.getContainer(), user)); - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - } - } - - public static List validateColumnProperties(ContainerUser context, Map properties) - { - List errors = new ArrayList<>(); - RemapCache cache = new RemapCache(); - for (Map.Entry entry : properties.entrySet()) - { - validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); - } - return errors; - } - - public static List validateProperties(ContainerUser context, Map properties) - { - List errors = new ArrayList<>(); - RemapCache cache = new RemapCache(); - for (Map.Entry entry : properties.entrySet()) - { - validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); - } - return errors; - } - - private static void validateProperty(ContainerUser context, ColumnInfo columnInfo, String value, RemapCache cache, List errors) - { - Lookup lookup = null; - if (columnInfo.isLookup()) - { - ForeignKey fk = columnInfo.getFk(); - lookup = new Lookup(fk.getLookupContainer(), fk.getLookupSchemaName(), fk.getLookupTableName()); - } - validateProperty(context, ColumnValidators.create(columnInfo, null), value, columnInfo.getName(), - false, lookup, columnInfo.getJavaClass(), cache, errors); - } - - private static void validateProperty(ContainerUser context, DomainProperty dp, String value, RemapCache cache, List errors) - { - String label = dp.getPropertyDescriptor().getNonBlankCaption(); - PropertyType type = dp.getPropertyDescriptor().getPropertyType(); - validateProperty(context, ColumnValidators.create(null, dp), value, label, dp.isRequired(), - dp.getLookup(), type.getJavaType(), cache, errors); - } - - private static void validateProperty( - ContainerUser context, - List validators, - String value, - String label, - Boolean required, - Lookup lookup, - Class type, - RemapCache cache, - List errors - ) - { - boolean missing = (value == null || value.isEmpty()); - int rowNum = 0; - - if (required && missing) - { - errors.add(new SimpleValidationError(label + " is required and must be of type " + ColumnInfo.getFriendlyTypeName(type) + ".")); - } - else if (!missing) - { - try - { - Object o; - if (type == File.class) - o = ExpDataFileConverter.convert(value); - else - o = ConvertUtils.convert(value, type); - ValidatorContext validatorContext = new ValidatorContext(context.getContainer(), context.getUser()); - for (ColumnValidator validator : validators) - { - String msg = validator.validate(rowNum, o, validatorContext); - if (msg != null) - errors.add(new PropertyValidationError(msg, label)); - } - } - catch (ConversionException e) - { - String message; - if (e instanceof ConvertHelper.FileConversionException fce) - message = fce.getMessage(); - else - { - message = ConvertHelper.getStandardConversionErrorMessage(value, label, type); - if (e.getCause() instanceof ArithmeticException) - message += ": " + e.getCause().getLocalizedMessage(); - else - message += "."; - } - - // Attempt to resolve lookups by display value - boolean skipError = false; - if (lookup != null) - { - Object remappedValue = OntologyManager.getRemappedValueForLookup(context.getUser(), context.getContainer(), cache, lookup, value); - if (remappedValue != null) - skipError = true; - } - - if (!skipError) - errors.add(new SimpleValidationError(message)); - } - } - } - - protected FileFilter getRelatedOutputDataFileFilter(final File primaryFile, final String baseName) - { - // baseName doesn't include the trailing '.', so add it here. We want to associate myRun.jpg - // with myRun.xls, but we don't want to associate myRun2.xls with myRun.xls (which will happen without - // the trailing dot in the check). - return f -> f.getName().startsWith(baseName + ".") && !primaryFile.equals(f); - } - - protected ProviderType getProvider() - { - return _provider; - } -} +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.assay; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.assay.actions.AssayRunUploadForm; +import org.labkey.api.assay.pipeline.AssayRunAsyncContext; +import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; +import org.labkey.api.assay.sample.AssaySampleLookupContext; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.assay.transform.TransformDataHandler; +import org.labkey.api.assay.transform.TransformResult; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.RemapCache; +import org.labkey.api.data.validator.ColumnValidator; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.ExperimentDataHandler; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.XarContext; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataRunInput; +import org.labkey.api.exp.api.ExpExperiment; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpObject; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpRunItem; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.study.assay.ParticipantVisitResolver; +import org.labkey.api.study.assay.ParticipantVisitResolverType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.writer.ContainerUser; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.unmodifiableCollection; +import static org.labkey.api.assay.AssayFileWriter.TEMP_DIR_NAME; + +public class DefaultAssayRunCreator implements AssayRunCreator +{ + private static final Logger LOG = LogManager.getLogger(DefaultAssayRunCreator.class); + public static final String CROSS_RUN_DATA_INPUT_ROLE = "cross run input"; + + private final ProviderType _provider; + + public DefaultAssayRunCreator(ProviderType provider) + { + _provider = provider; + } + + public TransformResult transform(AssayRunUploadContext context, ExpRun run) throws ValidationException + { + return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); + } + /** + * Create and save an experiment run synchronously or asynchronously in a background job depending upon the assay design. + * + * @param context The context used to create and save the batch and run. + * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. + * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. + */ + @Override + public Pair saveExperimentRun( + AssayRunUploadContext context, + @Nullable Long batchId, + boolean forceAsync, + Map transactionDetails + ) throws ExperimentException, ValidationException + { + ExpExperiment exp = null; + if (batchId != null) + { + exp = ExperimentService.get().getExpExperiment(batchId); + } + + AssayProvider provider = context.getProvider(); + ExpProtocol protocol = context.getProtocol(); + ExpRun run = null; + + try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) + { + if (transaction.getAuditId() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); + } + context.init(); + // Check if assay protocol is configured to import in the background. + // Issue 26811: If we don't have a view, assume that we are on a background job thread already. + boolean importInBackground = forceAsync || (provider.isBackgroundUpload(protocol) && HttpView.hasCurrentView()); + if (!importInBackground) + { + if ((Object) context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE) instanceof File errFile) + { + throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); + } + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); + run.setComments(context.getComments()); + run.setWorkflowTaskId(context.getWorkflowTask()); + + exp = saveExperimentRun(context, exp, run, false, transactionDetails); + + // re-fetch the run after it has been fully constructed + run = ExperimentService.get().getExpRun(run.getRowId()); + + context.uploadComplete(run); + } + else + { + context.uploadComplete(null); + context.setTransactionAuditId(transaction.getAuditId()); + exp = saveExperimentRunAsync(context, exp); + } + transaction.commit(); + } + + return Pair.of(exp, run); + } + + private ExpExperiment saveExperimentRunAsync(AssayRunUploadContext context, @Nullable ExpExperiment batch) throws ExperimentException + { + try + { + // Whether we need to save batch properties + boolean forceSaveBatchProps = false; + if (batch == null) + { + // No batch yet, so make one + batch = AssayService.get().createStandardBatch(context.getContainer(), null, context.getProtocol()); + batch.save(context.getUser()); + // It's brand new, so we need to eventually set its properties + forceSaveBatchProps = true; + } + + // Queue up a pipeline job to do the actual import in the background + ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); + + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + // Check if the primary file from the previous import is no longer present for a re-run + if (primaryFile == null && !context.getUploadedData().isEmpty()) + { + // Choose another file as the primary + primaryFile = context.getUploadedData().entrySet().iterator().next().getValue(); + } + primaryFile = Objects.requireNonNull(primaryFile); + AssayRunAsyncContext asyncContext = context.getProvider().createRunAsyncContext(context); + final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob( + asyncContext, + info, + batch, + forceSaveBatchProps, + PipelineService.get().getPipelineRootSetting(context.getContainer()), + primaryFile.toNioPathForRead().toFile() + ); + + context.setPipelineJobGUID(pipelineJob.getJobGUID()); + + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), null, pipelineJob.getJobGUID()); + resultsFileWriter.savePostedFiles(context); + + // Don't queue the job until the transaction is committed, since otherwise the thread + // that's running the job might start before it can access the job's row in the database. + ExperimentService.get().getSchema().getScope().addCommitTask(() -> { + try + { + PipelineService.get().queueJob(pipelineJob, asyncContext.getJobNotificationProvider()); + } + catch (PipelineValidationException e) + { + throw UnexpectedException.wrap(e); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + + return batch; + } + + /** + * @param batch if not null, the run group that's already created for this batch. If null, a new one needs to be created + * @param run The run to save + * @return the run and batch that were inserted + */ + @Override + public ExpExperiment saveExperimentRun( + final AssayRunUploadContext context, + @Nullable ExpExperiment batch, + @NotNull ExpRun run, + boolean forceSaveBatchProps, + @Nullable Map transactionDetails + ) throws ExperimentException, ValidationException + { + context.setAutoFillDefaultResultColumns(run.getRowId() > 0); // need to setAutoFillDefaultResultColumns before run is saved + + final Container container = context.getContainer(); + + Map inputMaterials = new HashMap<>(); + Map inputDatas = new HashMap<>(); + Map outputMaterials = new HashMap<>(); + Map outputDatas = new HashMap<>(); + Map transformedDatas = new HashMap<>(); + + Map runProperties = context.getRunProperties(); + Map unresolvedRunProperties = context.getUnresolvedRunProperties(); + Map batchProperties = context.getBatchProperties(); + + Map allProperties = new HashMap<>(); + allProperties.putAll(runProperties); + allProperties.putAll(batchProperties); + + ParticipantVisitResolverType resolverType = null; + for (Map.Entry entry : allProperties.entrySet()) + { + if (entry.getKey().getName().equals(AbstractAssayProvider.PARTICIPANT_VISIT_RESOLVER_PROPERTY_NAME)) + { + resolverType = AbstractAssayProvider.findType(entry.getValue(), getProvider().getParticipantVisitResolverTypes()); + if (resolverType != null) + { + resolverType.configureRun(context, run, inputDatas); + } + break; + } + } + + // TODO: Share these RemapCache and materialCache instances with AbstractAssayTsvDataHandler.checkData and ExpressionMatrixDataHandler.importFile + // Cache of resolved alternate lookup keys -> rowId + final RemapCache cache = new RemapCache(true); + // Cache of rowId -> ExpMaterial + final Map materialCache = new LongHashMap<>(); + + addInputMaterials(context, inputMaterials, cache, materialCache); + addInputDatas(context, inputDatas); + addOutputMaterials(context, outputMaterials, cache, materialCache); + addOutputDatas(context, inputDatas, outputDatas); + + boolean success = false; + DbScope scope = ExperimentService.get().getSchema().getScope(); + try (DbScope.Transaction transaction = scope.ensureTransaction(ExperimentService.get().getProtocolImportLock())) + { + if (transaction.getAuditId() == null) + { + var auditAction = context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT; + if (context.getTransactionAuditId() != null) + { + var auditEvent = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, context.getTransactionAuditId()); + transaction.setAuditEvent(auditEvent); + } + else + { + var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction, transactionDetails); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); + } + } + boolean saveBatchProps = forceSaveBatchProps; + + // Add any material/data inputs related to the specimen IDs, etc in the incoming data. + // Some subclasses may actually create ExpMaterials or do other database changes, so do this inside the + // overall transaction + resolveParticipantVisits(context, inputMaterials, inputDatas, outputMaterials, outputDatas, allProperties, resolverType); + + // Check for circular inputs/outputs + checkForCycles(inputMaterials, outputMaterials); + checkForCycles(inputDatas, outputDatas); + + // Create the batch, if needed + if (batch == null) + { + // Make sure that we have a batch to associate with this run + batch = AssayService.get().createStandardBatch(run.getContainer(), null, context.getProtocol()); + batch.save(context.getUser()); + saveBatchProps = true; + } + run.save(context.getUser()); + // Add the run to the batch so that we can find it when we're loading the data files + batch.addRuns(context.getUser(), run); + assert batch.equals(run.getBatch()) : "Run's batch should be the current batch"; + + ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); + XarContext xarContext = new AssayUploadXarContext("Simple Run Creation", context); + + run = ExperimentService.get().saveSimpleExperimentRun( + run, + inputMaterials, + inputDatas, + outputMaterials, + outputDatas, + transformedDatas, + info, + context.getLogger() != null ? context.getLogger() : LOG, + false + ); + + // handle data transformation + TransformResult transformResult = transform(context, run); + + if (transformResult.getWarnings() != null && context instanceof AssayRunUploadForm uploadForm) + { + context.setTransformResult(transformResult); + uploadForm.setName(run.getName()); + uploadForm.setComments(run.getComments()); + throw new ValidationException(" "); + } + + if (saveBatchProps) + saveProperties(context, batch, transformResult.getBatchProperties(), batchProperties); + if (null != transformResult.getAssayId()) + run.setName(transformResult.getAssayId()); + if (null != transformResult.getComments()) + run.setComments(transformResult.getComments()); + saveProperties(context, run, transformResult.getRunProperties(), runProperties); + + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); + resultsFileWriter.savePostedFiles(context); + + Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run); + if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir)) + { + FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir); + if (assayResultFileRoot != null) + QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot); + } + + importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult); + + var reRunId = context.getReRunId(); + if (reRunId != null && getProvider().getReRunSupport() == AssayProvider.ReRunSupport.ReRunAndReplace) + { + final ExpRun replacedRun = ExperimentService.get().getExpRun(reRunId); + if (replacedRun == null) + throw new ExperimentException(String.format("Unable to find run to be replaced (RowId %d)", reRunId)); + + if (replacedRun.getContainer().hasPermission(context.getUser(), UpdatePermission.class)) + { + replacedRun.setReplacedByRun(run); + replacedRun.save(context.getUser()); + } + + String auditMessage = String.format("Run id %d was replaced by run id %d", replacedRun.getRowId(), run.getRowId()); + ExperimentService.get().auditRunEvent(context.getUser(), context.getProtocol(), replacedRun, null, auditMessage, context.getAuditUserComment()); + + transaction.addCommitTask(() -> replacedRun.archiveDataFiles(context.getUser()), DbScope.CommitTaskOption.POSTCOMMIT); + // Issue 51710: Remove replaced assay runs from the search index + transaction.addCommitTask(() -> AssayService.get().deindexAssayRuns(List.of(replacedRun)), DbScope.CommitTaskOption.POSTCOMMIT); + } + + AssayService.get().ensureUniqueBatchName(batch, context.getProtocol(), context.getUser()); + + ExperimentService.get().onRunDataCreated(context.getProtocol(), run, container, context.getUser()); + + transaction.commit(); + success = true; + + // Inspect the run properties for a “prov:objectInputs” property that is a list of LSID strings. + // Attach run's starting protocol application with starting input LSIDs. + Object provInputsProperty = unresolvedRunProperties.get(ProvenanceService.PROVENANCE_INPUT_PROPERTY); + if (provInputsProperty != null) + { + ProvenanceService pvs = ProvenanceService.get(); + Set runInputLSIDs = null; + if (provInputsProperty instanceof String provInputs) + { + // parse as a JSONArray of values or a comma-separated list of values + if (provInputs.startsWith("[") && provInputs.endsWith("]")) + provInputsProperty = new JSONArray(provInputs); + else + runInputLSIDs = Set.of(provInputs.split(",")); + } + + if (provInputsProperty instanceof JSONArray jsonArray) + { + runInputLSIDs = jsonArray.toList().stream() + .map(String::valueOf) + .collect(Collectors.toSet()); + } + + if (runInputLSIDs != null && !runInputLSIDs.isEmpty()) + { + ExpProtocolApplication inputProtocolApp = run.getInputProtocolApplication(); + pvs.addProvenanceInputs(container, inputProtocolApp, runInputLSIDs); + } + } + + ExperimentService.get().queueSyncRunEdges(run); + + return batch; + } + catch (IOException | ConvertHelper.FileConversionException | BatchValidationException e) + { + // HACK: Rethrowing these as ApiUsageException avoids any upstream consequences of wrapping them in ExperimentException. + // Namely, that they are logged to the server/mothership. There has to be a better way. + if (e instanceof ConvertHelper.FileConversionException fce) + throw new ApiUsageException(fce.getMessage(), fce); + else if (e instanceof BatchValidationException bve) + throw new ApiUsageException(bve.getMessage(), bve); + + throw new ExperimentException(e); + } + finally + { + if (!success) + { + // clean up the run results file dir here if it was created, for non-async imports + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); + resultsFileWriter.cleanupPostedFiles(context.getContainer(), false); + + cleanPrimaryFile(context); + } + } + } + + private void cleanPrimaryFile(AssayRunUploadContext context) throws ExperimentException + { + // Do not clear the primary file for run re-imports + if (context.getReRunId() != null) + return; + + try + { + // Issue 51300: don't keep the primary file if the new run failed to save + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + + // If the uploaded file is in the temp directory, then do not delete it as it may be reused in the next import attempt. + if (primaryFile != null && primaryFile.exists() && !primaryFile.getPath().contains(TEMP_DIR_NAME)) + primaryFile.delete(); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + } + + private void resolveParticipantVisits( + AssayRunUploadContext context, + Map inputMaterials, + Map inputDatas, + Map outputMaterials, + Map outputDatas, + Map allProperties, + @Nullable ParticipantVisitResolverType resolverType + ) throws ExperimentException + { + try + { + ParticipantVisitResolver resolver = null; + if (resolverType != null) + { + String targetStudyId = null; + for (Map.Entry property : allProperties.entrySet()) + { + if (AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME.equals(property.getKey().getName())) + { + targetStudyId = property.getValue(); + break; + } + } + Container targetStudy = null; + if (targetStudyId != null && !targetStudyId.isEmpty()) + targetStudy = ContainerManager.getForId(targetStudyId); + + resolver = resolverType.createResolver( + unmodifiableCollection(inputMaterials.keySet()), + unmodifiableCollection(inputDatas.keySet()), + unmodifiableCollection(outputMaterials.keySet()), + unmodifiableCollection(outputDatas.keySet()), + context.getContainer(), + targetStudy, context.getUser()); + } + + resolveExtraRunData(resolver, context, inputMaterials, inputDatas, outputMaterials, outputDatas); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + } + + protected void importStandardResultData( + AssayRunUploadContext context, + ExpRun run, + Map inputDatas, + Map outputDatas, + ViewBackgroundInfo info, + XarContext xarContext + ) throws ExperimentException, BatchValidationException + { + DataIteratorBuilder rawData = context.getRawData(); + List insertedDatas = new ArrayList<>(); + + if (rawData != null) + { + insertedDatas.addAll(outputDatas.keySet()); + + ExpData primaryData = null; + // Decide which file to treat as the primary, to which the data rows will be attached + for (Map.Entry entry : outputDatas.entrySet()) + { + if (ExpDataRunInput.DEFAULT_ROLE.equalsIgnoreCase(entry.getValue())) + { + primaryData = entry.getKey(); + } + } + if (primaryData == null && !insertedDatas.isEmpty()) + primaryData = insertedDatas.get(0); + + if (primaryData != null) + { + TsvDataHandler dataHandler = new TsvDataHandler(); + dataHandler.setAllowEmptyData(true); + dataHandler.importRows(primaryData, context.getUser(), run, context.getProtocol(), getProvider(), rawData, null, context.shouldAutoFillDefaultResultColumns(), context); + } + } + else + { + for (Map.Entry entry : inputDatas.entrySet()) + { + // skip any of the cross run inputData that are already in the outputData + if (CROSS_RUN_DATA_INPUT_ROLE.equals(entry.getValue())) + continue; + + insertedDatas.add(entry.getKey()); + } + + insertedDatas.addAll(outputDatas.keySet()); + + Logger logger = context.getLogger() != null ? context.getLogger() : LOG; + for (ExpData insertedData : insertedDatas) + { + ExperimentDataHandler dataHandler = insertedData.findDataHandler(); + + FileLike fileLike = FileSystemLike.wrapFile(insertedData.getFile()); + if (dataHandler instanceof AbstractAssayTsvDataHandler tsvHandler) + { + tsvHandler.importFile(insertedData, fileLike, info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); + } + else + { + dataHandler.importFile(insertedData, fileLike, info, logger, xarContext); + } + } + } + } + + private void importResultData( + AssayRunUploadContext context, + ExpRun run, + Map inputDatas, + Map outputDatas, + ViewBackgroundInfo info, + XarContext xarContext, + TransformResult transformResult + ) throws ExperimentException, BatchValidationException + { + if (transformResult.getTransformedData().isEmpty()) + { + importStandardResultData(context, run, inputDatas, outputDatas, info, xarContext); + return; + } + + DataType dataType = context.getProvider().getDataType(); + if (dataType == null) + { + // we know that we are importing transformed data at this point + dataType = TsvDataHandler.RELATED_TRANSFORM_FILE_DATA_TYPE; + } + + ExpData data = ExperimentService.get().createData(context.getContainer(), dataType); + ExperimentDataHandler handler = data.findDataHandler(); + + // this should assert to always be true + if (handler instanceof TransformDataHandler transformDataHandler) + { + for (Map.Entry entry : transformResult.getTransformedData().entrySet()) + { + ExpData expData = entry.getKey(); + // The object may have already been claimed by + if (expData.getSourceApplication() == null) + { + expData.setSourceApplication(run.getOutputProtocolApplication()); + } + expData.save(context.getUser()); + + run.getOutputProtocolApplication().addDataInput(context.getUser(), expData, ExpDataRunInput.IMPORTED_DATA_ROLE); + // Add to the cached list of outputs + run.getDataOutputs().add(expData); + + transformDataHandler.importTransformDataMap(expData, context, run, entry.getValue()); + } + } + } + + protected void addInputMaterials( + AssayRunUploadContext context, + Map inputMaterials, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + addMaterials(context, inputMaterials, context.getInputMaterials(), null, cache, materialCache); + + // Find lookups to a SampleType and add the resolved material as an input sample + for (Map.Entry entry : context.getRunProperties().entrySet()) + { + String value = StringUtils.trimToNull(entry.getValue()); + if (value == null) + continue; + + // Lookup must point at "Samples.*", "exp.materials.*", or "exp.Materials" + DomainProperty dp = entry.getKey(); + var sampleLookup = AssaySampleLookupContext.checkSampleLookup(context.getContainer(), context.getUser(), dp); + if (!sampleLookup.isLookup()) + continue; + + String role = AssayService.get().getPropertyInputLineageRole(dp); + addMaterials(context, inputMaterials, Map.of(value, role), sampleLookup.expSampleType(), cache, materialCache); + } + } + + protected void addInputDatas( + AssayRunUploadContext context, + @NotNull Map inputDatas + ) throws ExperimentException, ValidationException + { + Logger log = context.getLogger() != null ? context.getLogger() : LOG; + + Map inputs = context.getInputDatas(); + addDatas(context.getContainer(), inputDatas, inputs, log); + + // Inspect the uploaded files which will be added as outputs of the run + if (context.isAllowCrossRunFileInputs()) + { + Map files = context.getUploadedData(); + for (Map.Entry entry : files.entrySet()) + { + String key = entry.getKey(); + if (AssayDataCollector.PRIMARY_FILE.equals(key)) + { + FileLike file = entry.getValue(); + + // Check if the file is created by a run + // Don't use getExpDataByURL(String). That method expects string in a very particular format. + ExpData existingData = ExperimentService.get().getExpDataByURL(file.toNioPathForRead(), context.getContainer()); + if (existingData != null && existingData.getRunId() != null && !inputDatas.containsKey(existingData)) + { + // Add this file as an input to the run. When we add the outputs to the run, we will detect + // that this file was already added as an input and create a new exp.data for the same file + // path and attach it as an output. + log.debug("found existing cross run file input: name={}, rowId={}, dataFileUrl={}", existingData.getName(), existingData.getRowId(), existingData.getDataFileUrl()); + inputDatas.put(existingData, CROSS_RUN_DATA_INPUT_ROLE); + } + } + } + } + } + + // CONSIDER: Move this to ExperimentService + // Resolve submitted values into ExpData objects + protected void addDatas(Container c, @NotNull Map resolved, @NotNull Map unresolved, @Nullable Logger log) throws ValidationException + { + for (Map.Entry entry : unresolved.entrySet()) + { + Object o = entry.getKey(); + String role = entry.getValue(); + + if (o instanceof ExpData expData) + { + resolved.put(expData, role); + } + else + { + File file = ExpDataFileConverter.convert(o); + if (file != null) + { + ExpData data = ExperimentService.get().getExpDataByURL(file, c); + if (data == null) + { + DataType dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + data = createData(c, file, file.getName(), dataType, false, true, log); + } + + resolved.put(data, role); + } + } + } + } + + public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData) throws ValidationException + { + return generateResultData(user, container, provider, dataArray, outputData, null); + } + + public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData, @Nullable Logger log) throws ValidationException + { + if (log == null) + log = LOG; + + ExpData newData = null; + + // Don't create an empty result data file if there are other outputs from this run, or if the user didn't + // include any data rows + if (!dataArray.isEmpty() && outputData.isEmpty()) + { + DataType dataType = provider.getDataType(); + if (dataType == null) + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + + newData = createData(container, "Analysis Results", dataType, log); + newData.save(user); + outputData.put(newData, ExpDataRunInput.DEFAULT_ROLE); + } + + return newData; + } + + // Find an existing ExpData for the File or null. + public static @Nullable ExpData findExistingData(Container c, @Nullable File file, @Nullable Logger log) + { + if (file == null) + return null; + + if (log == null) + log = LOG; + + List existing = ExperimentService.get().getAllExpDataByURL(file, c); + if (!existing.isEmpty()) + { + for (ExpData d : existing) + { + log.debug("found existing exp.data for file, rowId={}, runId={}, dataFileUrl={}", d.getRowId(), d.getRunId(), d.getDataFileUrl()); + } + + // pick the most recently created one + return existing.get(0); + } + + return null; + } + + public static @NotNull ExpData createData(Container c, String name, @NotNull DataType dataType, @Nullable Logger log) throws ValidationException + { + // NOTE: reuseExistingData and errorOnDataOwned flags are irrelevant when we aren't providing a File + return createData(c, null, name, dataType, false, false, log); + } + + public static @NotNull ExpData createData( + Container c, + File file, + String name, + @Nullable DataType dataType, + boolean reuseExistingData, + boolean errorIfDataOwned, + @Nullable Logger log + ) throws ValidationException + { + if (log == null) + log = LOG; + + ExpData data = findExistingData(c, file, log); + + ExpRun previousRun; + if (data != null && null != (previousRun = data.getRun())) + { + // There's an existing data, but it's already marked as being created by another run + String msg = "File '" + data.getName() + "' has been previously imported in run '" + previousRun.getName() + "' (" + previousRun.getRowId() + ")"; + if (reuseExistingData && errorIfDataOwned) + throw new ValidationException(msg); + + log.debug(msg); + + // Create a new one for the same path so the new run can claim it as its own + if (!reuseExistingData) + { + log.debug("ignoring existing exp.data, will create a new one"); + data = null; + } + } + + if (data == null) + { + if (dataType == null) + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + + log.debug("creating assay exp.data for file. dataType={}, file={}", dataType.getNamespacePrefix(), file); + data = ExperimentService.get().createData(c, dataType, name); + data.setLSID(ExperimentService.get().generateGuidLSID(c, dataType)); + if (file != null) + { + data.setDataFileURI(FileUtil.getAbsoluteCaseSensitiveFile(file).toURI()); + } + } + else + { + if (dataType != null && !dataType.matches(new Lsid(data.getLSID()))) + { + // Reset its LSID so that it's the correct type // CONSIDER: creating a new ExpData with the correct type instead + String newLsid = ExperimentService.get().generateGuidLSID(c, dataType); + log.debug("LSID doesn't match desired type. Changed the LSID from '{}' to '{}'", data.getLSID(), newLsid); + data.setLSID(newLsid); + } + } + return data; + } + + protected void addOutputMaterials( + AssayRunUploadContext context, + Map outputMaterials, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + addMaterials(context, outputMaterials, context.getOutputMaterials(), null, cache, materialCache); + } + + protected void addMaterials( + AssayRunUploadContext context, + @NotNull Map resolved, + @NotNull Map unresolved, + @Nullable ExpSampleType sampleType, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + for (Map.Entry entry : unresolved.entrySet()) + { + Object sampleIdentifier = entry.getKey(); + ExpMaterial material = ExperimentService.get().findExpMaterial(context.getContainer(), context.getUser(), sampleIdentifier, sampleType, cache, materialCache); + if (material == null) + throw new ExperimentException("Unable to resolve sample: " + sampleIdentifier); + + if (!resolved.containsKey(material)) + { + if (!material.isOperationPermitted(SampleTypeService.SampleOperations.AddAssayData)) + throw new ExperimentException(SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddAssayData)); + if (sampleType == null || sampleType.getLSID().equals(material.getCpasType())) + resolved.put(material, entry.getValue()); + } + } + } + + protected void addOutputDatas( + AssayRunUploadContext context, + Map inputDatas, + Map outputDatas + ) throws ExperimentException, ValidationException + { + Logger log = context.getLogger() != null ? context.getLogger() : LOG; + + // Create set of existing input files + Set inputFiles = new HashSet<>(); + for (ExpData inputData : inputDatas.keySet()) + { + FileLike f = inputData.getFileLike(); + if (f != null) + inputFiles.add(f); + } + + Map files = context.getUploadedData(); + + AssayDataType dataType; + for (Map.Entry entry : files.entrySet()) + { + FileLike file = entry.getValue(); + dataType = context.getProvider().getDataType(); + + // Reuse existing exp.data as the assay output file unless: + // - we are re-importing the run + // - or the output file is already one of the input files and if we are allowing cross-run file inputs + boolean reuseExistingData = true; + if (context.getReRunId() != null) + reuseExistingData = false; + if (context.isAllowCrossRunFileInputs() && inputFiles.contains(file)) + reuseExistingData = false; + + // For Luminex re-import, we want to reuse the existing exp.data but not + // throw an error when we discover that the exp.data is already owned. The + // original run will be duplicated for re-import and then will be deleted. + boolean errorIfDataOwned = getProvider().getReRunSupport() != AssayProvider.ReRunSupport.ReRunAndDelete; + + log.debug("adding output data: file={}", file.toNioPathForRead()); + log.debug(" context.getReRunId()={}", context.getReRunId()); + log.debug(" provider.getReRunSupport()={}", getProvider().getReRunSupport()); + log.debug(" context.allowCrossRunFileInputs={}", context.isAllowCrossRunFileInputs()); + log.debug(" inputFiles.contains(file)={}", inputFiles.contains(file)); + log.debug("==> reuseExistingData = {}", reuseExistingData); + log.debug("==> errorIfDataOwned = {}", errorIfDataOwned); + + ExpData data = DefaultAssayRunCreator.createData(context.getContainer(), file.toNioPathForRead().toFile(), file.getName(), dataType, reuseExistingData, errorIfDataOwned, log); + String role = ExpDataRunInput.DEFAULT_ROLE; + if (dataType != null && dataType.getFileType().isType(file)) + { + if (dataType.getRole() != null) + { + role = dataType.getRole(); + } + } + outputDatas.put(data, role); + } + + FileLike primaryFile = files.get(AssayDataCollector.PRIMARY_FILE); + if (primaryFile != null) + { + addRelatedOutputDatas(context, inputFiles, outputDatas, primaryFile); + } + + Map outputs = context.getOutputDatas(); + addDatas(context.getContainer(), outputDatas, outputs, log); + } + + /** + * Add files that follow the general naming convention (same basename) as the primary file + */ + public void addRelatedOutputDatas( + AssayRunUploadContext context, + Set inputFiles, + Map outputDatas, + final FileLike primaryFile + ) throws ValidationException + { + AssayDataType dataType = getProvider().getDataType(); + final String baseName = dataType == null ? null : dataType.getFileType().getBaseName(primaryFile.toNioPathForRead()); + if (baseName != null) + { + // Grab all the files that are related based on naming convention + File primary = primaryFile.toNioPathForRead().toFile(); + File parent = primary.getParentFile(); + // converting to File land to reuse the FileFilter + File[] relatedFiles = parent.listFiles(getRelatedOutputDataFileFilter(primary, baseName)); + if (relatedFiles != null) + { + for (File f : relatedFiles) + { + FileLike relatedFile = primaryFile.getParent().resolveChild(f.getName()); + // Ignore files already considered inputs to the run + if (inputFiles.contains(relatedFile)) + continue; + + Pair dataOutput = createdRelatedOutputData(context, baseName, f); + if (dataOutput != null) + { + outputDatas.put(dataOutput.getKey(), dataOutput.getValue()); + } + } + } + } + } + + protected void resolveExtraRunData( + ParticipantVisitResolver resolver, + AssayRunUploadContext context, + Map inputMaterials, + Map inputDatas, + Map outputMaterials, + Map outputDatas + ) throws ExperimentException + { + } + + /** + * Create an ExpData object for the file, and figure out what its role name should be + * @return null if the file is already linked to another run + */ + @Nullable + public static Pair createdRelatedOutputData(AssayRunUploadContext context, String baseName, File relatedFile) throws ValidationException + { + String roleName = null; + DataType dataType = null; + for (AssayDataType inputType : context.getProvider().getRelatedDataTypes()) + { + // Check if we recognize it as a specially handled file type + if (inputType.getFileType().isMatch(relatedFile.getName(), baseName)) + { + roleName = inputType.getRole(); + dataType = inputType; + break; + } + } + // If not, make up a new type and role for it + if (roleName == null) + { + roleName = relatedFile.getName().substring(baseName.length()); + while (!roleName.isEmpty() && (roleName.startsWith(".") || roleName.startsWith("-") || roleName.startsWith("_") || roleName.startsWith(" "))) + { + roleName = roleName.substring(1); + } + if (roleName.isEmpty()) + { + roleName = null; + } + } + if (dataType == null) + { + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + } + + // Find an existing data that isn't owned by another run or create a new own + ExpData data = findExistingData(context.getContainer(), relatedFile, context.getLogger()); + if (data != null) + { + if (data.getSourceApplication() == null) + return new Pair<>(data, roleName); + + // The file is already linked to another run, so this one must have not created it + return null; + } + + data = createData(context.getContainer(), relatedFile, relatedFile.getName(), dataType, true, true, context.getLogger()); + assert data.getSourceApplication() == null; + return Pair.of(data, roleName); + } + + // Disallow creating a run with inputs which are also outputs + protected void checkForCycles( + Map inputs, + Map outputs + ) throws ExperimentException + { + for (ExpRunItem input : inputs.keySet()) + { + if (outputs.containsKey(input)) + { + String role = outputs.get(input); + throw new ExperimentException("Circular input/output '" + input.getName() + "' with role '" + role + "'"); + } + } + } + + private void saveProperties( + final AssayRunUploadContext context, + ExpObject expObject, + Map transformResultProperties, + Map properties + ) throws ValidationException + { + Map propsToSave = transformResultProperties.isEmpty() ? properties : transformResultProperties; + List errors = validateProperties(context, propsToSave); + if (!errors.isEmpty()) + throw new ValidationException(errors); + + savePropertyObject(expObject, propsToSave, context.getUser()); + } + + protected void savePropertyObject(ExpObject object, Map properties, User user) throws ValidationException + { + for (Map.Entry entry : properties.entrySet()) + { + DomainProperty pd = entry.getKey(); + String value = entry.getValue(); + + // resolve any file links for batch or run properties + if (PropertyType.FILE_LINK.getTypeUri().equals(pd.getType().getTypeURI())) + { + File resolvedFile = ExpDataFileConverter.convert(value); + if (resolvedFile != null) + value = resolvedFile.getAbsolutePath(); + } + + // Treat the empty string as a null in the database, which is our normal behavior when receiving data + // from HTML forms. + if (StringUtils.trimToNull(value) == null) + { + value = null; + } + if (value != null) + { + object.setProperty(user, pd.getPropertyDescriptor(), value); + } + else + { + // We still need to validate blanks + List errors = new ArrayList<>(); + OntologyManager.validateProperty(pd.getValidators(), pd.getPropertyDescriptor(), new ObjectProperty(object.getLSID(), object.getContainer(), pd.getPropertyDescriptor(), value), errors, new ValidatorContext(pd.getContainer(), user)); + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + } + } + + public static List validateColumnProperties(ContainerUser context, Map properties) + { + List errors = new ArrayList<>(); + RemapCache cache = new RemapCache(); + for (Map.Entry entry : properties.entrySet()) + { + validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); + } + return errors; + } + + public static List validateProperties(ContainerUser context, Map properties) + { + List errors = new ArrayList<>(); + RemapCache cache = new RemapCache(); + for (Map.Entry entry : properties.entrySet()) + { + validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); + } + return errors; + } + + private static void validateProperty(ContainerUser context, ColumnInfo columnInfo, String value, RemapCache cache, List errors) + { + Lookup lookup = null; + if (columnInfo.isLookup()) + { + ForeignKey fk = columnInfo.getFk(); + lookup = new Lookup(fk.getLookupContainer(), fk.getLookupSchemaName(), fk.getLookupTableName()); + } + validateProperty(context, ColumnValidators.create(columnInfo, null), value, columnInfo.getName(), + false, lookup, columnInfo.getJavaClass(), cache, errors); + } + + private static void validateProperty(ContainerUser context, DomainProperty dp, String value, RemapCache cache, List errors) + { + String label = dp.getPropertyDescriptor().getNonBlankCaption(); + PropertyType type = dp.getPropertyDescriptor().getPropertyType(); + validateProperty(context, ColumnValidators.create(null, dp), value, label, dp.isRequired(), + dp.getLookup(), type.getJavaType(), cache, errors); + } + + private static void validateProperty( + ContainerUser context, + List validators, + String value, + String label, + Boolean required, + Lookup lookup, + Class type, + RemapCache cache, + List errors + ) + { + boolean missing = (value == null || value.isEmpty()); + int rowNum = 0; + + if (required && missing) + { + errors.add(new SimpleValidationError(label + " is required and must be of type " + ColumnInfo.getFriendlyTypeName(type) + ".")); + } + else if (!missing) + { + try + { + Object o; + if (type == File.class) + o = ExpDataFileConverter.convert(value); + else + o = ConvertUtils.convert(value, type); + ValidatorContext validatorContext = new ValidatorContext(context.getContainer(), context.getUser()); + for (ColumnValidator validator : validators) + { + String msg = validator.validate(rowNum, o, validatorContext); + if (msg != null) + errors.add(new PropertyValidationError(msg, label)); + } + } + catch (ConversionException e) + { + String message; + if (e instanceof ConvertHelper.FileConversionException fce) + message = fce.getMessage(); + else + { + message = ConvertHelper.getStandardConversionErrorMessage(value, label, type); + if (e.getCause() instanceof ArithmeticException) + message += ": " + e.getCause().getLocalizedMessage(); + else + message += "."; + } + + // Attempt to resolve lookups by display value + boolean skipError = false; + if (lookup != null) + { + Object remappedValue = OntologyManager.getRemappedValueForLookup(context.getUser(), context.getContainer(), cache, lookup, value); + if (remappedValue != null) + skipError = true; + } + + if (!skipError) + errors.add(new SimpleValidationError(message)); + } + } + } + + protected FileFilter getRelatedOutputDataFileFilter(final File primaryFile, final String baseName) + { + // baseName doesn't include the trailing '.', so add it here. We want to associate myRun.jpg + // with myRun.xls, but we don't want to associate myRun2.xls with myRun.xls (which will happen without + // the trailing dot in the check). + return f -> f.getName().startsWith(baseName + ".") && !primaryFile.equals(f); + } + + protected ProviderType getProvider() + { + return _provider; + } +} diff --git a/api/src/org/labkey/api/assay/actions/UploadWizardAction.java b/api/src/org/labkey/api/assay/actions/UploadWizardAction.java index e5520b9731c..b4564122006 100644 --- a/api/src/org/labkey/api/assay/actions/UploadWizardAction.java +++ b/api/src/org/labkey/api/assay/actions/UploadWizardAction.java @@ -1027,7 +1027,7 @@ public class RunStepHandler extends StepHandler public final ExpRun saveExperimentRun(FormType form) throws ExperimentException, ValidationException { - Pair pair = form.getProvider().getRunCreator().saveExperimentRun(form, form.getBatchId()); + Pair pair = form.getProvider().getRunCreator().saveExperimentRun(form, form.getBatchId(), false, getTransactionAuditDetails()); assert pair != null && pair.first != null; ExpExperiment exp = pair.first; ExpRun run = pair.second; diff --git a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java index a364fdbf70e..57fb8ba2823 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java @@ -22,6 +22,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; import org.labkey.api.assay.AssayUrls; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpRun; @@ -156,7 +157,11 @@ public void doWork() } // Do all the real work of the import - ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps); + Map transactionDetails = new HashMap<>(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, _primaryFile.getName()); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.APIAction, "AssayUploadPipelineJob"); + ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails/*TODO*/); setStatus(TaskStatus.complete); getLogger().info("Finished assay upload"); diff --git a/api/src/org/labkey/api/audit/AuditTypeEvent.java b/api/src/org/labkey/api/audit/AuditTypeEvent.java index af57a884112..48ad70b0e57 100644 --- a/api/src/org/labkey/api/audit/AuditTypeEvent.java +++ b/api/src/org/labkey/api/audit/AuditTypeEvent.java @@ -1,258 +1,267 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.security.Group; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ExceptionUtil; - -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Bean object to capture audit log entries. Will be used to populate the database tables via get/set methods that - * align with column names in the corresponding provisioned table. - */ -public class AuditTypeEvent -{ - protected static final String CREATED_BY_KEY = "auditEventCreatedBy"; - protected static final String IMPERSONATED_BY_KEY = "impersonatedBy"; - protected static final String CONTAINER_KEY = "container"; - protected static final String PROJECT_KEY = "project"; - protected static final String COMMENT_KEY = "comment"; - private static final String USER_COMMENT_KEY = "userComment"; - - // long type used here to allow for DbSequences to supply the rowId - private long _rowId; - private Integer _impersonatedBy; - private String _comment; - private Container _projectId; - private Container _container; - private String _eventType; - private Date _created; - private User _createdBy; - private Date _modified; - private User _modifiedBy; - private String _userComment; - private Long _transactionId; - - public AuditTypeEvent(@NotNull String eventType, @NotNull Container container, @Nullable String comment) - { - _eventType = eventType; - if (container == null) - { - ExceptionUtil.logExceptionToMothership(null, new IllegalStateException("Audit event container is null")); - } - _container = container; - _comment = comment; - _projectId = container.getProject(); - } - - /** Important for reflection-based instantiation */ - public AuditTypeEvent() - { - } - - public long getRowId() - { - return _rowId; - } - - public void setRowId(long rowId) - { - _rowId = rowId; - } - - public Integer getImpersonatedBy() - { - return _impersonatedBy; - } - - public void setImpersonatedBy(Integer impersonatedBy) - { - _impersonatedBy = impersonatedBy; - } - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public Container getProjectId() - { - return _projectId; - } - - public void setProjectId(Container projectId) - { - _projectId = projectId; - } - - public Container getContainer() - { - return _container; - } - - public void setContainer(Container container) - { - _container = container; - } - - public String getEventType() - { - return _eventType; - } - - public void setEventType(String eventType) - { - _eventType = eventType; - } - - public Date getCreated() - { - return _created; - } - - public void setCreated(Date created) - { - _created = created; - } - - public User getCreatedBy() - { - return _createdBy; - } - - public void setCreatedBy(User createdBy) - { - _createdBy = createdBy; - } - - public Date getModified() - { - return _modified; - } - - public void setModified(Date modified) - { - _modified = modified; - } - - public User getModifiedBy() - { - return _modifiedBy; - } - - public void setModifiedBy(User modifiedBy) - { - _modifiedBy = modifiedBy; - } - - public void setUserComment(String userComment) - { - _userComment = userComment; - } - - public String getUserComment() - { - return _userComment; - } - - public Long getTransactionId() - { - return _transactionId; - } - - public void setTransactionId(Long transactionId) - { - _transactionId = transactionId; - } - - protected String getContainerMessageElement(@NotNull Container container) - { - String value = " (" + container.getId() + ")"; - value = container.getPath() + value; - return value; - } - - protected String getUserMessageElement(@NotNull User user) - { - return user.getEmail() + " (" + user.getUserId() + ")"; - } - - protected String getUserMessageElement(int userId) - { - String value = " (" + userId + ")"; - User user = UserManager.getUser(userId); - if (user != null) - value = user.getEmail() + value; - return value; - } - - protected String getGroupMessageElement(@NotNull Group group) - { - return group.getName() + " (" + group.getUserId() + ")"; - } - - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - User createdBy = getCreatedBy(); - if (createdBy != null) - { - String message = createdBy.getEmail() != null ? createdBy.getEmail() : ""; - message += " (" + createdBy.getUserId() + ")"; - elements.put(CREATED_BY_KEY, message); - } - Integer impersonatorId = getImpersonatedBy(); - if (impersonatorId != null) - elements.put(IMPERSONATED_BY_KEY, getUserMessageElement(impersonatorId)); - Container container = getContainer(); - elements.put(CONTAINER_KEY, getContainerMessageElement(container)); - Container projectId = getProjectId(); - if (projectId != null) - elements.put(PROJECT_KEY, getContainerMessageElement(projectId)); - if (getComment() != null) - elements.put(COMMENT_KEY, getComment()); - if (getUserComment() != null) - elements.put(USER_COMMENT_KEY, getUserComment()); - - return elements; - } - - public String getAuditLogMessage() - { - StringBuilder builder = new StringBuilder(); - builder.append(getEventType()).append(" - "); - - Map messageElements = getAuditLogMessageElements(); - for (String key : messageElements.keySet()) - { - builder.append(key).append(": ").append(messageElements.get(key)).append(" | "); - } - return builder.toString(); - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.security.Group; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ExceptionUtil; + +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Bean object to capture audit log entries. Will be used to populate the database tables via get/set methods that + * align with column names in the corresponding provisioned table. + */ +public class AuditTypeEvent +{ + protected static final String CREATED_BY_KEY = "auditEventCreatedBy"; + protected static final String IMPERSONATED_BY_KEY = "impersonatedBy"; + protected static final String CONTAINER_KEY = "container"; + protected static final String PROJECT_KEY = "project"; + protected static final String COMMENT_KEY = "comment"; + private static final String USER_COMMENT_KEY = "userComment"; + + // long type used here to allow for DbSequences to supply the rowId + private long _rowId; + private Integer _impersonatedBy; + private String _comment; + private Container _projectId; + private Container _container; + private String _eventType; + private Date _created; + private User _createdBy; + private Date _modified; + private User _modifiedBy; + private String _userComment; + private Long _transactionId; + + public AuditTypeEvent(@NotNull String eventType, @NotNull Container container, @Nullable String comment) + { + _eventType = eventType; + if (container == null) + { + ExceptionUtil.logExceptionToMothership(null, new IllegalStateException("Audit event container is null")); + } + _container = container; + _comment = comment; + _projectId = container.getProject(); + } + + /** Important for reflection-based instantiation */ + public AuditTypeEvent() + { + } + + public long getRowId() + { + return _rowId; + } + + public void setRowId(long rowId) + { + _rowId = rowId; + } + + public Integer getImpersonatedBy() + { + return _impersonatedBy; + } + + public void setImpersonatedBy(Integer impersonatedBy) + { + _impersonatedBy = impersonatedBy; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public Container getProjectId() + { + return _projectId; + } + + public void setProjectId(Container projectId) + { + _projectId = projectId; + } + + public Container getContainer() + { + return _container; + } + + public void setContainer(Container container) + { + _container = container; + } + + public String getEventType() + { + return _eventType; + } + + public void setEventType(String eventType) + { + _eventType = eventType; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public User getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(User createdBy) + { + _createdBy = createdBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + + public User getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(User modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public void setUserComment(String userComment) + { + _userComment = userComment; + } + + public String getUserComment() + { + return _userComment; + } + + public Long getTransactionId() + { + return _transactionId; + } + + public void setTransactionId(Long transactionId) + { + _transactionId = transactionId; + } + + public void setTransactionEvent(@Nullable TransactionAuditProvider.TransactionAuditEvent transactionEvent, String auditEventType) + { + if (transactionEvent == null) + return; + + _transactionId = transactionEvent.getRowId(); + transactionEvent.addDetail(TransactionAuditProvider.TransactionDetail.AuditEvents, auditEventType); + } + + protected String getContainerMessageElement(@NotNull Container container) + { + String value = " (" + container.getId() + ")"; + value = container.getPath() + value; + return value; + } + + protected String getUserMessageElement(@NotNull User user) + { + return user.getEmail() + " (" + user.getUserId() + ")"; + } + + protected String getUserMessageElement(int userId) + { + String value = " (" + userId + ")"; + User user = UserManager.getUser(userId); + if (user != null) + value = user.getEmail() + value; + return value; + } + + protected String getGroupMessageElement(@NotNull Group group) + { + return group.getName() + " (" + group.getUserId() + ")"; + } + + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + User createdBy = getCreatedBy(); + if (createdBy != null) + { + String message = createdBy.getEmail() != null ? createdBy.getEmail() : ""; + message += " (" + createdBy.getUserId() + ")"; + elements.put(CREATED_BY_KEY, message); + } + Integer impersonatorId = getImpersonatedBy(); + if (impersonatorId != null) + elements.put(IMPERSONATED_BY_KEY, getUserMessageElement(impersonatorId)); + Container container = getContainer(); + elements.put(CONTAINER_KEY, getContainerMessageElement(container)); + Container projectId = getProjectId(); + if (projectId != null) + elements.put(PROJECT_KEY, getContainerMessageElement(projectId)); + if (getComment() != null) + elements.put(COMMENT_KEY, getComment()); + if (getUserComment() != null) + elements.put(USER_COMMENT_KEY, getUserComment()); + + return elements; + } + + public String getAuditLogMessage() + { + StringBuilder builder = new StringBuilder(); + builder.append(getEventType()).append(" - "); + + Map messageElements = getAuditLogMessageElements(); + for (String key : messageElements.keySet()) + { + builder.append(key).append(": ").append(messageElements.get(key)).append(" | "); + } + return builder.toString(); + } +} diff --git a/api/src/org/labkey/api/audit/ExperimentAuditEvent.java b/api/src/org/labkey/api/audit/ExperimentAuditEvent.java index 73ad958c9c7..1427e9d3531 100644 --- a/api/src/org/labkey/api/audit/ExperimentAuditEvent.java +++ b/api/src/org/labkey/api/audit/ExperimentAuditEvent.java @@ -21,13 +21,12 @@ public class ExperimentAuditEvent extends AuditTypeEvent public ExperimentAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public ExperimentAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getProtocolLsid() diff --git a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java index eb60d7424b1..3ed3637595f 100644 --- a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java +++ b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java @@ -104,13 +104,12 @@ public static SampleTimelineEventType getTypeFromName(@Nullable String name) public SampleTimelineAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public SampleTimelineAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getSampleLsid() diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 0dc6af6e889..94b764323d1 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -1,5 +1,6 @@ package org.labkey.api.audit; +import com.fasterxml.jackson.core.JsonProcessingException; import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; @@ -7,6 +8,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DbScope; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.JsonPrettyPrintDisplayColumnFactory; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.TableInfo; @@ -16,12 +18,17 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.UnexpectedException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -32,6 +39,7 @@ public class TransactionAuditProvider extends AbstractAuditTypeProvider implemen public static final String COLUMN_NAME_START_TIME = "StartTime"; public static final String COLUMN_NAME_TRANSACTION_TYPE = "TransactionType"; + public static final String COLUMN_NAME_TRANSACTION_DETAILS = "TransactionDetails"; static final List defaultVisibleColumns = new ArrayList<>(); @@ -81,6 +89,11 @@ protected void initColumn(MutableColumnInfo col) col.setLabel("Start Time"); else if (COLUMN_NAME_TRANSACTION_TYPE.equalsIgnoreCase(col.getName())) col.setLabel("Transaction Type"); + else if (COLUMN_NAME_TRANSACTION_DETAILS.equalsIgnoreCase(col.getName())) + { + col.setLabel("Transaction Details"); + col.setDisplayColumnFactory(new JsonPrettyPrintDisplayColumnFactory()); + } } }; } @@ -104,6 +117,64 @@ public static Long getCurrentTransactionAuditId() return DbScope.getLabKeyScope().getCurrentTransaction().getAuditId(); } + public static TransactionAuditEvent getCurrentTransactionAuditEvent() + { + if (null == DbScope.getLabKeyScope().getCurrentTransaction()) + return null; + return DbScope.getLabKeyScope().getCurrentTransaction().getAuditEvent(); + } + + public enum TransactionDetail + { + Operation(false), + AuditEvents(true), + ImportFileName(true), + //BackgroundImport(false), + ClientLibrary(false), + Product(false), + // CrossFolderImport(false), + // CrossTypeImport(false), + // AllowCreateStorage(false), + // UseTransactionAuditCache(false), + APIVersion(false), + APIAction(false), + QueryCommand(true), + BatchAction(false), + ImportOptions(true) + ; + + private final boolean multiValue; + TransactionDetail(boolean multiValue) + { + this.multiValue = multiValue; + } + + public void add(Map detailMap, Object value) + { + if (value == null) + return; + + if (!this.multiValue) + { + detailMap.put(this, value); + return; + } + Object existing = detailMap.get(this);; + Set values; + if (existing == null) + values = new HashSet<>(); + else if (existing instanceof Set) + values = (Set) existing; + else + values = new HashSet<>(List.of(existing.toString())); + if (value instanceof String) + values.add((String) value); + else if (value instanceof Collections) + values.addAll((Set) value); + detailMap.put(this, values); + } + } + public static class TransactionAuditEvent extends AuditTypeEvent { private Date _startTime; @@ -112,6 +183,8 @@ public static class TransactionAuditEvent extends AuditTypeEvent private int _commentCount = 0; // the audit event comment might have been updated/appended multiple times, for example, the original insert triggers additional insert/update via trigger scripts + private final Map _detailMap = new HashMap<>(); + public TransactionAuditEvent() { super(); @@ -146,6 +219,28 @@ public void setTransactionType(String transactionType) _transactionType = transactionType; } + public String getTransactionDetails() + { + try + { + return JsonUtil.DEFAULT_MAPPER.writeValueAsString(_detailMap); + } + catch (JsonProcessingException e) + { + throw UnexpectedException.wrap(e); + } + } + + public void addDetail(TransactionDetail key, Object value) + { + key.add(_detailMap, value); + } + + public void addDetails(Map details) + { + _detailMap.putAll(details); + } + @Override public void setComment(String comment) { @@ -197,6 +292,7 @@ public TransactionAuditDomainKind() Set fields = new LinkedHashSet<>(); fields.add(createPropertyDescriptor(COLUMN_NAME_START_TIME, PropertyType.DATE_TIME)); fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_TYPE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_DETAILS, PropertyType.STRING, -1)); _fields = Collections.unmodifiableSet(fields); // We override the base fields so we can use a DbSequence as the RowId and make it available diff --git a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java index b103f6ac0b8..83195e9e0a4 100644 --- a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java +++ b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java @@ -1,235 +1,234 @@ -/* - * Copyright (c) 2015-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit.provider; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.data.Container; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/19/13 - */ -public class FileSystemAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String EVENT_TYPE = "FileSystem"; - - public static final String COLUMN_NAME_DIRECTORY = "Directory"; - public static final String COLUMN_NAME_FILE = "File"; - public static final String COLUMN_NAME_PROVIDED_FILE = "ProvidedFileName"; - public static final String COLUMN_NAME_FIELD_NAME = "FieldName"; - public static final String COLUMN_NAME_RESOURCE_PATH = "ResourcePath"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_DIRECTORY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FILE)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROVIDED_FILE)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FIELD_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - } - - public FileSystemAuditProvider() - { - super(new FileSystemAuditDomainKind()); - } - - @Override - public String getEventName() - { - return EVENT_TYPE; - } - - @Override - public String getLabel() - { - return "File events"; - } - - @Override - public String getDescription() - { - return "Displays information about file uploads and modifications."; - } - - @Override - public Map legacyNameMap() - { - Map legacyNames = super.legacyNameMap(); - legacyNames.put(FieldKey.fromParts("key1"), COLUMN_NAME_DIRECTORY); - legacyNames.put(FieldKey.fromParts("key2"), COLUMN_NAME_FILE); - legacyNames.put(FieldKey.fromParts("key3"), COLUMN_NAME_RESOURCE_PATH); - return legacyNames; - } - - @Override - public Class getEventClass() - { - return (Class)FileSystemAuditEvent.class; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - public static class FileSystemAuditEvent extends AuditTypeEvent - { - private String _directory; // the directory name - private String _file; // the file name - private String _resourcePath; // the webdav resource path - private String _providedFileName; // the name of the file as provided by the user, before renaming to make it unique and/or legal - private String _fieldName; // name of the field associated with the file, if any - - /** Important for reflection-based instantiation */ - public FileSystemAuditEvent() - { - super(); - setEventType(EVENT_TYPE); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public FileSystemAuditEvent(Container container, String comment) - { - super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public String getDirectory() - { - return _directory; - } - - public void setDirectory(String directory) - { - _directory = directory; - } - - public String getFile() - { - return _file; - } - - public void setFile(String file) - { - _file = file; - } - - public String getResourcePath() - { - return _resourcePath; - } - - public void setResourcePath(String resourcePath) - { - _resourcePath = resourcePath; - } - - public String getProvidedFileName() - { - return _providedFileName; - } - - public void setProvidedFileName(String providedFileName) - { - _providedFileName = providedFileName; - } - - public String getFieldName() - { - return _fieldName; - } - - public void setFieldName(String fieldName) - { - _fieldName = fieldName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("directory", getDirectory()); - elements.put("file", getFile()); - elements.put("resourcePath", getResourcePath()); - elements.put("providedFileName", getProvidedFileName()); - elements.put("fieldName", getFieldName()); - elements.put("transactionId", getTransactionId()); - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class FileSystemAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "FileSystemAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public FileSystemAuditDomainKind() - { - super(EVENT_TYPE); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_DIRECTORY, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_FILE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_PROVIDED_FILE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_FIELD_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_RESOURCE_PATH, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2015-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit.provider; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.data.Container; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: 7/19/13 + */ +public class FileSystemAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String EVENT_TYPE = "FileSystem"; + + public static final String COLUMN_NAME_DIRECTORY = "Directory"; + public static final String COLUMN_NAME_FILE = "File"; + public static final String COLUMN_NAME_PROVIDED_FILE = "ProvidedFileName"; + public static final String COLUMN_NAME_FIELD_NAME = "FieldName"; + public static final String COLUMN_NAME_RESOURCE_PATH = "ResourcePath"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_DIRECTORY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FILE)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROVIDED_FILE)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FIELD_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + } + + public FileSystemAuditProvider() + { + super(new FileSystemAuditDomainKind()); + } + + @Override + public String getEventName() + { + return EVENT_TYPE; + } + + @Override + public String getLabel() + { + return "File events"; + } + + @Override + public String getDescription() + { + return "Displays information about file uploads and modifications."; + } + + @Override + public Map legacyNameMap() + { + Map legacyNames = super.legacyNameMap(); + legacyNames.put(FieldKey.fromParts("key1"), COLUMN_NAME_DIRECTORY); + legacyNames.put(FieldKey.fromParts("key2"), COLUMN_NAME_FILE); + legacyNames.put(FieldKey.fromParts("key3"), COLUMN_NAME_RESOURCE_PATH); + return legacyNames; + } + + @Override + public Class getEventClass() + { + return (Class)FileSystemAuditEvent.class; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + public static class FileSystemAuditEvent extends AuditTypeEvent + { + private String _directory; // the directory name + private String _file; // the file name + private String _resourcePath; // the webdav resource path + private String _providedFileName; // the name of the file as provided by the user, before renaming to make it unique and/or legal + private String _fieldName; // name of the field associated with the file, if any + + /** Important for reflection-based instantiation */ + public FileSystemAuditEvent() + { + super(); + setEventType(EVENT_TYPE); + } + + public FileSystemAuditEvent(Container container, String comment) + { + super(EVENT_TYPE, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); + } + + public String getDirectory() + { + return _directory; + } + + public void setDirectory(String directory) + { + _directory = directory; + } + + public String getFile() + { + return _file; + } + + public void setFile(String file) + { + _file = file; + } + + public String getResourcePath() + { + return _resourcePath; + } + + public void setResourcePath(String resourcePath) + { + _resourcePath = resourcePath; + } + + public String getProvidedFileName() + { + return _providedFileName; + } + + public void setProvidedFileName(String providedFileName) + { + _providedFileName = providedFileName; + } + + public String getFieldName() + { + return _fieldName; + } + + public void setFieldName(String fieldName) + { + _fieldName = fieldName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("directory", getDirectory()); + elements.put("file", getFile()); + elements.put("resourcePath", getResourcePath()); + elements.put("providedFileName", getProvidedFileName()); + elements.put("fieldName", getFieldName()); + elements.put("transactionId", getTransactionId()); + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class FileSystemAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "FileSystemAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public FileSystemAuditDomainKind() + { + super(EVENT_TYPE); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_DIRECTORY, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_FILE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_PROVIDED_FILE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_FIELD_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_RESOURCE_PATH, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 136e9d9186a..98cd0be2b6d 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -51,13 +51,11 @@ import org.labkey.api.resource.Resource; import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CPUTimer; import org.labkey.api.util.FileStream; import org.labkey.api.util.FileUtil; -import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.Path; @@ -75,7 +73,6 @@ import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.servlet.ModelAndView; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -83,6 +80,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; import static org.labkey.api.query.AbstractQueryUpdateService.addTransactionAuditEvent; @@ -334,6 +332,23 @@ protected Map getOptionParamsMap() return _optionParamsMap; } + protected Set getTransactionImportParams(String insertOption, boolean useAsync) + { + Set importParams = new TreeSet<>(); + importParams.add(insertOption); + if (useAsync) + importParams.add("backgroundImport"); + if (Boolean.valueOf(getParam(Params.crossTypeImport))) + importParams.add(Params.crossTypeImport.name()); + if (Boolean.valueOf(getParam(Params.crossFolderImport))) + importParams.add(Params.crossFolderImport.name()); + if (Boolean.valueOf(getParam(Params.useTransactionAuditCache))) + importParams.add(Params.useTransactionAuditCache.name()); + if (Boolean.valueOf(getParam(Params.allowCreateStorage))) + importParams.add(Params.allowCreateStorage.name()); + return importParams; + } + protected boolean getOptionParamValue(Params p) { return getOptionParamsMap().getOrDefault(p, false); @@ -441,6 +456,9 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except // If not defined there, check for the audit behavior defined in the action form (getAuditBehaviorType()). AuditBehaviorType behaviorType = (_target != null) ? _target.getEffectiveAuditBehavior(getAuditBehaviorType()) : getAuditBehaviorType(); + Map transactionDetails = getTransactionAuditDetails(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, getTransactionImportParams(_insertOption.name(), _useAsync)); + try { if (null != StringUtils.trimToNull(text)) @@ -471,6 +489,7 @@ else if (StringUtils.isNotBlank(path)) loader = DataLoader.get().createLoader(resource, _hasColumnHeaders, null, null); file = resource.getFileStream(user); originalName = resource.getName(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); } if (!hasPostData) @@ -514,6 +533,7 @@ else if (_target != null) hasPostData = true; loader = DataLoader.get().createLoader(r, _hasColumnHeaders, null, TabLoader.TSV_FILE_TYPE); originalName = p.getName(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); // Set file to null so assay import doesn't copy the file file = null; } @@ -525,6 +545,7 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) MultipartFile multipartfile = null==files ? null : files.get("file"); if (null != multipartfile && multipartfile.getSize() > 0) { + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename()); hasPostData = true; originalName = multipartfile.getOriginalFilename(); // can't read the multipart file twice so create temp file (12800) @@ -582,6 +603,7 @@ else if (!dataFileDir.exists()) .setJobDescription(getQueryImportDescription()) .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); + importContextBuilder.setTransactionDetails(transactionDetails); QueryImportPipelineJob job = new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName()); @@ -613,7 +635,7 @@ else if (!dataFileDir.exists()) TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE)) - auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction); + auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction, transactionDetails); int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); @@ -862,7 +884,10 @@ public static int importData(DataLoader dl, TableInfo target, QueryUpdateService if (context.getErrors().hasErrors()) return 0; if (auditEvent != null) + { + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true /* qus.loadRows always use DIB*/); auditEvent.addComment(auditAction, count); + } incrementRowCountMetric(count, context.getInsertOption(), getMetricPrefix(target)); transaction.commit(); diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index b6d9256fba2..0afea156a79 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -1,1512 +1,1520 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.assay.AssayFileWriter; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.collections.ArrayListMap; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.ImportAliasable; -import org.labkey.api.data.MultiValuedForeignKey; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.AttachmentDataIterator; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.ExistingRecordDataIterator; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.dataiterator.Pump; -import org.labkey.api.dataiterator.StandardDataIteratorBuilder; -import org.labkey.api.dataiterator.TriggerDataBuilderHelper; -import org.labkey.api.dataiterator.WrapperDataIterator; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URIUtil; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.vfs.FileLike; -import org.springframework.web.multipart.MultipartFile; - -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.function.Function; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.audit.TransactionAuditProvider.DB_SEQUENCE_NAME; -import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior; -import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment; -import static org.labkey.api.files.FileContentService.UPLOADED_FILE; -import static org.labkey.api.util.FileUtil.toFileForRead; -import static org.labkey.api.util.FileUtil.toFileForWrite; - -public abstract class AbstractQueryUpdateService implements QueryUpdateService -{ - protected final TableInfo _queryTable; - - private boolean _bulkLoad = false; - private CaseInsensitiveHashMap _columnImportMap = null; - private VirtualFile _att = null; - - /* AbstractQueryUpdateService is generally responsible for some shared functionality - * - triggers - * - coercion/validation - * - detailed logging - * - attachments - * - * If a subclass wants to disable some of these features (w/o subclassing), put flags here... - */ - protected boolean _enableExistingRecordsDataIterator = true; - protected Set _previouslyUpdatedRows = new HashSet<>(); - - protected AbstractQueryUpdateService(TableInfo queryTable) - { - if (queryTable == null) - throw new IllegalArgumentException(); - _queryTable = queryTable; - } - - protected TableInfo getQueryTable() - { - return _queryTable; - } - - public @NotNull Set getPreviouslyUpdatedRows() - { - return _previouslyUpdatedRows == null ? new HashSet<>() : _previouslyUpdatedRows; - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) - { - return getQueryTable().hasPermission(user, acl); - } - - protected Map getRow(User user, Container container, Map keys, boolean allowCrossContainer) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - return getRow(user, container, keys); - } - - protected abstract Map getRow(User user, Container container, Map keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException; - - @Override - public List> getRows(User user, Container container, List> keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - - List> result = new ArrayList<>(); - for (Map rowKeys : keys) - { - Map row = getRow(user, container, rowKeys); - if (row != null) - result.add(row); - } - return result; - } - - @Override - public Map> getExistingRows(User user, Container container, Map> keys, boolean verifyNoCrossFolderData, boolean verifyExisting, @Nullable Set columns) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - - Map> result = new LinkedHashMap<>(); - for (Map.Entry> key : keys.entrySet()) - { - Map row = getRow(user, container, key.getValue(), verifyNoCrossFolderData); - if (row != null && !row.isEmpty()) - { - result.put(key.getKey(), row); - if (verifyNoCrossFolderData) - { - String dataContainer = (String) row.get("container"); - if (StringUtils.isEmpty(dataContainer)) - dataContainer = (String) row.get("folder"); - if (!container.getId().equals(dataContainer)) - throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values()); - } - } - else if (verifyExisting) - throw new InvalidKeyException("Data not found for " + key.getValue().values()); - } - return result; - } - - @Override - public boolean hasExistingRowsInOtherContainers(Container container, Map> keys) - { - return false; - } - - public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction) - { - long auditId = DbSequenceManager.get(ContainerManager.getRoot(), DB_SEQUENCE_NAME).next(); - return new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); - } - - public static void addTransactionAuditEvent(DbScope.Transaction transaction, User user, TransactionAuditProvider.TransactionAuditEvent auditEvent) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, ContainerManager.getRoot()); - - if (schema != null) - { - // This is a little hack to ensure that the audit table has actually been created and gets put into the table cache by the time the - // pre-commit task is executed. Otherwise, since the creation of the table happens while within the commit for the - // outermost transaction, it looks like there is a close that hasn't happened when trying to commit the transaction for creating the - // table. - schema.getTable(auditEvent.getEventType(), false); - - transaction.addCommitTask(() -> AuditLogService.get().addEvent(user, auditEvent), DbScope.CommitTaskOption.PRECOMMIT); - - transaction.setAuditEvent(auditEvent); - } - } - - protected final DataIteratorContext getDataIteratorContext(BatchValidationException errors, InsertOption forImport, Map configParameters) - { - if (null == errors) - errors = new BatchValidationException(); - DataIteratorContext context = new DataIteratorContext(errors); - context.setInsertOption(forImport); - context.setConfigParameters(configParameters); - configureDataIteratorContext(context); - return context; - } - - /** - * If QUS wants to use something other than PKs to select existing rows for merge, it can override this method. - * Used only for generating ExistingRecordDataIterator at the moment. - */ - protected Set getSelectKeys(DataIteratorContext context) - { - if (!context.getAlternateKeys().isEmpty()) - return context.getAlternateKeys(); - return null; - } - - /* - * construct the core DataIterator transformation pipeline for this table, may be just StandardDataIteratorBuilder. - * does NOT handle triggers or the insert/update iterator. - */ - public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) - { - DataIteratorBuilder dib = StandardDataIteratorBuilder.forInsert(getQueryTable(), data, container, user); - - if (_enableExistingRecordsDataIterator || context.getInsertOption().updateOnly) - { - // some tables need to generate PKs, so they need to add ExistingRecordDataIterator in persistRows() (after generating PK, before inserting) - dib = ExistingRecordDataIterator.createBuilder(dib, getQueryTable(), getSelectKeys(context)); - } - - dib = ((UpdateableTableInfo) getQueryTable()).persistRows(dib, context); - dib = AttachmentDataIterator.getAttachmentDataIteratorBuilder(getQueryTable(), dib, user, context.getInsertOption().batch ? getAttachmentDirectory() : null, container, getAttachmentParentFactory()); - dib = DetailedAuditLogDataIterator.getDataIteratorBuilder(getQueryTable(), dib, context.getInsertOption(), user, container, null); - return dib; - } - - - /** - * Implementation to use insertRows() while we migrate to using DIB for all code paths - *

- * DataIterator should/must use the same error collection as passed in - */ - @Deprecated - protected int _importRowsUsingInsertRows(User user, Container container, DataIterator rows, BatchValidationException errors, Map extraScriptContext) - { - MapDataIterator mapIterator = DataIteratorUtil.wrapMap(rows, true); - List> list = new ArrayList<>(); - List> ret; - Exception rowException; - - try - { - while (mapIterator.next()) - list.add(mapIterator.getMap()); - ret = insertRows(user, container, list, errors, null, extraScriptContext); - if (errors.hasErrors()) - return 0; - return ret.size(); - } - catch (BatchValidationException x) - { - assert x == errors; - assert x.hasErrors(); - return 0; - } - catch (QueryUpdateServiceException | DuplicateKeyException | SQLException x) - { - rowException = x; - } - finally - { - DataIteratorUtil.closeQuietly(mapIterator); - } - errors.addRowError(new ValidationException(rowException.getMessage())); - return 0; - } - - protected boolean hasImportRowsPermission(User user, Container container, DataIteratorContext context) - { - return hasPermission(user, context.getInsertOption().updateOnly ? UpdatePermission.class : InsertPermission.class); - } - - // override this - protected void preImportDIBValidation(@Nullable DataIteratorBuilder in, @Nullable Collection inputColumns) - { - } - - protected int _importRowsUsingDIB(User user, Container container, DataIteratorBuilder in, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasImportRowsPermission(user, container, context)) - throw new UnauthorizedException("You do not have permission to " + (context.getInsertOption().updateOnly ? "update data in this table." : "insert data into this table.")); - - if (!context.getConfigParameterBoolean(ConfigParameters.SkipInsertOptionValidation)) - assert(getQueryTable().supportsInsertOption(context.getInsertOption())); - - context.getErrors().setExtraContext(extraScriptContext); - if (extraScriptContext != null) - { - context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE)); - } - - preImportDIBValidation(in, null); - - boolean skipTriggers = context.getConfigParameterBoolean(ConfigParameters.SkipTriggers) || context.isCrossTypeImport() || context.isCrossFolderImport(); - boolean hasTableScript = hasTableScript(container); - TriggerDataBuilderHelper helper = new TriggerDataBuilderHelper(getQueryTable(), container, user, extraScriptContext, context.getInsertOption().useImportAliases); - if (!skipTriggers) - { - in = preTriggerDataIterator(in, context); - if (hasTableScript) - in = helper.before(in); - } - DataIteratorBuilder importDIB = createImportDIB(user, container, in, context); - DataIteratorBuilder out = importDIB; - - if (!skipTriggers) - { - if (hasTableScript) - out = helper.after(importDIB); - - out = postTriggerDataIterator(out, context); - } - - if (hasTableScript) - { - context.setFailFast(false); - context.setMaxRowErrors(Math.max(context.getMaxRowErrors(),1000)); - } - int count = _pump(out, outputRows, context); - - if (context.getErrors().hasErrors()) - return 0; - else - { - if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level - { - AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); - String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); - boolean skipAuditLevelCheck = false; - if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) - { - if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad - skipAuditLevelCheck = true; - } - getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); - } - return count; - } - } - - protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) - { - return in; - } - - protected DataIteratorBuilder postTriggerDataIterator(DataIteratorBuilder out, DataIteratorContext context) - { - return out; - } - - - /** this is extracted so subclasses can add wrap */ - protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList> rows, DataIteratorContext context) - { - DataIterator it = etl.getDataIterator(context); - - try - { - if (null != rows) - { - MapDataIterator maps = DataIteratorUtil.wrapMap(it, false); - it = new WrapperDataIterator(maps) - { - @Override - public boolean next() throws BatchValidationException - { - boolean ret = super.next(); - if (ret) - rows.add(((MapDataIterator)_delegate).getMap()); - return ret; - } - }; - } - - Pump pump = new Pump(it, context); - pump.run(); - - return pump.getRowCount(); - } - finally - { - DataIteratorUtil.closeQuietly(it); - } - } - - /* can be used for simple bookkeeping tasks, per row processing belongs in a data iterator */ - protected void afterInsertUpdate(int count, BatchValidationException errors, boolean isUpdate) - { - afterInsertUpdate(count, errors); - } - - protected void afterInsertUpdate(int count, BatchValidationException errors) - {} - - @Override - public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - return loadRows(user, container, rows, null, context, extraScriptContext); - } - - public int loadRows(User user, Container container, DataIteratorBuilder rows, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - configureDataIteratorContext(context); - int count = _importRowsUsingDIB(user, container, rows, outputRows, context, extraScriptContext); - afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); - return count; - } - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, @Nullable Map extraScriptContext) - { - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingInsertRows(user, container, rows.getDataIterator(context), errors, extraScriptContext); - afterInsertUpdate(count, errors, context.getInsertOption().updateOnly); - return count; - } - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - { - throw new UnsupportedOperationException("merge is not supported for all tables"); - } - - private boolean hasTableScript(Container container) - { - return getQueryTable().hasTriggers(container); - } - - - protected Map insertRow(User user, Container container, Map row) - throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException("Not implemented by this QueryUpdateService"); - } - - - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, InsertPermission.class)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); - } - - protected @Nullable List> _insertUpdateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - DataIteratorBuilder dib = _toDataIteratorBuilder(getClass().getSimpleName() + (context.getInsertOption().updateOnly ? ".updateRows" : ".insertRows()"), rows); - ArrayList> outputRows = new ArrayList<>(); - int count = _importRowsUsingDIB(user, container, dib, outputRows, context, extraScriptContext); - afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); - - if (context.getErrors().hasErrors()) - return null; - - return outputRows; - } - - // not yet supported - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); - } - - - protected DataIteratorBuilder _toDataIteratorBuilder(String debugName, List> rows) - { - // TODO probably can't assume all rows have all columns - // TODO can we assume that all rows refer to columns consistently? (not PTID and MouseId for the same column) - // TODO optimize ArrayListMap? - Set colNames; - - if (!rows.isEmpty() && rows.get(0) instanceof ArrayListMap) - { - colNames = ((ArrayListMap)rows.get(0)).getFindMap().keySet(); - } - else - { - // Preserve casing by using wrapped CaseInsensitiveHashMap instead of CaseInsensitiveHashSet - colNames = Sets.newCaseInsensitiveHashSet(); - for (Map row : rows) - colNames.addAll(row.keySet()); - } - - preImportDIBValidation(null, colNames); - return MapDataIterator.of(colNames, rows, debugName); - } - - - /** @deprecated switch to using DIB based method */ - @Deprecated - protected List> _insertRowsUsingInsertRow(User user, Container container, List> rows, BatchValidationException errors, Map extraScriptContext) - throws DuplicateKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, InsertPermission.class)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - assert(getQueryTable().supportsInsertOption(InsertOption.INSERT)); - - boolean hasTableScript = hasTableScript(container); - - errors.setExtraContext(extraScriptContext); - if (hasTableScript) - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, true, errors, extraScriptContext); - - List> result = new ArrayList<>(rows.size()); - List> providedValues = new ArrayList<>(rows.size()); - for (int i = 0; i < rows.size(); i++) - { - Map row = rows.get(i); - row = normalizeColumnNames(row); - try - { - providedValues.add(new CaseInsensitiveHashMap<>()); - row = coerceTypes(row, providedValues.get(i), false); - if (hasTableScript) - { - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, true, i, row, null, extraScriptContext); - } - row = insertRow(user, container, row); - if (row == null) - continue; - - if (hasTableScript) - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, false, i, row, null, extraScriptContext); - result.add(row); - } - catch (SQLException sqlx) - { - if (StringUtils.startsWith(sqlx.getSQLState(), "22") || RuntimeSQLException.isConstraintException(sqlx)) - { - ValidationException vex = new ValidationException(sqlx.getMessage()); - vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i+1); - errors.addRowError(vex); - } - else if (SqlDialect.isTransactionException(sqlx) && errors.hasErrors()) - { - // if we already have some errors, just break - break; - } - else - { - throw sqlx; - } - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - } - - if (hasTableScript) - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, false, errors, extraScriptContext); - - addAuditEvent(user, container, QueryService.AuditAction.INSERT, null, result, null, providedValues); - - return result; - } - - protected void addAuditEvent(User user, Container container, QueryService.AuditAction auditAction, @Nullable Map configParameters, @Nullable List> rows, @Nullable List> existingRows, @Nullable List> providedValues) - { - if (!isBulkLoad()) - { - AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(AuditBehavior) : null; - String userComment = configParameters == null ? null : (String) configParameters.get(AuditUserComment); - getQueryTable().getAuditHandler(auditBehavior) - .addAuditEvent(user, container, getQueryTable(), auditBehavior, userComment, auditAction, rows, existingRows, providedValues); - } - } - - private Map normalizeColumnNames(Map row) - { - if(_columnImportMap == null) - { - _columnImportMap = (CaseInsensitiveHashMap)ImportAliasable.Helper.createImportMap(getQueryTable().getColumns(), false); - } - - Map newRow = new CaseInsensitiveHashMap<>(); - CaseInsensitiveHashSet columns = new CaseInsensitiveHashSet(); - columns.addAll(row.keySet()); - - String newName; - for(String key : row.keySet()) - { - if(_columnImportMap.containsKey(key)) - { - //it is possible for a normalized name to conflict with an existing property. if so, defer to the original - newName = _columnImportMap.get(key).getName(); - if(!columns.contains(newName)){ - newRow.put(newName, row.get(key)); - continue; - } - } - newRow.put(key, row.get(key)); - } - - return newRow; - } - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws DuplicateKeyException, QueryUpdateServiceException, SQLException - { - try - { - List> ret = _insertRowsUsingInsertRow(user, container, rows, errors, extraScriptContext); - afterInsertUpdate(null==ret?0:ret.size(), errors); - if (errors.hasErrors()) - return null; - return ret; - } - catch (BatchValidationException x) - { - assert x == errors; - assert x.hasErrors(); - } - return null; - } - - - /** Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */ - @Deprecated - protected Map coerceTypes(Map row, Map providedValues, boolean isUpdate) - { - Map result = new CaseInsensitiveHashMap<>(row.size()); - Map columnMap = ImportAliasable.Helper.createImportMap(_queryTable.getColumns(), true); - for (Map.Entry entry : row.entrySet()) - { - ColumnInfo col = columnMap.get(entry.getKey()); - - Object value = entry.getValue(); - if (col != null && value != null && - !col.getJavaObjectClass().isInstance(value) && - !(value instanceof AttachmentFile) && - !(value instanceof MultipartFile) && - !(value instanceof String[]) && - !(col.getFk() instanceof MultiValuedForeignKey)) - { - try - { - if (PropertyType.FILE_LINK.equals(col.getPropertyType())) - value = ExpDataFileConverter.convert(value); - else if (col.getKindOfQuantity() != null) - { - providedValues.put(entry.getKey(), value); - value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit()); - } - else - value = col.getConvertFn().apply(value); - } - catch (ConvertHelper.FileConversionException e) - { - throw e; - } - catch (ConversionException e) - { - // That's OK, the transformation script may be able to fix up the value before it gets inserted - } - } - result.put(entry.getKey(), value); - } - return result; - } - - - protected abstract Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; - - - protected boolean firstUpdateRow = true; - Function,Map> updateTransform = Function.identity(); - - /* Do standard AQUS stuff here, then call the subclass specific implementation of updateRow() */ - final protected Map updateOneRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - if (firstUpdateRow) - { - firstUpdateRow = false; - if (null != OntologyService.get()) - { - var t = OntologyService.get().getConceptUpdateHandler(_queryTable); - if (null != t) - updateTransform = t; - } - } - row = updateTransform.apply(row); - return updateRow(user, container, row, oldRow, configParameters); - } - - // used by updateRows to check if all rows have the same set of keys - // prepared statement can only be used to updateRows if all rows have the same set of keys - protected boolean hasUniformKeys(List> rowsToUpdate) - { - if (rowsToUpdate == null || rowsToUpdate.isEmpty()) - return false; - - if (rowsToUpdate.size() == 1) - return true; - - Set keys = rowsToUpdate.get(0).keySet(); - int keySize = keys.size(); - - for (int i = 1 ; i < rowsToUpdate.size(); i ++) - { - Set otherKeys = rowsToUpdate.get(i).keySet(); - if (otherKeys.size() != keySize) - return false; - if (!otherKeys.containsAll(keys)) - return false; - } - - return true; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - if (oldKeys != null && rows.size() != oldKeys.size()) - throw new IllegalArgumentException("rows and oldKeys are required to be the same length, but were " + rows.size() + " and " + oldKeys + " in length, respectively"); - - assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); - - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, true, errors, extraScriptContext); - - List> result = new ArrayList<>(rows.size()); - List> oldRows = new ArrayList<>(rows.size()); - List> providedValues = new ArrayList<>(rows.size()); - // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container - boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; - - for (int i = 0; i < rows.size(); i++) - { - Map row = rows.get(i); - providedValues.add(new CaseInsensitiveHashMap<>()); - row = coerceTypes(row, providedValues.get(i), true); - try - { - Map oldKey = oldKeys == null ? row : oldKeys.get(i); - Map oldRow = null; - if (!streaming) - { - oldRow = getRow(user, container, oldKey); - if (oldRow == null) - throw new NotFoundException("The existing row was not found."); - } - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, true, i, row, oldRow, extraScriptContext); - Map updatedRow = updateOneRow(user, container, row, oldRow, configParameters); - if (!streaming && updatedRow == null) - continue; - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, false, i, updatedRow, oldRow, extraScriptContext); - if (!streaming) - { - result.add(updatedRow); - oldRows.add(oldRow); - } - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (OptimisticConflictException e) - { - errors.addRowError(new ValidationException("Unable to update. Row may have been deleted.")); - } - } - - // Fire triggers, if any, and also throw if there are any errors - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, false, errors, extraScriptContext); - afterInsertUpdate(null==result?0:result.size(), errors, true); - - if (errors.hasErrors()) - throw errors; - - addAuditEvent(user, container, QueryService.AuditAction.UPDATE, configParameters, result, oldRows, providedValues); - - return result; - } - - protected void checkDuplicateUpdate(Object pkVals) throws ValidationException - { - if (pkVals == null) - return; - - Set updatedRows = getPreviouslyUpdatedRows(); - - Object[] keysObj; - if (pkVals.getClass().isArray()) - keysObj = (Object[]) pkVals; - else if (pkVals instanceof Map map) - { - List orderedKeyVals = new ArrayList<>(); - SortedSet sortedKeys = new TreeSet<>(map.keySet()); - for (String key : sortedKeys) - orderedKeyVals.add(map.get(key)); - keysObj = orderedKeyVals.toArray(); - } - else - keysObj = new Object[]{pkVals}; - - if (keysObj.length == 1) - { - if (updatedRows.contains(keysObj[0])) - throw new ValidationException("Duplicate key provided: " + keysObj[0]); - updatedRows.add(keysObj[0]); - return; - } - - List keys = new ArrayList<>(); - for (Object key : keysObj) - keys.add(String.valueOf(key)); - if (updatedRows.contains(keys)) - throw new ValidationException("Duplicate key provided: " + StringUtils.join(keys, ", ")); - updatedRows.add(keys); - } - - @Override - public Map moveRows(User user, Container container, Container targetContainer, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException("Move is not supported for this table type."); - } - - protected abstract Map deleteRow(User user, Container container, Map oldRow) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; - - protected Map deleteRow(User user, Container container, Map oldRow, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - return deleteRow(user, container, oldRow); - } - - @Override - public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, DeletePermission.class)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - BatchValidationException errors = new BatchValidationException(); - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, true, errors, extraScriptContext); - - // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container - boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; - - List> result = new ArrayList<>(keys.size()); - for (int i = 0; i < keys.size(); i++) - { - Map key = keys.get(i); - try - { - Map oldRow = null; - if (!streaming) - { - oldRow = getRow(user, container, key); - // if row doesn't exist, bail early - if (oldRow == null) - continue; - } - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, true, i, null, oldRow, extraScriptContext); - Map updatedRow = deleteRow(user, container, oldRow, configParameters, extraScriptContext); - if (!streaming && updatedRow == null) - continue; - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, false, i, null, updatedRow, extraScriptContext); - result.add(updatedRow); - } - catch (InvalidKeyException ex) - { - ValidationException vex = new ValidationException(ex.getMessage()); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - } - - // Fire triggers, if any, and also throw if there are any errors - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, false, errors, extraScriptContext); - - addAuditEvent(user, container, QueryService.AuditAction.DELETE, configParameters, result, null, null); - - return result; - } - - protected int truncateRows(User user, Container container) - throws QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException(); - } - - @Override - public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!container.hasPermission(user, AdminPermission.class) && !hasPermission(user, DeletePermission.class)) - throw new UnauthorizedException("You do not have permission to truncate this table."); - - BatchValidationException errors = new BatchValidationException(); - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, true, errors, extraScriptContext); - - int result = truncateRows(user, container); - - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, false, errors, extraScriptContext); - addAuditEvent(user, container, QueryService.AuditAction.TRUNCATE, configParameters, null, null, null); - - return result; - } - - @Override - public void setBulkLoad(boolean bulkLoad) - { - _bulkLoad = bulkLoad; - } - - @Override - public boolean isBulkLoad() - { - return _bulkLoad; - } - - public static Object saveFile(User user, Container container, String name, Object value, @Nullable String dirName) throws ValidationException, QueryUpdateServiceException - { - FileLike dirPath = AssayFileWriter.getUploadDirectoryPath(container, dirName); - return saveFile(user, container, name, value, dirPath); - } - - /** - * Save uploaded file to dirName directory under file or pipeline root. - */ - public static Object saveFile(User user, Container container, String name, Object value, @Nullable FileLike dirPath) throws ValidationException, QueryUpdateServiceException - { - if (!(value instanceof MultipartFile) && !(value instanceof SpringAttachmentFile)) - throw new ValidationException("Invalid file value"); - - String auditMessageFormat = "Saved file '%s' for field '%s' in folder %s."; - FileLike file = null; - try - { - FileLike dir = AssayFileWriter.ensureUploadDirectory(dirPath); - - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(container, null); - if (value instanceof MultipartFile multipartFile) - { - // Once we've found one, write it to disk and replace the row's value with just the File reference to it - if (multipartFile.isEmpty()) - { - throw new ValidationException("File " + multipartFile.getOriginalFilename() + " for field " + name + " has no content"); - } - file = FileUtil.findUniqueFileName(multipartFile.getOriginalFilename(), dir); - checkFileUnderRoot(container, file); - multipartFile.transferTo(toFileForWrite(file)); - event.setComment(String.format(auditMessageFormat, multipartFile.getOriginalFilename(), name, container.getPath())); - event.setProvidedFileName(multipartFile.getOriginalFilename()); - } - else - { - SpringAttachmentFile saf = (SpringAttachmentFile) value; - file = FileUtil.findUniqueFileName(saf.getFilename(), dir); - checkFileUnderRoot(container, file); - saf.saveTo(file); - event.setComment(String.format(auditMessageFormat, saf.getFilename(), name, container.getPath())); - event.setProvidedFileName(saf.getFilename()); - } - event.setFile(file.getName()); - event.setFieldName(name); - event.setDirectory(file.getParent().toURI().getPath()); - AuditLogService.get().addEvent(user, event); - } - catch (IOException | ExperimentException e) - { - throw new QueryUpdateServiceException(e); - } - - ensureExpData(user, container, file.toNioPathForRead().toFile()); - return file; - } - - public static ExpData ensureExpData(User user, Container container, File file) - { - ExpData existingData = ExperimentService.get().getExpDataByURL(file, container); - // create exp.data record - if (existingData == null) - { - File canonicalFile = FileUtil.getAbsoluteCaseSensitiveFile(file); - ExpData data = ExperimentService.get().createData(container, UPLOADED_FILE); - data.setName(file.getName()); - data.setDataFileURI(canonicalFile.toPath().toUri()); - if (data.getDataFileUrl() != null && data.getDataFileUrl().length() <= ExperimentService.get().getTinfoData().getColumn("DataFileURL").getScale()) - { - // If the path is too long to store, bail out without creating an exp.data row - data.save(user); - } - - return data; - } - - return existingData; - } - - // For security reasons, make sure the user hasn't tried to reference a file that's not under - // the pipeline root or @assayfiles root. Otherwise, they could get access to any file on the server - static FileLike checkFileUnderRoot(Container container, FileLike file) throws ExperimentException - { - Path assayFilesRoot = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.assayfiles); - if (assayFilesRoot != null && URIUtil.isDescendant(assayFilesRoot.toUri(), file.toURI())) - return file; - - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null) - throw new ExperimentException("Pipeline root not available in container " + container.getPath()); - - if (!root.isUnderRoot(toFileForRead(file))) - { - throw new ExperimentException("Cannot reference file '" + file + "' from " + container.getPath()); - } - - return file; - } - - /** - * Is used by the AttachmentDataIterator to point to the location of the serialized - * attachment files. - */ - public void setAttachmentDirectory(VirtualFile att) - { - _att = att; - } - - @Nullable - protected VirtualFile getAttachmentDirectory() - { - return _att; - } - - /** - * QUS instances that allow import of attachments through the AttachmentDataIterator should furnish a factory - * implementation in order to resolve the attachment parent on incoming attachment files. - */ - @Nullable - protected AttachmentParentFactory getAttachmentParentFactory() - { - return null; - } - - /** Translate between the column name that query is exposing to the column name that actually lives in the database */ - protected static void aliasColumns(Map columnMapping, Map row) - { - for (Map.Entry entry : columnMapping.entrySet()) - { - if (row.containsKey(entry.getValue()) && !row.containsKey(entry.getKey())) - { - row.put(entry.getKey(), row.get(entry.getValue())); - } - } - } - - /** - * The database table has underscores for MV column names, but we expose a column without the underscore. - * Therefore, we need to translate between the two sets of column names. - * @return database column name -> exposed TableInfo column name - */ - protected static Map createMVMapping(Domain domain) - { - Map result = new CaseInsensitiveHashMap<>(); - if (domain != null) - { - for (DomainProperty domainProperty : domain.getProperties()) - { - if (domainProperty.isMvEnabled()) - { - result.put(PropertyStorageSpec.getMvIndicatorStorageColumnName(domainProperty.getPropertyDescriptor()), domainProperty.getName() + MvColumn.MV_INDICATOR_SUFFIX); - } - } - } - return result; - } - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert - { - private boolean _useAlias = false; - - static TabLoader getTestData() throws IOException - { - TabLoader testData = new TabLoader(new StringReader("pk,i,s\n0,0,zero\n1,1,one\n2,2,two"),true); - testData.parseAsCSV(); - testData.getColumns()[0].clazz = Integer.class; - testData.getColumns()[1].clazz = Integer.class; - testData.getColumns()[2].clazz = String.class; - return testData; - } - - @BeforeClass - public static void createList() throws Exception - { - if (null == ListService.get()) - return; - deleteList(); - - TabLoader testData = getTestData(); - String hash = GUID.makeHash(); - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - ListService s = ListService.get(); - UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); - assertNotNull(lists); - - ListDefinition R = s.createList(c, "R", ListDefinition.KeyType.Integer); - R.setKeyName("pk"); - Domain d = requireNonNull(R.getDomain()); - for (int i=0 ; i> getRows() - { - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); - TableInfo rTableInfo = requireNonNull(lists.getTable("R", null)); - return Arrays.asList(new TableSelector(rTableInfo, TableSelector.ALL_COLUMNS, null, new Sort("PK")).getMapArray()); - } - - @Before - public void resetList() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - qus.truncateRows(user, c, null, null); - } - - @AfterClass - public static void deleteList() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - ListService s = ListService.get(); - Map m = s.getLists(c); - if (m.containsKey("R")) - m.get("R").delete(user); - } - - void validateDefaultData(List> rows) - { - assertEquals(3, rows.size()); - - assertEquals(0, rows.get(0).get("pk")); - assertEquals(1, rows.get(1).get("pk")); - assertEquals(2, rows.get(2).get("pk")); - - assertEquals(0, rows.get(0).get("i")); - assertEquals(1, rows.get(1).get("i")); - assertEquals(2, rows.get(2).get("i")); - - assertEquals("zero", rows.get(0).get("s")); - assertEquals("one", rows.get(1).get("s")); - assertEquals("two", rows.get(2).get("s")); - } - - @Test - public void INSERT() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - assert(getRows().isEmpty()); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - BatchValidationException errors = new BatchValidationException(); - var rows = qus.insertRows(user, c, getTestData().load(), errors, null, null); - assertFalse(errors.hasErrors()); - validateDefaultData(rows); - validateDefaultData(getRows()); - - qus.insertRows(user, c, getTestData().load(), errors, null, null); - assertTrue(errors.hasErrors()); - } - - @Test - public void UPSERT() throws Exception - { - if (null == ListService.get()) - return; - /* not sure how you use/test ImportOptions.UPSERT - * the only row returning QUS method is insertRows(), which doesn't let you specify the InsertOption? - */ - } - - @Test - public void IMPORT() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - assert(getRows().isEmpty()); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - BatchValidationException errors = new BatchValidationException(); - var count = qus.importRows(user, c, getTestData(), errors, null, null); - assertFalse(errors.hasErrors()); - assert(count == 3); - validateDefaultData(getRows()); - - qus.importRows(user, c, getTestData(), errors, null, null); - assertTrue(errors.hasErrors()); - } - - @Test - public void MERGE() throws Exception - { - if (null == ListService.get()) - return; - INSERT(); - assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var mergeRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); - BatchValidationException errors = new BatchValidationException() - { - @Override - public void addRowError(ValidationException vex) - { - LogManager.getLogger(AbstractQueryUpdateService.class).error("test error", vex); - fail(vex.getMessage()); - } - }; - int count=0; - try (var tx = rTableInfo.getSchema().getScope().ensureTransaction()) - { - var ret = qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); - if (!errors.hasErrors()) - { - tx.commit(); - count = ret; - } - } - assertFalse("mergeRows error(s): " + errors.getMessage(), errors.hasErrors()); - assertEquals(2, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO", rows.get(2).get("s")); - // test existing row value is not updated - assertEquals(2, rows.get(2).get("i")); - // test new row - assertEquals("THREE", rows.get(3).get("s")); - assertNull(rows.get(3).get("i")); - - // merge should fail if duplicate keys are provided - errors = new BatchValidationException(); - mergeRows = new ArrayList<>(); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); - qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); - assertTrue(errors.hasErrors()); - assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); - } - - @Test - public void UPDATE() throws Exception - { - if (null == ListService.get()) - return; - INSERT(); - assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var updateRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - - // update using data iterator - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP")); - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(InsertOption.UPDATE); - var count = qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertFalse(context.getErrors().hasErrors()); - assertEquals(1, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO-UP", rows.get(2).get("s")); - // test existing row value is not updated/erased - assertEquals(2, rows.get(2).get("i")); - - // update should fail if a new record is provided - updateRows = new ArrayList<>(); - updateRows.add(CaseInsensitiveHashMap.of(pkName,123,colName,"NEW")); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertTrue(context.getErrors().hasErrors()); - - // Issue 52728: update should fail if duplicate key is provide - updateRows = new ArrayList<>(); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); - - // use DIB - context = new DataIteratorContext(); - context.setInsertOption(InsertOption.UPDATE); - qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertTrue(context.getErrors().hasErrors()); - assertTrue("Duplicate key error: " + context.getErrors().getMessage(), context.getErrors().getMessage().contains("Duplicate key provided: 2")); - - // use updateRows - if (!_useAlias) // _update using alias is not supported - { - BatchValidationException errors = new BatchValidationException(); - try - { - qus.updateRows(user, c, updateRows, null, errors, null, null); - } - catch (Exception e) - { - - } - assertTrue(errors.hasErrors()); - assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); - - } - } - - @Test - public void REPLACE() throws Exception - { - if (null == ListService.get()) - return; - assert(getRows().isEmpty()); - INSERT(); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var mergeRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(InsertOption.REPLACE); - var count = qus.loadRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), context, null); - assertFalse(context.getErrors().hasErrors()); - assertEquals(2, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO", rows.get(2).get("s")); - // test existing row value is updated - assertNull(rows.get(2).get("i")); - // test new row - assertEquals("THREE", rows.get(3).get("s")); - assertNull(rows.get(3).get("i")); - } - - @Test - public void IMPORT_IDENTITY() - { - if (null == ListService.get()) - return; - // TODO - } - - @Test - public void ALIAS_MERGE() throws Exception - { - _useAlias = true; - MERGE(); - } - - @Test - public void ALIAS_REPLACE() throws Exception - { - _useAlias = true; - REPLACE(); - } - - @Test - public void ALIAS_UPDATE() throws Exception - { - _useAlias = true; - UPDATE(); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.collections.ArrayListMap; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.MultiValuedForeignKey; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.AttachmentDataIterator; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.ExistingRecordDataIterator; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.dataiterator.Pump; +import org.labkey.api.dataiterator.StandardDataIteratorBuilder; +import org.labkey.api.dataiterator.TriggerDataBuilderHelper; +import org.labkey.api.dataiterator.WrapperDataIterator; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.reader.TabLoader; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URIUtil; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.vfs.FileLike; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.audit.TransactionAuditProvider.DB_SEQUENCE_NAME; +import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior; +import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment; +import static org.labkey.api.files.FileContentService.UPLOADED_FILE; +import static org.labkey.api.util.FileUtil.toFileForRead; +import static org.labkey.api.util.FileUtil.toFileForWrite; + +public abstract class AbstractQueryUpdateService implements QueryUpdateService +{ + protected final TableInfo _queryTable; + + private boolean _bulkLoad = false; + private CaseInsensitiveHashMap _columnImportMap = null; + private VirtualFile _att = null; + + /* AbstractQueryUpdateService is generally responsible for some shared functionality + * - triggers + * - coercion/validation + * - detailed logging + * - attachments + * + * If a subclass wants to disable some of these features (w/o subclassing), put flags here... + */ + protected boolean _enableExistingRecordsDataIterator = true; + protected Set _previouslyUpdatedRows = new HashSet<>(); + + protected AbstractQueryUpdateService(TableInfo queryTable) + { + if (queryTable == null) + throw new IllegalArgumentException(); + _queryTable = queryTable; + } + + protected TableInfo getQueryTable() + { + return _queryTable; + } + + public @NotNull Set getPreviouslyUpdatedRows() + { + return _previouslyUpdatedRows == null ? new HashSet<>() : _previouslyUpdatedRows; + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) + { + return getQueryTable().hasPermission(user, acl); + } + + protected Map getRow(User user, Container container, Map keys, boolean allowCrossContainer) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + return getRow(user, container, keys); + } + + protected abstract Map getRow(User user, Container container, Map keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException; + + @Override + public List> getRows(User user, Container container, List> keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + + List> result = new ArrayList<>(); + for (Map rowKeys : keys) + { + Map row = getRow(user, container, rowKeys); + if (row != null) + result.add(row); + } + return result; + } + + @Override + public Map> getExistingRows(User user, Container container, Map> keys, boolean verifyNoCrossFolderData, boolean verifyExisting, @Nullable Set columns) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + + Map> result = new LinkedHashMap<>(); + for (Map.Entry> key : keys.entrySet()) + { + Map row = getRow(user, container, key.getValue(), verifyNoCrossFolderData); + if (row != null && !row.isEmpty()) + { + result.put(key.getKey(), row); + if (verifyNoCrossFolderData) + { + String dataContainer = (String) row.get("container"); + if (StringUtils.isEmpty(dataContainer)) + dataContainer = (String) row.get("folder"); + if (!container.getId().equals(dataContainer)) + throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values()); + } + } + else if (verifyExisting) + throw new InvalidKeyException("Data not found for " + key.getValue().values()); + } + return result; + } + + @Override + public boolean hasExistingRowsInOtherContainers(Container container, Map> keys) + { + return false; + } + + public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction) + { + return createTransactionAuditEvent(container, auditAction, null); + } + + public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction, @Nullable Map details) + { + long auditId = DbSequenceManager.get(ContainerManager.getRoot(), DB_SEQUENCE_NAME).next(); + TransactionAuditProvider.TransactionAuditEvent event = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); + if (details != null) + event.addDetails(details); + return event; + } + + public static void addTransactionAuditEvent(DbScope.Transaction transaction, User user, TransactionAuditProvider.TransactionAuditEvent auditEvent) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, ContainerManager.getRoot()); + + if (schema != null) + { + // This is a little hack to ensure that the audit table has actually been created and gets put into the table cache by the time the + // pre-commit task is executed. Otherwise, since the creation of the table happens while within the commit for the + // outermost transaction, it looks like there is a close that hasn't happened when trying to commit the transaction for creating the + // table. + schema.getTable(auditEvent.getEventType(), false); + + transaction.addCommitTask(() -> AuditLogService.get().addEvent(user, auditEvent), DbScope.CommitTaskOption.PRECOMMIT); + + transaction.setAuditEvent(auditEvent); + } + } + + protected final DataIteratorContext getDataIteratorContext(BatchValidationException errors, InsertOption forImport, Map configParameters) + { + if (null == errors) + errors = new BatchValidationException(); + DataIteratorContext context = new DataIteratorContext(errors); + context.setInsertOption(forImport); + context.setConfigParameters(configParameters); + configureDataIteratorContext(context); + return context; + } + + /** + * If QUS wants to use something other than PKs to select existing rows for merge, it can override this method. + * Used only for generating ExistingRecordDataIterator at the moment. + */ + protected Set getSelectKeys(DataIteratorContext context) + { + if (!context.getAlternateKeys().isEmpty()) + return context.getAlternateKeys(); + return null; + } + + /* + * construct the core DataIterator transformation pipeline for this table, may be just StandardDataIteratorBuilder. + * does NOT handle triggers or the insert/update iterator. + */ + public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) + { + DataIteratorBuilder dib = StandardDataIteratorBuilder.forInsert(getQueryTable(), data, container, user); + + if (_enableExistingRecordsDataIterator || context.getInsertOption().updateOnly) + { + // some tables need to generate PKs, so they need to add ExistingRecordDataIterator in persistRows() (after generating PK, before inserting) + dib = ExistingRecordDataIterator.createBuilder(dib, getQueryTable(), getSelectKeys(context)); + } + + dib = ((UpdateableTableInfo) getQueryTable()).persistRows(dib, context); + dib = AttachmentDataIterator.getAttachmentDataIteratorBuilder(getQueryTable(), dib, user, context.getInsertOption().batch ? getAttachmentDirectory() : null, container, getAttachmentParentFactory()); + dib = DetailedAuditLogDataIterator.getDataIteratorBuilder(getQueryTable(), dib, context.getInsertOption(), user, container, null); + return dib; + } + + + /** + * Implementation to use insertRows() while we migrate to using DIB for all code paths + *

+ * DataIterator should/must use the same error collection as passed in + */ + @Deprecated + protected int _importRowsUsingInsertRows(User user, Container container, DataIterator rows, BatchValidationException errors, Map extraScriptContext) + { + MapDataIterator mapIterator = DataIteratorUtil.wrapMap(rows, true); + List> list = new ArrayList<>(); + List> ret; + Exception rowException; + + try + { + while (mapIterator.next()) + list.add(mapIterator.getMap()); + ret = insertRows(user, container, list, errors, null, extraScriptContext); + if (errors.hasErrors()) + return 0; + return ret.size(); + } + catch (BatchValidationException x) + { + assert x == errors; + assert x.hasErrors(); + return 0; + } + catch (QueryUpdateServiceException | DuplicateKeyException | SQLException x) + { + rowException = x; + } + finally + { + DataIteratorUtil.closeQuietly(mapIterator); + } + errors.addRowError(new ValidationException(rowException.getMessage())); + return 0; + } + + protected boolean hasImportRowsPermission(User user, Container container, DataIteratorContext context) + { + return hasPermission(user, context.getInsertOption().updateOnly ? UpdatePermission.class : InsertPermission.class); + } + + // override this + protected void preImportDIBValidation(@Nullable DataIteratorBuilder in, @Nullable Collection inputColumns) + { + } + + protected int _importRowsUsingDIB(User user, Container container, DataIteratorBuilder in, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasImportRowsPermission(user, container, context)) + throw new UnauthorizedException("You do not have permission to " + (context.getInsertOption().updateOnly ? "update data in this table." : "insert data into this table.")); + + if (!context.getConfigParameterBoolean(ConfigParameters.SkipInsertOptionValidation)) + assert(getQueryTable().supportsInsertOption(context.getInsertOption())); + + context.getErrors().setExtraContext(extraScriptContext); + if (extraScriptContext != null) + { + context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE)); + } + + preImportDIBValidation(in, null); + + boolean skipTriggers = context.getConfigParameterBoolean(ConfigParameters.SkipTriggers) || context.isCrossTypeImport() || context.isCrossFolderImport(); + boolean hasTableScript = hasTableScript(container); + TriggerDataBuilderHelper helper = new TriggerDataBuilderHelper(getQueryTable(), container, user, extraScriptContext, context.getInsertOption().useImportAliases); + if (!skipTriggers) + { + in = preTriggerDataIterator(in, context); + if (hasTableScript) + in = helper.before(in); + } + DataIteratorBuilder importDIB = createImportDIB(user, container, in, context); + DataIteratorBuilder out = importDIB; + + if (!skipTriggers) + { + if (hasTableScript) + out = helper.after(importDIB); + + out = postTriggerDataIterator(out, context); + } + + if (hasTableScript) + { + context.setFailFast(false); + context.setMaxRowErrors(Math.max(context.getMaxRowErrors(),1000)); + } + int count = _pump(out, outputRows, context); + + if (context.getErrors().hasErrors()) + return 0; + else + { + if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level + { + AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); + String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); + boolean skipAuditLevelCheck = false; + if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) + { + if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad + skipAuditLevelCheck = true; + } + getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); + } + return count; + } + } + + protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) + { + return in; + } + + protected DataIteratorBuilder postTriggerDataIterator(DataIteratorBuilder out, DataIteratorContext context) + { + return out; + } + + + /** this is extracted so subclasses can add wrap */ + protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList> rows, DataIteratorContext context) + { + DataIterator it = etl.getDataIterator(context); + + try + { + if (null != rows) + { + MapDataIterator maps = DataIteratorUtil.wrapMap(it, false); + it = new WrapperDataIterator(maps) + { + @Override + public boolean next() throws BatchValidationException + { + boolean ret = super.next(); + if (ret) + rows.add(((MapDataIterator)_delegate).getMap()); + return ret; + } + }; + } + + Pump pump = new Pump(it, context); + pump.run(); + + return pump.getRowCount(); + } + finally + { + DataIteratorUtil.closeQuietly(it); + } + } + + /* can be used for simple bookkeeping tasks, per row processing belongs in a data iterator */ + protected void afterInsertUpdate(int count, BatchValidationException errors, boolean isUpdate) + { + afterInsertUpdate(count, errors); + } + + protected void afterInsertUpdate(int count, BatchValidationException errors) + {} + + @Override + public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + return loadRows(user, container, rows, null, context, extraScriptContext); + } + + public int loadRows(User user, Container container, DataIteratorBuilder rows, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + configureDataIteratorContext(context); + int count = _importRowsUsingDIB(user, container, rows, outputRows, context, extraScriptContext); + afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); + return count; + } + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, @Nullable Map extraScriptContext) + { + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingInsertRows(user, container, rows.getDataIterator(context), errors, extraScriptContext); + afterInsertUpdate(count, errors, context.getInsertOption().updateOnly); + return count; + } + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + { + throw new UnsupportedOperationException("merge is not supported for all tables"); + } + + private boolean hasTableScript(Container container) + { + return getQueryTable().hasTriggers(container); + } + + + protected Map insertRow(User user, Container container, Map row) + throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException("Not implemented by this QueryUpdateService"); + } + + + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, InsertPermission.class)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); + } + + protected @Nullable List> _insertUpdateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + DataIteratorBuilder dib = _toDataIteratorBuilder(getClass().getSimpleName() + (context.getInsertOption().updateOnly ? ".updateRows" : ".insertRows()"), rows); + ArrayList> outputRows = new ArrayList<>(); + int count = _importRowsUsingDIB(user, container, dib, outputRows, context, extraScriptContext); + afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); + + if (context.getErrors().hasErrors()) + return null; + + return outputRows; + } + + // not yet supported + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); + } + + + protected DataIteratorBuilder _toDataIteratorBuilder(String debugName, List> rows) + { + // TODO probably can't assume all rows have all columns + // TODO can we assume that all rows refer to columns consistently? (not PTID and MouseId for the same column) + // TODO optimize ArrayListMap? + Set colNames; + + if (!rows.isEmpty() && rows.get(0) instanceof ArrayListMap) + { + colNames = ((ArrayListMap)rows.get(0)).getFindMap().keySet(); + } + else + { + // Preserve casing by using wrapped CaseInsensitiveHashMap instead of CaseInsensitiveHashSet + colNames = Sets.newCaseInsensitiveHashSet(); + for (Map row : rows) + colNames.addAll(row.keySet()); + } + + preImportDIBValidation(null, colNames); + return MapDataIterator.of(colNames, rows, debugName); + } + + + /** @deprecated switch to using DIB based method */ + @Deprecated + protected List> _insertRowsUsingInsertRow(User user, Container container, List> rows, BatchValidationException errors, Map extraScriptContext) + throws DuplicateKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, InsertPermission.class)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + assert(getQueryTable().supportsInsertOption(InsertOption.INSERT)); + + boolean hasTableScript = hasTableScript(container); + + errors.setExtraContext(extraScriptContext); + if (hasTableScript) + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, true, errors, extraScriptContext); + + List> result = new ArrayList<>(rows.size()); + List> providedValues = new ArrayList<>(rows.size()); + for (int i = 0; i < rows.size(); i++) + { + Map row = rows.get(i); + row = normalizeColumnNames(row); + try + { + providedValues.add(new CaseInsensitiveHashMap<>()); + row = coerceTypes(row, providedValues.get(i), false); + if (hasTableScript) + { + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, true, i, row, null, extraScriptContext); + } + row = insertRow(user, container, row); + if (row == null) + continue; + + if (hasTableScript) + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, false, i, row, null, extraScriptContext); + result.add(row); + } + catch (SQLException sqlx) + { + if (StringUtils.startsWith(sqlx.getSQLState(), "22") || RuntimeSQLException.isConstraintException(sqlx)) + { + ValidationException vex = new ValidationException(sqlx.getMessage()); + vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i+1); + errors.addRowError(vex); + } + else if (SqlDialect.isTransactionException(sqlx) && errors.hasErrors()) + { + // if we already have some errors, just break + break; + } + else + { + throw sqlx; + } + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + } + + if (hasTableScript) + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, false, errors, extraScriptContext); + + addAuditEvent(user, container, QueryService.AuditAction.INSERT, null, result, null, providedValues); + + return result; + } + + protected void addAuditEvent(User user, Container container, QueryService.AuditAction auditAction, @Nullable Map configParameters, @Nullable List> rows, @Nullable List> existingRows, @Nullable List> providedValues) + { + if (!isBulkLoad()) + { + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(AuditBehavior) : null; + String userComment = configParameters == null ? null : (String) configParameters.get(AuditUserComment); + getQueryTable().getAuditHandler(auditBehavior) + .addAuditEvent(user, container, getQueryTable(), auditBehavior, userComment, auditAction, rows, existingRows, providedValues); + } + } + + private Map normalizeColumnNames(Map row) + { + if(_columnImportMap == null) + { + _columnImportMap = (CaseInsensitiveHashMap)ImportAliasable.Helper.createImportMap(getQueryTable().getColumns(), false); + } + + Map newRow = new CaseInsensitiveHashMap<>(); + CaseInsensitiveHashSet columns = new CaseInsensitiveHashSet(); + columns.addAll(row.keySet()); + + String newName; + for(String key : row.keySet()) + { + if(_columnImportMap.containsKey(key)) + { + //it is possible for a normalized name to conflict with an existing property. if so, defer to the original + newName = _columnImportMap.get(key).getName(); + if(!columns.contains(newName)){ + newRow.put(newName, row.get(key)); + continue; + } + } + newRow.put(key, row.get(key)); + } + + return newRow; + } + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws DuplicateKeyException, QueryUpdateServiceException, SQLException + { + try + { + List> ret = _insertRowsUsingInsertRow(user, container, rows, errors, extraScriptContext); + afterInsertUpdate(null==ret?0:ret.size(), errors); + if (errors.hasErrors()) + return null; + return ret; + } + catch (BatchValidationException x) + { + assert x == errors; + assert x.hasErrors(); + } + return null; + } + + + /** Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */ + @Deprecated + protected Map coerceTypes(Map row, Map providedValues, boolean isUpdate) + { + Map result = new CaseInsensitiveHashMap<>(row.size()); + Map columnMap = ImportAliasable.Helper.createImportMap(_queryTable.getColumns(), true); + for (Map.Entry entry : row.entrySet()) + { + ColumnInfo col = columnMap.get(entry.getKey()); + + Object value = entry.getValue(); + if (col != null && value != null && + !col.getJavaObjectClass().isInstance(value) && + !(value instanceof AttachmentFile) && + !(value instanceof MultipartFile) && + !(value instanceof String[]) && + !(col.getFk() instanceof MultiValuedForeignKey)) + { + try + { + if (PropertyType.FILE_LINK.equals(col.getPropertyType())) + value = ExpDataFileConverter.convert(value); + else if (col.getKindOfQuantity() != null) + { + providedValues.put(entry.getKey(), value); + value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit()); + } + else + value = col.getConvertFn().apply(value); + } + catch (ConvertHelper.FileConversionException e) + { + throw e; + } + catch (ConversionException e) + { + // That's OK, the transformation script may be able to fix up the value before it gets inserted + } + } + result.put(entry.getKey(), value); + } + return result; + } + + + protected abstract Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; + + + protected boolean firstUpdateRow = true; + Function,Map> updateTransform = Function.identity(); + + /* Do standard AQUS stuff here, then call the subclass specific implementation of updateRow() */ + final protected Map updateOneRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + if (firstUpdateRow) + { + firstUpdateRow = false; + if (null != OntologyService.get()) + { + var t = OntologyService.get().getConceptUpdateHandler(_queryTable); + if (null != t) + updateTransform = t; + } + } + row = updateTransform.apply(row); + return updateRow(user, container, row, oldRow, configParameters); + } + + // used by updateRows to check if all rows have the same set of keys + // prepared statement can only be used to updateRows if all rows have the same set of keys + protected boolean hasUniformKeys(List> rowsToUpdate) + { + if (rowsToUpdate == null || rowsToUpdate.isEmpty()) + return false; + + if (rowsToUpdate.size() == 1) + return true; + + Set keys = rowsToUpdate.get(0).keySet(); + int keySize = keys.size(); + + for (int i = 1 ; i < rowsToUpdate.size(); i ++) + { + Set otherKeys = rowsToUpdate.get(i).keySet(); + if (otherKeys.size() != keySize) + return false; + if (!otherKeys.containsAll(keys)) + return false; + } + + return true; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + if (oldKeys != null && rows.size() != oldKeys.size()) + throw new IllegalArgumentException("rows and oldKeys are required to be the same length, but were " + rows.size() + " and " + oldKeys + " in length, respectively"); + + assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); + + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, true, errors, extraScriptContext); + + List> result = new ArrayList<>(rows.size()); + List> oldRows = new ArrayList<>(rows.size()); + List> providedValues = new ArrayList<>(rows.size()); + // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container + boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; + + for (int i = 0; i < rows.size(); i++) + { + Map row = rows.get(i); + providedValues.add(new CaseInsensitiveHashMap<>()); + row = coerceTypes(row, providedValues.get(i), true); + try + { + Map oldKey = oldKeys == null ? row : oldKeys.get(i); + Map oldRow = null; + if (!streaming) + { + oldRow = getRow(user, container, oldKey); + if (oldRow == null) + throw new NotFoundException("The existing row was not found."); + } + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, true, i, row, oldRow, extraScriptContext); + Map updatedRow = updateOneRow(user, container, row, oldRow, configParameters); + if (!streaming && updatedRow == null) + continue; + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, false, i, updatedRow, oldRow, extraScriptContext); + if (!streaming) + { + result.add(updatedRow); + oldRows.add(oldRow); + } + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (OptimisticConflictException e) + { + errors.addRowError(new ValidationException("Unable to update. Row may have been deleted.")); + } + } + + // Fire triggers, if any, and also throw if there are any errors + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, false, errors, extraScriptContext); + afterInsertUpdate(null==result?0:result.size(), errors, true); + + if (errors.hasErrors()) + throw errors; + + addAuditEvent(user, container, QueryService.AuditAction.UPDATE, configParameters, result, oldRows, providedValues); + + return result; + } + + protected void checkDuplicateUpdate(Object pkVals) throws ValidationException + { + if (pkVals == null) + return; + + Set updatedRows = getPreviouslyUpdatedRows(); + + Object[] keysObj; + if (pkVals.getClass().isArray()) + keysObj = (Object[]) pkVals; + else if (pkVals instanceof Map map) + { + List orderedKeyVals = new ArrayList<>(); + SortedSet sortedKeys = new TreeSet<>(map.keySet()); + for (String key : sortedKeys) + orderedKeyVals.add(map.get(key)); + keysObj = orderedKeyVals.toArray(); + } + else + keysObj = new Object[]{pkVals}; + + if (keysObj.length == 1) + { + if (updatedRows.contains(keysObj[0])) + throw new ValidationException("Duplicate key provided: " + keysObj[0]); + updatedRows.add(keysObj[0]); + return; + } + + List keys = new ArrayList<>(); + for (Object key : keysObj) + keys.add(String.valueOf(key)); + if (updatedRows.contains(keys)) + throw new ValidationException("Duplicate key provided: " + StringUtils.join(keys, ", ")); + updatedRows.add(keys); + } + + @Override + public Map moveRows(User user, Container container, Container targetContainer, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException("Move is not supported for this table type."); + } + + protected abstract Map deleteRow(User user, Container container, Map oldRow) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; + + protected Map deleteRow(User user, Container container, Map oldRow, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + return deleteRow(user, container, oldRow); + } + + @Override + public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, DeletePermission.class)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + BatchValidationException errors = new BatchValidationException(); + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, true, errors, extraScriptContext); + + // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container + boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; + + List> result = new ArrayList<>(keys.size()); + for (int i = 0; i < keys.size(); i++) + { + Map key = keys.get(i); + try + { + Map oldRow = null; + if (!streaming) + { + oldRow = getRow(user, container, key); + // if row doesn't exist, bail early + if (oldRow == null) + continue; + } + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, true, i, null, oldRow, extraScriptContext); + Map updatedRow = deleteRow(user, container, oldRow, configParameters, extraScriptContext); + if (!streaming && updatedRow == null) + continue; + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, false, i, null, updatedRow, extraScriptContext); + result.add(updatedRow); + } + catch (InvalidKeyException ex) + { + ValidationException vex = new ValidationException(ex.getMessage()); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + } + + // Fire triggers, if any, and also throw if there are any errors + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, false, errors, extraScriptContext); + + addAuditEvent(user, container, QueryService.AuditAction.DELETE, configParameters, result, null, null); + + return result; + } + + protected int truncateRows(User user, Container container) + throws QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException(); + } + + @Override + public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!container.hasPermission(user, AdminPermission.class) && !hasPermission(user, DeletePermission.class)) + throw new UnauthorizedException("You do not have permission to truncate this table."); + + BatchValidationException errors = new BatchValidationException(); + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, true, errors, extraScriptContext); + + int result = truncateRows(user, container); + + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, false, errors, extraScriptContext); + addAuditEvent(user, container, QueryService.AuditAction.TRUNCATE, configParameters, null, null, null); + + return result; + } + + @Override + public void setBulkLoad(boolean bulkLoad) + { + _bulkLoad = bulkLoad; + } + + @Override + public boolean isBulkLoad() + { + return _bulkLoad; + } + + public static Object saveFile(User user, Container container, String name, Object value, @Nullable String dirName) throws ValidationException, QueryUpdateServiceException + { + FileLike dirPath = AssayFileWriter.getUploadDirectoryPath(container, dirName); + return saveFile(user, container, name, value, dirPath); + } + + /** + * Save uploaded file to dirName directory under file or pipeline root. + */ + public static Object saveFile(User user, Container container, String name, Object value, @Nullable FileLike dirPath) throws ValidationException, QueryUpdateServiceException + { + if (!(value instanceof MultipartFile) && !(value instanceof SpringAttachmentFile)) + throw new ValidationException("Invalid file value"); + + String auditMessageFormat = "Saved file '%s' for field '%s' in folder %s."; + FileLike file = null; + try + { + FileLike dir = AssayFileWriter.ensureUploadDirectory(dirPath); + + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(container, null); + if (value instanceof MultipartFile multipartFile) + { + // Once we've found one, write it to disk and replace the row's value with just the File reference to it + if (multipartFile.isEmpty()) + { + throw new ValidationException("File " + multipartFile.getOriginalFilename() + " for field " + name + " has no content"); + } + file = FileUtil.findUniqueFileName(multipartFile.getOriginalFilename(), dir); + checkFileUnderRoot(container, file); + multipartFile.transferTo(toFileForWrite(file)); + event.setComment(String.format(auditMessageFormat, multipartFile.getOriginalFilename(), name, container.getPath())); + event.setProvidedFileName(multipartFile.getOriginalFilename()); + } + else + { + SpringAttachmentFile saf = (SpringAttachmentFile) value; + file = FileUtil.findUniqueFileName(saf.getFilename(), dir); + checkFileUnderRoot(container, file); + saf.saveTo(file); + event.setComment(String.format(auditMessageFormat, saf.getFilename(), name, container.getPath())); + event.setProvidedFileName(saf.getFilename()); + } + event.setFile(file.getName()); + event.setFieldName(name); + event.setDirectory(file.getParent().toURI().getPath()); + AuditLogService.get().addEvent(user, event); + } + catch (IOException | ExperimentException e) + { + throw new QueryUpdateServiceException(e); + } + + ensureExpData(user, container, file.toNioPathForRead().toFile()); + return file; + } + + public static ExpData ensureExpData(User user, Container container, File file) + { + ExpData existingData = ExperimentService.get().getExpDataByURL(file, container); + // create exp.data record + if (existingData == null) + { + File canonicalFile = FileUtil.getAbsoluteCaseSensitiveFile(file); + ExpData data = ExperimentService.get().createData(container, UPLOADED_FILE); + data.setName(file.getName()); + data.setDataFileURI(canonicalFile.toPath().toUri()); + if (data.getDataFileUrl() != null && data.getDataFileUrl().length() <= ExperimentService.get().getTinfoData().getColumn("DataFileURL").getScale()) + { + // If the path is too long to store, bail out without creating an exp.data row + data.save(user); + } + + return data; + } + + return existingData; + } + + // For security reasons, make sure the user hasn't tried to reference a file that's not under + // the pipeline root or @assayfiles root. Otherwise, they could get access to any file on the server + static FileLike checkFileUnderRoot(Container container, FileLike file) throws ExperimentException + { + Path assayFilesRoot = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.assayfiles); + if (assayFilesRoot != null && URIUtil.isDescendant(assayFilesRoot.toUri(), file.toURI())) + return file; + + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null) + throw new ExperimentException("Pipeline root not available in container " + container.getPath()); + + if (!root.isUnderRoot(toFileForRead(file))) + { + throw new ExperimentException("Cannot reference file '" + file + "' from " + container.getPath()); + } + + return file; + } + + /** + * Is used by the AttachmentDataIterator to point to the location of the serialized + * attachment files. + */ + public void setAttachmentDirectory(VirtualFile att) + { + _att = att; + } + + @Nullable + protected VirtualFile getAttachmentDirectory() + { + return _att; + } + + /** + * QUS instances that allow import of attachments through the AttachmentDataIterator should furnish a factory + * implementation in order to resolve the attachment parent on incoming attachment files. + */ + @Nullable + protected AttachmentParentFactory getAttachmentParentFactory() + { + return null; + } + + /** Translate between the column name that query is exposing to the column name that actually lives in the database */ + protected static void aliasColumns(Map columnMapping, Map row) + { + for (Map.Entry entry : columnMapping.entrySet()) + { + if (row.containsKey(entry.getValue()) && !row.containsKey(entry.getKey())) + { + row.put(entry.getKey(), row.get(entry.getValue())); + } + } + } + + /** + * The database table has underscores for MV column names, but we expose a column without the underscore. + * Therefore, we need to translate between the two sets of column names. + * @return database column name -> exposed TableInfo column name + */ + protected static Map createMVMapping(Domain domain) + { + Map result = new CaseInsensitiveHashMap<>(); + if (domain != null) + { + for (DomainProperty domainProperty : domain.getProperties()) + { + if (domainProperty.isMvEnabled()) + { + result.put(PropertyStorageSpec.getMvIndicatorStorageColumnName(domainProperty.getPropertyDescriptor()), domainProperty.getName() + MvColumn.MV_INDICATOR_SUFFIX); + } + } + } + return result; + } + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert + { + private boolean _useAlias = false; + + static TabLoader getTestData() throws IOException + { + TabLoader testData = new TabLoader(new StringReader("pk,i,s\n0,0,zero\n1,1,one\n2,2,two"),true); + testData.parseAsCSV(); + testData.getColumns()[0].clazz = Integer.class; + testData.getColumns()[1].clazz = Integer.class; + testData.getColumns()[2].clazz = String.class; + return testData; + } + + @BeforeClass + public static void createList() throws Exception + { + if (null == ListService.get()) + return; + deleteList(); + + TabLoader testData = getTestData(); + String hash = GUID.makeHash(); + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + ListService s = ListService.get(); + UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); + assertNotNull(lists); + + ListDefinition R = s.createList(c, "R", ListDefinition.KeyType.Integer); + R.setKeyName("pk"); + Domain d = requireNonNull(R.getDomain()); + for (int i=0 ; i> getRows() + { + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); + TableInfo rTableInfo = requireNonNull(lists.getTable("R", null)); + return Arrays.asList(new TableSelector(rTableInfo, TableSelector.ALL_COLUMNS, null, new Sort("PK")).getMapArray()); + } + + @Before + public void resetList() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + qus.truncateRows(user, c, null, null); + } + + @AfterClass + public static void deleteList() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + ListService s = ListService.get(); + Map m = s.getLists(c); + if (m.containsKey("R")) + m.get("R").delete(user); + } + + void validateDefaultData(List> rows) + { + assertEquals(3, rows.size()); + + assertEquals(0, rows.get(0).get("pk")); + assertEquals(1, rows.get(1).get("pk")); + assertEquals(2, rows.get(2).get("pk")); + + assertEquals(0, rows.get(0).get("i")); + assertEquals(1, rows.get(1).get("i")); + assertEquals(2, rows.get(2).get("i")); + + assertEquals("zero", rows.get(0).get("s")); + assertEquals("one", rows.get(1).get("s")); + assertEquals("two", rows.get(2).get("s")); + } + + @Test + public void INSERT() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + assert(getRows().isEmpty()); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + BatchValidationException errors = new BatchValidationException(); + var rows = qus.insertRows(user, c, getTestData().load(), errors, null, null); + assertFalse(errors.hasErrors()); + validateDefaultData(rows); + validateDefaultData(getRows()); + + qus.insertRows(user, c, getTestData().load(), errors, null, null); + assertTrue(errors.hasErrors()); + } + + @Test + public void UPSERT() throws Exception + { + if (null == ListService.get()) + return; + /* not sure how you use/test ImportOptions.UPSERT + * the only row returning QUS method is insertRows(), which doesn't let you specify the InsertOption? + */ + } + + @Test + public void IMPORT() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + assert(getRows().isEmpty()); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + BatchValidationException errors = new BatchValidationException(); + var count = qus.importRows(user, c, getTestData(), errors, null, null); + assertFalse(errors.hasErrors()); + assert(count == 3); + validateDefaultData(getRows()); + + qus.importRows(user, c, getTestData(), errors, null, null); + assertTrue(errors.hasErrors()); + } + + @Test + public void MERGE() throws Exception + { + if (null == ListService.get()) + return; + INSERT(); + assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var mergeRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); + BatchValidationException errors = new BatchValidationException() + { + @Override + public void addRowError(ValidationException vex) + { + LogManager.getLogger(AbstractQueryUpdateService.class).error("test error", vex); + fail(vex.getMessage()); + } + }; + int count=0; + try (var tx = rTableInfo.getSchema().getScope().ensureTransaction()) + { + var ret = qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); + if (!errors.hasErrors()) + { + tx.commit(); + count = ret; + } + } + assertFalse("mergeRows error(s): " + errors.getMessage(), errors.hasErrors()); + assertEquals(2, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO", rows.get(2).get("s")); + // test existing row value is not updated + assertEquals(2, rows.get(2).get("i")); + // test new row + assertEquals("THREE", rows.get(3).get("s")); + assertNull(rows.get(3).get("i")); + + // merge should fail if duplicate keys are provided + errors = new BatchValidationException(); + mergeRows = new ArrayList<>(); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); + qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); + assertTrue(errors.hasErrors()); + assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); + } + + @Test + public void UPDATE() throws Exception + { + if (null == ListService.get()) + return; + INSERT(); + assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var updateRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + + // update using data iterator + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP")); + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(InsertOption.UPDATE); + var count = qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertFalse(context.getErrors().hasErrors()); + assertEquals(1, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO-UP", rows.get(2).get("s")); + // test existing row value is not updated/erased + assertEquals(2, rows.get(2).get("i")); + + // update should fail if a new record is provided + updateRows = new ArrayList<>(); + updateRows.add(CaseInsensitiveHashMap.of(pkName,123,colName,"NEW")); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertTrue(context.getErrors().hasErrors()); + + // Issue 52728: update should fail if duplicate key is provide + updateRows = new ArrayList<>(); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); + + // use DIB + context = new DataIteratorContext(); + context.setInsertOption(InsertOption.UPDATE); + qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertTrue(context.getErrors().hasErrors()); + assertTrue("Duplicate key error: " + context.getErrors().getMessage(), context.getErrors().getMessage().contains("Duplicate key provided: 2")); + + // use updateRows + if (!_useAlias) // _update using alias is not supported + { + BatchValidationException errors = new BatchValidationException(); + try + { + qus.updateRows(user, c, updateRows, null, errors, null, null); + } + catch (Exception e) + { + + } + assertTrue(errors.hasErrors()); + assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); + + } + } + + @Test + public void REPLACE() throws Exception + { + if (null == ListService.get()) + return; + assert(getRows().isEmpty()); + INSERT(); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var mergeRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(InsertOption.REPLACE); + var count = qus.loadRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), context, null); + assertFalse(context.getErrors().hasErrors()); + assertEquals(2, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO", rows.get(2).get("s")); + // test existing row value is updated + assertNull(rows.get(2).get("i")); + // test new row + assertEquals("THREE", rows.get(3).get("s")); + assertNull(rows.get(3).get("i")); + } + + @Test + public void IMPORT_IDENTITY() + { + if (null == ListService.get()) + return; + // TODO + } + + @Test + public void ALIAS_MERGE() throws Exception + { + _useAlias = true; + MERGE(); + } + + @Test + public void ALIAS_REPLACE() throws Exception + { + _useAlias = true; + REPLACE(); + } + + @Test + public void ALIAS_UPDATE() throws Exception + { + _useAlias = true; + UPDATE(); + } + } +} diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index 3534b4a8c70..a4820826fab 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -67,6 +67,8 @@ public static class QueryImportAsyncContextBuilder String _jobNotificationProvider; + Map _transactionDetails; + public QueryImportAsyncContextBuilder() { @@ -233,6 +235,17 @@ public QueryImportAsyncContextBuilder setJobDescription(String jobDescription) return this; } + public QueryImportAsyncContextBuilder setTransactionDetails(Map transactionDetails) + { + _transactionDetails = transactionDetails; + return this; + } + + public Map getTransactionDetails() + { + return _transactionDetails; + } + } @Override @@ -300,7 +313,7 @@ public void run() TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (diContext.isCrossTypeImport() || (_importContextBuilder.getAuditBehaviorType() != null && _importContextBuilder.getAuditBehaviorType() != AuditBehaviorType.NONE)) - auditEvent = createTransactionAuditEvent(getContainer(), diContext.getInsertOption().auditAction); + auditEvent = createTransactionAuditEvent(getContainer(), diContext.getInsertOption().auditAction, _importContextBuilder.getTransactionDetails()); int importedCount = AbstractQueryImportAction.importData(loader, target, updateService, diContext, auditEvent, getInfo().getUser(), getInfo().getContainer()); diff --git a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java index 30240d4c353..456ba2298e1 100644 --- a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java +++ b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.DatabaseTableType; @@ -61,6 +62,8 @@ public SimpleQueryUpdateService(final SimpleTable queryTable, TableInfo dbTable, @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.IMPORT, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -69,6 +72,8 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -77,6 +82,8 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B @Override public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws DuplicateKeyException, QueryUpdateServiceException, SQLException { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); List> result = super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); afterInsertUpdate(result == null ? 0 : result.size(), errors); return result; @@ -138,6 +145,9 @@ public List> updateRows(User user, Container container, List { if (shouldUpdateUsingDIB(container, rows, oldKeys, configParameters)) { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, configParameters); context.putConfigParameter(PreferPKOverObjectUriAsKey, shouldPreferPKOverObjectUriAsUpdateKey(rows)); List> result = super._updateRowsUsingDIB(user, container, rows, context, extraScriptContext); diff --git a/api/src/org/labkey/api/query/UserSchemaAction.java b/api/src/org/labkey/api/query/UserSchemaAction.java index 12f94df02c1..3479b23bc72 100644 --- a/api/src/org/labkey/api/query/UserSchemaAction.java +++ b/api/src/org/labkey/api/query/UserSchemaAction.java @@ -1,335 +1,335 @@ -/* - * Copyright (c) 2012-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.NullSafeBindException; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.data.ActionButton; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.TableInfo; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.springframework.beans.PropertyValues; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.multipart.MultipartFile; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: 9/29/12 - */ -public abstract class UserSchemaAction extends FormViewAction -{ - protected QueryForm _form; - protected UserSchema _schema; - protected TableInfo _table; - - @Override - public BindException bindParameters(PropertyValues m) throws Exception - { - _form = createQueryForm(getViewContext()); - _schema = _form.getSchema(); - if (null == _schema) - { - throw new NotFoundException("Schema not found"); - } - _table = _schema.getTable(_form.getQueryName(), getBindParametersContainerFilter(), true, true); - if (null == _table) - { - throw new NotFoundException("Query not found"); - } - QueryUpdateForm command = new QueryUpdateForm(_table, getViewContext(), null); - if (command.isBulkUpdate()) - command.setValidateRequired(false); - BindException errors = new NullSafeBindException(new BeanUtilsPropertyBindingResult(command, "form")); - command.validateBind(errors); - return errors; - } - - // This ContainerFilter is applied to the underlying table that backs this UserSchemaAction. - // As a result all lookup fields, that respect container filters, in these views will populate - // with this container filter applied. - protected @Nullable ContainerFilter getBindParametersContainerFilter() - { - return QueryService.get().getContainerFilterForLookups(getContainer(), getUser()); - } - - protected QueryForm createQueryForm(ViewContext context) - { - QueryForm form = new QueryForm(); - form.setViewContext(getViewContext()); - form.bindParameters(getViewContext().getBindPropertyValues()); - - return form; - } - - @Override - public void validateCommand(QueryUpdateForm target, Errors errors) - { - } - - protected ButtonBar createSubmitCancelButtonBar(QueryUpdateForm form) - { - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - - ActionButton btnSubmit = new ActionButton(getViewContext().getActionURL(), "Submit") - .setActionType(ActionButton.Action.POST) - .setDisableOnClick(true); - - return bb.add( - btnSubmit, - new ButtonBuilder("Cancel").href(getCancelURL(form)).build() - ); - } - - private ActionURL getActionURLParam(ActionURL.Param param) - { - String url = getViewContext().getActionURL().getParameter(param); - if (url != null) - { - try - { - return new ActionURL(url); - } - catch (IllegalArgumentException ignored) {} - } - return null; - } - - /* - * NOTE (MAB) UserSchemaAction.appendNavTrail() uses getSuccessURL(null) for the nav trail link. - * That's not really right, since the success url and the back/cancel url could be different. - * - * I changed getSuccessURL(null) to return cancelUrl if it is provided. - */ - @Override - public ActionURL getSuccessURL(QueryUpdateForm form) - { - return resolveReturnUrl(form == null ? getActionURLParam(ActionURL.Param.cancelUrl) : null, form); - } - - private ActionURL resolveReturnUrl(@Nullable ActionURL returnUrl, QueryUpdateForm form) - { - if (null == returnUrl) - returnUrl = getActionURLParam(ActionURL.Param.returnUrl); - if (null == returnUrl) - { - if (_schema != null && _table != null) - returnUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); - else - returnUrl = QueryService.get().urlDefault(form.getContainer(), QueryAction.executeQuery, null, null); - } - return returnUrl; - } - - public ActionURL getCancelURL(QueryUpdateForm form) - { - ActionURL cancelURL = getActionURLParam(ActionURL.Param.cancelUrl); - return resolveReturnUrl(cancelURL, form); - } - - @Override - public void addNavTrail(NavTree root) - { - if (_table != null) - root.addChild(_table.getName(), getSuccessURL(null)); - } - - protected List> doInsertUpdate(QueryUpdateForm form, BindException errors, boolean insert) - { - TableInfo table = form.getTable(); - if (!table.hasPermission(form.getUser(), insert ? InsertPermission.class : UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - Map values = form.getTypedColumns(); - - // Allow for attachment-based columns - Map fileMap = getFileMap(); - if (null != fileMap) - { - for (String key : fileMap.keySet()) - { - // Check if the column has already been processed - if (!values.containsKey(key)) - { - SpringAttachmentFile file = new SpringAttachmentFile(fileMap.get(key)); - form.setTypedValue(key, file.isEmpty() ? null : file); - } - } - } - - values = form.getTypedColumns(); - - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - throw new IllegalArgumentException("The query '" + _table.getName() + "' in the schema '" + _schema.getName() + "' is not updatable."); - - - List> rows; - List> ret = null; - - if (form.isBulkUpdate()) - { - rows = new ArrayList<>(); - - // Merge the bulk edits back into the selected rows + validate - String[] pkValues = form.getSelectedRows(); - - if (pkValues == null || pkValues.length == 0) - errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Please reselect the rows to update."); - else if (table.getPkColumnNames().size() > 1) - { - errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Does not support update for multi-keyed tables."); - } - else - { - Map row; - String pkName = table.getPkColumnNames().get(0); - for (String pkValue : pkValues) - { - row = new CaseInsensitiveHashMap<>(); - for (Map.Entry entry : values.entrySet()) - { - // If a value is left as null it is considered untouched for a given row - if (entry.getValue() != null) - row.put(entry.getKey(), entry.getValue()); - } - - row.put(pkName, pkValue); - rows.add(row); - } - } - } - else - { - rows = Collections.singletonList(values); - } - - DbSchema dbschema = table.getSchema(); - try - { - try (DbScope.Transaction transaction = dbschema.getScope().ensureTransaction()) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; - QueryService.AuditAction auditAction = insert ? QueryService.AuditAction.INSERT : QueryService.AuditAction.UPDATE; - - // transaction audit - BatchValidationException batchErrors = new BatchValidationException(); - - AuditBehaviorType auditBehaviorType = table.getEffectiveAuditBehavior(); - if (auditBehaviorType != AuditBehaviorType.NONE) - { - if (transaction.getAuditEvent() != null) - auditEvent = transaction.getAuditEvent(); - else - { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); - } - } - - if (insert) - { - ret = qus.insertRows(form.getUser(), form.getContainer(), rows, batchErrors, null, null); - } - else - { - // Currently, bulkUpdate doesn't support oldValues due to the need to re-query... - if (form.isBulkUpdate()) - { - ret = qus.updateRows(form.getUser(), form.getContainer(), rows, null, batchErrors, null, null); - } - else - { - Map oldValues = null; - if (form.getOldValues() instanceof Map) - { - oldValues = (Map) form.getOldValues(); - if (!(oldValues instanceof CaseInsensitiveMapWrapper)) - oldValues = new CaseInsensitiveMapWrapper<>(oldValues); - } - - // 18292 - updateRows expects a null list in the case of an "empty" or null map. - List> oldKeys = (oldValues == null || oldValues.isEmpty()) ? null : Collections.singletonList(oldValues); - ret = qus.updateRows(form.getUser(), form.getContainer(), rows, oldKeys, batchErrors, null, null); - } - } - if (batchErrors.hasErrors()) - batchErrors.addToErrors(errors); - - if (!errors.hasErrors()) - { - if (auditEvent != null) - auditEvent.addComment(auditAction, ret.size()); - transaction.commit(); // Only commit if there were no errors - } - - } - catch (SQLException x) - { - if (!RuntimeSQLException.isConstraintException(x)) - throw x; - errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); - } - catch (InvalidKeyException | DuplicateKeyException | DataIntegrityViolationException | RuntimeSQLException x) - { - errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); - } - catch (BatchValidationException x) - { - x.addToErrors(errors); - } - return ret; - } - catch (Exception x) - { - // Do this in a separate, outer try/catch so that we will have already committed or rolled back - // the transaction we started. Otherwise, our database connection is likely in a bad state and can't be - // reused when submitting the exception report. - errors.reject(SpringActionController.ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); - return null; - } - } -} +/* + * Copyright (c) 2012-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.NullSafeBindException; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.data.ActionButton; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.TableInfo; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.springframework.beans.PropertyValues; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: 9/29/12 + */ +public abstract class UserSchemaAction extends FormViewAction +{ + protected QueryForm _form; + protected UserSchema _schema; + protected TableInfo _table; + + @Override + public BindException bindParameters(PropertyValues m) throws Exception + { + _form = createQueryForm(getViewContext()); + _schema = _form.getSchema(); + if (null == _schema) + { + throw new NotFoundException("Schema not found"); + } + _table = _schema.getTable(_form.getQueryName(), getBindParametersContainerFilter(), true, true); + if (null == _table) + { + throw new NotFoundException("Query not found"); + } + QueryUpdateForm command = new QueryUpdateForm(_table, getViewContext(), null); + if (command.isBulkUpdate()) + command.setValidateRequired(false); + BindException errors = new NullSafeBindException(new BeanUtilsPropertyBindingResult(command, "form")); + command.validateBind(errors); + return errors; + } + + // This ContainerFilter is applied to the underlying table that backs this UserSchemaAction. + // As a result all lookup fields, that respect container filters, in these views will populate + // with this container filter applied. + protected @Nullable ContainerFilter getBindParametersContainerFilter() + { + return QueryService.get().getContainerFilterForLookups(getContainer(), getUser()); + } + + protected QueryForm createQueryForm(ViewContext context) + { + QueryForm form = new QueryForm(); + form.setViewContext(getViewContext()); + form.bindParameters(getViewContext().getBindPropertyValues()); + + return form; + } + + @Override + public void validateCommand(QueryUpdateForm target, Errors errors) + { + } + + protected ButtonBar createSubmitCancelButtonBar(QueryUpdateForm form) + { + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + + ActionButton btnSubmit = new ActionButton(getViewContext().getActionURL(), "Submit") + .setActionType(ActionButton.Action.POST) + .setDisableOnClick(true); + + return bb.add( + btnSubmit, + new ButtonBuilder("Cancel").href(getCancelURL(form)).build() + ); + } + + private ActionURL getActionURLParam(ActionURL.Param param) + { + String url = getViewContext().getActionURL().getParameter(param); + if (url != null) + { + try + { + return new ActionURL(url); + } + catch (IllegalArgumentException ignored) {} + } + return null; + } + + /* + * NOTE (MAB) UserSchemaAction.appendNavTrail() uses getSuccessURL(null) for the nav trail link. + * That's not really right, since the success url and the back/cancel url could be different. + * + * I changed getSuccessURL(null) to return cancelUrl if it is provided. + */ + @Override + public ActionURL getSuccessURL(QueryUpdateForm form) + { + return resolveReturnUrl(form == null ? getActionURLParam(ActionURL.Param.cancelUrl) : null, form); + } + + private ActionURL resolveReturnUrl(@Nullable ActionURL returnUrl, QueryUpdateForm form) + { + if (null == returnUrl) + returnUrl = getActionURLParam(ActionURL.Param.returnUrl); + if (null == returnUrl) + { + if (_schema != null && _table != null) + returnUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); + else + returnUrl = QueryService.get().urlDefault(form.getContainer(), QueryAction.executeQuery, null, null); + } + return returnUrl; + } + + public ActionURL getCancelURL(QueryUpdateForm form) + { + ActionURL cancelURL = getActionURLParam(ActionURL.Param.cancelUrl); + return resolveReturnUrl(cancelURL, form); + } + + @Override + public void addNavTrail(NavTree root) + { + if (_table != null) + root.addChild(_table.getName(), getSuccessURL(null)); + } + + protected List> doInsertUpdate(QueryUpdateForm form, BindException errors, boolean insert) + { + TableInfo table = form.getTable(); + if (!table.hasPermission(form.getUser(), insert ? InsertPermission.class : UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + Map values = form.getTypedColumns(); + + // Allow for attachment-based columns + Map fileMap = getFileMap(); + if (null != fileMap) + { + for (String key : fileMap.keySet()) + { + // Check if the column has already been processed + if (!values.containsKey(key)) + { + SpringAttachmentFile file = new SpringAttachmentFile(fileMap.get(key)); + form.setTypedValue(key, file.isEmpty() ? null : file); + } + } + } + + values = form.getTypedColumns(); + + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + throw new IllegalArgumentException("The query '" + _table.getName() + "' in the schema '" + _schema.getName() + "' is not updatable."); + + + List> rows; + List> ret = null; + + if (form.isBulkUpdate()) + { + rows = new ArrayList<>(); + + // Merge the bulk edits back into the selected rows + validate + String[] pkValues = form.getSelectedRows(); + + if (pkValues == null || pkValues.length == 0) + errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Please reselect the rows to update."); + else if (table.getPkColumnNames().size() > 1) + { + errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Does not support update for multi-keyed tables."); + } + else + { + Map row; + String pkName = table.getPkColumnNames().get(0); + for (String pkValue : pkValues) + { + row = new CaseInsensitiveHashMap<>(); + for (Map.Entry entry : values.entrySet()) + { + // If a value is left as null it is considered untouched for a given row + if (entry.getValue() != null) + row.put(entry.getKey(), entry.getValue()); + } + + row.put(pkName, pkValue); + rows.add(row); + } + } + } + else + { + rows = Collections.singletonList(values); + } + + DbSchema dbschema = table.getSchema(); + try + { + try (DbScope.Transaction transaction = dbschema.getScope().ensureTransaction()) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = null; + QueryService.AuditAction auditAction = insert ? QueryService.AuditAction.INSERT : QueryService.AuditAction.UPDATE; + + // transaction audit + BatchValidationException batchErrors = new BatchValidationException(); + + AuditBehaviorType auditBehaviorType = table.getEffectiveAuditBehavior(); + if (auditBehaviorType != AuditBehaviorType.NONE) + { + if (transaction.getAuditEvent() != null) + auditEvent = transaction.getAuditEvent(); + else + { + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction, getTransactionAuditDetails()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); + } + } + + if (insert) + { + ret = qus.insertRows(form.getUser(), form.getContainer(), rows, batchErrors, null, null); + } + else + { + // Currently, bulkUpdate doesn't support oldValues due to the need to re-query... + if (form.isBulkUpdate()) + { + ret = qus.updateRows(form.getUser(), form.getContainer(), rows, null, batchErrors, null, null); + } + else + { + Map oldValues = null; + if (form.getOldValues() instanceof Map) + { + oldValues = (Map) form.getOldValues(); + if (!(oldValues instanceof CaseInsensitiveMapWrapper)) + oldValues = new CaseInsensitiveMapWrapper<>(oldValues); + } + + // 18292 - updateRows expects a null list in the case of an "empty" or null map. + List> oldKeys = (oldValues == null || oldValues.isEmpty()) ? null : Collections.singletonList(oldValues); + ret = qus.updateRows(form.getUser(), form.getContainer(), rows, oldKeys, batchErrors, null, null); + } + } + if (batchErrors.hasErrors()) + batchErrors.addToErrors(errors); + + if (!errors.hasErrors()) + { + if (auditEvent != null) + auditEvent.addComment(auditAction, ret.size()); + transaction.commit(); // Only commit if there were no errors + } + + } + catch (SQLException x) + { + if (!RuntimeSQLException.isConstraintException(x)) + throw x; + errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); + } + catch (InvalidKeyException | DuplicateKeyException | DataIntegrityViolationException | RuntimeSQLException x) + { + errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); + } + catch (BatchValidationException x) + { + x.addToErrors(errors); + } + return ret; + } + catch (Exception x) + { + // Do this in a separate, outer try/catch so that we will have already committed or rolled back + // the transaction we started. Otherwise, our database connection is likely in a bad state and can't be + // reused when submitting the exception report. + errors.reject(SpringActionController.ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); + return null; + } + } +} diff --git a/api/src/org/labkey/api/util/HttpUtil.java b/api/src/org/labkey/api/util/HttpUtil.java index 2ee8ef53aa5..9094f3a8f71 100644 --- a/api/src/org/labkey/api/util/HttpUtil.java +++ b/api/src/org/labkey/api/util/HttpUtil.java @@ -36,6 +36,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; import org.labkey.api.view.BadRequestException; import org.springframework.web.servlet.mvc.Controller; import org.w3c.dom.Document; @@ -257,7 +258,7 @@ public static void trackClientApiRequests(HttpServletRequest request) SimpleMetricsService.get().increment(DefaultModule.CORE_MODULE_NAME, "clientApiRequests", clientLibrary); } - private static @Nullable String getClientLibrary(HttpServletRequest request) + public static @Nullable String getClientLibrary(HttpServletRequest request) { String userAgent = getUserAgent(request); if (null != userAgent) @@ -299,4 +300,28 @@ else if (userAgent.startsWith("LabKey SAS API")) } return request.getRemoteAddr(); } + + public static @Nullable String getProductName(HttpServletRequest request) + { + if (!isBrowser(request)) + return null; + + String referer = request.getHeader("Referer"); + if (referer != null) + { + try + { + ActionURL url = new ActionURL(referer); + String actionName = url.getAction(); + if ("app".equalsIgnoreCase(actionName) || "appdev".equalsIgnoreCase(actionName)) + return url.getController(); + } + catch (Exception e) + { + return null; + } + } + + return null; + } } diff --git a/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java b/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java index 2edcdca159f..7505d817d26 100644 --- a/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java +++ b/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java @@ -22,6 +22,8 @@ import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; +import org.labkey.api.action.BaseViewAction; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.MvUtil; @@ -219,7 +221,7 @@ protected ExpExperiment saveExperimentRun( AssayRunUploadContext uploadContext = createRunUploadContext(context, protocol, runJson, dataRows, inputData, outputData, inputMaterial, outputMaterial); - return saveAssayRun(uploadContext, batch, run); + return saveAssayRun(uploadContext, batch, run, BaseViewAction.getTransactionAuditDetails(context)); } return null; } @@ -316,10 +318,10 @@ protected AssayRunUploadContext createRunUploadContext( return provider.createRunUploadFactory(protocol, context); } - protected ExpExperiment saveAssayRun(AssayRunUploadContext context, ExpExperiment batch, ExpRun run) throws ExperimentException, ValidationException + protected ExpExperiment saveAssayRun(AssayRunUploadContext context, ExpExperiment batch, ExpRun run, Map transactionDetail) throws ExperimentException, ValidationException { AssayRunCreator runCreator = getProvider().getRunCreator(); - return runCreator.saveExperimentRun(context, batch, run, false); + return runCreator.saveExperimentRun(context, batch, run, false, transactionDetail); } @Override diff --git a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java index a4bc9a84ac8..9eac8ca22c6 100644 --- a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java +++ b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java @@ -319,15 +319,16 @@ else if (rawData != null && !rawData.isEmpty()) try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT); + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, getTransactionAuditDetails()); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); - Long auditTransactionId = transaction.getAuditId(); + var auditTransactionEvent = transaction.getAuditEvent(); + Long auditTransactionId = auditTransactionEvent == null ? null : auditTransactionEvent.getRowId(); // Bind file property values and persist files to the file system. { Map fileMap = getFileMap(); - bindAndPersistFilePropertyValues(AssayJSONConverter.BATCH_PROPERTIES, batchProperties, fileMap, filePropertyWriter, auditTransactionId, "Assay batch property file uploaded."); - bindAndPersistFilePropertyValues(AssayJSONConverter.RUN_PROPERTIES, runProperties, fileMap, filePropertyWriter, auditTransactionId, "Assay run property file uploaded."); + bindAndPersistFilePropertyValues(AssayJSONConverter.BATCH_PROPERTIES, batchProperties, fileMap, filePropertyWriter, auditTransactionEvent, "Assay batch property file uploaded."); + bindAndPersistFilePropertyValues(AssayJSONConverter.RUN_PROPERTIES, runProperties, fileMap, filePropertyWriter, auditTransactionEvent, "Assay run property file uploaded."); } AssayRunUploadContext uploadContext = factory.setOutputDatas(outputData) @@ -337,7 +338,7 @@ else if (rawData != null && !rawData.isEmpty()) .setUploadedFiles(filePropertyWriter.getUploadedFiles()) .create(); - Pair result = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, forceAsync); + Pair result = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, forceAsync, getTransactionAuditDetails()); ExpRun run = result.second; transaction.commit(); @@ -377,7 +378,7 @@ private void bindAndPersistFilePropertyValues( CaseInsensitiveHashMap properties, Map fileMap, AssayFilePropertyWriter fileWriter, - Long auditTransactionId, + TransactionAuditProvider.TransactionAuditEvent auditTransactionEvent, String auditComment ) throws ExperimentException, ValidationException { @@ -388,7 +389,7 @@ private void bindAndPersistFilePropertyValues( if (filePropertyMap.isEmpty()) return; - var fileProperties = fileWriter.savePostedFiles(getContainer(), getUser(), filePropertyMap, auditTransactionId, auditComment); + var fileProperties = fileWriter.savePostedFiles(getContainer(), getUser(), filePropertyMap, auditTransactionEvent, auditComment); for (var entry : fileProperties.entrySet()) properties.put(entry.getKey(), entry.getValue().toNioPathForRead().toString()); } diff --git a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java index a13e34b7293..b33271297d4 100644 --- a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java +++ b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java @@ -24,6 +24,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.assay.AssayService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.DbScope; @@ -837,7 +838,11 @@ public RecordedActionSet run() throws PipelineJobException Long batchId = null; // Import the assay run - Pair pair = provider.getRunCreator().saveExperimentRun(uploadContext, batchId); + Map transactionDetails = new HashMap<>(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, uploadedData.getName()); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.APIAction, "AssayImportRunTask"); + Pair pair = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, false, transactionDetails); ExpRun run = pair.second; if (getJob() instanceof FileAnalysisJobSupport) diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 81f3b709ea4..9c055d795e7 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -2918,7 +2918,7 @@ public PlateSetImpl createPlateSet( // Example comment: "Plate set was created. Created via reformat. Initially contains 5 plates." int plateCount = plates == null ? 0 : plates.size(); String comment = StringUtilsLabKey.joinNonBlank(" ", StringUtils.trimToEmpty(additionalAuditComment), String.format("Initially contains %s.", StringUtilsLabKey.pluralize(plateCount, "plate"))); - PlateSetAuditEvent auditEvent = PlateSetAuditProvider.EventFactory.plateSetCreated(container, tx.getAuditId(), newPlateSet, comment); + PlateSetAuditEvent auditEvent = PlateSetAuditProvider.EventFactory.plateSetCreated(container, tx.getAuditEvent(), newPlateSet, comment); AuditLogService.get().addEvent(user, auditEvent); } @@ -3060,7 +3060,7 @@ public void archive(Container container, User user, @Nullable List plateSe archive(container, user, AssayDbSchema.getInstance().getTableInfoPlateSet(), "plate sets", plateSetIds, archive); tx.addCommitTask(() -> clearPlateSetCache(container, plateSetIds), DbScope.CommitTaskOption.POSTCOMMIT); - List auditEvents = PlateSetAuditProvider.EventFactory.plateSetsArchived(container, tx.getAuditId(), plateSetIds, archive); + List auditEvents = PlateSetAuditProvider.EventFactory.plateSetsArchived(container, tx.getAuditEvent(), plateSetIds, archive); AuditLogService.get().addEvents(user, auditEvents, true); } @@ -5162,17 +5162,17 @@ private void addPlateAuditEvents(User user, Collection plates, Function

plates, @Nullable String additionalComment) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateCreated(container, tx.getAuditId(), plate, additionalComment)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateCreated(container, tx.getAuditEvent(), plate, additionalComment)); } public void addPlateDeletedAuditEvents(Container container, User user, DbScope.Transaction tx, Collection plates) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateDeleted(container, tx.getAuditId(), plate)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateDeleted(container, tx.getAuditEvent(), plate)); } public void addPlateImportAuditEvents(Container container, User user, DbScope.Transaction tx, Collection plates, ExpRun run, boolean isReimport) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateImported(container, tx.getAuditId(), plate, run, isReimport)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateImported(container, tx.getAuditEvent(), plate, run, isReimport)); } public void ensureTransactionAuditId(DbScope.Transaction tx, Container container, User user, QueryService.AuditAction auditAction) diff --git a/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java b/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java index 5bfafa4e892..f6c8fbd1be3 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java @@ -2,6 +2,7 @@ import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.Container; @@ -36,7 +37,7 @@ protected PlateAuditEvent( PlateAuditProvider.PlateEventType eventType, Container container, PlateImpl plate, - Long transactionAuditId + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent ) { super(EVENT_NAME, container, eventType.getComment(plate.isTemplate())); @@ -47,7 +48,7 @@ protected PlateAuditEvent( setPlateTypeRowId(plate.getPlateType().getRowId()); setSourcePlateRowId(plate.getSourcePlateRowId()); setTemplate(plate.isTemplate()); - setTransactionId(transactionAuditId); + setTransactionEvent(transactionAuditEvent, EVENT_NAME); } public String getPlateEventType() diff --git a/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java b/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java index 9c9697f226b..ff5bad3d5bc 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java @@ -3,6 +3,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; @@ -185,12 +186,12 @@ public static class EventFactory { public static PlateAuditEvent plateCreated( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate, @Nullable String additionalComment ) { - var event = new PlateAuditEvent(PlateEventType.CREATE_PLATE, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.CREATE_PLATE, container, plate, transactionAuditEvent); event.setNewRecordMap(container, plate); if (additionalComment != null) @@ -199,17 +200,17 @@ public static PlateAuditEvent plateCreated( return event; } - public static PlateAuditEvent plateDeleted(Container container, Long transactionAuditId, PlateImpl plate) + public static PlateAuditEvent plateDeleted(Container container, TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate) { - var event = new PlateAuditEvent(PlateEventType.DELETE_PLATE, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.DELETE_PLATE, container, plate, transactionAuditEvent); event.setOldRecordMap(container, plate); return event; } - public static PlateAuditEvent plateImported(Container container, Long transactionAuditId, PlateImpl plate, ExpRun run, boolean isReimport) + public static PlateAuditEvent plateImported(Container container, TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate, ExpRun run, boolean isReimport) { - var event = new PlateAuditEvent(PlateEventType.PLATE_IMPORT, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.PLATE_IMPORT, container, plate, transactionAuditEvent); event.setImportRunId(run.getRowId()); event.setReimport(isReimport); diff --git a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java index 4398a8bef3b..9ddfb49f2f9 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java @@ -2,6 +2,7 @@ import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.Container; @@ -33,7 +34,7 @@ public PlateSetAuditEvent( PlateSetAuditProvider.PlateSetEventType eventType, Container container, PlateSetImpl plateSet, - Long transactionAuditId + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent ) { super(EVENT_NAME, container, eventType.getComment()); @@ -45,7 +46,7 @@ public PlateSetAuditEvent( setPrimaryPlateSetRowId(plateSet.getPrimaryPlateSetId()); setParentPlateSetRowId(plateSet.getParentPlateSetId()); setRootPlateSetRowId(plateSet.getRootPlateSetId()); - setTransactionId(transactionAuditId); + setTransactionEvent(transactionAuditEvent, EVENT_NAME); } public Boolean getArchived() diff --git a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java index b9e1ccc175a..651757c9ee1 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java @@ -3,6 +3,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.collections.CaseInsensitiveHashMap; @@ -185,12 +186,12 @@ public static class EventFactory { public static PlateSetAuditEvent plateSetCreated( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateSetImpl plateSet, @Nullable String additionalComment ) { - var event = new PlateSetAuditEvent(PlateSetEventType.CREATE_PLATE_SET, container, plateSet, transactionAuditId); + var event = new PlateSetAuditEvent(PlateSetEventType.CREATE_PLATE_SET, container, plateSet, transactionAuditEvent); event.setNewRecordMap(container, plateSet); if (additionalComment != null) @@ -201,7 +202,7 @@ public static PlateSetAuditEvent plateSetCreated( public static List plateSetsArchived( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, List plateSetIds, boolean archive ) throws ValidationException @@ -220,7 +221,7 @@ public static List plateSetsArchived( plateSet.setArchived(archive); - var event = new PlateSetAuditEvent(eventType, container, plateSet, transactionAuditId); + var event = new PlateSetAuditEvent(eventType, container, plateSet, transactionAuditEvent); event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of(PlateTable.Column.Archived.name(), String.valueOf(!archive)))); event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of(PlateTable.Column.Archived.name(), String.valueOf(archive)))); events.add(event); diff --git a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java index 0ad0a24832a..41730f6a38e 100644 --- a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java +++ b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java @@ -1,221 +1,220 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/21/13 - */ -public class SampleTypeAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String EVENT_TYPE = "SampleSetAuditEvent"; - - public static final String COLUMN_NAME_SOURCE_LSID = "SourceLsid"; - public static final String COLUMN_NAME_SAMPLE_TYPE_NAME = "SampleSetName"; - public static final String COLUMN_NAME_INSERT_UPDATE_CHOICE = "InsertUpdateChoice"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SAMPLE_TYPE_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public SampleTypeAuditProvider() - { - super(new SampleTypeAuditDomainKind()); - } - - @Override - public String getEventName() - { - return EVENT_TYPE; - } - - @Override - public String getLabel() - { - return "Sample Type events"; - } - - @Override - public String getDescription() - { - return "Summarizes events from sample type inserts or updates"; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SAMPLE_TYPE_NAME.equalsIgnoreCase(col.getName())) - col.setLabel("Sample Type"); - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - col.setLabel("Reason"); - } - }; - } - - @Override - public Class getEventClass() - { - return (Class) SampleTypeAuditEvent.class; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - public static class SampleTypeAuditEvent extends AuditTypeEvent - { - private String _sourceLsid; - private String _sampleSetName; - private String _insertUpdateChoice; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public SampleTypeAuditEvent() - { - super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public SampleTypeAuditEvent(Container container, String comment) - { - super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public String getSourceLsid() - { - return _sourceLsid; - } - - public void setSourceLsid(String sourceLsid) - { - _sourceLsid = sourceLsid; - } - - public String getSampleSetName() - { - return _sampleSetName; - } - - public void setSampleSetName(String sampleSetName) - { - _sampleSetName = sampleSetName; - } - - public String getInsertUpdateChoice() - { - return _insertUpdateChoice; - } - - public void setInsertUpdateChoice(String insertUpdateChoice) - { - _insertUpdateChoice = insertUpdateChoice; - } - - - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("sourceLsid", getSourceLsid()); - elements.put("sampleSetName", getSampleSetName()); - elements.put("insertUpdateChoice", getInsertUpdateChoice()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class SampleTypeAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "SampleSetAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - private final Set _fields; - - public SampleTypeAuditDomainKind() - { - super(EVENT_TYPE); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_SOURCE_LSID, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SAMPLE_TYPE_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_INSERT_UPDATE_CHOICE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: 7/21/13 + */ +public class SampleTypeAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String EVENT_TYPE = "SampleSetAuditEvent"; + + public static final String COLUMN_NAME_SOURCE_LSID = "SourceLsid"; + public static final String COLUMN_NAME_SAMPLE_TYPE_NAME = "SampleSetName"; + public static final String COLUMN_NAME_INSERT_UPDATE_CHOICE = "InsertUpdateChoice"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SAMPLE_TYPE_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public SampleTypeAuditProvider() + { + super(new SampleTypeAuditDomainKind()); + } + + @Override + public String getEventName() + { + return EVENT_TYPE; + } + + @Override + public String getLabel() + { + return "Sample Type events"; + } + + @Override + public String getDescription() + { + return "Summarizes events from sample type inserts or updates"; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SAMPLE_TYPE_NAME.equalsIgnoreCase(col.getName())) + col.setLabel("Sample Type"); + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + col.setLabel("Reason"); + } + }; + } + + @Override + public Class getEventClass() + { + return (Class) SampleTypeAuditEvent.class; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + public static class SampleTypeAuditEvent extends AuditTypeEvent + { + private String _sourceLsid; + private String _sampleSetName; + private String _insertUpdateChoice; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public SampleTypeAuditEvent() + { + super(); + } + + public SampleTypeAuditEvent(Container container, String comment) + { + super(EVENT_TYPE, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); + } + + public String getSourceLsid() + { + return _sourceLsid; + } + + public void setSourceLsid(String sourceLsid) + { + _sourceLsid = sourceLsid; + } + + public String getSampleSetName() + { + return _sampleSetName; + } + + public void setSampleSetName(String sampleSetName) + { + _sampleSetName = sampleSetName; + } + + public String getInsertUpdateChoice() + { + return _insertUpdateChoice; + } + + public void setInsertUpdateChoice(String insertUpdateChoice) + { + _insertUpdateChoice = insertUpdateChoice; + } + + + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("sourceLsid", getSourceLsid()); + elements.put("sampleSetName", getSampleSetName()); + elements.put("insertUpdateChoice", getInsertUpdateChoice()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class SampleTypeAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "SampleSetAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + private final Set _fields; + + public SampleTypeAuditDomainKind() + { + super(EVENT_TYPE); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_SOURCE_LSID, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SAMPLE_TYPE_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_INSERT_UPDATE_CHOICE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 9b3bd485f40..6dd2c93ebc4 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -28,6 +28,7 @@ import org.labkey.api.attachments.AttachmentParent; import org.labkey.api.attachments.AttachmentParentFactory; import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.Sets; @@ -1506,6 +1507,10 @@ public List> updateRows(User user, Container container, List { Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + results = super._updateRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters), extraScriptContext); } else diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 23ba91d58dc..f3d51bf42b7 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9817,7 +9817,7 @@ public Map moveAssayRuns(@NotNull List assayR for (List runFileRenameData : assayMoveData.fileMovesByRunId().values()) { for (AbstractAssayProvider.AssayFileMoveData renameData : runFileRenameData) - moveFile(renameData, container, user, transaction.getAuditId()); + moveFile(renameData, container, user, transaction.getAuditEvent()); } }, POSTCOMMIT); @@ -9827,7 +9827,7 @@ public Map moveAssayRuns(@NotNull List assayR return assayMoveData.counts(); } - private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Container sourceContainer, User user, Long txAuditId) + private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) { String fieldName = renameData.fieldName() == null ? "datafileurl" : renameData.fieldName(); File targetFile = renameData.targetFile(); @@ -9850,7 +9850,7 @@ private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Con String changeDetail = String.format("assay '%s' run '%s'", assayName, runName); - return moveFileLinkFile(sourceFile, targetFile, sourceContainer, user, changeDetail, txAuditId, fieldName); + return moveFileLinkFile(sourceFile, targetFile, sourceContainer, user, changeDetail, txAuditEvent, fieldName); } @Override @@ -10030,7 +10030,7 @@ public boolean isLookupToMaterials(DomainProperty dp) * * Move file (post-commit) after moving sample/assay data */ - public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sourceFileContainer, User user, String actionComment, Long txAuditId, String fieldName) + public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sourceFileContainer, User user, String actionComment, TransactionAuditProvider.TransactionAuditEvent txAuditEvent, String fieldName) { if (!sourceFile.exists()) return false; @@ -10062,8 +10062,8 @@ public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sour return false; } } - if (txAuditId != null && event.getTransactionId() == null) - event.setTransactionId(txAuditId); + if (txAuditEvent != null && event.getTransactionId() == null) + event.setTransactionEvent(txAuditEvent, FileSystemAuditProvider.EVENT_TYPE); event.setDirectory(sourceFile.getParentFile().getAbsolutePath()); event.setFile(targetFile.getName()); event.setProvidedFileName(sourceFile.getName()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 9109ecd0e27..80c2cdf9d4d 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2370 +1,2370 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container, false); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container, - boolean useRootMaterialLSID - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates, useRootMaterialLSID); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append(useRootMaterialLSID ? "a.rootMaterialLsid = m.lsid" : "a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql; - if (useRootMaterialLSID) - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialLSID as rootLsid, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialLSID <> LSID AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialLSID - ) AS c ON m.lsid = c.rootLsid - WHERE m.rootmateriallsid = m.LSID AND m.rowid\s"""); - } - else - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - } - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append(useRootMaterialLSID ? "parent.lsid = aliquot.rootmateriallsid" : "parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append(useRootMaterialLSID ? "aliquot.rootmateriallsid <> aliquot.lsid" : "aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditId()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, Long txAuditId) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditId, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container, false); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container, + boolean useRootMaterialLSID + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents, useRootMaterialLSID); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates, useRootMaterialLSID); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates, useRootMaterialLSID); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds, boolean useRootMaterialLSID) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append(useRootMaterialLSID ? "a.rootMaterialLsid = m.lsid" : "a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates, boolean useRootMaterialLSID) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql; + if (useRootMaterialLSID) + { + sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialLSID as rootLsid, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialLSID <> LSID AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialLSID + ) AS c ON m.lsid = c.rootLsid + WHERE m.rootmateriallsid = m.LSID AND m.rowid\s"""); + } + else + { + sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + } + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates, boolean useRootMaterialLSID) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append(useRootMaterialLSID ? "parent.lsid = aliquot.rootmateriallsid" : "parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append(useRootMaterialLSID ? "aliquot.rootmateriallsid <> aliquot.lsid" : "aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 8417fd01071..3fe4ac3893b 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -28,6 +28,7 @@ import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.attachments.AttachmentFile; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -560,6 +561,9 @@ public List> updateRows(User user, Container container, List Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + try { results = scope.executeWithRetry(transaction -> diff --git a/list/src/org/labkey/list/model/ListAuditProvider.java b/list/src/org/labkey/list/model/ListAuditProvider.java index e91414869f8..f2294fa07bf 100644 --- a/list/src/org/labkey/list/model/ListAuditProvider.java +++ b/list/src/org/labkey/list/model/ListAuditProvider.java @@ -1,274 +1,273 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.util.StringExpressionFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class ListAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String COLUMN_NAME_LIST_ID = "ListId"; - public static final String COLUMN_NAME_LIST_DOMAIN_URI = "ListDomainUri"; - public static final String COLUMN_NAME_LIST_ITEM_ENTITY_ID = "ListItemEntityId"; - public static final String COLUMN_NAME_LIST_NAME = "ListName"; - - private static final List defaultVisibleColumns = new ArrayList<>(); - - static - { - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_LIST_DOMAIN_URI)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - } - - public ListAuditProvider() - { - super(new ListAuditDomainKind()); - } - - @Override - public String getEventName() - { - return ListManager.LIST_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "List events"; - } - - @Override - public String getDescription() - { - return "Data about list creation, deletion, insertion, etc."; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_LIST_DOMAIN_URI.equalsIgnoreCase(col.getName())) - { - col.setLabel("List"); - col.setDisplayColumnFactory(colInfo -> new DomainAuditProvider.DomainColumn(colInfo, COLUMN_NAME_CONTAINER, COLUMN_NAME_LIST_NAME)); - } - } - }; - appendValueMapColumns(table, null, true); - - // Render a details URL only for rows that have a listItemEntityId - DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&name=${listName}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); - table.setDetailsURL(url); - - return table; - } - - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("intKey1"), COLUMN_NAME_LIST_ID); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_LIST_DOMAIN_URI); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_LIST_ITEM_ENTITY_ID); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_LIST_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - // Unused Property/oldRecord and Property/newRecord columns should just be migrated to the oldRecordMap and newRecordMap columns - legacyMap.put(FieldKey.fromParts("Property", "OldRecord"), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", "NewRecord"), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)ListAuditEvent.class; - } - - public int moveEvents(Container targetContainer, List listRowEntityIds) - { - return moveEvents(targetContainer, COLUMN_NAME_LIST_ITEM_ENTITY_ID, listRowEntityIds); - } - - public static class ListAuditEvent extends DetailedAuditTypeEvent - { - private int _listId; - private String _listDomainUri; - private String _listItemEntityId; - private String _listName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public ListAuditEvent() - { - super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public ListAuditEvent(Container container, String comment, ListDefinitionImpl list) - { - super(ListManager.LIST_AUDIT_EVENT, container, comment); - setListDomainUri(list.getDomain().getTypeURI()); - setListId(list.getListId()); - setListName(list.getName()); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public int getListId() - { - return _listId; - } - - public void setListId(int listId) - { - _listId = listId; - } - - public String getListDomainUri() - { - return _listDomainUri; - } - - public void setListDomainUri(String listDomainUri) - { - _listDomainUri = listDomainUri; - } - - public String getListItemEntityId() - { - return _listItemEntityId; - } - - public void setListItemEntityId(String listItemEntityId) - { - _listItemEntityId = listItemEntityId; - } - - public String getListName() - { - return _listName; - } - - public void setListName(String listName) - { - _listName = listName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("list", getListName() + " (" + getListId() + ")"); - elements.put("listDomainUri", getListDomainUri()); - elements.put("listItemEntityId", getListItemEntityId()); - // N.B. oldRecordMap and newRecordMap can be very large and are not included here - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class ListAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "ListAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public ListAuditDomainKind() - { - super(ListManager.LIST_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ID, PropertyType.INTEGER)); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_DOMAIN_URI, PropertyType.STRING)); - // Choose a length that should be much larger than necessary to give extra buffer, but still small enough - // to be indexed - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ITEM_ENTITY_ID, PropertyType.STRING, 300)); // UNDONE: is needed ? .setEntityId(true)); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - - @Override - public Set getPropertyIndices(Domain domain) - { - Set indexes = super.getPropertyIndices(domain); - indexes.add(new PropertyStorageSpec.Index(false, COLUMN_NAME_LIST_ITEM_ENTITY_ID)); - return indexes; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.util.StringExpressionFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ListAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String COLUMN_NAME_LIST_ID = "ListId"; + public static final String COLUMN_NAME_LIST_DOMAIN_URI = "ListDomainUri"; + public static final String COLUMN_NAME_LIST_ITEM_ENTITY_ID = "ListItemEntityId"; + public static final String COLUMN_NAME_LIST_NAME = "ListName"; + + private static final List defaultVisibleColumns = new ArrayList<>(); + + static + { + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_LIST_DOMAIN_URI)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + } + + public ListAuditProvider() + { + super(new ListAuditDomainKind()); + } + + @Override + public String getEventName() + { + return ListManager.LIST_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "List events"; + } + + @Override + public String getDescription() + { + return "Data about list creation, deletion, insertion, etc."; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_LIST_DOMAIN_URI.equalsIgnoreCase(col.getName())) + { + col.setLabel("List"); + col.setDisplayColumnFactory(colInfo -> new DomainAuditProvider.DomainColumn(colInfo, COLUMN_NAME_CONTAINER, COLUMN_NAME_LIST_NAME)); + } + } + }; + appendValueMapColumns(table, null, true); + + // Render a details URL only for rows that have a listItemEntityId + DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&name=${listName}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); + table.setDetailsURL(url); + + return table; + } + + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("intKey1"), COLUMN_NAME_LIST_ID); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_LIST_DOMAIN_URI); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_LIST_ITEM_ENTITY_ID); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_LIST_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + // Unused Property/oldRecord and Property/newRecord columns should just be migrated to the oldRecordMap and newRecordMap columns + legacyMap.put(FieldKey.fromParts("Property", "OldRecord"), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", "NewRecord"), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)ListAuditEvent.class; + } + + public int moveEvents(Container targetContainer, List listRowEntityIds) + { + return moveEvents(targetContainer, COLUMN_NAME_LIST_ITEM_ENTITY_ID, listRowEntityIds); + } + + public static class ListAuditEvent extends DetailedAuditTypeEvent + { + private int _listId; + private String _listDomainUri; + private String _listItemEntityId; + private String _listName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public ListAuditEvent() + { + super(); + } + + public ListAuditEvent(Container container, String comment, ListDefinitionImpl list) + { + super(ListManager.LIST_AUDIT_EVENT, container, comment); + setListDomainUri(list.getDomain().getTypeURI()); + setListId(list.getListId()); + setListName(list.getName()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), ListManager.LIST_AUDIT_EVENT); + } + + public int getListId() + { + return _listId; + } + + public void setListId(int listId) + { + _listId = listId; + } + + public String getListDomainUri() + { + return _listDomainUri; + } + + public void setListDomainUri(String listDomainUri) + { + _listDomainUri = listDomainUri; + } + + public String getListItemEntityId() + { + return _listItemEntityId; + } + + public void setListItemEntityId(String listItemEntityId) + { + _listItemEntityId = listItemEntityId; + } + + public String getListName() + { + return _listName; + } + + public void setListName(String listName) + { + _listName = listName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("list", getListName() + " (" + getListId() + ")"); + elements.put("listDomainUri", getListDomainUri()); + elements.put("listItemEntityId", getListItemEntityId()); + // N.B. oldRecordMap and newRecordMap can be very large and are not included here + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class ListAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "ListAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public ListAuditDomainKind() + { + super(ListManager.LIST_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ID, PropertyType.INTEGER)); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_DOMAIN_URI, PropertyType.STRING)); + // Choose a length that should be much larger than necessary to give extra buffer, but still small enough + // to be indexed + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ITEM_ENTITY_ID, PropertyType.STRING, 300)); // UNDONE: is needed ? .setEntityId(true)); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + + @Override + public Set getPropertyIndices(Domain domain) + { + Set indexes = super.getPropertyIndices(domain); + indexes.add(new PropertyStorageSpec.Index(false, COLUMN_NAME_LIST_ITEM_ENTITY_ID)); + return indexes; + } + } +} diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 75b679af3ae..c5f84eea273 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -1,884 +1,890 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.LookupResolutionType; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.Selector.ForEachBatchBlock; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListImportProgress; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.lists.permissions.ManagePicklistsPermission; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.ElevatedUser; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.list.view.ListItemAttachmentParent; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.labkey.api.util.IntegerUtils.isIntegral; - -/** - * Implementation of QueryUpdateService for Lists - */ -public class ListQueryUpdateService extends DefaultQueryUpdateService -{ - private final ListDefinitionImpl _list; - private static final String ID = "entityId"; - - public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) - { - super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); - _list = (ListDefinitionImpl) list; - } - - @Override - public void configureDataIteratorContext(DataIteratorContext context) - { - if (context.getInsertOption().batch) - { - context.setMaxRowErrors(100); - context.setFailFast(false); - } - - context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); - } - - @Override - protected @Nullable AttachmentParentFactory getAttachmentParentFactory() - { - return new ListItemAttachmentParentFactory(); - } - - @Override - protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException - { - Map ret = null; - - if (null != listRow) - { - SimpleFilter keyFilter = getKeyFilter(listRow); - - if (null != keyFilter) - { - TableInfo queryTable = getQueryTable(); - Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); - - if (null != raw && !raw.isEmpty()) - { - ret = new CaseInsensitiveHashMap<>(); - - // EntityId - ret.put("EntityId", raw.get("entityid")); - - for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) - { - String propName = prop.getName(); - ColumnInfo column = queryTable.getColumn(propName); - Object value = column.getValue(raw); - if (value != null) - ret.put(propName, value); - } - } - } - } - - return ret; - } - - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - for (Map row : rows) - { - aliasColumns(getColumnMapping(), row); - } - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - - if (null != result) - { - ListManager mgr = ListManager.get(); - - for (Map row : result) - { - if (null != row.get(ID)) - { - // Audit each row - String entityId = (String) row.get(ID); - String newRecord = mgr.formatAuditItem(_list, user, row); - - mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); - } - } - - if (!result.isEmpty() && !errors.hasErrors()) - mgr.indexList(_list); - } - - return result; - } - - private User getListUser(User user, Container container) - { - if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) - { - // if the list is a picklist and you have permission to manage picklists, that equates - // to having editor permission. - return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); - } - return user; - } - - @Override - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - @Override - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, - @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - User updatedUser = getListUser(user, container); - DataIteratorContext context = new DataIteratorContext(errors); - context.setFailFast(false); - context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter - context.setSupportAutoIncrementKey(supportAutoIncrementKey); - context.setLookupResolutionType(lookupResolutionType); - setAttachmentDirectory(attachmentDir); - TableInfo ti = _list.getTable(updatedUser); - - if (null != ti) - { - try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) - { - int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); - - if (!errors.hasErrors()) - { - //Make entry to audit log if anything was inserted - if (imported > 0) - ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); - - transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - - return imported; - } - - return 0; - } - } - - return 0; - } - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - } - - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); - if (count > 0 && !errors.hasErrors()) - ListManager.get().indexList(_list); - return count; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data into this table."); - - List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (!result.isEmpty()) - ListManager.get().indexList(_list); - return result; - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - // TODO: Check for equivalency so that attachments can be deleted etc. - - Map dps = new HashMap<>(); - for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) - { - dps.put(dp.getPropertyURI(), dp); - } - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - ListItm itm = new ListItm(); - itm.setEntityId((String) oldRow.get(ID)); - itm.setListId(_list.getListId()); - itm.setKey(oldRow.get(_list.getKeyName())); - - ListItem item = new ListItemImpl(_list, itm); - - if (item.getProperties() != null) - { - List errors = new ArrayList<>(); - for (Map.Entry entry : dps.entrySet()) - { - Object value = row.get(entry.getValue().getName()); - validateProperty(entry.getValue(), value, row, errors, validatorCache); - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - - // MVIndicators - Map rowCopy = new CaseInsensitiveHashMap<>(); - ArrayList modifiedAttachmentColumns = new ArrayList<>(); - ArrayList attachmentFiles = new ArrayList<>(); - - TableInfo qt = getQueryTable(); - for (Map.Entry r : row.entrySet()) - { - ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); - rowCopy.put(r.getKey(), r.getValue()); - - // 22747: Attachment columns - if (null != column) - { - DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); - if (null != dp && isAttachmentProperty(dp)) - { - modifiedAttachmentColumns.add(column); - - // setup any new attachments - if (r.getValue() instanceof AttachmentFile file) - { - if (null != file.getFilename()) - attachmentFiles.add(file); - } - else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) - { - throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); - } - } - } - } - - // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) - Object newRowKey = getField(rowCopy, _list.getKeyName()); - Object oldRowKey = getField(oldRow, _list.getKeyName()); - - if (null == newRowKey && null != oldRowKey) - rowCopy.put(_list.getKeyName(), oldRowKey); - - Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); - - if (null != result) - { - result = getRow(user, container, result); - - if (null != result && null != result.get(ID)) - { - ListManager mgr = ListManager.get(); - String entityId = (String) result.get(ID); - - try - { - // Remove prior attachment -- only includes columns which are modified in this update - for (ColumnInfo col : modifiedAttachmentColumns) - { - Object value = oldRow.get(col.getName()); - if (null != value) - { - AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); - } - } - - // Update attachments - if (!attachmentFiles.isEmpty()) - AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); - } - catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) - { - // issues 21503, 28633: turn these into a validation exception to get a nicer error - throw new ValidationException(e.getMessage()); - } - catch (IOException e) - { - throw UnexpectedException.wrap(e); - } - finally - { - for (AttachmentFile attachmentFile : attachmentFiles) - { - try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} - } - } - - String oldRecord = mgr.formatAuditItem(_list, user, oldRow); - String newRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); - } - } - - return result; - } - - // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() - private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) - { - //check for isRequired - if (prop.isRequired()) - { - // for mv indicator columns either an indicator or a field value is sufficient - boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); - if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) - { - if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) - { - errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); - return false; - } - } - } - - if (null != value) - { - for (IPropertyValidator validator : prop.getValidators()) - { - if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) - return false; - } - } - - return true; - } - - private record ListRecord(Object key, String entityId) { } - - @Override - public Map moveRows( - User _user, - Container container, - Container targetContainer, - List> rows, - BatchValidationException errors, - @Nullable Map configParameters, - @Nullable Map extraScriptContext - ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException - { - // Ensure the list is in scope for the target container - if (null == ListService.get().getList(targetContainer, _list.getName(), true)) - { - errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); - throw errors; - } - - User user = getListUser(_user, container); - Map> containerRows = getListRowsForMoveRows(container, user, targetContainer, rows, errors); - if (errors.hasErrors()) - throw errors; - - int fileAttachmentsMovedCount = 0; - int listAuditEventsCreatedCount = 0; - int listAuditEventsMovedCount = 0; - int listRecordsCount = 0; - int queryAuditEventsMovedCount = 0; - - if (containerRows.isEmpty()) - return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); - - AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; - String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; - boolean hasAttachmentProperties = _list.getDomainOrThrow() - .getProperties() - .stream() - .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); - - ListAuditProvider listAuditProvider = new ListAuditProvider(); - final int BATCH_SIZE = 5_000; - boolean isAuditEnabled = auditBehavior != null && AuditBehaviorType.NONE != auditBehavior; - - try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) - { - if (isAuditEnabled && tx.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); - AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); - } - - List listAuditEvents = new ArrayList<>(); - - for (GUID containerId : containerRows.keySet()) - { - Container sourceContainer = ContainerManager.getForId(containerId); - if (sourceContainer == null) - throw new InvalidKeyException("Container '" + containerId + "' does not exist."); - - if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) - throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); - - TableInfo listTable = _list.getTable(user, sourceContainer); - if (listTable == null) - throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); - - List records = containerRows.get(containerId); - int numRecords = records.size(); - - for (int start = 0; start < numRecords; start += BATCH_SIZE) - { - int end = Math.min(start + BATCH_SIZE, numRecords); - List batch = records.subList(start, end); - List rowPks = batch.stream().map(ListRecord::key).toList(); - - // Before trigger per batch - Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); - if (errors.hasErrors()) - throw errors; - - listRecordsCount += Table.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); - if (errors.hasErrors()) - throw errors; - - if (hasAttachmentProperties) - { - fileAttachmentsMovedCount += moveAttachments(user, sourceContainer, targetContainer, batch, errors); - if (errors.hasErrors()) - throw errors; - } - - queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, ListQuerySchema.NAME, _list.getName()); - listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, batch.stream().map(ListRecord::entityId).toList()); - - // Detailed audit events per row - if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) - listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, batch); - - // After trigger per batch - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); - if (errors.hasErrors()) - throw errors; - } - - // Create a summary audit event for the source container - if (isAuditEnabled) - { - String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - - // Create a summary audit event for the target container - if (isAuditEnabled) - { - String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - } - - if (!listAuditEvents.isEmpty()) - { - AuditLogService.get().addEvents(user, listAuditEvents, true); - listAuditEventsCreatedCount += listAuditEvents.size(); - } - - tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - - tx.commit(); - - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); - } - - return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); - } - - private Map moveRowsCounts(int fileAttachmentsMovedCount, int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) - { - return Map.of( - "fileAttachmentsMoved", fileAttachmentsMovedCount, - "listAuditEventsCreated", listAuditEventsCreated, - "listAuditEventsMoved", listAuditEventsMoved, - "listRecords", listRecords, - "queryAuditEventsMoved", queryAuditEventsMoved - ); - } - - private Map> getListRowsForMoveRows( - Container container, - User user, - Container targetContainer, - List> rows, - BatchValidationException errors - ) throws QueryUpdateServiceException - { - if (rows.isEmpty()) - return Collections.emptyMap(); - - String keyName = _list.getKeyName(); - List keys = new ArrayList<>(); - for (var row : rows) - { - Object key = getField(row, keyName); - if (key == null) - { - errors.addRowError(new ValidationException("Key field value required for moving list rows.")); - return Collections.emptyMap(); - } - - keys.add(getKeyFilterValue(key)); - } - - SimpleFilter filter = new SimpleFilter(); - FieldKey fieldKey = FieldKey.fromParts(keyName); - filter.addInClause(fieldKey, keys); - filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); - - // Request all rows without a container filter so that rows are more easily resolved across the list scope. - // Read permissions are subsequently checked upon loading a row. - TableInfo table = _list.getTable(user, container, ContainerFilter.getUnsafeEverythingFilter()); - if (table == null) - throw new QueryUpdateServiceException(String.format("Failed to resolve table for list %s in %s", _list.getName(), container.getPath())); - - Map> containerRows = new HashMap<>(); - try (var result = new TableSelector(table, PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) - { - while (result.next()) - { - GUID containerId = new GUID(result.getString("Container")); - if (!containerRows.containsKey(containerId)) - { - var c = ContainerManager.getForId(containerId); - if (c == null) - throw new QueryUpdateServiceException(String.format("Failed to resolve container for row in list %s in %s.", _list.getName(), container.getPath())); - else if (!c.hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read list rows in all source containers."); - } - - containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); - containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return containerRows; - } - - private int moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) - { - List parents = new ArrayList<>(); - for (ListRecord record : records) - parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); - - int count = 0; - try - { - count = AttachmentService.get().moveAttachments(targetContainer, parents, user); - } - catch (IOException e) - { - errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); - } - - return count; - } - - private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) - { - List auditEvents = new ArrayList<>(records.size()); - String keyName = _list.getKeyName(); - String sourcePath = sourceContainer.getPath(); - String targetPath = targetContainer.getPath(); - - for (ListRecord record : records) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); - event.setListItemEntityId(record.entityId); - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); - auditEvents.add(event); - } - - AuditLogService.get().addEvents(user, auditEvents, true); - - return auditEvents.size(); - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); - - if (null != result) - { - String entityId = (String) result.get(ID); - - if (null != entityId) - { - ListManager mgr = ListManager.get(); - String deletedRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); - - // Remove attachments - if (hasAttachmentProperties()) - AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); - - // Clean up Search indexer - if (!result.isEmpty()) - mgr.deleteItemIndex(_list, entityId); - } - } - - return result; - } - - - // Deletes attachments & discussions, and removes list documents from full-text search index. - public void deleteRelatedListData(final User user, final Container container) - { - // Unindex all item docs and the entire list doc - ListManager.get().deleteIndexedList(_list); - - // Delete attachments and discussions associated with a list in batches of 1,000 - new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() - { - @Override - public boolean accept(String entityId) - { - return null != entityId; - } - - @Override - public void exec(List entityIds) - { - // delete the related list data for this block - deleteRelatedListData(user, container, entityIds); - } - }); - } - - // delete the related list data for this block of entityIds - private void deleteRelatedListData(User user, Container container, List entityIds) - { - // Build up set of entityIds and AttachmentParents - List attachmentParents = new ArrayList<>(); - - // Delete Attachments - if (hasAttachmentProperties()) - { - for (String entityId : entityIds) - { - attachmentParents.add(new ListItemAttachmentParent(entityId, container)); - } - AttachmentService.get().deleteAttachments(attachmentParents); - } - } - - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - int result; - try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) - { - deleteRelatedListData(user, container); - result = super.truncateRows(getListUser(user, container), container); - transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - } - - return result; - } - - @Nullable - public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException - { - String keyName = _list.getKeyName(); - Object key = getField(map, keyName); - - if (null == key) - { - // Auto-increment lists might not provide a key so allow them to pass through - if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) - return null; - throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); - } - - return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); - } - - @NotNull - private Object getKeyFilterValue(@NotNull Object key) - { - ListDefinition.KeyType type = _list.getKeyType(); - - // Check the type of the list to ensure proper casting of the key type - if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) - return isIntegral(key) ? key : Integer.valueOf(key.toString()); - - return key.toString(); - } - - @Nullable - private Object getField(Map map, String key) - { - /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ - Object value = map.get(key); - - if (null == value) - value = map.get(key + "_"); - - if (null == value) - value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); - - return value; - } - - /** - * Delegate class to generate an AttachmentParent - */ - public static class ListItemAttachmentParentFactory implements AttachmentParentFactory - { - @Override - public AttachmentParent generateAttachmentParent(String entityId, Container c) - { - return new ListItemAttachmentParent(entityId, c); - } - } - - /** - * Get Domain from list definition, unless null then get from super - */ - @Override - protected Domain getDomain() - { - return _list != null? - _list.getDomain() : - super.getDomain(); - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.LookupResolutionType; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Selector.ForEachBatchBlock; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListImportProgress; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.ElevatedUser; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.list.view.ListItemAttachmentParent; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.labkey.api.util.IntegerUtils.isIntegral; + +/** + * Implementation of QueryUpdateService for Lists + */ +public class ListQueryUpdateService extends DefaultQueryUpdateService +{ + private final ListDefinitionImpl _list; + private static final String ID = "entityId"; + + public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) + { + super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); + _list = (ListDefinitionImpl) list; + } + + @Override + public void configureDataIteratorContext(DataIteratorContext context) + { + if (context.getInsertOption().batch) + { + context.setMaxRowErrors(100); + context.setFailFast(false); + } + + context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); + } + + @Override + protected @Nullable AttachmentParentFactory getAttachmentParentFactory() + { + return new ListItemAttachmentParentFactory(); + } + + @Override + protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException + { + Map ret = null; + + if (null != listRow) + { + SimpleFilter keyFilter = getKeyFilter(listRow); + + if (null != keyFilter) + { + TableInfo queryTable = getQueryTable(); + Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); + + if (null != raw && !raw.isEmpty()) + { + ret = new CaseInsensitiveHashMap<>(); + + // EntityId + ret.put("EntityId", raw.get("entityid")); + + for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) + { + String propName = prop.getName(); + ColumnInfo column = queryTable.getColumn(propName); + Object value = column.getValue(raw); + if (value != null) + ret.put(propName, value); + } + } + } + } + + return ret; + } + + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + for (Map row : rows) + { + aliasColumns(getColumnMapping(), row); + } + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + + if (null != result) + { + ListManager mgr = ListManager.get(); + + for (Map row : result) + { + if (null != row.get(ID)) + { + // Audit each row + String entityId = (String) row.get(ID); + String newRecord = mgr.formatAuditItem(_list, user, row); + + mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); + } + } + + if (!result.isEmpty() && !errors.hasErrors()) + mgr.indexList(_list); + } + + return result; + } + + private User getListUser(User user, Container container) + { + if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) + { + // if the list is a picklist and you have permission to manage picklists, that equates + // to having editor permission. + return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); + } + return user; + } + + @Override + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + @Override + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, + @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + User updatedUser = getListUser(user, container); + DataIteratorContext context = new DataIteratorContext(errors); + context.setFailFast(false); + context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter + context.setSupportAutoIncrementKey(supportAutoIncrementKey); + context.setLookupResolutionType(lookupResolutionType); + setAttachmentDirectory(attachmentDir); + TableInfo ti = _list.getTable(updatedUser); + + if (null != ti) + { + try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) + { + int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); + + if (!errors.hasErrors()) + { + //Make entry to audit log if anything was inserted + if (imported > 0) + ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); + + transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + + return imported; + } + + return 0; + } + } + + return 0; + } + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + } + + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); + if (count > 0 && !errors.hasErrors()) + ListManager.get().indexList(_list); + return count; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data into this table."); + + List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (!result.isEmpty()) + ListManager.get().indexList(_list); + return result; + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + // TODO: Check for equivalency so that attachments can be deleted etc. + + Map dps = new HashMap<>(); + for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) + { + dps.put(dp.getPropertyURI(), dp); + } + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + ListItm itm = new ListItm(); + itm.setEntityId((String) oldRow.get(ID)); + itm.setListId(_list.getListId()); + itm.setKey(oldRow.get(_list.getKeyName())); + + ListItem item = new ListItemImpl(_list, itm); + + if (item.getProperties() != null) + { + List errors = new ArrayList<>(); + for (Map.Entry entry : dps.entrySet()) + { + Object value = row.get(entry.getValue().getName()); + validateProperty(entry.getValue(), value, row, errors, validatorCache); + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + + // MVIndicators + Map rowCopy = new CaseInsensitiveHashMap<>(); + ArrayList modifiedAttachmentColumns = new ArrayList<>(); + ArrayList attachmentFiles = new ArrayList<>(); + + TableInfo qt = getQueryTable(); + for (Map.Entry r : row.entrySet()) + { + ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); + rowCopy.put(r.getKey(), r.getValue()); + + // 22747: Attachment columns + if (null != column) + { + DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); + if (null != dp && isAttachmentProperty(dp)) + { + modifiedAttachmentColumns.add(column); + + // setup any new attachments + if (r.getValue() instanceof AttachmentFile file) + { + if (null != file.getFilename()) + attachmentFiles.add(file); + } + else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) + { + throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); + } + } + } + } + + // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) + Object newRowKey = getField(rowCopy, _list.getKeyName()); + Object oldRowKey = getField(oldRow, _list.getKeyName()); + + if (null == newRowKey && null != oldRowKey) + rowCopy.put(_list.getKeyName(), oldRowKey); + + Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); + + if (null != result) + { + result = getRow(user, container, result); + + if (null != result && null != result.get(ID)) + { + ListManager mgr = ListManager.get(); + String entityId = (String) result.get(ID); + + try + { + // Remove prior attachment -- only includes columns which are modified in this update + for (ColumnInfo col : modifiedAttachmentColumns) + { + Object value = oldRow.get(col.getName()); + if (null != value) + { + AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); + } + } + + // Update attachments + if (!attachmentFiles.isEmpty()) + AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); + } + catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) + { + // issues 21503, 28633: turn these into a validation exception to get a nicer error + throw new ValidationException(e.getMessage()); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + finally + { + for (AttachmentFile attachmentFile : attachmentFiles) + { + try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} + } + } + + String oldRecord = mgr.formatAuditItem(_list, user, oldRow); + String newRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); + } + } + + return result; + } + + // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() + private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) + { + //check for isRequired + if (prop.isRequired()) + { + // for mv indicator columns either an indicator or a field value is sufficient + boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); + if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) + { + if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) + { + errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); + return false; + } + } + } + + if (null != value) + { + for (IPropertyValidator validator : prop.getValidators()) + { + if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) + return false; + } + } + + return true; + } + + private record ListRecord(Object key, String entityId) { } + + @Override + public Map moveRows( + User _user, + Container container, + Container targetContainer, + List> rows, + BatchValidationException errors, + @Nullable Map configParameters, + @Nullable Map extraScriptContext + ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException + { + // Ensure the list is in scope for the target container + if (null == ListService.get().getList(targetContainer, _list.getName(), true)) + { + errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); + throw errors; + } + + User user = getListUser(_user, container); + Map> containerRows = getListRowsForMoveRows(container, user, targetContainer, rows, errors); + if (errors.hasErrors()) + throw errors; + + int fileAttachmentsMovedCount = 0; + int listAuditEventsCreatedCount = 0; + int listAuditEventsMovedCount = 0; + int listRecordsCount = 0; + int queryAuditEventsMovedCount = 0; + + if (containerRows.isEmpty()) + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; + String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; + boolean hasAttachmentProperties = _list.getDomainOrThrow() + .getProperties() + .stream() + .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); + + ListAuditProvider listAuditProvider = new ListAuditProvider(); + final int BATCH_SIZE = 5_000; + boolean isAuditEnabled = auditBehavior != null && AuditBehaviorType.NONE != auditBehavior; + + try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) + { + if (isAuditEnabled && tx.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); + } + + List listAuditEvents = new ArrayList<>(); + + for (GUID containerId : containerRows.keySet()) + { + Container sourceContainer = ContainerManager.getForId(containerId); + if (sourceContainer == null) + throw new InvalidKeyException("Container '" + containerId + "' does not exist."); + + if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) + throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); + + TableInfo listTable = _list.getTable(user, sourceContainer); + if (listTable == null) + throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); + + List records = containerRows.get(containerId); + int numRecords = records.size(); + + for (int start = 0; start < numRecords; start += BATCH_SIZE) + { + int end = Math.min(start + BATCH_SIZE, numRecords); + List batch = records.subList(start, end); + List rowPks = batch.stream().map(ListRecord::key).toList(); + + // Before trigger per batch + Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); + if (errors.hasErrors()) + throw errors; + + listRecordsCount += Table.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); + if (errors.hasErrors()) + throw errors; + + if (hasAttachmentProperties) + { + fileAttachmentsMovedCount += moveAttachments(user, sourceContainer, targetContainer, batch, errors); + if (errors.hasErrors()) + throw errors; + } + + queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, ListQuerySchema.NAME, _list.getName()); + listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, batch.stream().map(ListRecord::entityId).toList()); + + // Detailed audit events per row + if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) + listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, batch); + + // After trigger per batch + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); + if (errors.hasErrors()) + throw errors; + } + + // Create a summary audit event for the source container + if (isAuditEnabled) + { + String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + // Create a summary audit event for the target container + if (isAuditEnabled) + { + String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + } + + if (!listAuditEvents.isEmpty()) + { + AuditLogService.get().addEvents(user, listAuditEvents, true); + listAuditEventsCreatedCount += listAuditEvents.size(); + } + + tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + + tx.commit(); + + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); + } + + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + } + + private Map moveRowsCounts(int fileAttachmentsMovedCount, int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) + { + return Map.of( + "fileAttachmentsMoved", fileAttachmentsMovedCount, + "listAuditEventsCreated", listAuditEventsCreated, + "listAuditEventsMoved", listAuditEventsMoved, + "listRecords", listRecords, + "queryAuditEventsMoved", queryAuditEventsMoved + ); + } + + private Map> getListRowsForMoveRows( + Container container, + User user, + Container targetContainer, + List> rows, + BatchValidationException errors + ) throws QueryUpdateServiceException + { + if (rows.isEmpty()) + return Collections.emptyMap(); + + String keyName = _list.getKeyName(); + List keys = new ArrayList<>(); + for (var row : rows) + { + Object key = getField(row, keyName); + if (key == null) + { + errors.addRowError(new ValidationException("Key field value required for moving list rows.")); + return Collections.emptyMap(); + } + + keys.add(getKeyFilterValue(key)); + } + + SimpleFilter filter = new SimpleFilter(); + FieldKey fieldKey = FieldKey.fromParts(keyName); + filter.addInClause(fieldKey, keys); + filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); + + // Request all rows without a container filter so that rows are more easily resolved across the list scope. + // Read permissions are subsequently checked upon loading a row. + TableInfo table = _list.getTable(user, container, ContainerFilter.getUnsafeEverythingFilter()); + if (table == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve table for list %s in %s", _list.getName(), container.getPath())); + + Map> containerRows = new HashMap<>(); + try (var result = new TableSelector(table, PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) + { + while (result.next()) + { + GUID containerId = new GUID(result.getString("Container")); + if (!containerRows.containsKey(containerId)) + { + var c = ContainerManager.getForId(containerId); + if (c == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve container for row in list %s in %s.", _list.getName(), container.getPath())); + else if (!c.hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read list rows in all source containers."); + } + + containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); + containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return containerRows; + } + + private int moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) + { + List parents = new ArrayList<>(); + for (ListRecord record : records) + parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); + + int count = 0; + try + { + count = AttachmentService.get().moveAttachments(targetContainer, parents, user); + } + catch (IOException e) + { + errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); + } + + return count; + } + + private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) + { + List auditEvents = new ArrayList<>(records.size()); + String keyName = _list.getKeyName(); + String sourcePath = sourceContainer.getPath(); + String targetPath = targetContainer.getPath(); + + for (ListRecord record : records) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); + event.setListItemEntityId(record.entityId); + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); + auditEvents.add(event); + } + + AuditLogService.get().addEvents(user, auditEvents, true); + + return auditEvents.size(); + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); + + if (null != result) + { + String entityId = (String) result.get(ID); + + if (null != entityId) + { + ListManager mgr = ListManager.get(); + String deletedRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); + + // Remove attachments + if (hasAttachmentProperties()) + AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); + + // Clean up Search indexer + if (!result.isEmpty()) + mgr.deleteItemIndex(_list, entityId); + } + } + + return result; + } + + + // Deletes attachments & discussions, and removes list documents from full-text search index. + public void deleteRelatedListData(final User user, final Container container) + { + // Unindex all item docs and the entire list doc + ListManager.get().deleteIndexedList(_list); + + // Delete attachments and discussions associated with a list in batches of 1,000 + new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() + { + @Override + public boolean accept(String entityId) + { + return null != entityId; + } + + @Override + public void exec(List entityIds) + { + // delete the related list data for this block + deleteRelatedListData(user, container, entityIds); + } + }); + } + + // delete the related list data for this block of entityIds + private void deleteRelatedListData(User user, Container container, List entityIds) + { + // Build up set of entityIds and AttachmentParents + List attachmentParents = new ArrayList<>(); + + // Delete Attachments + if (hasAttachmentProperties()) + { + for (String entityId : entityIds) + { + attachmentParents.add(new ListItemAttachmentParent(entityId, container)); + } + AttachmentService.get().deleteAttachments(attachmentParents); + } + } + + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + int result; + try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) + { + deleteRelatedListData(user, container); + result = super.truncateRows(getListUser(user, container), container); + transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } + + return result; + } + + @Nullable + public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException + { + String keyName = _list.getKeyName(); + Object key = getField(map, keyName); + + if (null == key) + { + // Auto-increment lists might not provide a key so allow them to pass through + if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) + return null; + throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); + } + + return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); + } + + @NotNull + private Object getKeyFilterValue(@NotNull Object key) + { + ListDefinition.KeyType type = _list.getKeyType(); + + // Check the type of the list to ensure proper casting of the key type + if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) + return isIntegral(key) ? key : Integer.valueOf(key.toString()); + + return key.toString(); + } + + @Nullable + private Object getField(Map map, String key) + { + /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ + Object value = map.get(key); + + if (null == value) + value = map.get(key + "_"); + + if (null == value) + value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); + + return value; + } + + /** + * Delegate class to generate an AttachmentParent + */ + public static class ListItemAttachmentParentFactory implements AttachmentParentFactory + { + @Override + public AttachmentParent generateAttachmentParent(String entityId, Container c) + { + return new ListItemAttachmentParent(entityId, c); + } + } + + /** + * Get Domain from list definition, unless null then get from super + */ + @Override + protected Domain getDomain() + { + return _list != null? + _list.getDomain() : + super.getDomain(); + } +} diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 76f970a6e05..58d659275f1 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -1,305 +1,304 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query.audit; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.view.ViewContext; -import org.labkey.query.controllers.QueryController; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; - - public static final String COLUMN_NAME_ROW_PK = "RowPk"; - public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; - public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public QueryUpdateAuditProvider() - { - super(new QueryUpdateAuditDomainKind()); - } - - @Override - public String getEventName() - { - return QUERY_UPDATE_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "Query update events"; - } - - @Override - public String getDescription() - { - return "Data about insert and update queries."; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)QueryUpdateAuditEvent.class; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Schema Name"); - } - else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Query Name"); - } - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - { - col.setLabel("User Comment"); - } - } - }; - appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); - - return table; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - - public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) - { - return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); - } - - public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) - { - SimpleFilter filter = new SimpleFilter(); - filter.addInClause(FieldKey.fromParts(COLUMN_NAME_ROW_PK), rowPks.stream().map(Object::toString).toList()); - filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); - - return Table.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); - } - - public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent - { - private String _rowPk; - private String _schemaName; - private String _queryName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public QueryUpdateAuditEvent() - { - super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public QueryUpdateAuditEvent(Container container, String comment) - { - super(QUERY_UPDATE_AUDIT_EVENT, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); - } - - public String getRowPk() - { - return _rowPk; - } - - public void setRowPk(String rowPk) - { - _rowPk = rowPk; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("rowPk", getRowPk()); - elements.put("schemaName", getSchemaName()); - elements.put("queryName", getQueryName()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "QueryUpdateAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public QueryUpdateAuditDomainKind() - { - super(QUERY_UPDATE_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query.audit; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.view.ViewContext; +import org.labkey.query.controllers.QueryController; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; + + public static final String COLUMN_NAME_ROW_PK = "RowPk"; + public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; + public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public QueryUpdateAuditProvider() + { + super(new QueryUpdateAuditDomainKind()); + } + + @Override + public String getEventName() + { + return QUERY_UPDATE_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "Query update events"; + } + + @Override + public String getDescription() + { + return "Data about insert and update queries."; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)QueryUpdateAuditEvent.class; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Schema Name"); + } + else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Query Name"); + } + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + { + col.setLabel("User Comment"); + } + } + }; + appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); + + return table; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + + public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) + { + return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); + } + + public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) + { + SimpleFilter filter = new SimpleFilter(); + filter.addInClause(FieldKey.fromParts(COLUMN_NAME_ROW_PK), rowPks.stream().map(Object::toString).toList()); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); + + return Table.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); + } + + public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent + { + private String _rowPk; + private String _schemaName; + private String _queryName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public QueryUpdateAuditEvent() + { + super(); + } + + public QueryUpdateAuditEvent(Container container, String comment) + { + super(QUERY_UPDATE_AUDIT_EVENT, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), QUERY_UPDATE_AUDIT_EVENT); + } + + public String getRowPk() + { + return _rowPk; + } + + public void setRowPk(String rowPk) + { + _rowPk = rowPk; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("rowPk", getRowPk()); + elements.put("schemaName", getSchemaName()); + elements.put("queryName", getQueryName()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "QueryUpdateAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public QueryUpdateAuditDomainKind() + { + super(QUERY_UPDATE_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 4b37da7c7dd..0a168d9aa21 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -1,8770 +1,8778 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.query.controllers; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.servlet.http.HttpSession; -import org.antlr.runtime.tree.Tree; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.apache.commons.collections4.multimap.HashSetValuedHashMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.xmlbeans.XmlError; -import org.apache.xmlbeans.XmlException; -import org.apache.xmlbeans.XmlOptions; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONParserConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.action.Action; -import org.labkey.api.action.ActionType; -import org.labkey.api.action.ApiJsonForm; -import org.labkey.api.action.ApiJsonWriter; -import org.labkey.api.action.ApiQueryResponse; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiResponseWriter; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.ApiVersion; -import org.labkey.api.action.ConfirmAction; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.ExportException; -import org.labkey.api.action.ExtendedApiQueryResponse; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasBindParameters; -import org.labkey.api.action.JsonInputLimit; -import org.labkey.api.action.LabKeyError; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.NullSafeBindException; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReportingApiQueryResponse; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.collections.RowMapFactory; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.AbstractTableInfo; -import org.labkey.api.data.ActionButton; -import org.labkey.api.data.Aggregate; -import org.labkey.api.data.AnalyticsProviderItem; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.CachedResultSets; -import org.labkey.api.data.ColumnHeaderType; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.ExcelWriter; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.JdbcMetaDataSelector; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.JsonWriter; -import org.labkey.api.data.PHI; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.PropertyMap; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.QueryLogging; -import org.labkey.api.data.ResultSetView; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.ShowRows; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.VirtualTable; -import org.labkey.api.data.dialect.JdbcMetaDataLocator; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.ListofMapsDataIterator; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ProvenanceRecordingParams; -import org.labkey.api.exp.api.ProvenanceService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.AbstractQueryImportAction; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.CustomView; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.DuplicateKeyException; -import org.labkey.api.query.ExportScriptModel; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryAction; -import org.labkey.api.query.QueryDefinition; -import org.labkey.api.query.QueryException; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QueryParam; -import org.labkey.api.query.QueryParseException; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryUpdateForm; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.QueryUrls; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RuntimeValidationException; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleSchemaTreeVisitor; -import org.labkey.api.query.TempQuerySettings; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.UserSchemaAction; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.report.ReportDescriptor; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.CSRF; -import org.labkey.api.security.IgnoresTermsOfUse; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.RequiresAllOf; -import org.labkey.api.security.RequiresAnyOf; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.permissions.AbstractActionPermissionTest; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.EditSharedViewPermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; -import org.labkey.api.stats.ColumnAnalyticsProvider; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.DOM; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.JavaScriptFragment; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.ReturnURLString; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.util.XmlBeansUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.DetailsView; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.InsertView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.UpdateView; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.writer.HtmlWriter; -import org.labkey.api.writer.ZipFile; -import org.labkey.data.xml.ColumnType; -import org.labkey.data.xml.ImportTemplateType; -import org.labkey.data.xml.TableType; -import org.labkey.data.xml.TablesDocument; -import org.labkey.data.xml.TablesType; -import org.labkey.data.xml.externalSchema.TemplateSchemaType; -import org.labkey.data.xml.queryCustomView.FilterType; -import org.labkey.query.AutoGeneratedDetailsCustomView; -import org.labkey.query.AutoGeneratedInsertCustomView; -import org.labkey.query.AutoGeneratedUpdateCustomView; -import org.labkey.query.CustomViewImpl; -import org.labkey.query.CustomViewUtil; -import org.labkey.query.EditQueriesPermission; -import org.labkey.query.EditableCustomView; -import org.labkey.query.LinkedTableInfo; -import org.labkey.query.MetadataTableJSON; -import org.labkey.query.ModuleCustomQueryDefinition; -import org.labkey.query.ModuleCustomView; -import org.labkey.query.QueryServiceImpl; -import org.labkey.query.TableXML; -import org.labkey.query.audit.QueryExportAuditProvider; -import org.labkey.query.audit.QueryUpdateAuditProvider; -import org.labkey.query.model.MetadataTableJSONMixin; -import org.labkey.query.persist.AbstractExternalSchemaDef; -import org.labkey.query.persist.CstmView; -import org.labkey.query.persist.ExternalSchemaDef; -import org.labkey.query.persist.ExternalSchemaDefCache; -import org.labkey.query.persist.LinkedSchemaDef; -import org.labkey.query.persist.QueryDef; -import org.labkey.query.persist.QueryManager; -import org.labkey.query.reports.ReportsController; -import org.labkey.query.reports.getdata.DataRequest; -import org.labkey.query.sql.QNode; -import org.labkey.query.sql.Query; -import org.labkey.query.sql.SqlParser; -import org.labkey.query.xml.ApiTestsDocument; -import org.labkey.query.xml.TestCaseType; -import org.labkey.remoteapi.RemoteConnections; -import org.labkey.remoteapi.SelectRowsStreamHack; -import org.labkey.remoteapi.query.SelectRowsCommand; -import org.labkey.vfs.FileLike; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.PropertyValue; -import org.springframework.beans.PropertyValues; -import org.springframework.dao.DataAccessException; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.ModelAndView; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.trimToEmpty; -import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; -import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; -import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; -import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; -import static org.labkey.api.util.DOM.BR; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.FONT; -import static org.labkey.api.util.DOM.Renderable; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.query.MetadataTableJSON.getTableType; -import static org.labkey.query.MetadataTableJSON.parseDocument; - -@SuppressWarnings("DefaultAnnotationParam") - -public class QueryController extends SpringActionController -{ - private static final Logger LOG = LogManager.getLogger(QueryController.class); - private static final String ROW_ATTACHMENT_INDEX_DELIM = "::"; - - private static final Set RESERVED_VIEW_NAMES = CaseInsensitiveHashSet.of( - "Default", - AutoGeneratedDetailsCustomView.NAME, - AutoGeneratedInsertCustomView.NAME, - AutoGeneratedUpdateCustomView.NAME - ); - - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(QueryController.class, - ValidateQueryAction.class, - ValidateQueriesAction.class, - GetSchemaQueryTreeAction.class, - GetQueryDetailsAction.class, - ViewQuerySourceAction.class - ); - - public QueryController() - { - setActionResolver(_actionResolver); - } - - public static void registerAdminConsoleLinks() - { - AdminConsole.addLink(AdminConsole.SettingsLinkType.Diagnostics, "data sources", new ActionURL(DataSourceAdminAction.class, ContainerManager.getRoot())); - } - - public static class RemoteQueryConnectionUrls - { - public static ActionURL urlManageRemoteConnection(Container c) - { - return new ActionURL(ManageRemoteConnectionsAction.class, c); - } - - public static ActionURL urlCreateRemoteConnection(Container c) - { - return new ActionURL(EditRemoteConnectionAction.class, c); - } - - public static ActionURL urlEditRemoteConnection(Container c, String connectionName) - { - ActionURL url = new ActionURL(EditRemoteConnectionAction.class, c); - url.addParameter("connectionName", connectionName); - return url; - } - - public static ActionURL urlSaveRemoteConnection(Container c) - { - return new ActionURL(EditRemoteConnectionAction.class, c); - } - - public static ActionURL urlDeleteRemoteConnection(Container c, @Nullable String connectionName) - { - ActionURL url = new ActionURL(DeleteRemoteConnectionAction.class, c); - if (connectionName != null) - url.addParameter("connectionName", connectionName); - return url; - } - - public static ActionURL urlTestRemoteConnection(Container c, String connectionName) - { - ActionURL url = new ActionURL(TestRemoteConnectionAction.class, c); - url.addParameter("connectionName", connectionName); - return url; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); - if (!errors.hasErrors()) - { - String name = remoteConnectionForm.getConnectionName(); - // package the remote-connection properties into the remoteConnectionForm and pass them along - Map map1 = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); - remoteConnectionForm.setUrl(map1.get("URL")); - remoteConnectionForm.setUserEmail(map1.get("user")); - remoteConnectionForm.setPassword(map1.get("password")); - remoteConnectionForm.setFolderPath(map1.get("container")); - } - setHelpTopic("remoteConnection"); - return new JspView<>("/org/labkey/query/view/createRemoteConnection.jsp", remoteConnectionForm, errors); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - return RemoteConnections.createOrEditRemoteConnection(remoteConnectionForm, getContainer(), errors); - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Create/Edit Remote Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class DeleteRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/query/view/confirmDeleteConnection.jsp", remoteConnectionForm, errors); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); - return RemoteConnections.deleteRemoteConnection(remoteConnectionForm, getContainer()); - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Confirm Delete Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - String name = remoteConnectionForm.getConnectionName(); - String schemaName = "core"; // test Schema Name - String queryName = "Users"; // test Query Name - - // Extract the username, password, and container from the secure property store - Map singleConnectionMap = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); - if (singleConnectionMap.isEmpty()) - throw new NotFoundException(); - String url = singleConnectionMap.get(RemoteConnections.FIELD_URL); - String user = singleConnectionMap.get(RemoteConnections.FIELD_USER); - String password = singleConnectionMap.get(RemoteConnections.FIELD_PASSWORD); - String container = singleConnectionMap.get(RemoteConnections.FIELD_CONTAINER); - - // connect to the remote server and retrieve an input stream - org.labkey.remoteapi.Connection cn = new org.labkey.remoteapi.Connection(url, user, password); - final SelectRowsCommand cmd = new SelectRowsCommand(schemaName, queryName); - try - { - DataIteratorBuilder source = SelectRowsStreamHack.go(cn, container, cmd, getContainer()); - // immediately close the source after opening it, this is a test. - source.getDataIterator(new DataIteratorContext()).close(); - } - catch (Exception e) - { - errors.addError(new LabKeyError("The listed credentials for this remote connection failed to connect.")); - return new JspView<>("/org/labkey/query/view/testRemoteConnectionsFailure.jsp", remoteConnectionForm); - } - - return new JspView<>("/org/labkey/query/view/testRemoteConnectionsSuccess.jsp", remoteConnectionForm); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - return true; - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - public static class QueryUrlsImpl implements QueryUrls - { - @Override - public ActionURL urlSchemaBrowser(Container c) - { - return new ActionURL(BeginAction.class, c); - } - - @Override - public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName) - { - ActionURL ret = urlSchemaBrowser(c); - if (schemaName != null) - { - ret.addParameter(QueryParam.schemaName.toString(), schemaName); - } - return ret; - } - - @Override - public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName, @Nullable String queryName) - { - if (StringUtils.isEmpty(queryName)) - return urlSchemaBrowser(c, schemaName); - ActionURL ret = urlSchemaBrowser(c); - ret.addParameter(QueryParam.schemaName.toString(), trimToEmpty(schemaName)); - ret.addParameter(QueryParam.queryName.toString(), trimToEmpty(queryName)); - return ret; - } - - public ActionURL urlExternalSchemaAdmin(Container c) - { - return urlExternalSchemaAdmin(c, null); - } - - public ActionURL urlExternalSchemaAdmin(Container c, @Nullable String message) - { - ActionURL url = new ActionURL(AdminAction.class, c); - - if (null != message) - url.addParameter("message", message); - - return url; - } - - public ActionURL urlInsertExternalSchema(Container c) - { - return new ActionURL(InsertExternalSchemaAction.class, c); - } - - public ActionURL urlNewQuery(Container c) - { - return new ActionURL(NewQueryAction.class, c); - } - - public ActionURL urlUpdateExternalSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(EditExternalSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - public ActionURL urlReloadExternalSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(ReloadExternalSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - public ActionURL urlDeleteSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(DeleteSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - @Override - public ActionURL urlStartBackgroundRReport(@NotNull ActionURL baseURL, String reportId) - { - ActionURL result = baseURL.clone(); - result.setAction(ReportsController.StartBackgroundRReportAction.class); - result.replaceParameter(ReportDescriptor.Prop.reportId, reportId); - return result; - } - - @Override - public ActionURL urlExecuteQuery(@NotNull ActionURL baseURL) - { - ActionURL result = baseURL.clone(); - result.setAction(ExecuteQueryAction.class); - return result; - } - - @Override - public ActionURL urlExecuteQuery(Container c, String schemaName, String queryName) - { - return new ActionURL(ExecuteQueryAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter(QueryParam.queryName, queryName); - } - - @Override - public @NotNull ActionURL urlCreateExcelTemplate(Container c, String schemaName, String queryName) - { - return new ActionURL(ExportExcelTemplateAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter("query.queryName", queryName); - } - - @Override - public ActionURL urlMetadataQuery(Container c, String schemaName, String queryName) - { - return new ActionURL(MetadataQueryAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter(QueryParam.queryName, queryName); - } - } - - @Override - public PageConfig defaultPageConfig() - { - // set default help topic for query controller - PageConfig config = super.defaultPageConfig(); - config.setHelpTopic("querySchemaBrowser"); - return config; - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class DataSourceAdminAction extends SimpleViewAction - { - public DataSourceAdminAction() - { - } - - public DataSourceAdminAction(ViewContext viewContext) - { - setViewContext(viewContext); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Site Admin or Troubleshooter? Troubleshooters can see all the information but can't test data sources. - boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); - List allDefs = QueryManager.get().getExternalSchemaDefs(null); - - MultiValuedMap byDataSourceName = new ArrayListValuedHashMap<>(); - - for (ExternalSchemaDef def : allDefs) - byDataSourceName.put(def.getDataSource(), def); - - MutableInt row = new MutableInt(); - - Renderable r = DOM.DIV( - DIV("This page lists all the data sources defined in your " + AppProps.getInstance().getWebappConfigurationFilename() + " file that were available when first referenced and the external schemas defined in each."), - BR(), - TABLE(cl("labkey-data-region"), - TR(cl("labkey-show-borders"), - hasAdminOpsPerms ? TD(cl("labkey-column-header"), "Test") : null, - TD(cl("labkey-column-header"), "Data Source"), - TD(cl("labkey-column-header"), "Current Status"), - TD(cl("labkey-column-header"), "URL"), - TD(cl("labkey-column-header"), "Database Name"), - TD(cl("labkey-column-header"), "Product Name"), - TD(cl("labkey-column-header"), "Product Version"), - TD(cl("labkey-column-header"), "Max Connections"), - TD(cl("labkey-column-header"), "Active Connections"), - TD(cl("labkey-column-header"), "Idle Connections"), - TD(cl("labkey-column-header"), "Max Wait (ms)") - ), - DbScope.getDbScopes().stream() - .flatMap(scope -> { - String rowStyle = row.getAndIncrement() % 2 == 0 ? "labkey-alternate-row labkey-show-borders" : "labkey-row labkey-show-borders"; - Object status; - boolean connected = false; - try (Connection ignore = scope.getConnection()) - { - status = "connected"; - connected = true; - } - catch (Exception e) - { - status = FONT(cl("labkey-error"), "disconnected"); - } - - return Stream.of( - TR( - cl(rowStyle), - hasAdminOpsPerms ? TD(connected ? new ButtonBuilder("Test").href(new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", scope.getDataSourceName())) : "") : null, - TD(HtmlString.NBSP, scope.getDisplayName()), - TD(status), - TD(scope.getDatabaseUrl()), - TD(scope.getDatabaseName()), - TD(scope.getDatabaseProductName()), - TD(scope.getDatabaseProductVersion()), - TD(scope.getDataSourceProperties().getMaxTotal()), - TD(scope.getDataSourceProperties().getNumActive()), - TD(scope.getDataSourceProperties().getNumIdle()), - TD(scope.getDataSourceProperties().getMaxWaitMillis()) - ), - TR( - cl(rowStyle), - TD(HtmlString.NBSP), - TD(at(DOM.Attribute.colspan, 10), getDataSourceTable(byDataSourceName.get(scope.getDataSourceName()))) - ) - ); - }) - ) - ); - - return new HtmlView(r); - } - - private Renderable getDataSourceTable(Collection dsDefs) - { - if (dsDefs.isEmpty()) - return TABLE(TR(TD(HtmlString.NBSP))); - - MultiValuedMap byContainerPath = new ArrayListValuedHashMap<>(); - - for (ExternalSchemaDef def : dsDefs) - byContainerPath.put(def.getContainerPath(), def); - - TreeSet paths = new TreeSet<>(byContainerPath.keySet()); - - return TABLE(paths.stream() - .map(path -> TR(TD(at(DOM.Attribute.colspan, 4), getDataSourcePath(path, byContainerPath.get(path))))) - ); - } - - private Renderable getDataSourcePath(String path, Collection unsorted) - { - List defs = new ArrayList<>(unsorted); - defs.sort(Comparator.comparing(AbstractExternalSchemaDef::getUserSchemaName, String.CASE_INSENSITIVE_ORDER)); - Container c = ContainerManager.getForPath(path); - - if (null == c) - return TD(); - - boolean hasRead = c.hasPermission(getUser(), ReadPermission.class); - QueryUrlsImpl urls = new QueryUrlsImpl(); - - return - TD(TABLE( - TR(TD( - at(DOM.Attribute.colspan, 3), - hasRead ? LinkBuilder.simpleLink(path, urls.urlExternalSchemaAdmin(c)) : path - )), - TR(TD(TABLE( - defs.stream() - .map(def -> TR(TD( - at(DOM.Attribute.style, "padding-left:20px"), - hasRead ? LinkBuilder.simpleLink(def.getUserSchemaName() + - (!Strings.CS.equals(def.getSourceSchemaName(), def.getUserSchemaName()) ? " (" + def.getSourceSchemaName() + ")" : ""), urls.urlUpdateExternalSchema(c, def)) - : def.getUserSchemaName() - ))) - ))) - )); - } - - @Override - public void addNavTrail(NavTree root) - { - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Data Source Administration", getClass(), getContainer()); - } - } - - public static class TestDataSourceForm - { - private String _dataSource; - - public String getDataSource() - { - return _dataSource; - } - - @SuppressWarnings("unused") - public void setDataSource(String dataSource) - { - _dataSource = dataSource; - } - } - - public static class TestDataSourceConfirmForm extends TestDataSourceForm - { - private String _excludeSchemas; - private String _excludeTables; - - public String getExcludeSchemas() - { - return _excludeSchemas; - } - - @SuppressWarnings("unused") - public void setExcludeSchemas(String excludeSchemas) - { - _excludeSchemas = excludeSchemas; - } - - public String getExcludeTables() - { - return _excludeTables; - } - - @SuppressWarnings("unused") - public void setExcludeTables(String excludeTables) - { - _excludeTables = excludeTables; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestDataSourceConfirmAction extends FormViewAction - { - private DbScope _scope; - - @Override - public ModelAndView getView(TestDataSourceConfirmForm form, boolean reshow, BindException errors) throws Exception - { - validateCommand(form, errors); - return new JspView<>("/org/labkey/query/view/testDataSourceConfirm.jsp", _scope); - } - - @Override - public void validateCommand(TestDataSourceConfirmForm form, Errors errors) - { - _scope = DbScope.getDbScope(form.getDataSource()); - - if (null == _scope) - throw new NotFoundException("Could not resolve data source " + form.getDataSource()); - } - - @Override - public boolean handlePost(TestDataSourceConfirmForm form, BindException errors) throws Exception - { - saveTestDataSourceProperties(form); - return true; - } - - @Override - public URLHelper getSuccessURL(TestDataSourceConfirmForm form) - { - return new ActionURL(TestDataSourceAction.class, getContainer()).addParameter("dataSource", _scope.getDataSourceName()); - } - - @Override - public void addNavTrail(NavTree root) - { - new DataSourceAdminAction(getViewContext()).addNavTrail(root); - root.addChild("Prepare Test of " + _scope.getDataSourceName()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestDataSourceAction extends SimpleViewAction - { - private DbScope _scope; - - @Override - public ModelAndView getView(TestDataSourceForm form, BindException errors) - { - _scope = DbScope.getDbScope(form.getDataSource()); - - if (null == _scope) - throw new NotFoundException("Could not resolve data source " + form.getDataSource()); - - return new JspView<>("/org/labkey/query/view/testDataSource.jsp", _scope); - } - - @Override - public void addNavTrail(NavTree root) - { - new DataSourceAdminAction(getViewContext()).addNavTrail(root); - root.addChild("Test " + _scope.getDataSourceName()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ResetDataSourcePropertiesAction extends FormHandlerAction - { - @Override - public void validateCommand(TestDataSourceForm target, Errors errors) - { - } - - @Override - public boolean handlePost(TestDataSourceForm form, BindException errors) throws Exception - { - WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), false); - if (map != null) - map.delete(); - return true; - } - - @Override - public URLHelper getSuccessURL(TestDataSourceForm form) - { - return new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", form.getDataSource()) ; - } - } - - private static final String TEST_DATA_SOURCE_CATEGORY = "testDataSourceProperties"; - private static final String TEST_DATA_SOURCE_SCHEMAS_PROPERTY = "excludeSchemas"; - private static final String TEST_DATA_SOURCE_TABLES_PROPERTY = "excludeTables"; - - private static String getCategory(String dataSourceName) - { - return TEST_DATA_SOURCE_CATEGORY + "|" + dataSourceName; - } - - public static void saveTestDataSourceProperties(TestDataSourceConfirmForm form) - { - WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), true); - // Save empty entries as empty string to distinguish from null (which results in default values) - map.put(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, StringUtils.trimToEmpty(form.getExcludeSchemas())); - map.put(TEST_DATA_SOURCE_TABLES_PROPERTY, StringUtils.trimToEmpty(form.getExcludeTables())); - map.save(); - } - - public static TestDataSourceConfirmForm getTestDataSourceProperties(DbScope scope) - { - TestDataSourceConfirmForm form = new TestDataSourceConfirmForm(); - PropertyMap map = PropertyManager.getProperties(getCategory(scope.getDataSourceName())); - form.setExcludeSchemas(map.getOrDefault(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, scope.getSqlDialect().getDefaultSchemasToExcludeFromTesting())); - form.setExcludeTables(map.getOrDefault(TEST_DATA_SOURCE_TABLES_PROPERTY, scope.getSqlDialect().getDefaultTablesToExcludeFromTesting())); - - return form; - } - - @RequiresPermission(ReadPermission.class) - public static class BrowseAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/query/view/browse.jsp", null); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Schema Browser"); - } - } - - @RequiresPermission(ReadPermission.class) - public static class BeginAction extends QueryViewAction - { - @SuppressWarnings("UnusedDeclaration") - public BeginAction() - { - } - - public BeginAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - JspView view = new JspView<>("/org/labkey/query/view/browse.jsp", form); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Query Schema Browser", new QueryUrlsImpl().urlSchemaBrowser(getContainer())); - } - } - - @RequiresPermission(ReadPermission.class) - public class SchemaAction extends QueryViewAction - { - public SchemaAction() {} - - SchemaAction(QueryForm form) - { - _form = form; - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - _form = form; - return new JspView<>("/org/labkey/query/view/browse.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - if (_form != null && _form.getSchema() != null) - addSchemaActionNavTrail(root, _form.getSchema().getSchemaPath(), _form.getQueryName()); - } - } - - - void addSchemaActionNavTrail(NavTree root, SchemaKey schemaKey, String queryName) - { - if (getContainer().hasOneOf(getUser(), AdminPermission.class, PlatformDeveloperPermission.class)) - { - // Don't show the full query nav trail to non-admin/non-developer users as they almost certainly don't - // want it - try - { - String schemaName = schemaKey.toDisplayString(); - ActionURL url = new ActionURL(BeginAction.class, getContainer()); - url.addParameter("schemaName", schemaKey.toString()); - url.addParameter("queryName", queryName); - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild(schemaName + " Schema", url); - } - catch (NullPointerException e) - { - LOG.error("NullPointerException in addNavTrail", e); - } - } - } - - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.SelectData.class) - public class NewQueryAction extends FormViewAction - { - private NewQueryForm _form; - private ActionURL _successUrl; - - @Override - public void validateCommand(NewQueryForm target, org.springframework.validation.Errors errors) - { - target.ff_newQueryName = StringUtils.trimToNull(target.ff_newQueryName); - if (null == target.ff_newQueryName) - errors.reject(ERROR_MSG, "QueryName is required"); - } - - @Override - public ModelAndView getView(NewQueryForm form, boolean reshow, BindException errors) - { - form.ensureSchemaExists(); - - if (!form.getSchema().canCreate()) - { - throw new UnauthorizedException(); - } - - getPageConfig().setFocusId("ff_newQueryName"); - _form = form; - setHelpTopic("sqlTutorial"); - return new JspView<>("/org/labkey/query/view/newQuery.jsp", form, errors); - } - - @Override - public boolean handlePost(NewQueryForm form, BindException errors) - { - form.ensureSchemaExists(); - - if (!form.getSchema().canCreate()) - { - throw new UnauthorizedException(); - } - - try - { - if (StringUtils.isEmpty(form.ff_baseTableName)) - { - errors.reject(ERROR_MSG, "You must select a base table or query name."); - return false; - } - - UserSchema schema = form.getSchema(); - String newQueryName = form.ff_newQueryName; - QueryDef existing = QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), newQueryName, true); - if (existing != null) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); - return false; - } - TableInfo existingTable = form.getSchema().getTable(newQueryName, null); - if (existingTable != null) - { - errors.reject(ERROR_MSG, "A table with the name '" + newQueryName + "' already exists."); - return false; - } - // bug 6095 -- conflicting query and dataset names - if (form.getSchema().getTableNames().contains(newQueryName)) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists as a table"); - return false; - } - QueryDefinition newDef = QueryService.get().createQueryDef(getUser(), getContainer(), form.getSchemaKey(), form.ff_newQueryName); - Query query = new Query(schema); - query.setRootTable(FieldKey.fromParts(form.ff_baseTableName)); - String sql = query.getQueryText(); - if (null == sql) - sql = "SELECT * FROM \"" + form.ff_baseTableName + "\""; - newDef.setSql(sql); - - try - { - newDef.save(getUser(), getContainer()); - } - catch (SQLException x) - { - if (RuntimeSQLException.isConstraintException(x)) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); - return false; - } - else - { - throw x; - } - } - - _successUrl = newDef.urlFor(form.ff_redirect); - return true; - } - catch (Exception e) - { - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); - errors.reject(ERROR_MSG, Objects.toString(e.getMessage(), e.toString())); - return false; - } - } - - @Override - public ActionURL getSuccessURL(NewQueryForm newQueryForm) - { - return _successUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - root.addChild("New Query", new QueryUrlsImpl().urlNewQuery(getContainer())); - } - } - - // CONSIDER : deleting this action after the SQL editor UI changes are finalized, keep in mind that built-in views - // use this view as well via the edit metadata page. - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) // Note: This action deals with just meta data; it AJAXes data into place using GetWebPartAction - public class SourceQueryAction extends SimpleViewAction - { - public SourceForm _form; - public UserSchema _schema; - public QueryDefinition _queryDef; - - - @Override - public void validate(SourceForm target, BindException errors) - { - _form = target; - if (StringUtils.isEmpty(target.getSchemaName())) - throw new NotFoundException("schema name not specified"); - if (StringUtils.isEmpty(target.getQueryName())) - throw new NotFoundException("query name not specified"); - - QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); - if (null == querySchema) - throw new NotFoundException("schema not found: " + _form.getSchemaKey().toDisplayString()); - if (!(querySchema instanceof UserSchema)) - throw new NotFoundException("Could not find the schema '" + _form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); - _schema = (UserSchema)querySchema; - } - - - @Override - public ModelAndView getView(SourceForm form, BindException errors) - { - _queryDef = _schema.getQueryDef(form.getQueryName()); - if (null == _queryDef) - _queryDef = _schema.getQueryDefForTable(form.getQueryName()); - if (null == _queryDef) - throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - try - { - if (form.ff_queryText == null) - { - form.ff_queryText = _queryDef.getSql(); - form.ff_metadataText = _queryDef.getMetadataXml(); - if (null == form.ff_metadataText) - form.ff_metadataText = form.getDefaultMetadataText(); - } - - for (QueryException qpe : _queryDef.getParseErrors(_schema)) - { - errors.reject(ERROR_MSG, Objects.toString(qpe.getMessage(), qpe.toString())); - } - } - catch (Exception e) - { - try - { - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); - } - catch (Throwable t) - { - // - } - errors.reject("ERROR_MSG", e.toString()); - LOG.error("Error", e); - } - - Renderable moduleWarning = null; - if (_queryDef instanceof ModuleCustomQueryDefinition mcqd && _queryDef.canEdit(getUser())) - { - moduleWarning = DIV(cl("labkey-warning-messages"), - "This SQL query is defined in the '" + mcqd.getModuleName() + "' module in directory '" + mcqd.getSqlFile().getParent() + "'.", - BR(), - "Changes to this query will be reflected in all usages across different folders on the server." - ); - } - - var sourceQueryView = new JspView<>("/org/labkey/query/view/sourceQuery.jsp", this, errors); - WebPartView ret = sourceQueryView; - if (null != moduleWarning) - ret = new VBox(new HtmlView(moduleWarning), sourceQueryView); - return ret; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("useSqlEditor"); - - addSchemaActionNavTrail(root, _form.getSchemaKey(), _form.getQueryName()); - - root.addChild("Edit " + _form.getQueryName()); - } - } - - - /** - * Ajax action to save a query. If the save is successful the request will return successfully. A query - * with SQL syntax errors can still be saved successfully. - * - * If the SQL contains parse errors, a parseErrors object will be returned which contains an array of - * JSON serialized error information. - */ - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.Configure.class) - public static class SaveSourceQueryAction extends MutatingApiAction - { - private UserSchema _schema; - - @Override - public void validateForm(SourceForm form, Errors errors) - { - if (StringUtils.isEmpty(form.getSchemaName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - if (StringUtils.isEmpty(form.getQueryName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - - QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), form.getSchemaKey()); - if (null == querySchema) - throw new NotFoundException("schema not found: " + form.getSchemaKey().toDisplayString()); - if (!(querySchema instanceof UserSchema)) - throw new NotFoundException("Could not find the schema '" + form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); - _schema = (UserSchema)querySchema; - - XmlOptions options = XmlBeansUtil.getDefaultParseOptions(); - List xmlErrors = new ArrayList<>(); - options.setErrorListener(xmlErrors); - try - { - // had a couple of real-world failures due to null pointers in this code, so it's time to be paranoid - if (form.ff_metadataText != null) - { - TablesDocument tablesDoc = TablesDocument.Factory.parse(form.ff_metadataText, options); - if (tablesDoc != null) - { - tablesDoc.validate(options); - TablesType tablesType = tablesDoc.getTables(); - if (tablesType != null) - { - for (TableType tableType : tablesType.getTableArray()) - { - if (null != tableType) - { - if (!Objects.equals(tableType.getTableName(), form.getQueryName())) - { - errors.reject(ERROR_MSG, "Table name in the XML metadata must match the table/query name: " + form.getQueryName()); - } - - TableType.Columns tableColumns = tableType.getColumns(); - if (null != tableColumns) - { - ColumnType[] tableColumnArray = tableColumns.getColumnArray(); - for (ColumnType column : tableColumnArray) - { - if (column.isSetPhi() || column.isSetProtected()) - { - throw new IllegalArgumentException("PHI/protected metadata must not be set here."); - } - - ColumnType.Fk fk = column.getFk(); - if (null != fk) - { - try - { - validateForeignKey(fk, column, errors); - validateLookupFilter(AbstractTableInfo.parseXMLLookupFilters(fk.getFilters()), errors); - } - catch (ValidationException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - } - } - } - } - } - } - } - } - } - catch (XmlException e) - { - throw new RuntimeValidationException(e); - } - - for (XmlError xmle : xmlErrors) - { - errors.reject(ERROR_MSG, XmlBeansUtil.getErrorMessage(xmle)); - } - } - - private void validateForeignKey(ColumnType.Fk fk, ColumnType column, Errors errors) - { - if (fk.isSetFkMultiValued()) - { - // issue 51695 : don't let users create unsupported MVFK types - String type = fk.getFkMultiValued(); - if (!AbstractTableInfo.MultiValuedFkType.junction.name().equals(type)) - { - errors.reject(ERROR_MSG, String.format("Column : \"%s\" has an invalid fkMultiValued value : \"%s\" is not supported.", column.getColumnName(), type)); - } - } - } - - private void validateLookupFilter(Map> filterMap, Errors errors) - { - filterMap.forEach((operation, filters) -> { - - String displayStr = "Filter for operation : " + operation.name(); - for (FilterType filter : filters) - { - if (isBlank(filter.getColumn())) - errors.reject(ERROR_MSG, displayStr + " requires columnName"); - - if (null == filter.getOperator()) - { - errors.reject(ERROR_MSG, displayStr + " requires operator"); - } - else - { - CompareType compareType = CompareType.getByURLKey(filter.getOperator().toString()); - if (null == compareType) - { - errors.reject(ERROR_MSG, displayStr + " operator is invalid"); - } - else - { - if (compareType.isDataValueRequired() && null == filter.getValue()) - errors.reject(ERROR_MSG, displayStr + " requires a value but none is specified"); - } - } - } - - try - { - // attempt to convert to something we can query against - SimpleFilter.fromXml(filters.toArray(new FilterType[0])); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - }); - } - - @Override - public ApiResponse execute(SourceForm form, BindException errors) - { - var queryDef = _schema.getQueryDef(form.getQueryName()); - if (null == queryDef) - queryDef = _schema.getQueryDefForTable(form.getQueryName()); - if (null == queryDef) - throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - ApiSimpleResponse response = new ApiSimpleResponse(); - - try - { - if (form.ff_queryText != null) - { - if (!queryDef.isSqlEditable()) - throw new UnauthorizedException("Query SQL is not editable."); - - if (!queryDef.canEdit(getUser())) - throw new UnauthorizedException("Edit permissions are required."); - - queryDef.setSql(form.ff_queryText); - } - - String metadataText = StringUtils.trimToNull(form.ff_metadataText); - if (!Objects.equals(metadataText, queryDef.getMetadataXml())) - { - if (queryDef.isMetadataEditable()) - { - if (!queryDef.canEditMetadata(getUser())) - throw new UnauthorizedException("Edit metadata permissions are required."); - - if (!getUser().isTrustedBrowserDev()) - { - JavaScriptFragment.ensureXMLMetadataNoJavaScript(metadataText); - } - - queryDef.setMetadataXml(metadataText); - } - else - { - if (metadataText != null) - throw new UnsupportedOperationException("Query metadata is not editable."); - } - } - - queryDef.save(getUser(), getContainer()); - - // the query was successfully saved, validate the query but return any errors in the success response - List parseErrors = new ArrayList<>(); - List parseWarnings = new ArrayList<>(); - queryDef.validateQuery(_schema, parseErrors, parseWarnings); - if (!parseErrors.isEmpty()) - { - JSONArray errorArray = new JSONArray(); - - for (QueryException e : parseErrors) - { - errorArray.put(e.toJSON(form.ff_queryText)); - } - response.put("parseErrors", errorArray); - } - else if (!parseWarnings.isEmpty()) - { - JSONArray errorArray = new JSONArray(); - - for (QueryException e : parseWarnings) - { - errorArray.put(e.toJSON(form.ff_queryText)); - } - response.put("parseWarnings", errorArray); - } - } - catch (SQLException e) - { - errors.reject(ERROR_MSG, "An exception occurred: " + e); - LOG.error("Error", e); - } - catch (RuntimeException e) - { - errors.reject(ERROR_MSG, "An exception occurred: " + e.getMessage()); - LOG.error("Error", e); - } - - if (errors.hasErrors()) - return null; - - //if we got here, the query is OK - response.put("success", true); - return response; - } - - } - - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, DeletePermission.class}) - @Action(ActionType.Configure.class) - public static class DeleteQueryAction extends ConfirmAction - { - public SourceForm _form; - public QuerySchema _baseSchema; - public QueryDefinition _queryDef; - - - @Override - public void validateCommand(SourceForm target, Errors errors) - { - _form = target; - if (StringUtils.isEmpty(target.getSchemaName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - if (StringUtils.isEmpty(target.getQueryName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - - _baseSchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); - if (null == _baseSchema) - throw new NotFoundException("Schema not found: " + _form.getSchemaKey().toDisplayString()); - } - - - @Override - public ModelAndView getConfirmView(SourceForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Query"); - _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); - - if (null == _queryDef) - throw new NotFoundException("Query not found: " + form.getQueryName()); - - if (!_queryDef.canDelete(getUser())) - { - errors.reject(ERROR_MSG, "Sorry, this query can not be deleted"); - } - - return new JspView<>("/org/labkey/query/view/deleteQuery.jsp", this, errors); - } - - - @Override - public boolean handlePost(SourceForm form, BindException errors) throws Exception - { - _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); - - if (null == _queryDef) - return false; - try - { - _queryDef.delete(getUser()); - } - catch (OptimisticConflictException x) - { - /* reshow will throw NotFound, so just ignore */ - } - return true; - } - - @Override - @NotNull - public ActionURL getSuccessURL(SourceForm queryForm) - { - return ((UserSchema)_baseSchema).urlFor(QueryAction.schema); - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public class ExecuteQueryAction extends QueryViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _form = form; - - if (errors.hasErrors()) - return new SimpleErrorView(errors, true); - - QueryView queryView = Objects.requireNonNull(form.getQueryView()); - - var t = queryView.getTable(); - if (null != t && !t.allowRobotsIndex()) - { - getPageConfig().setRobotsNone(); - } - - if (isPrint()) - { - queryView.setPrintView(true); - getPageConfig().setTemplate(PageConfig.Template.Print); - getPageConfig().setShowPrintDialog(true); - } - - queryView.setShadeAlternatingRows(true); - queryView.setShowBorders(true); - setHelpTopic("customSQL"); - _queryView = queryView; - return queryView; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - TableInfo ti = null; - try - { - if (null != _queryView) - ti = _queryView.getTable(); - } - catch (QueryParseException x) - { - /* */ - } - String display = ti == null ? _form.getQueryName() : ti.getTitle(); - root.addChild(display); - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class RawTableMetaDataAction extends QueryViewAction - { - private String _dbSchemaName; - private String _dbTableName; - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _form = form; - - QueryView queryView = form.getQueryView(); - String userSchemaName = queryView.getSchema().getName(); - TableInfo ti = queryView.getTable(); - if (null == ti) - throw new NotFoundException(); - - DbScope scope = ti.getSchema().getScope(); - - // Test for provisioned table - if (ti.getDomain() != null) - { - Domain domain = ti.getDomain(); - if (domain.getStorageTableName() != null) - { - // Use the real table and schema names for getting the metadata - _dbTableName = domain.getStorageTableName(); - _dbSchemaName = domain.getDomainKind().getStorageSchemaName(); - } - } - - // No domain or domain with non-provisioned storage (e.g., core.Users) - if (null == _dbSchemaName || null == _dbTableName) - { - DbSchema dbSchema = ti.getSchema(); - _dbSchemaName = dbSchema.getName(); - - // Try to get the underlying schema table and use the meta data name, #12015 - if (ti instanceof FilteredTable fti) - ti = fti.getRealTable(); - - if (ti instanceof SchemaTableInfo) - _dbTableName = ti.getMetaDataIdentifier().getId(); - else if (ti instanceof LinkedTableInfo) - _dbTableName = ti.getName(); - - if (null == _dbTableName) - { - TableInfo tableInfo = dbSchema.getTable(ti.getName()); - if (null != tableInfo) - _dbTableName = tableInfo.getMetaDataIdentifier().getId(); - } - } - - if (null != _dbTableName) - { - VBox result = new VBox(); - - ActionURL url = null; - QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(userSchemaName); - if (qs != null) - { - url = new ActionURL(RawSchemaMetaDataAction.class, getContainer()); - url.addParameter("schemaName", userSchemaName); - } - - SqlDialect dialect = scope.getSqlDialect(); - ScopeView scopeInfo = new ScopeView("Scope and Schema Information", scope, _dbSchemaName, url, _dbTableName); - - result.addView(scopeInfo); - - try (JdbcMetaDataLocator locator = dialect.getTableResolver().getSingleTableLocator(scope, _dbSchemaName, _dbTableName)) - { - JdbcMetaDataSelector columnSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getColumns(l.getCatalogName(), l.getSchemaNamePattern(), l.getTableNamePattern(), null)); - result.addView(new ResultSetView(CachedResultSets.create(columnSelector.getResultSet(), true, Table.ALL_ROWS), "Table Meta Data")); - - JdbcMetaDataSelector pkSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getPrimaryKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(pkSelector.getResultSet(), true, Table.ALL_ROWS), "Primary Key Meta Data")); - - if (dialect.canCheckIndices(ti)) - { - JdbcMetaDataSelector indexSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getIndexInfo(l.getCatalogName(), l.getSchemaName(), l.getTableName(), false, false)); - result.addView(new ResultSetView(CachedResultSets.create(indexSelector.getResultSet(), true, Table.ALL_ROWS), "Other Index Meta Data")); - } - - JdbcMetaDataSelector ikSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getImportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(ikSelector.getResultSet(), true, Table.ALL_ROWS), "Imported Keys Meta Data")); - - JdbcMetaDataSelector ekSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getExportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(ekSelector.getResultSet(), true, Table.ALL_ROWS), "Exported Keys Meta Data")); - } - return result; - } - else - { - errors.reject(ERROR_MSG, "Raw metadata not accessible for table " + ti.getName()); - return new SimpleErrorView(errors); - } - } - - @Override - public void addNavTrail(NavTree root) - { - (new SchemaAction(_form)).addNavTrail(root); - if (null != _dbTableName) - root.addChild("JDBC Meta Data For Table \"" + _dbSchemaName + "." + _dbTableName + "\""); - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class RawSchemaMetaDataAction extends SimpleViewAction - { - private String _schemaName; - - @Override - public ModelAndView getView(Object form, BindException errors) throws Exception - { - _schemaName = getViewContext().getActionURL().getParameter("schemaName"); - if (null == _schemaName) - throw new NotFoundException(); - QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(_schemaName); - if (null == qs) - throw new NotFoundException(_schemaName); - DbSchema schema = qs.getDbSchema(); - String dbSchemaName = schema.getName(); - DbScope scope = schema.getScope(); - SqlDialect dialect = scope.getSqlDialect(); - - HttpView scopeInfo = new ScopeView("Scope Information", scope); - - ModelAndView tablesView; - - try (JdbcMetaDataLocator locator = dialect.getTableResolver().getAllTablesLocator(scope, dbSchemaName)) - { - JdbcMetaDataSelector selector = new JdbcMetaDataSelector(locator, - (dbmd, locator1) -> dbmd.getTables(locator1.getCatalogName(), locator1.getSchemaNamePattern(), locator1.getTableNamePattern(), null)); - Set tableNames = Sets.newCaseInsensitiveHashSet(qs.getTableNames()); - - ActionURL url = new ActionURL(RawTableMetaDataAction.class, getContainer()) - .addParameter("schemaName", _schemaName) - .addParameter("query.queryName", null); - tablesView = new ResultSetView(CachedResultSets.create(selector.getResultSet(), true, Table.ALL_ROWS), "Tables", "TABLE_NAME", url) - { - @Override - protected boolean shouldLink(ResultSet rs) throws SQLException - { - // Only link to tables and views (not indexes or sequences). And only if they're defined in the query schema. - String name = rs.getString("TABLE_NAME"); - String type = rs.getString("TABLE_TYPE"); - return ("TABLE".equalsIgnoreCase(type) || "VIEW".equalsIgnoreCase(type)) && tableNames.contains(name); - } - }; - } - - return new VBox(scopeInfo, tablesView); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("JDBC Meta Data For Schema \"" + _schemaName + "\""); - } - } - - - public static class ScopeView extends WebPartView - { - private final DbScope _scope; - private final String _schemaName; - private final String _tableName; - private final ActionURL _url; - - private ScopeView(String title, DbScope scope) - { - this(title, scope, null, null, null); - } - - private ScopeView(String title, DbScope scope, String schemaName, ActionURL url, String tableName) - { - super(title); - _scope = scope; - _schemaName = schemaName; - _tableName = tableName; - _url = url; - } - - @Override - protected void renderView(Object model, HtmlWriter out) - { - TABLE( - null != _schemaName ? getLabelAndContents("Schema", _url == null ? _schemaName : LinkBuilder.simpleLink(_schemaName, _url)) : null, - null != _tableName ? getLabelAndContents("Table", _tableName) : null, - getLabelAndContents("Scope", _scope.getDisplayName()), - getLabelAndContents("Dialect", _scope.getSqlDialect().getClass().getSimpleName()), - getLabelAndContents("URL", _scope.getDatabaseUrl()) - ).appendTo(out); - } - - // Return a single row (TR) with styled label and contents in separate TDs - private Renderable getLabelAndContents(String label, Object contents) - { - return TR( - TD( - cl("labkey-form-label"), - label - ), - TD( - contents - ) - ); - } - } - - // for backwards compat same as _executeQuery.view ?_print=1 - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public class PrintRowsAction extends ExecuteQueryAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _print = true; - ModelAndView result = super.getView(form, errors); - String title = form.getQueryName(); - if (StringUtils.isEmpty(title)) - title = form.getSchemaName(); - getPageConfig().setTitle(title, true); - return result; - } - } - - - abstract static class _ExportQuery extends SimpleViewAction - { - @Override - public ModelAndView getView(K form, BindException errors) throws Exception - { - QueryView view = form.getQueryView(); - getPageConfig().setTemplate(PageConfig.Template.None); - HttpServletResponse response = getViewContext().getResponse(); - response.setHeader("X-Robots-Tag", "noindex"); - try - { - _export(form, view); - return null; - } - catch (QueryService.NamedParameterNotProvided | QueryParseException x) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - throw x; - } - } - - abstract void _export(K form, QueryView view) throws Exception; - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportScriptForm extends QueryForm - { - private String _type; - - public String getScriptType() - { - return _type; - } - - public void setScriptType(String type) - { - _type = type; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) // This is called "export" but it doesn't export any data - @CSRF(CSRF.Method.ALL) - public static class ExportScriptAction extends SimpleViewAction - { - @Override - public void validate(ExportScriptForm form, BindException errors) - { - // calling form.getQueryView() as a validation check as it will throw if schema/query missing - form.getQueryView(); - - if (StringUtils.isEmpty(form.getScriptType())) - throw new NotFoundException("Missing required parameter: scriptType."); - } - - @Override - public ModelAndView getView(ExportScriptForm form, BindException errors) - { - return ExportScriptModel.getExportScriptView(QueryView.create(form, errors), form.getScriptType(), getPageConfig(), getViewContext().getResponse()); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsExcelAction extends _ExportQuery - { - @Override - void _export(ExportQueryForm form, QueryView view) throws Exception - { - view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xls, form.getRenameColumnMap()); - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsXLSXAction extends _ExportQuery - { - @Override - void _export(ExportQueryForm form, QueryView view) throws Exception - { - view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xlsx, form.getRenameColumnMap()); - } - } - - public static class ExportQueriesForm extends ExportQueryForm implements ApiJsonForm - { - private String filename; - private List queryForms; - - public void setFilename(String filename) - { - this.filename = filename; - } - - public String getFilename() - { - return filename; - } - - public void setQueryForms(List queryForms) - { - this.queryForms = queryForms; - } - - public List getQueryForms() - { - return queryForms; - } - - /** - * Map JSON to Spring PropertyValue objects. - * @param json the properties - */ - private MutablePropertyValues getPropertyValues(JSONObject json) - { - // Collecting mapped properties as a list because adding them to an existing MutablePropertyValues object replaces existing values - List properties = new ArrayList<>(); - - for (String key : json.keySet()) - { - Object value = json.get(key); - if (value instanceof JSONArray val) - { - // Split arrays into individual pairs to be bound (Issue #45452) - for (int i = 0; i < val.length(); i++) - { - properties.add(new PropertyValue(key, val.get(i).toString())); - } - } - else - { - properties.add(new PropertyValue(key, value)); - } - } - - return new MutablePropertyValues(properties); - } - - @Override - public void bindJson(JSONObject json) - { - setFilename(json.get("filename").toString()); - List forms = new ArrayList<>(); - - JSONArray models = json.optJSONArray("queryForms"); - if (models == null) - { - QueryController.LOG.error("No models to export; Form's `queryForms` property was null"); - throw new RuntimeValidationException("No queries to export; Form's `queryForms` property was null"); - } - - for (JSONObject queryModel : JsonUtil.toJSONObjectList(models)) - { - ExportQueryForm qf = new ExportQueryForm(); - qf.setViewContext(getViewContext()); - - qf.bindParameters(getPropertyValues(queryModel)); - forms.add(qf); - } - - setQueryForms(forms); - } - } - - /** - * Export multiple query forms - */ - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportQueriesXLSXAction extends ReadOnlyApiAction - { - @Override - public Object execute(ExportQueriesForm form, BindException errors) throws Exception - { - getPageConfig().setTemplate(PageConfig.Template.None); - HttpServletResponse response = getViewContext().getResponse(); - response.setHeader("X-Robots-Tag", "noindex"); - ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment); - ViewContext viewContext = getViewContext(); - - Map> nameFormMap = new CaseInsensitiveHashMap<>(); - Map sheetNames = new HashMap<>(); - form.getQueryForms().forEach(qf -> { - String sheetName = qf.getSheetName(); - QueryView qv = qf.getQueryView(); - // use the given sheet name if provided, otherwise try the query definition name - String name = StringUtils.isNotBlank(sheetName) ? sheetName : qv.getQueryDef().getName(); - // if there is no sheet name or queryDefinition name, use a data region name if provided. Otherwise, use "Data" - name = StringUtils.isNotBlank(name) ? name : StringUtils.isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; - // clean it to remove undesirable characters and make it of an acceptable length - name = ExcelWriter.cleanSheetName(name); - nameFormMap.computeIfAbsent(name, k -> new ArrayList<>()).add(qf); - }); - // Issue 53722: Need to assure unique names for the sheets in the presence of really long names - for (Map.Entry> entry : nameFormMap.entrySet()) { - String name = entry.getKey(); - if (entry.getValue().size() > 1) - { - List queryForms = entry.getValue(); - int countLength = String.valueOf(queryForms.size()).length() + 2; - if (countLength > name.length()) - throw new IllegalArgumentException("Cannot create sheet names from overlapping query names."); - for (int i = 0; i < queryForms.size(); i++) - { - sheetNames.put(entry.getValue().get(i), name.substring(0, name.length() - countLength) + "(" + i + ")"); - } - } - else - { - sheetNames.put(entry.getValue().get(0), name); - } - } - ExcelWriter writer = new ExcelWriter(ExcelWriter.ExcelDocumentType.xlsx) { - @Override - protected void renderSheets(Workbook workbook) - { - for (ExportQueryForm qf : form.getQueryForms()) - { - qf.setViewContext(viewContext); - qf.getSchema(); - - QueryView qv = qf.getQueryView(); - QueryView.ExcelExportConfig config = new QueryView.ExcelExportConfig(response, qf.getHeaderType()) - .setExcludeColumns(qf.getExcludeColumns()) - .setRenamedColumns(qf.getRenameColumnMap()); - qv.configureExcelWriter(this, config); - setSheetName(sheetNames.get(qf)); - setAutoSize(true); - renderNewSheet(workbook); - qv.logAuditEvent("Exported to Excel", getDataRowCount()); - } - - workbook.setActiveSheet(0); - } - }; - writer.setFilenamePrefix(form.getFilename()); - writer.renderWorkbook(response); - return null; //Returning anything here will cause error as excel writer will close the response stream - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class TemplateForm extends ExportQueryForm - { - boolean insertColumnsOnly = true; - String filenamePrefix; - FieldKey[] includeColumn; - String fileType; - - public TemplateForm() - { - _headerType = ColumnHeaderType.Caption; - } - - // "captionType" field backwards compatibility - public void setCaptionType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public ColumnHeaderType getCaptionType() - { - return _headerType; - } - - public List getIncludeColumns() - { - if (includeColumn == null || includeColumn.length == 0) - return Collections.emptyList(); - return Arrays.asList(includeColumn); - } - - public FieldKey[] getIncludeColumn() - { - return includeColumn; - } - - public void setIncludeColumn(FieldKey[] includeColumn) - { - this.includeColumn = includeColumn; - } - - @NotNull - public String getFilenamePrefix() - { - return filenamePrefix == null ? getQueryName() : filenamePrefix; - } - - public void setFilenamePrefix(String prefix) - { - filenamePrefix = prefix; - } - - public String getFileType() - { - return fileType; - } - - public void setFileType(String fileType) - { - this.fileType = fileType; - } - } - - - /** - * Can be used to generate an Excel template for import into a table. Supported URL params include: - *
- *
filenamePrefix
- *
the prefix of the excel file that is generated, defaults to '_data'
- * - *
query.viewName
- *
if provided, the resulting excel file will use the fields present in this view. - * Non-usereditable columns will be skipped. - * Non-existent columns (like a lookup) unless includeMissingColumns is true. - * Any required columns missing from this view will be appended to the end of the query. - *
- * - *
includeColumn
- *
List of column names to include, even if the column doesn't exist or is non-userEditable. - * For example, this can be used to add a fake column that is only supported during the import process. - *
- * - *
excludeColumn
- *
List of column names to exclude. - *
- * - *
exportAlias.columns
- *
Use alternative column name in excel: exportAlias.originalColumnName=aliasColumnName - *
- * - *
captionType
- *
determines which column property is used in the header, either Label or Name
- *
- */ - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportExcelTemplateAction extends _ExportQuery - { - public ExportExcelTemplateAction() - { - setCommandClass(TemplateForm.class); - } - - @Override - void _export(TemplateForm form, QueryView view) throws Exception - { - boolean respectView = form.getViewName() != null; - ExcelWriter.ExcelDocumentType fileType = ExcelWriter.ExcelDocumentType.xlsx; - if (form.getFileType() != null) - { - try - { - fileType = ExcelWriter.ExcelDocumentType.valueOf(form.getFileType().toLowerCase()); - } - catch (IllegalArgumentException ignored) {} - } - view.exportToExcel( new QueryView.ExcelExportConfig(getViewContext().getResponse(), form.getHeaderType()) - .setTemplateOnly(true) - .setInsertColumnsOnly(form.insertColumnsOnly) - .setDocType(fileType) - .setRespectView(respectView) - .setIncludeColumns(form.getIncludeColumns()) - .setExcludeColumns(form.getExcludeColumns()) - .setRenamedColumns(form.getRenameColumnMap()) - .setPrefix((StringUtils.isEmpty(form.getFilenamePrefix()) ? "Import" : form.getFilenamePrefix()) + "_Template") // Issue 48028: Change template file names - ); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportQueryForm extends QueryForm - { - protected ColumnHeaderType _headerType = null; // QueryView will provide a default header type if the user doesn't select one - FieldKey[] excludeColumn; - Map renameColumns = null; - private String sheetName; - - public void setSheetName(String sheetName) - { - this.sheetName = sheetName; - } - - public String getSheetName() - { - return sheetName; - } - - public ColumnHeaderType getHeaderType() - { - return _headerType; - } - - public void setHeaderType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public List getExcludeColumns() - { - if (excludeColumn == null || excludeColumn.length == 0) - return Collections.emptyList(); - return Arrays.asList(excludeColumn); - } - - public void setExcludeColumn(FieldKey[] excludeColumn) - { - this.excludeColumn = excludeColumn; - } - - public Map getRenameColumnMap() - { - if (renameColumns != null) - return renameColumns; - - renameColumns = new CaseInsensitiveHashMap<>(); - final String renameParamPrefix = "exportAlias."; - PropertyValue[] pvs = getInitParameters().getPropertyValues(); - for (PropertyValue pv : pvs) - { - String paramName = pv.getName(); - if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) - continue; - - renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); - } - - return renameColumns; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportRowsTsvForm extends ExportQueryForm - { - private TSVWriter.DELIM _delim = TSVWriter.DELIM.TAB; - private TSVWriter.QUOTE _quote = TSVWriter.QUOTE.DOUBLE; - - public TSVWriter.DELIM getDelim() - { - return _delim; - } - - public void setDelim(TSVWriter.DELIM delim) - { - _delim = delim; - } - - public TSVWriter.QUOTE getQuote() - { - return _quote; - } - - public void setQuote(TSVWriter.QUOTE quote) - { - _quote = quote; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsTsvAction extends _ExportQuery - { - public ExportRowsTsvAction() - { - setCommandClass(ExportRowsTsvForm.class); - } - - @Override - void _export(ExportRowsTsvForm form, QueryView view) throws Exception - { - view.exportToTsv(getViewContext().getResponse(), form.getDelim(), form.getQuote(), form.getHeaderType(), form.getRenameColumnMap()); - } - } - - - @RequiresNoPermission - @IgnoresTermsOfUse - @Action(ActionType.Export.class) - public static class ExcelWebQueryAction extends ExportRowsTsvAction - { - @Override - public ModelAndView getView(ExportRowsTsvForm form, BindException errors) throws Exception - { - if (!getContainer().hasPermission(getUser(), ReadPermission.class)) - { - if (!getUser().isGuest()) - { - throw new UnauthorizedException(); - } - getViewContext().getResponse().setHeader("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); - getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); - return null; - } - - // Bug 5610. Excel web queries don't work over SSL if caching is disabled, - // so we need to allow caching so that Excel can read from IE on Windows. - HttpServletResponse response = getViewContext().getResponse(); - // Set the headers to allow the client to cache, but not proxies - ResponseHelper.setPrivate(response); - - QueryView view = form.getQueryView(); - getPageConfig().setTemplate(PageConfig.Template.None); - view.exportToExcelWebQuery(getViewContext().getResponse()); - return null; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExcelWebQueryDefinitionAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - getPageConfig().setTemplate(PageConfig.Template.None); - form.getQueryView(); - String queryViewActionURL = form.getQueryViewActionURL(); - ActionURL url; - if (queryViewActionURL != null) - { - url = new ActionURL(queryViewActionURL); - } - else - { - url = getViewContext().cloneActionURL(); - url.setAction(ExcelWebQueryAction.class); - } - getViewContext().getResponse().setContentType("text/x-ms-iqy"); - String filename = FileUtil.makeFileNameWithTimestamp(form.getQueryName(), "iqy"); - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, filename); - PrintWriter writer = getViewContext().getResponse().getWriter(); - writer.println("WEB"); - writer.println("1"); - writer.println(url.getURIString()); - - QueryService.get().addAuditEvent(getUser(), getContainer(), form.getSchemaName(), form.getQueryName(), url, "Exported to Excel Web Query definition", null); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.SelectMetaData.class) - public class MetadataQueryAction extends SimpleViewAction - { - QueryForm _form = null; - - @Override - public ModelAndView getView(QueryForm queryForm, BindException errors) throws Exception - { - String schemaName = queryForm.getSchemaName(); - String queryName = queryForm.getQueryName(); - - _form = queryForm; - - if (schemaName.isEmpty() && (null == queryName || queryName.isEmpty())) - { - throw new NotFoundException("Must provide schemaName and queryName."); - } - - if (schemaName.isEmpty()) - { - throw new NotFoundException("Must provide schemaName."); - } - - if (null == queryName || queryName.isEmpty()) - { - throw new NotFoundException("Must provide queryName."); - } - - if (!queryForm.getQueryDef().isMetadataEditable()) - throw new UnauthorizedException("Query metadata is not editable"); - - if (!queryForm.canEditMetadata()) - throw new UnauthorizedException("You do not have permission to edit the query metadata"); - - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("queryMetadataEditor")); - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - var metadataQuery = _form.getQueryDef().getName(); - if (null != metadataQuery) - root.addChild("Edit Metadata: " + _form.getQueryName(), metadataQuery); - else - root.addChild("Edit Metadata: " + _form.getQueryName()); - } - } - - // Uck. Supports the old and new view designer. - protected JSONObject saveCustomView(Container container, QueryDefinition queryDef, - String regionName, String viewName, boolean replaceExisting, - boolean share, boolean inherit, - boolean session, boolean saveFilter, - boolean hidden, JSONObject jsonView, - ActionURL returnUrl, - BindException errors) - { - User owner = getUser(); - boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); - if (share && canSaveForAllUsers && !session) - { - owner = null; - } - String name = StringUtils.trimToNull(viewName); - - if (name != null && RESERVED_VIEW_NAMES.contains(name.toLowerCase())) - errors.reject(ERROR_MSG, "The grid view name '" + name + "' is not allowed."); - - boolean isHidden = hidden; - CustomView view; - if (owner == null) - view = queryDef.getSharedCustomView(name); - else - view = queryDef.getCustomView(owner, getViewContext().getRequest(), name); - - if (view != null && !replaceExisting && !StringUtils.isEmpty(name)) - errors.reject(ERROR_MSG, "A saved view by the name \"" + viewName + "\" already exists. "); - - // 11179: Allow editing the view if we're saving to session. - // NOTE: Check for session flag first otherwise the call to canEdit() will add errors to the errors collection. - boolean canEdit = view == null || session || view.canEdit(container, errors); - if (errors.hasErrors()) - return null; - - if (canEdit) - { - // Issue 13594: Disallow setting of the customview inherit bit for query views - // that have no available container filter types. Unfortunately, the only way - // to get the container filters is from the QueryView. Ideally, the query def - // would know if it was container filterable or not instead of using the QueryView. - if (inherit && canSaveForAllUsers && !session) - { - UserSchema schema = queryDef.getSchema(); - QueryView queryView = schema.createView(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, queryDef.getName(), errors); - if (queryView != null) - { - Set allowableContainerFilterTypes = queryView.getAllowableContainerFilterTypes(); - if (allowableContainerFilterTypes.size() <= 1) - { - errors.reject(ERROR_MSG, "QueryView doesn't support inherited custom views"); - return null; - } - } - } - - // Create a new view if none exists or the current view is a shared view - // and the user wants to override the shared view with a personal view. - if (view == null || (owner != null && view.isShared())) - { - if (owner == null) - view = queryDef.createSharedCustomView(name); - else - view = queryDef.createCustomView(owner, name); - - if (owner != null && session) - ((CustomViewImpl) view).isSession(true); - view.setIsHidden(hidden); - } - else if (session != view.isSession()) - { - if (session) - { - assert !view.isSession(); - if (owner == null) - { - errors.reject(ERROR_MSG, "Session views can't be saved for all users"); - return null; - } - - // The form is saving to session but the view is in the database. - // Make a copy in case it's a read-only version from an XML file - view = queryDef.createCustomView(owner, name); - ((CustomViewImpl) view).isSession(true); - } - else - { - // Remove the session view and call saveCustomView again to either create a new view or update an existing view. - assert view.isSession(); - boolean success = false; - try - { - view.delete(getUser(), getViewContext().getRequest()); - JSONObject ret = saveCustomView(container, queryDef, regionName, viewName, replaceExisting, share, inherit, session, saveFilter, hidden, jsonView, returnUrl, errors); - success = !errors.hasErrors() && ret != null; - return success ? ret : null; - } - finally - { - if (!success) - { - // dirty the view then save the deleted session view back in session state - view.setName(view.getName()); - view.save(getUser(), getViewContext().getRequest()); - } - } - } - } - - // NOTE: Updating, saving, and deleting the view may throw an exception - CustomViewImpl cview = null; - if (view instanceof EditableCustomView && view.isOverridable()) - { - cview = ((EditableCustomView)view).getEditableViewInfo(owner, session); - } - if (null == cview) - { - throw new IllegalArgumentException("View cannot be edited"); - } - - cview.update(jsonView, saveFilter); - if (canSaveForAllUsers && !session) - { - cview.setCanInherit(inherit); - } - isHidden = view.isHidden(); - cview.setContainer(container); - cview.save(getUser(), getViewContext().getRequest()); - if (owner == null) - { - // New view is shared so delete any previous custom view owned by the user with the same name. - CustomView personalView = queryDef.getCustomView(getUser(), getViewContext().getRequest(), name); - if (personalView != null && !personalView.isShared()) - { - personalView.delete(getUser(), getViewContext().getRequest()); - } - } - } - - if (null == returnUrl) - { - returnUrl = getViewContext().cloneActionURL().setAction(ExecuteQueryAction.class); - } - else - { - returnUrl = returnUrl.clone(); - if (name == null || !canEdit) - { - returnUrl.deleteParameter(regionName + "." + QueryParam.viewName); - } - else if (!isHidden) - { - returnUrl.replaceParameter(regionName + "." + QueryParam.viewName, name); - } - returnUrl.deleteParameter(regionName + "." + QueryParam.ignoreFilter); - if (saveFilter) - { - for (String key : returnUrl.getKeysByPrefix(regionName + ".")) - { - if (isFilterOrSort(regionName, key)) - returnUrl.deleteFilterParameters(key); - } - } - } - - JSONObject ret = new JSONObject(); - ret.put("redirect", returnUrl); - Map viewAsMap = CustomViewUtil.toMap(view, getUser(), true); - try - { - ret.put("view", new JSONObject(viewAsMap, new JSONParserConfiguration().withMaxNestingDepth(10))); - } - catch (JSONException e) - { - LOG.error("Failed to save view: {}", jsonView, e); - } - return ret; - } - - private boolean isFilterOrSort(String dataRegionName, String param) - { - assert param.startsWith(dataRegionName + "."); - String check = param.substring(dataRegionName.length() + 1); - if (check.contains("~")) - return true; - if ("sort".equals(check)) - return true; - if (check.equals("containerFilterName")) - return true; - return false; - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Configure.class) - @JsonInputLimit(100_000) - public class SaveQueryViewsAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SimpleApiJsonForm form, BindException errors) - { - JSONObject json = form.getJsonObject(); - if (json == null) - throw new NotFoundException("Empty request"); - - String schemaName = json.optString(QueryParam.schemaName.toString(), null); - String queryName = json.optString(QueryParam.queryName.toString(), null); - if (schemaName == null || queryName == null) - throw new NotFoundException("schemaName and queryName are required"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - if (schema == null) - throw new NotFoundException("schema not found"); - - QueryDefinition queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), schemaName, queryName); - if (queryDef == null) - queryDef = schema.getQueryDefForTable(queryName); - if (queryDef == null) - throw new NotFoundException("query not found"); - - JSONObject response = new JSONObject(); - response.put(QueryParam.schemaName.toString(), schemaName); - response.put(QueryParam.queryName.toString(), queryName); - JSONArray views = new JSONArray(); - response.put("views", views); - - ActionURL redirect = null; - JSONArray jsonViews = json.getJSONArray("views"); - for (int i = 0; i < jsonViews.length(); i++) - { - final JSONObject jsonView = jsonViews.getJSONObject(i); - String viewName = jsonView.optString("name", null); - if (viewName == null) - throw new NotFoundException("'name' is required all views'"); - - boolean shared = jsonView.optBoolean("shared", false); - boolean replace = jsonView.optBoolean("replace", true); // "replace" was the default before the flag is introduced - boolean inherit = jsonView.optBoolean("inherit", false); - boolean session = jsonView.optBoolean("session", false); - boolean hidden = jsonView.optBoolean("hidden", false); - // Users may save views to a location other than the current container - String containerPath = jsonView.optString("containerPath", getContainer().getPath()); - Container container; - if (inherit) - { - // Only respect this request if it's a view that is inheritable in subfolders - container = ContainerManager.getForPath(containerPath); - } - else - { - // Otherwise, save it in the current container - container = getContainer().getContainerFor(ContainerType.DataType.customQueryViews); - } - - if (container == null) - { - throw new NotFoundException("No such container: " + containerPath); - } - - JSONObject savedView = saveCustomView( - container, queryDef, QueryView.DATAREGIONNAME_DEFAULT, viewName, replace, - shared, inherit, session, true, hidden, jsonView, null, errors); - - if (savedView != null) - { - if (redirect == null) - redirect = (ActionURL)savedView.get("redirect"); - views.put(savedView.getJSONObject("view")); - } - } - - if (redirect != null) - response.put("redirect", redirect); - - if (errors.hasErrors()) - return null; - else - return new ApiSimpleResponse(response); - } - } - - public static class RenameQueryViewForm extends QueryForm - { - private String newName; - - public String getNewName() - { - return newName; - } - - public void setNewName(String newName) - { - this.newName = newName; - } - } - - @RequiresPermission(ReadPermission.class) - public class RenameQueryViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RenameQueryViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - - Container container = getContainer(); - User user = getUser(); - - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - if (schemaName == null || queryName == null) - throw new NotFoundException("schemaName and queryName are required"); - - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); - if (schema == null) - throw new NotFoundException("schema not found"); - - QueryDefinition queryDef = QueryService.get().getQueryDef(user, container, schemaName, queryName); - if (queryDef == null) - queryDef = schema.getQueryDefForTable(queryName); - if (queryDef == null) - throw new NotFoundException("query not found"); - - renameCustomView(container, queryDef, view, form.getNewName(), errors); - - if (errors.hasErrors()) - return null; - else - return new ApiSimpleResponse("success", true); - } - } - - protected void renameCustomView(Container container, QueryDefinition queryDef, CustomView fromView, String newViewName, BindException errors) - { - if (newViewName != null && RESERVED_VIEW_NAMES.contains(newViewName.toLowerCase())) - errors.reject(ERROR_MSG, "The grid view name '" + newViewName + "' is not allowed."); - - String newName = StringUtils.trimToNull(newViewName); - if (StringUtils.isEmpty(newName)) - errors.reject(ERROR_MSG, "View name cannot be blank."); - - if (errors.hasErrors()) - return; - - User owner = getUser(); - boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); - - if (!fromView.canEdit(container, errors)) - return; - - if (fromView.isSession()) - { - errors.reject(ERROR_MSG, "Cannot rename a session view."); - return; - } - - CustomView duplicateView = queryDef.getCustomView(owner, getViewContext().getRequest(), newName); - if (duplicateView == null && canSaveForAllUsers) - duplicateView = queryDef.getSharedCustomView(newName); - if (duplicateView != null) - { - // only allow duplicate view name if creating a new private view to shadow an existing shared view - if (!(!fromView.isShared() && duplicateView.isShared())) - { - errors.reject(ERROR_MSG, "Another saved view by the name \"" + newName + "\" already exists. "); - return; - } - } - - fromView.setName(newViewName); - fromView.save(getUser(), getViewContext().getRequest()); - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Configure.class) - public class PropertiesQueryAction extends FormViewAction - { - PropertiesForm _form = null; - private String _queryName; - - @Override - public void validateCommand(PropertiesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(PropertiesForm form, boolean reshow, BindException errors) - { - // assertQueryExists requires that it be well-formed - // assertQueryExists(form); - QueryDefinition queryDef = form.getQueryDef(); - _form = form; - _form.setDescription(queryDef.getDescription()); - _form.setInheritable(queryDef.canInherit()); - _form.setHidden(queryDef.isHidden()); - setHelpTopic("editQueryProperties"); - _queryName = form.getQueryName(); - - return new JspView<>("/org/labkey/query/view/propertiesQuery.jsp", form, errors); - } - - @Override - public boolean handlePost(PropertiesForm form, BindException errors) throws Exception - { - // assertQueryExists requires that it be well-formed - // assertQueryExists(form); - if (!form.canEdit()) - { - throw new UnauthorizedException(); - } - QueryDefinition queryDef = form.getQueryDef(); - _queryName = form.getQueryName(); - if (!queryDef.getDefinitionContainer().getId().equals(getContainer().getId())) - throw new NotFoundException("Query not found"); - - _form = form; - - if (!StringUtils.isEmpty(form.rename) && !form.rename.equalsIgnoreCase(queryDef.getName())) - { - // issue 17766: check if query or table exist with this name - if (null != QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), form.rename, true) - || null != form.getSchema().getTable(form.rename,null)) - { - errors.reject(ERROR_MSG, "A query or table with the name \"" + form.rename + "\" already exists."); - return false; - } - - // Issue 40895: update queryName in xml metadata - updateXmlMetadata(queryDef); - queryDef.setName(form.rename); - // update form so getSuccessURL() works - _form = new PropertiesForm(form.getSchemaName(), form.rename); - _form.setViewContext(form.getViewContext()); - _queryName = form.rename; - } - - queryDef.setDescription(form.description); - queryDef.setCanInherit(form.inheritable); - queryDef.setIsHidden(form.hidden); - queryDef.save(getUser(), getContainer()); - return true; - } - - private void updateXmlMetadata(QueryDefinition queryDef) throws XmlException - { - if (null != queryDef.getMetadataXml()) - { - TablesDocument doc = TablesDocument.Factory.parse(queryDef.getMetadataXml()); - if (null != doc) - { - for (TableType tableType : doc.getTables().getTableArray()) - { - if (tableType.getTableName().equalsIgnoreCase(queryDef.getName())) - { - // update tableName in xml - tableType.setTableName(_form.rename); - } - } - XmlOptions xmlOptions = new XmlOptions(); - xmlOptions.setSavePrettyPrint(); - // Don't use an explicit namespace, making the XML much more readable - xmlOptions.setUseDefaultNamespace(); - queryDef.setMetadataXml(doc.xmlText(xmlOptions)); - } - } - } - - @Override - public ActionURL getSuccessURL(PropertiesForm propertiesForm) - { - ActionURL url = new ActionURL(BeginAction.class, propertiesForm.getViewContext().getContainer()); - url.addParameter("schemaName", propertiesForm.getSchemaName()); - if (null != _queryName) - url.addParameter("queryName", _queryName); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - root.addChild("Edit query properties"); - } - } - - @ActionNames("truncateTable") - @RequiresPermission(AdminPermission.class) - public static class TruncateTableAction extends MutatingApiAction - { - UserSchema schema; - TableInfo table; - - @Override - public void validateForm(QueryForm form, Errors errors) - { - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - - if (isBlank(schemaName) || isBlank(queryName)) - throw new NotFoundException("schemaName and queryName are required"); - - schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - if (null == schema) - throw new NotFoundException("The schema '" + schemaName + "' does not exist."); - - table = schema.getTable(queryName, null); - if (null == table) - throw new NotFoundException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); - } - - @Override - public ApiResponse execute(QueryForm form, BindException errors) throws Exception - { - int deletedRows; - QueryUpdateService qus = table.getUpdateService(); - - if (null == qus) - throw new IllegalArgumentException("The query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "' is not truncatable."); - - try (DbScope.Transaction transaction = table.getSchema().getScope().ensureTransaction()) - { - deletedRows = qus.truncateRows(getUser(), getContainer(), null, null); - transaction.commit(); - } - - ApiSimpleResponse response = new ApiSimpleResponse(); - - response.put("success", true); - response.put(BaseSaveRowsAction.PROP_SCHEMA_NAME, form.getSchemaName()); - response.put(BaseSaveRowsAction.PROP_QUERY_NAME, form.getQueryName()); - response.put("deletedRows", deletedRows); - - return response; - } - } - - - @RequiresPermission(DeletePermission.class) - public static class DeleteQueryRowsAction extends FormHandlerAction - { - @Override - public void validateCommand(QueryForm target, Errors errors) - { - } - - @Override - public boolean handlePost(QueryForm form, BindException errors) - { - TableInfo table = form.getQueryView().getTable(); - - if (!table.hasPermission(getUser(), DeletePermission.class)) - { - throw new UnauthorizedException(); - } - - QueryUpdateService updateService = table.getUpdateService(); - if (updateService == null) - throw new UnsupportedOperationException("Unable to delete - no QueryUpdateService registered for " + form.getSchemaName() + "." + form.getQueryName()); - - Set ids = DataRegionSelection.getSelected(form.getViewContext(), null, true); - List pks = table.getPkColumns(); - int numPks = pks.size(); - - //normalize the pks to arrays of correctly-typed objects - List> keyValues = new ArrayList<>(ids.size()); - for (String id : ids) - { - String[] stringValues; - if (numPks > 1) - { - stringValues = id.split(","); - if (stringValues.length != numPks) - throw new IllegalStateException("This table has " + numPks + " primary-key columns, but " + stringValues.length + " primary-key values were provided!"); - } - else - stringValues = new String[]{id}; - - Map rowKeyValues = new CaseInsensitiveHashMap<>(); - for (int idx = 0; idx < numPks; ++idx) - { - ColumnInfo keyColumn = pks.get(idx); - Object keyValue = keyColumn.getJavaClass() == String.class ? stringValues[idx] : ConvertUtils.convert(stringValues[idx], keyColumn.getJavaClass()); - rowKeyValues.put(keyColumn.getName(), keyValue); - } - keyValues.add(rowKeyValues); - } - - DbSchema dbSchema = table.getSchema(); - try - { - dbSchema.getScope().executeWithRetry(tx -> - { - try - { - updateService.deleteRows(getUser(), getContainer(), keyValues, null, null); - } - catch (SQLException x) - { - if (!RuntimeSQLException.isConstraintException(x)) - throw new RuntimeSQLException(x); - errors.reject(ERROR_MSG, getMessage(table.getSchema().getSqlDialect(), x)); - } - catch (DataIntegrityViolationException | OptimisticConflictException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - catch (BatchValidationException x) - { - x.addToErrors(errors); - } - catch (Exception x) - { - errors.reject(ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); - } - // need to throw here to avoid committing tx - if (errors.hasErrors()) - throw new DbScope.RetryPassthroughException(errors); - return true; - }); - } - catch (DbScope.RetryPassthroughException x) - { - if (x.getCause() != errors) - x.throwRuntimeException(); - } - return !errors.hasErrors(); - } - - @Override - public ActionURL getSuccessURL(QueryForm form) - { - return form.getReturnActionURL(); - } - } - - @RequiresPermission(ReadPermission.class) - public static class DetailsQueryRowAction extends UserSchemaAction - { - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - - if (_schema != null && _table != null) - { - if (_table.hasPermission(getUser(), UpdatePermission.class)) - { - StringExpression updateExpr = _form.getQueryDef().urlExpr(QueryAction.updateQueryRow, _schema.getContainer()); - if (updateExpr != null) - { - String url = updateExpr.eval(tableForm.getTypedValues()); - if (url != null) - { - ActionURL updateUrl = new ActionURL(url); - ActionButton editButton = new ActionButton("Edit", updateUrl); - bb.add(editButton); - } - } - } - - - ActionURL gridUrl; - if (_form.getReturnActionURL() != null) - { - // If we have a specific return URL requested, use that - gridUrl = _form.getReturnActionURL(); - } - else - { - // Otherwise go back to the default grid view - gridUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); - } - if (gridUrl != null) - { - ActionButton gridButton = new ActionButton("Show Grid", gridUrl); - bb.add(gridButton); - } - } - - DetailsView detailsView = new DetailsView(tableForm); - detailsView.setFrame(WebPartView.FrameType.PORTAL); - detailsView.getDataRegion().setButtonBar(bb); - - VBox view = new VBox(detailsView); - - DetailsURL detailsURL = QueryService.get().getAuditDetailsURL(getUser(), getContainer(), _table); - - if (detailsURL != null) - { - String url = detailsURL.eval(tableForm.getTypedValues()); - if (url != null) - { - ActionURL auditURL = new ActionURL(url); - - QueryView historyView = QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), - auditURL.getParameter(QueryParam.schemaName), - auditURL.getParameter(QueryParam.queryName), - auditURL.getParameter("keyValue"), errors); - - if (null != historyView) - { - historyView.setFrame(WebPartView.FrameType.PORTAL); - historyView.setTitle("History"); - - view.addView(historyView); - } - } - } - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - return false; - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Details"); - } - } - - @RequiresPermission(InsertPermission.class) - public static class InsertQueryRowAction extends UserSchemaAction - { - @Override - public BindException bindParameters(PropertyValues m) throws Exception - { - BindException bind = super.bindParameters(m); - - // what is going on with UserSchemaAction and form binding? Why doesn't successUrl bind? - QueryUpdateForm form = (QueryUpdateForm)bind.getTarget(); - if (null == form.getSuccessUrl() && null != m.getPropertyValue(ActionURL.Param.successUrl.name())) - form.setSuccessUrl(new ReturnURLString(m.getPropertyValue(ActionURL.Param.successUrl.name()).getValue().toString())); - return bind; - } - - Map insertedRow = null; - - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Insert Row"); - - InsertView view = new InsertView(tableForm, errors); - view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - List> list = doInsertUpdate(tableForm, errors, true); - if (null != list && list.size() == 1) - insertedRow = list.get(0); - return 0 == errors.getErrorCount(); - } - - /** - * NOTE: UserSchemaAction.addNavTrail() uses this method getSuccessURL() for the nav trail link (form==null). - * It is used for where to go on success, and also as a "back" link in the nav trail - * If there is a setSuccessUrl specified, we will use that for successful submit - */ - @Override - public ActionURL getSuccessURL(QueryUpdateForm form) - { - if (null == form) - return super.getSuccessURL(null); - - String str = null; - if (form.getSuccessUrl() != null) - str = form.getSuccessUrl().toString(); - if (isBlank(str)) - str = form.getReturnUrl(); - - if ("details.view".equals(str)) - { - if (null == insertedRow) - return super.getSuccessURL(form); - StringExpression se = form.getTable().getDetailsURL(null, getContainer()); - if (null == se) - return super.getSuccessURL(form); - str = se.eval(insertedRow); - } - try - { - if (!isBlank(str)) - return new ActionURL(str); - } - catch (IllegalArgumentException x) - { - // pass - } - return super.getSuccessURL(form); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Insert " + _table.getName()); - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateQueryRowAction extends UserSchemaAction - { - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - ButtonBar bb = createSubmitCancelButtonBar(tableForm); - UpdateView view = new UpdateView(tableForm, errors); - view.getDataRegion().setButtonBar(bb); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception - { - doInsertUpdate(tableForm, errors, false); - return 0 == errors.getErrorCount(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Edit " + _table.getName()); - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateQueryRowsAction extends UpdateQueryRowAction - { - @Override - public ModelAndView handleRequest(QueryUpdateForm tableForm, BindException errors) throws Exception - { - tableForm.setBulkUpdate(true); - return super.handleRequest(tableForm, errors); - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception - { - boolean ret; - - if (tableForm.isDataSubmit()) - { - ret = super.handlePost(tableForm, errors); - if (ret) - DataRegionSelection.clearAll(getViewContext(), null); // in case we altered primary keys, see issue #35055 - return ret; - } - - return false; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Edit Multiple " + _table.getName()); - } - } - - // alias - public static class DeleteAction extends DeleteQueryRowsAction - { - } - - public abstract static class QueryViewAction extends SimpleViewAction - { - QueryForm _form; - QueryView _queryView; - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class APIQueryForm extends ContainerFilterQueryForm - { - private Integer _start; - private Integer _limit; - private boolean _includeDetailsColumn = false; - private boolean _includeUpdateColumn = false; - private boolean _includeTotalCount = true; - private boolean _includeStyle = false; - private boolean _includeDisplayValues = false; - private boolean _minimalColumns = true; - private boolean _includeMetadata = true; - - public Integer getStart() - { - return _start; - } - - public void setStart(Integer start) - { - _start = start; - } - - public Integer getLimit() - { - return _limit; - } - - public void setLimit(Integer limit) - { - _limit = limit; - } - - public boolean isIncludeTotalCount() - { - return _includeTotalCount; - } - - public void setIncludeTotalCount(boolean includeTotalCount) - { - _includeTotalCount = includeTotalCount; - } - - public boolean isIncludeStyle() - { - return _includeStyle; - } - - public void setIncludeStyle(boolean includeStyle) - { - _includeStyle = includeStyle; - } - - public boolean isIncludeDetailsColumn() - { - return _includeDetailsColumn; - } - - public void setIncludeDetailsColumn(boolean includeDetailsColumn) - { - _includeDetailsColumn = includeDetailsColumn; - } - - public boolean isIncludeUpdateColumn() - { - return _includeUpdateColumn; - } - - public void setIncludeUpdateColumn(boolean includeUpdateColumn) - { - _includeUpdateColumn = includeUpdateColumn; - } - - public boolean isIncludeDisplayValues() - { - return _includeDisplayValues; - } - - public void setIncludeDisplayValues(boolean includeDisplayValues) - { - _includeDisplayValues = includeDisplayValues; - } - - public boolean isMinimalColumns() - { - return _minimalColumns; - } - - public void setMinimalColumns(boolean minimalColumns) - { - _minimalColumns = minimalColumns; - } - - public boolean isIncludeMetadata() - { - return _includeMetadata; - } - - public void setIncludeMetadata(boolean includeMetadata) - { - _includeMetadata = includeMetadata; - } - - @Override - protected QuerySettings createQuerySettings(UserSchema schema) - { - QuerySettings results = super.createQuerySettings(schema); - - // See dataintegration/202: The java client api / remote ETL calls selectRows with showRows=all. We need to test _initParameters to properly read this - boolean missingShowRows = null == getViewContext().getRequest().getParameter(getDataRegionName() + "." + QueryParam.showRows) && null == _initParameters.getPropertyValue(getDataRegionName() + "." + QueryParam.showRows); - if (null == getLimit() && !results.isMaxRowsSet() && missingShowRows) - { - results.setShowRows(ShowRows.PAGINATED); - results.setMaxRows(DEFAULT_API_MAX_ROWS); - } - - if (getLimit() != null) - { - results.setShowRows(ShowRows.PAGINATED); - results.setMaxRows(getLimit()); - } - if (getStart() != null) - results.setOffset(getStart()); - - return results; - } - } - - public static final int DEFAULT_API_MAX_ROWS = 100000; - - @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 - @ActionNames("selectRows, getQuery") - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.1) - @Action(ActionType.SelectData.class) - public class SelectRowsAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(APIQueryForm form, BindException errors) - { - // Issue 12233: add implicit maxRows=100k when using client API - QueryView view = form.getQueryView(); - - view.setShowPagination(form.isIncludeTotalCount()); - - //if viewName was specified, ensure that it was actually found and used - //QueryView.create() will happily ignore an invalid view name and just return the default view - if (null != StringUtils.trimToNull(form.getViewName()) && - null == view.getQueryDef().getCustomView(getUser(), getViewContext().getRequest(), form.getViewName())) - { - throw new NotFoundException("The requested view '" + form.getViewName() + "' does not exist for this user."); - } - - TableInfo t = view.getTable(); - if (null == t) - { - List qpes = view.getParseErrors(); - if (!qpes.isEmpty()) - throw qpes.get(0); - throw new NotFoundException(form.getQueryName()); - } - - boolean isEditable = isQueryEditable(view.getTable()); - boolean metaDataOnly = form.getQuerySettings().getMaxRows() == 0; - boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; - boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; - - ApiQueryResponse response; - - // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. - if (getRequestedApiVersion() >= 13.2) - { - ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, true, view.getQueryDef().getName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); - fancyResponse.includeFormattedValue(includeFormattedValue); - response = fancyResponse; - } - //if requested version is >= 9.1, use the extended api query response - else if (getRequestedApiVersion() >= 9.1) - { - response = new ExtendedApiQueryResponse(view, isEditable, true, - form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - } - else - { - response = new ApiQueryResponse(view, isEditable, true, - form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), - form.isIncludeDisplayValues(), form.isIncludeMetadata()); - } - response.includeStyle(form.isIncludeStyle()); - - // Issues 29515 and 32269 - force key and other non-requested columns to be sent back, but only if the client has - // requested minimal columns, as we now do for ExtJS stores - if (form.isMinimalColumns()) - { - // Be sure to use the settings from the view, as it may have swapped it out with a customized version. - // See issue 38747. - response.setColumnFilter(view.getSettings().getFieldKeys()); - } - - return response; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class GetDataAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - ObjectMapper mapper = JsonUtil.createDefaultMapper(); - mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - JSONObject object = form.getJsonObject(); - if (object == null) - { - object = new JSONObject(); - } - DataRequest builder = mapper.readValue(object.toString(), DataRequest.class); - - return builder.render(getViewContext(), errors); - } - } - - protected boolean isQueryEditable(TableInfo table) - { - if (!getContainer().hasPermission("isQueryEditable", getUser(), DeletePermission.class)) - return false; - QueryUpdateService updateService = null; - try - { - updateService = table.getUpdateService(); - } - catch(Exception ignore) {} - return null != table && null != updateService; - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExecuteSqlForm extends APIQueryForm - { - private String _sql; - private Integer _maxRows; - private Integer _offset; - private boolean _saveInSession; - - public String getSql() - { - return _sql; - } - - public void setSql(String sql) - { - _sql = PageFlowUtil.wafDecode(StringUtils.trim(sql)); - } - - public Integer getMaxRows() - { - return _maxRows; - } - - public void setMaxRows(Integer maxRows) - { - _maxRows = maxRows; - } - - public Integer getOffset() - { - return _offset; - } - - public void setOffset(Integer offset) - { - _offset = offset; - } - - @Override - public void setLimit(Integer limit) - { - _maxRows = limit; - } - - @Override - public void setStart(Integer start) - { - _offset = start; - } - - public boolean isSaveInSession() - { - return _saveInSession; - } - - public void setSaveInSession(boolean saveInSession) - { - _saveInSession = saveInSession; - } - - @Override - public String getQueryName() - { - // ExecuteSqlAction doesn't allow setting query name parameter. - return null; - } - - @Override - public void setQueryName(String name) - { - // ExecuteSqlAction doesn't allow setting query name parameter. - } - } - - @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.1) - @Action(ActionType.SelectData.class) - public class ExecuteSqlAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(ExecuteSqlForm form, BindException errors) - { - form.ensureSchemaExists(); - - String schemaName = StringUtils.trimToNull(form.getQuerySettings().getSchemaName()); - if (null == schemaName) - throw new IllegalArgumentException("No value was supplied for the required parameter 'schemaName'."); - String sql = form.getSql(); - if (StringUtils.isBlank(sql)) - throw new IllegalArgumentException("No value was supplied for the required parameter 'sql'."); - - //create a temp query settings object initialized with the posted LabKey SQL - //this will provide a temporary QueryDefinition to Query - QuerySettings settings = form.getQuerySettings(); - if (form.isSaveInSession()) - { - HttpSession session = getViewContext().getSession(); - if (session == null) - throw new IllegalStateException("Session required"); - - QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), schemaName, sql); - settings.setDataRegionName("executeSql"); - settings.setQueryName(def.getName()); - } - else - { - settings = new TempQuerySettings(getViewContext(), sql, settings); - } - - //need to explicitly turn off various UI options that will try to refer to the - //current URL and query string - settings.setAllowChooseView(false); - settings.setAllowCustomizeView(false); - - // Issue 12233: add implicit maxRows=100k when using client API - settings.setShowRows(ShowRows.PAGINATED); - settings.setMaxRows(DEFAULT_API_MAX_ROWS); - - // 16961: ExecuteSql API without maxRows parameter defaults to returning 100 rows - //apply optional settings (maxRows, offset) - boolean metaDataOnly = false; - if (null != form.getMaxRows() && (form.getMaxRows() >= 0 || form.getMaxRows() == Table.ALL_ROWS)) - { - settings.setMaxRows(form.getMaxRows()); - metaDataOnly = Table.NO_ROWS == form.getMaxRows(); - } - - int offset = 0; - if (null != form.getOffset()) - { - settings.setOffset(form.getOffset().longValue()); - offset = form.getOffset(); - } - - //build a query view using the schema and settings - QueryView view = new QueryView(form.getSchema(), settings, errors); - view.setShowRecordSelectors(false); - view.setShowExportButtons(false); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - view.setShowPagination(form.isIncludeTotalCount()); - - TableInfo t = view.getTable(); - boolean isEditable = null != t && isQueryEditable(view.getTable()); - boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; - boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; - - ApiQueryResponse response; - - // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. - if (getRequestedApiVersion() >= 13.2) - { - ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, false, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); - fancyResponse.includeFormattedValue(includeFormattedValue); - response = fancyResponse; - } - else if (getRequestedApiVersion() >= 9.1) - { - response = new ExtendedApiQueryResponse(view, isEditable, - false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - } - else - { - response = new ApiQueryResponse(view, isEditable, - false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), - form.isIncludeDisplayValues()); - } - response.includeStyle(form.isIncludeStyle()); - - return response; - } - } - - public static class ContainerFilterQueryForm extends QueryForm - { - private String _containerFilter; - - public String getContainerFilter() - { - return _containerFilter; - } - - public void setContainerFilter(String containerFilter) - { - _containerFilter = containerFilter; - } - - @Override - protected QuerySettings createQuerySettings(UserSchema schema) - { - var result = super.createQuerySettings(schema); - if (getContainerFilter() != null) - { - // If the user specified an incorrect filter, throw an IllegalArgumentException - try - { - ContainerFilter.Type containerFilterType = ContainerFilter.Type.valueOf(getContainerFilter()); - result.setContainerFilterName(containerFilterType.name()); - } - catch (IllegalArgumentException e) - { - // Remove bogus value from error message, Issue 45567 - throw new IllegalArgumentException("'containerFilter' parameter is not valid"); - } - } - return result; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public class SelectDistinctAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(ContainerFilterQueryForm form, BindException errors) throws Exception - { - TableInfo table = form.getQueryView().getTable(); - if (null == table) - throw new NotFoundException(); - SqlSelector sqlSelector = getDistinctSql(table, form, errors); - - if (errors.hasErrors() || null == sqlSelector) - return null; - - ApiResponseWriter writer = new ApiJsonWriter(getViewContext().getResponse()); - - try (ResultSet rs = sqlSelector.getResultSet()) - { - writer.startResponse(); - writer.writeProperty("schemaName", form.getSchemaName()); - writer.writeProperty("queryName", form.getQueryName()); - writer.startList("values"); - - while (rs.next()) - { - writer.writeListEntry(rs.getObject(1)); - } - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - catch (DataAccessException x) // Spring error translator can return various subclasses of this - { - throw new RuntimeException(x); - } - writer.endList(); - writer.endResponse(); - - return null; - } - - @Nullable - private SqlSelector getDistinctSql(TableInfo table, ContainerFilterQueryForm form, BindException errors) - { - QuerySettings settings = form.getQuerySettings(); - QueryService service = QueryService.get(); - - if (null == getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())) - { - settings.setMaxRows(DEFAULT_API_MAX_ROWS); - } - else - { - try - { - int maxRows = Integer.parseInt(getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())); - settings.setMaxRows(maxRows); - } - catch (NumberFormatException e) - { - // Standard exception message, Issue 45567 - QuerySettings.throwParameterParseException(QueryParam.maxRows); - } - } - - List fieldKeys = settings.getFieldKeys(); - if (null == fieldKeys || fieldKeys.size() != 1) - { - errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); - return null; - } - Map columns = service.getColumns(table, fieldKeys); - if (columns.size() != 1) - { - errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); - return null; - } - - ColumnInfo col = columns.get(settings.getFieldKeys().get(0)); - if (col == null) - { - errors.reject(ERROR_MSG, "\"" + settings.getFieldKeys().get(0).getName() + "\" is not a valid column."); - return null; - } - - try - { - SimpleFilter filter = getFilterFromQueryForm(form); - - // Strip out filters on columns that don't exist - issue 21669 - service.ensureRequiredColumns(table, columns.values(), filter, null, new HashSet<>()); - QueryLogging queryLogging = new QueryLogging(); - QueryService.SelectBuilder builder = service.getSelectBuilder(table) - .columns(columns.values()) - .filter(filter) - .queryLogging(queryLogging) - .distinct(true); - SQLFragment selectSql = builder.buildSqlFragment(); - - // TODO: queryLogging.isShouldAudit() is always false at this point. - // The only place that seems to set this is ComplianceQueryLoggingProfileListener.queryInvoked() - if (queryLogging.isShouldAudit() && null != queryLogging.getExceptionToThrowIfLoggingIsEnabled()) - { - // this is probably a more helpful message - errors.reject(ERROR_MSG, "Cannot choose values from a column that requires logging."); - return null; - } - - // Regenerate the column since the alias may have changed after call to getSelectSQL() - columns = service.getColumns(table, settings.getFieldKeys()); - var colGetAgain = columns.get(settings.getFieldKeys().get(0)); - // I don't believe the above comment, so here's an assert - assert(colGetAgain.getAlias().equals(col.getAlias())); - - SQLFragment sql = new SQLFragment("SELECT ").appendIdentifier(col.getAlias()).append(" AS value FROM ("); - sql.append(selectSql); - sql.append(") S ORDER BY value"); - - sql = table.getSqlDialect().limitRows(sql, settings.getMaxRows()); - - // 18875: Support Parameterized queries in Select Distinct - Map _namedParameters = settings.getQueryParameters(); - - service.bindNamedParameters(sql, _namedParameters); - service.validateNamedParameters(sql); - - return new SqlSelector(table.getSchema().getScope(), sql, queryLogging); - } - catch (ConversionException | QueryService.NamedParameterNotProvided e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return null; - } - } - } - - private SimpleFilter getFilterFromQueryForm(QueryForm form) - { - QuerySettings settings = form.getQuerySettings(); - SimpleFilter filter = null; - - // 21032: Respect 'ignoreFilter' - if (settings != null && !settings.getIgnoreUserFilter()) - { - // Attach any URL-based filters. This would apply to 'filterArray' from the JavaScript API. - filter = new SimpleFilter(settings.getBaseFilter()); - - String dataRegionName = form.getDataRegionName(); - if (StringUtils.trimToNull(dataRegionName) == null) - dataRegionName = QueryView.DATAREGIONNAME_DEFAULT; - - // Support for 'viewName' - CustomView view = settings.getCustomView(getViewContext(), form.getQueryDef()); - if (null != view && view.hasFilterOrSort() && !settings.getIgnoreViewFilter()) - { - ActionURL url = new ActionURL(SelectDistinctAction.class, getContainer()); - view.applyFilterAndSortToURL(url, dataRegionName); - filter.addAllClauses(new SimpleFilter(url, dataRegionName)); - } - - filter.addUrlFilters(settings.getSortFilterURL(), dataRegionName, Collections.emptyList(), getUser(), getContainer()); - } - - return filter; - } - - @RequiresPermission(ReadPermission.class) - public class GetColumnSummaryStatsAction extends ReadOnlyApiAction - { - private FieldKey _colFieldKey; - - @Override - public void validateForm(QueryForm form, Errors errors) - { - QuerySettings settings = form.getQuerySettings(); - List fieldKeys = settings != null ? settings.getFieldKeys() : null; - if (null == fieldKeys || fieldKeys.size() != 1) - errors.reject(ERROR_MSG, "GetColumnSummaryStats requires that only one column be requested."); - else - _colFieldKey = fieldKeys.get(0); - } - - @Override - public ApiResponse execute(QueryForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - QueryView view = form.getQueryView(); - DisplayColumn displayColumn = null; - - for (DisplayColumn dc : view.getDisplayColumns()) - { - if (dc.getColumnInfo() != null && _colFieldKey.equals(dc.getColumnInfo().getFieldKey())) - { - displayColumn = dc; - break; - } - } - - if (displayColumn != null && displayColumn.getColumnInfo() != null) - { - // get the map of the analytics providers to their relevant aggregates and add the information to the response - Map> analyticsProviders = new LinkedHashMap<>(); - Set colAggregates = new HashSet<>(); - for (ColumnAnalyticsProvider analyticsProvider : displayColumn.getAnalyticsProviders()) - { - if (analyticsProvider instanceof BaseAggregatesAnalyticsProvider baseAggProvider) - { - Map props = new HashMap<>(); - props.put("label", baseAggProvider.getLabel()); - - List aggregateNames = new ArrayList<>(); - for (Aggregate aggregate : AnalyticsProviderItem.createAggregates(baseAggProvider, _colFieldKey, null)) - { - aggregateNames.add(aggregate.getType().getName()); - colAggregates.add(aggregate); - } - props.put("aggregates", aggregateNames); - - analyticsProviders.put(baseAggProvider.getName(), props); - } - } - - // get the filter set from the queryform and verify that they resolve - SimpleFilter filter = getFilterFromQueryForm(form); - if (filter != null) - { - Map resolvedCols = QueryService.get().getColumns(view.getTable(), filter.getAllFieldKeys()); - for (FieldKey filterFieldKey : filter.getAllFieldKeys()) - { - if (!resolvedCols.containsKey(filterFieldKey)) - filter.deleteConditions(filterFieldKey); - } - } - - // query the table/view for the aggregate results - Collection columns = Collections.singleton(displayColumn.getColumnInfo()); - TableSelector selector = new TableSelector(view.getTable(), columns, filter, null).setNamedParameters(form.getQuerySettings().getQueryParameters()); - Map> aggResults = selector.getAggregates(new ArrayList<>(colAggregates)); - - // create a response object mapping the analytics providers to their relevant aggregate results - Map> aggregateResults = new HashMap<>(); - if (aggResults.containsKey(_colFieldKey.toString())) - { - for (Aggregate.Result r : aggResults.get(_colFieldKey.toString())) - { - Map props = new HashMap<>(); - Aggregate.Type type = r.getAggregate().getType(); - props.put("label", type.getFullLabel()); - props.put("description", type.getDescription()); - props.put("value", r.getFormattedValue(displayColumn, getContainer()).value()); - aggregateResults.put(type.getName(), props); - } - - response.put("success", true); - response.put("analyticsProviders", analyticsProviders); - response.put("aggregateResults", aggregateResults); - } - else - { - response.put("success", false); - response.put("message", "Unable to get aggregate results for " + _colFieldKey); - } - } - else - { - response.put("success", false); - response.put("message", "Unable to find ColumnInfo for " + _colFieldKey); - } - - return response; - } - } - - @RequiresPermission(ReadPermission.class) - public class ImportAction extends AbstractQueryImportAction - { - private QueryForm _form; - - @Override - protected void initRequest(QueryForm form) throws ServletException - { - _form = form; - - _insertOption = form.getInsertOption(); - QueryDefinition query = form.getQueryDef(); - List qpe = new ArrayList<>(); - TableInfo t = query.getTable(form.getSchema(), qpe, true); - if (!qpe.isEmpty()) - throw qpe.get(0); - if (null != t) - setTarget(t); - _auditBehaviorType = form.getAuditBehavior(); - _auditUserComment = form.getAuditUserComment(); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - initRequest(form); - return super.getDefaultImportView(form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - var executeQuery = _form.urlFor(QueryAction.executeQuery); - if (null == executeQuery) - root.addChild(_form.getQueryName()); - else - root.addChild(_form.getQueryName(), executeQuery); - root.addChild("Import Data"); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportSqlForm - { - private String _sql; - private String _schemaName; - private String _containerFilter; - private String _format = "excel"; - - public String getSql() - { - return _sql; - } - - public void setSql(String sql) - { - _sql = PageFlowUtil.wafDecode(sql); - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getContainerFilter() - { - return _containerFilter; - } - - public void setContainerFilter(String containerFilter) - { - _containerFilter = containerFilter; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.2) - @Action(ActionType.Export.class) - public static class ExportSqlAction extends ExportAction - { - @Override - public void export(ExportSqlForm form, HttpServletResponse response, BindException errors) throws IOException, ExportException - { - String schemaName = StringUtils.trimToNull(form.getSchemaName()); - if (null == schemaName) - throw new NotFoundException("No value was supplied for the required parameter 'schemaName'"); - String sql = StringUtils.trimToNull(form.getSql()); - if (null == sql) - throw new NotFoundException("No value was supplied for the required parameter 'sql'"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - - if (null == schema) - throw new NotFoundException("Schema '" + schemaName + "' not found in this folder"); - - //create a temp query settings object initialized with the posted LabKey SQL - //this will provide a temporary QueryDefinition to Query - TempQuerySettings settings = new TempQuerySettings(getViewContext(), sql); - - //need to explicitly turn off various UI options that will try to refer to the - //current URL and query string - settings.setAllowChooseView(false); - settings.setAllowCustomizeView(false); - - //return all rows - settings.setShowRows(ShowRows.ALL); - - //add container filter if supplied - if (form.getContainerFilter() != null && !form.getContainerFilter().isEmpty()) - { - ContainerFilter.Type containerFilterType = - ContainerFilter.Type.valueOf(form.getContainerFilter()); - settings.setContainerFilterName(containerFilterType.name()); - } - - //build a query view using the schema and settings - QueryView view = new QueryView(schema, settings, errors); - view.setShowRecordSelectors(false); - view.setShowExportButtons(false); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - - //export it - ResponseHelper.setPrivate(response); - response.setHeader("X-Robots-Tag", "noindex"); - - if ("excel".equalsIgnoreCase(form.getFormat())) - view.exportToExcel(response); - else if ("tsv".equalsIgnoreCase(form.getFormat())) - view.exportToTsv(response); - else - errors.reject(null, "Invalid format specified; must be 'excel' or 'tsv'"); - - for (QueryException qe : view.getParseErrors()) - errors.reject(null, qe.getMessage()); - - if (errors.hasErrors()) - throw new ExportException(new SimpleErrorView(errors, false)); - } - } - - public static class ApiSaveRowsForm extends SimpleApiJsonForm - { - } - - private enum CommandType - { - insert(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException - { - BatchValidationException errors = new BatchValidationException(); - List> insertedRows = qus.insertRows(user, container, rows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - // Issue 42519: Submitter role not able to insert - // as per the definition of submitter, should allow insert without read - if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) - { - return qus.getRows(user, container, insertedRows); - } - else - { - return insertedRows; - } - } - }, - insertWithKeys(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException - { - List> newRows = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - for (Map row : rows) - { - //issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null - CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); - newRows.add(newMap); - - CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); - oldKeys.add(oldMap); - } - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.insertRows(user, container, newRows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - // Issue 42519: Submitter role not able to insert - // as per the definition of submitter, should allow insert without read - if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) - { - updatedRows = qus.getRows(user, container, updatedRows); - } - List> results = new ArrayList<>(); - for (int i = 0; i < updatedRows.size(); i++) - { - Map result = new HashMap<>(); - result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); - result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); - results.add(result); - } - return results; - } - }, - importRows(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - DataIteratorBuilder it = new ListofMapsDataIterator.Builder(rows.get(0).keySet(), rows); - qus.importRows(user, container, it, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return Collections.emptyList(); - } - }, - moveRows(MoveEntitiesPermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - - Container targetContainer = (Container) configParameters.get(QueryUpdateService.ConfigParameters.TargetContainer); - Map updatedCounts = qus.moveRows(user, container, targetContainer, rows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return Collections.singletonList(updatedCounts); - } - }, - update(UpdatePermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.updateRows(user, container, rows, null, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return shouldReselect(configParameters) ? qus.getRows(user, container, updatedRows) : updatedRows; - } - }, - updateChangingKeys(UpdatePermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - List> newRows = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - for (Map row : rows) - { - // issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null. - // this should never happen on an update, but we will let it fail later with a better error message instead of the NPE here - CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); - newRows.add(newMap); - - CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); - oldKeys.add(oldMap); - } - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.updateRows(user, container, newRows, oldKeys, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - if (shouldReselect(configParameters)) - updatedRows = qus.getRows(user, container, updatedRows); - List> results = new ArrayList<>(); - for (int i = 0; i < updatedRows.size(); i++) - { - Map result = new HashMap<>(); - result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); - result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); - results.add(result); - } - return results; - } - }, - delete(DeletePermission.class, QueryService.AuditAction.DELETE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - return qus.deleteRows(user, container, rows, configParameters, extraContext); - } - }; - - private final Class _permission; - private final QueryService.AuditAction _auditAction; - - CommandType(Class permission, QueryService.AuditAction auditAction) - { - _permission = permission; - _auditAction = auditAction; - } - - public Class getPermission() - { - return _permission; - } - - public QueryService.AuditAction getAuditAction() - { - return _auditAction; - } - - public static boolean shouldReselect(Map configParameters) - { - if (configParameters == null || !configParameters.containsKey(QueryUpdateService.ConfigParameters.SkipReselectRows)) - return true; - - return Boolean.TRUE != configParameters.get(QueryUpdateService.ConfigParameters.SkipReselectRows); - } - - public abstract List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException; - } - - /** - * Base action class for insert/update/delete actions - */ - protected abstract static class BaseSaveRowsAction extends MutatingApiAction - { - public static final String PROP_SCHEMA_NAME = "schemaName"; - public static final String PROP_QUERY_NAME = "queryName"; - public static final String PROP_CONTAINER_PATH = "containerPath"; - public static final String PROP_TARGET_CONTAINER_PATH = "targetContainerPath"; - public static final String PROP_COMMAND = "command"; - public static final String PROP_ROWS = "rows"; - - private JSONObject _json; - - @Override - public void validateForm(FORM apiSaveRowsForm, Errors errors) - { - _json = apiSaveRowsForm.getJsonObject(); - - // if the POST was done using FormData, the apiSaveRowsForm would not have bound the json data, so - // we'll instead look for that data in the request param directly - if (_json == null && getViewContext().getRequest() != null && getViewContext().getRequest().getParameter("json") != null) - _json = new JSONObject(getViewContext().getRequest().getParameter("json")); - } - - protected JSONObject getJsonObject() - { - return _json; - } - - protected Container getContainerForCommand(JSONObject json) - { - return getContainerForCommand(json, PROP_CONTAINER_PATH, getContainer()); - } - - protected Container getContainerForCommand(JSONObject json, String containerPathProp, @Nullable Container defaultContainer) - { - Container container; - String containerPath = StringUtils.trimToNull(json.optString(containerPathProp)); - if (containerPath == null) - { - if (defaultContainer != null) - container = defaultContainer; - else - throw new IllegalArgumentException(containerPathProp + " is required but was not provided."); - } - else - { - container = ContainerManager.getForPath(containerPath); - if (container == null) - { - throw new IllegalArgumentException("Unknown container: " + containerPath); - } - } - - // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more downstream - if (!container.hasPermission(getUser(), ReadPermission.class) && - !container.hasPermission(getUser(), DeletePermission.class) && - !container.hasPermission(getUser(), InsertPermission.class) && - !container.hasPermission(getUser(), UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - return container; - } - - protected String getTargetContainerProp() - { - JSONObject json = getJsonObject(); - return json.optString(PROP_TARGET_CONTAINER_PATH, null); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors) throws Exception - { - return executeJson(json, commandType, allowTransaction, errors, false); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction) throws Exception - { - return executeJson(json, commandType, allowTransaction, errors, isNestedTransaction, null); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction, @Nullable Integer commandIndex) throws Exception - { - JSONObject response = new JSONObject(); - Container container = getContainerForCommand(json); - User user = getUser(); - - if (json == null) - throw new ValidationException("Empty request"); - - JSONArray rows; - try - { - rows = json.getJSONArray(PROP_ROWS); - if (rows.isEmpty()) - throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); - } - catch (JSONException x) - { - throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); - } - - String schemaName = json.getString(PROP_SCHEMA_NAME); - String queryName = json.getString(PROP_QUERY_NAME); - TableInfo table = getTableInfo(container, user, schemaName, queryName); - - if (!table.hasPermission(user, commandType.getPermission())) - throw new UnauthorizedException(); - - if (commandType != CommandType.insert && table.getPkColumns().isEmpty()) - throw new IllegalArgumentException("The table '" + table.getPublicSchemaName() + "." + - table.getPublicName() + "' cannot be updated because it has no primary key defined!"); - - QueryUpdateService qus = table.getUpdateService(); - if (null == qus) - throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + - "' is not updatable via the HTTP-based APIs."); - - int rowsAffected = 0; - - List> rowsToProcess = new ArrayList<>(); - - // NOTE RowMapFactory is faster, but for update it's important to preserve missing v explicit NULL values - // Do we need to support some sort of UNDEFINED and NULL instance of MvFieldWrapper? - RowMapFactory f = null; - if (commandType == CommandType.insert || commandType == CommandType.insertWithKeys || commandType == CommandType.delete) - f = new RowMapFactory<>(); - CaseInsensitiveHashMap referenceCasing = new CaseInsensitiveHashMap<>(); - - for (int idx = 0; idx < rows.length(); ++idx) - { - JSONObject jsonObj; - try - { - jsonObj = rows.getJSONObject(idx); - } - catch (JSONException x) - { - throw new IllegalArgumentException("rows[" + idx + "] is not an object."); - } - if (null != jsonObj) - { - Map rowMap = null == f ? new CaseInsensitiveHashMap<>(new HashMap<>(), referenceCasing) : f.getRowMap(); - // Use shallow copy since jsonObj.toMap() will translate contained JSONObjects into Maps, which we don't want - boolean conflictingCasing = JsonUtil.fillMapShallow(jsonObj, rowMap); - if (conflictingCasing) - { - // Issue 52616 - LOG.error("Row contained conflicting casing for key names in the incoming row: {}", jsonObj); - } - if (allowRowAttachments()) - addRowAttachments(table, rowMap, idx, commandIndex); - - rowsToProcess.add(rowMap); - rowsAffected++; - } - } - - Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); - - Map configParameters = new HashMap<>(); - - // Check first if the audit behavior has been defined for the table either in code or through XML. - // If not defined there, check for the audit behavior defined in the action form (json). - AuditBehaviorType behaviorType = table.getEffectiveAuditBehavior(json.optString("auditBehavior", null)); - if (behaviorType != null) - { - configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, behaviorType); - String auditComment = json.optString("auditUserComment", null); - if (!StringUtils.isEmpty(auditComment)) - configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment, auditComment); - } - - boolean skipReselectRows = json.optBoolean("skipReselectRows", false); - if (skipReselectRows) - configParameters.put(QueryUpdateService.ConfigParameters.SkipReselectRows, true); - - if (getTargetContainerProp() != null) - { - Container targetContainer = getContainerForCommand(json, PROP_TARGET_CONTAINER_PATH, null); - configParameters.put(QueryUpdateService.ConfigParameters.TargetContainer, targetContainer); - } - - //set up the response, providing the schema name, query name, and operation - //so that the client can sort out which request this response belongs to - //(clients often submit these async) - response.put(PROP_SCHEMA_NAME, schemaName); - response.put(PROP_QUERY_NAME, queryName); - response.put("command", commandType.name()); - response.put("containerPath", container.getPath()); - - //we will transact operations by default, but the user may - //override this by sending a "transacted" property set to false - // 11741: A transaction may already be active if we're trying to - // insert/update/delete from within a transformation/validation script. - boolean transacted = allowTransaction && json.optBoolean("transacted", true); - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; - try (DbScope.Transaction transaction = transacted ? table.getSchema().getScope().ensureTransaction() : NO_OP_TRANSACTION) - { - if (behaviorType != null && behaviorType != AuditBehaviorType.NONE) - { - DbScope.Transaction auditTransaction = !transacted && isNestedTransaction ? table.getSchema().getScope().getCurrentTransaction() : transaction; - if (auditTransaction == null) - auditTransaction = NO_OP_TRANSACTION; - - if (auditTransaction.getAuditEvent() != null) - auditEvent = auditTransaction.getAuditEvent(); - else - { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction()); - AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); - } - } - - QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); - List> responseRows = - commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); - if (auditEvent != null) - auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); - - if (commandType == CommandType.moveRows) - { - // moveRows returns a single map of updateCounts - response.put("updateCounts", responseRows.get(0)); - } - else if (commandType != CommandType.importRows) - { - response.put("rows", responseRows.stream() - .map(JsonUtil::toMapPreserveNonFinite) - .map(JsonUtil::toJsonPreserveNulls) - .collect(LabKeyCollectors.toJSONArray())); - } - - // if there is any provenance information, save it here - ProvenanceService svc = ProvenanceService.get(); - if (json.has("provenance")) - { - JSONObject provenanceJSON = json.getJSONObject("provenance"); - ProvenanceRecordingParams params = svc.createRecordingParams(getViewContext(), provenanceJSON, ProvenanceService.ADD_RECORDING); - RecordedAction action = svc.createRecordedAction(getViewContext(), params); - if (action != null && params.getRecordingId() != null) - { - // check for any row level provenance information - if (json.has("rows")) - { - Object rowObject = json.get("rows"); - if (rowObject instanceof JSONArray jsonArray) - { - // we need to match any provenance object inputs to the object outputs from the response rows, this typically would - // be the row lsid but it configurable in the provenance recording params - // - List> provenanceMap = svc.createProvenanceMapFromRows(getViewContext(), params, jsonArray, responseRows); - if (!provenanceMap.isEmpty()) - { - action.getProvenanceMap().addAll(provenanceMap); - } - svc.addRecordingStep(getViewContext().getRequest(), params.getRecordingId(), action); - } - else - { - errors.reject(SpringActionController.ERROR_MSG, "Unable to process provenance information, the rows object was not an array"); - } - } - } - } - transaction.commit(); - } - catch (OptimisticConflictException e) - { - //issue 13967: provide better message for OptimisticConflictException - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - catch (QueryUpdateServiceException | ConversionException | DuplicateKeyException | DataIntegrityViolationException e) - { - //Issue 14294: improve handling of ConversionException (and DuplicateKeyException (Issue 28037), and DataIntegrity (uniqueness) (Issue 22779) - errors.reject(SpringActionController.ERROR_MSG, e.getMessage() == null ? e.toString() : e.getMessage()); - } - catch (BatchValidationException e) - { - if (isSuccessOnValidationError()) - { - response.put("errors", createResponseWriter().toJSON(e)); - } - else - { - ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - throw e; - } - } - if (auditEvent != null) - { - response.put("transactionAuditId", auditEvent.getRowId()); - response.put("reselectRowCount", auditEvent.hasMultiActions()); - } - - response.put("rowsAffected", rowsAffected); - - return response; - } - - protected boolean allowRowAttachments() - { - return false; - } - - private void addRowAttachments(TableInfo tableInfo, Map rowMap, int rowIndex, @Nullable Integer commandIndex) - { - if (getFileMap() != null) - { - for (Map.Entry fileEntry : getFileMap().entrySet()) - { - // Allow for the fileMap key to include the row index, and optionally command index, for defining - // which row to attach this file to - String fullKey = fileEntry.getKey(); - String fieldKey = fullKey; - // Issue 52827: Cannot attach a file if the field name contains :: - // use lastIndexOf instead of split to get the proper parts - int lastDelimIndex = fullKey.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); - if (lastDelimIndex > -1) - { - String fieldKeyExcludeIndex = fullKey.substring(0, lastDelimIndex); - String fieldRowIndex = fullKey.substring(lastDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); - if (!fieldRowIndex.equals(rowIndex+"")) continue; - - if (commandIndex == null) - { - // Single command, so we're parsing file names in the format of: FileField::0 - fieldKey = fieldKeyExcludeIndex; - } - else - { - // Multi-command, so we're parsing file names in the format of: FileField::0::1 - int subDelimIndex = fieldKeyExcludeIndex.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); - if (subDelimIndex > -1) - { - fieldKey = fieldKeyExcludeIndex.substring(0, subDelimIndex); - String fieldCommandIndex = fieldKeyExcludeIndex.substring(subDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); - if (!fieldCommandIndex.equals(commandIndex+"")) - continue; - } - else - continue; - } - } - - SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); - rowMap.put(fieldKey, file.isEmpty() ? null : file); - } - } - - for (ColumnInfo col : tableInfo.getColumns()) - DataIteratorUtil.MatchType.multiPartFormData.updateRowMap(col, rowMap); - } - - protected boolean isSuccessOnValidationError() - { - return getRequestedApiVersion() >= 13.2; - } - - @NotNull - protected TableInfo getTableInfo(Container container, User user, String schemaName, String queryName) - { - if (null == schemaName || null == queryName) - throw new IllegalArgumentException("You must supply a schemaName and queryName!"); - - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); - if (null == schema) - throw new IllegalArgumentException("The schema '" + schemaName + "' does not exist."); - - TableInfo table = schema.getTableForInsert(queryName); - if (table == null) - throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); - return table; - } - } - - // Issue: 20522 - require read access to the action but executeJson will check for update privileges from the table - // - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class UpdateRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.update, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - } - - @RequiresAnyOf({ReadPermission.class, InsertPermission.class}) //will check below - @ApiVersion(8.3) - public static class InsertRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.insert, true, errors); - if (response == null || errors.hasErrors()) - return null; - - return new ApiSimpleResponse(response); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - } - - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class ImportRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.importRows, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - } - - @ActionNames("deleteRows, delRows") - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class DeleteRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.delete, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - } - - @RequiresPermission(ReadPermission.class) //will check below - public static class MoveRowsAction extends BaseSaveRowsAction - { - private Container _targetContainer; - - @Override - public void validateForm(MoveRowsForm form, Errors errors) - { - super.validateForm(form, errors); - - JSONObject json = getJsonObject(); - if (json == null) - { - errors.reject(ERROR_GENERIC, "Empty request"); - } - else - { - // Since we are moving between containers, we know we have product folders enabled - if (getContainer().getProject().getAuditCommentsRequired() && StringUtils.isBlank(json.optString("auditUserComment"))) - errors.reject(ERROR_GENERIC, "A reason for the move of data is required."); - else - { - String queryName = json.optString(PROP_QUERY_NAME, null); - _targetContainer = ContainerManager.getMoveTargetContainer(queryName, getContainer(), getUser(), getTargetContainerProp(), errors); - } - } - } - - @Override - public ApiResponse execute(MoveRowsForm form, BindException errors) throws Exception - { - // if JSON does not have rows array, see if they were provided via selectionKey - if (!getJsonObject().has(PROP_ROWS)) - setRowsFromSelectionKey(form); - - JSONObject response = executeJson(getJsonObject(), CommandType.moveRows, true, errors); - if (response == null || errors.hasErrors()) - return null; - - updateSelections(form); - - response.put("success", true); - response.put("containerPath", _targetContainer.getPath()); - return new ApiSimpleResponse(response); - } - - private void updateSelections(MoveRowsForm form) - { - String selectionKey = form.getDataRegionSelectionKey(); - if (selectionKey != null) - { - Set rowIds = form.getIds(getViewContext(), false) - .stream().map(Object::toString).collect(Collectors.toSet()); - DataRegionSelection.setSelected(getViewContext(), selectionKey, rowIds, false); - - // if moving entities from a type, the selections from other selectionKeys in that container will - // possibly be holding onto invalid keys after the move, so clear them based on the containerPath and selectionKey suffix - String[] keyParts = selectionKey.split("|"); - if (keyParts.length > 1) - DataRegionSelection.clearRelatedByContainerPath(getViewContext(), keyParts[keyParts.length - 1]); - } - } - - private void setRowsFromSelectionKey(MoveRowsForm form) - { - Set rowIds = form.getIds(getViewContext(), false); // handle clear of selectionKey after move complete - - // convert rowIds to a JSONArray of JSONObjects with a single property "RowId" - JSONArray rows = new JSONArray(); - for (Long rowId : rowIds) - { - JSONObject row = new JSONObject(); - row.put("RowId", rowId); - rows.put(row); - } - getJsonObject().put(PROP_ROWS, rows); - } - } - - public static class MoveRowsForm extends ApiSaveRowsForm - { - private String _dataRegionSelectionKey; - private boolean _useSnapshotSelection; - - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - public void setDataRegionSelectionKey(String dataRegionSelectionKey) - { - _dataRegionSelectionKey = dataRegionSelectionKey; - } - - public boolean isUseSnapshotSelection() - { - return _useSnapshotSelection; - } - - public void setUseSnapshotSelection(boolean useSnapshotSelection) - { - _useSnapshotSelection = useSnapshotSelection; - } - - @Override - public void bindJson(JSONObject json) - { - super.bindJson(json); - _dataRegionSelectionKey = json.optString("dataRegionSelectionKey", null); - _useSnapshotSelection = json.optBoolean("useSnapshotSelection", false); - } - - public Set getIds(ViewContext context, boolean clear) - { - if (_useSnapshotSelection) - return new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(context, getDataRegionSelectionKey())); - else - return DataRegionSelection.getSelectedIntegers(context, getDataRegionSelectionKey(), clear); - } - } - - @RequiresNoPermission //will check below - public static class SaveRowsAction extends BaseSaveRowsAction - { - public static final String PROP_VALUES = "values"; - public static final String PROP_OLD_KEYS = "oldKeys"; - - @Override - protected boolean isFailure(BindException errors) - { - return !isSuccessOnValidationError() && super.isFailure(errors); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more - // specific permissions later once we've figured out exactly what they're trying to do. This helps us - // give a better HTTP response code when they're trying to access a resource that's not available to guests - if (!getContainer().hasPermission(getUser(), ReadPermission.class) && - !getContainer().hasPermission(getUser(), DeletePermission.class) && - !getContainer().hasPermission(getUser(), InsertPermission.class) && - !getContainer().hasPermission(getUser(), UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - JSONObject json = getJsonObject(); - if (json == null) - throw new IllegalArgumentException("Empty request"); - - JSONArray commands = json.optJSONArray("commands"); - if (commands == null || commands.isEmpty()) - { - throw new NotFoundException("Empty request"); - } - - boolean validateOnly = json.optBoolean("validateOnly", false); - // If we are going to validate and not commit, we need to be sure we're transacted as well. Otherwise, - // respect the client's request. - boolean transacted = validateOnly || json.optBoolean("transacted", true); - - // Keep track of whether we end up committing or not - boolean committed = false; - - DbScope scope = null; - if (transacted) - { - for (int i = 0; i < commands.length(); i++) - { - JSONObject commandJSON = commands.getJSONObject(i); - String schemaName = commandJSON.getString(PROP_SCHEMA_NAME); - String queryName = commandJSON.getString(PROP_QUERY_NAME); - Container container = getContainerForCommand(commandJSON); - TableInfo tableInfo = getTableInfo(container, getUser(), schemaName, queryName); - if (scope == null) - { - scope = tableInfo.getSchema().getScope(); - } - else if (scope != tableInfo.getSchema().getScope()) - { - throw new IllegalArgumentException("All queries must be from the same source database"); - } - } - assert scope != null; - } - - JSONArray resultArray = new JSONArray(); - JSONObject extraContext = json.optJSONObject("extraContext"); - - int startingErrorIndex = 0; - int errorCount = 0; - // 11741: A transaction may already be active if we're trying to - // insert/update/delete from within a transformation/validation script. - - try (DbScope.Transaction transaction = transacted ? scope.ensureTransaction() : NO_OP_TRANSACTION) - { - for (int i = 0; i < commands.length(); i++) - { - JSONObject commandObject = commands.getJSONObject(i); - String commandName = commandObject.getString(PROP_COMMAND); - if (commandName == null) - { - throw new ApiUsageException(PROP_COMMAND + " is required but was missing"); - } - CommandType command = CommandType.valueOf(commandName); - - // Copy the top-level 'extraContext' and merge in the command-level extraContext. - Map commandExtraContext = new HashMap<>(); - if (extraContext != null) - commandExtraContext.putAll(extraContext.toMap()); - if (commandObject.has("extraContext")) - { - commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); - } - commandObject.put("extraContext", commandExtraContext); - - JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); - // Bail out immediately if we're going to return a failure-type response message - if (commandResponse == null || (errors.hasErrors() && !isSuccessOnValidationError())) - return null; - - //this would be populated in executeJson when a BatchValidationException is thrown - if (commandResponse.has("errors")) - { - errorCount += commandResponse.getJSONObject("errors").getInt("errorCount"); - } - - // If we encountered errors with this particular command and the client requested that don't treat - // the whole request as a failure (non-200 HTTP status code), stash the errors for this particular - // command in its response section. - // NOTE: executeJson should handle and serialize BatchValidationException - // these errors upstream - if (errors.getErrorCount() > startingErrorIndex && isSuccessOnValidationError()) - { - commandResponse.put("errors", ApiResponseWriter.convertToJSON(errors, startingErrorIndex).getValue()); - startingErrorIndex = errors.getErrorCount(); - } - - resultArray.put(commandResponse); - } - - // Don't commit if we had errors or if the client requested that we only validate (and not commit) - if (!errors.hasErrors() && !validateOnly && errorCount == 0) - { - transaction.commit(); - committed = true; - } - } - - errorCount += errors.getErrorCount(); - JSONObject result = new JSONObject(); - result.put("result", resultArray); - result.put("committed", committed); - result.put("errorCount", errorCount); - - return new ApiSimpleResponse(result); - } - } - - @RequiresPermission(ReadPermission.class) - public static class ApiTestAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/query/view/apitest.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("API Test"); - } - } - - - @RequiresPermission(AdminPermission.class) - public static class AdminAction extends SimpleViewAction - { - @SuppressWarnings("UnusedDeclaration") - public AdminAction() - { - } - - public AdminAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/admin.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Schema Administration", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ResetRemoteConnectionsForm - { - private boolean _reset; - - public boolean isReset() - { - return _reset; - } - - public void setReset(boolean reset) - { - _reset = reset; - } - } - - - @RequiresPermission(AdminPermission.class) - public static class ManageRemoteConnectionsAction extends FormViewAction - { - @Override - public void validateCommand(ResetRemoteConnectionsForm target, Errors errors) {} - - @Override - public boolean handlePost(ResetRemoteConnectionsForm form, BindException errors) - { - if (form.isReset()) - { - PropertyManager.getEncryptedStore().deletePropertySet(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); - } - return true; - } - - @Override - public URLHelper getSuccessURL(ResetRemoteConnectionsForm queryForm) - { - return new ActionURL(ManageRemoteConnectionsAction.class, getContainer()); - } - - @Override - public ModelAndView getView(ResetRemoteConnectionsForm queryForm, boolean reshow, BindException errors) - { - Map connectionMap; - try - { - // if the encrypted property store is configured but no values have yet been set, and empty map is returned - connectionMap = PropertyManager.getEncryptedStore().getProperties(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); - } - catch (Exception e) - { - connectionMap = null; // render the failure page - } - setHelpTopic("remoteConnection"); - return new JspView<>("/org/labkey/query/view/manageRemoteConnections.jsp", connectionMap, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - private abstract static class BaseInsertExternalSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction - { - protected BaseInsertExternalSchemaAction(Class commandClass) - { - super(commandClass); - } - - @Override - public void validateCommand(F form, Errors errors) - { - form.validate(errors); - } - - @Override - public boolean handlePost(F form, BindException errors) throws Exception - { - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - form.doInsert(); - auditSchemaAdminActivity(form.getBean(), "created", getContainer(), getUser()); - QueryManager.get().updateExternalSchemas(getContainer()); - - t.commit(); - } - catch (RuntimeSQLException e) - { - if (e.isConstraintException()) - { - errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); - return false; - } - - throw e; - } - - return true; - } - - @Override - public ActionURL getSuccessURL(F form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext()).addNavTrail(root); - root.addChild("Define Schema", new ActionURL(getClass(), getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class InsertLinkedSchemaAction extends BaseInsertExternalSchemaAction - { - public InsertLinkedSchemaAction() - { - super(LinkedSchemaForm.class); - } - - @Override - public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) - { - setHelpTopic("filterSchema"); - return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), form.getBean(), true), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class InsertExternalSchemaAction extends BaseInsertExternalSchemaAction - { - public InsertExternalSchemaAction() - { - super(ExternalSchemaForm.class); - } - - @Override - public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) - { - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), form.getBean(), true), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class DeleteSchemaAction extends ConfirmAction - { - @Override - public String getConfirmText() - { - return "Delete"; - } - - @Override - public ModelAndView getConfirmView(SchemaForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Schema"); - - AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - String schemaName = isBlank(def.getUserSchemaName()) ? "this schema" : "the schema '" + def.getUserSchemaName() + "'"; - return new HtmlView(HtmlString.of("Are you sure you want to delete " + schemaName + "? The tables and queries defined in this schema will no longer be accessible.")); - } - - @Override - public boolean handlePost(SchemaForm form, BindException errors) - { - AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - auditSchemaAdminActivity(def, "deleted", getContainer(), getUser()); - QueryManager.get().delete(def); - t.commit(); - } - return true; - } - - @Override - public void validateCommand(SchemaForm form, Errors errors) - { - } - - @Override - @NotNull - public ActionURL getSuccessURL(SchemaForm form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - } - - private static void auditSchemaAdminActivity(AbstractExternalSchemaDef def, String action, Container container, User user) - { - String comment = StringUtils.capitalize(def.getSchemaType().toString()) + " schema '" + def.getUserSchemaName() + "' " + action; - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, comment); - AuditLogService.get().addEvent(user, event); - } - - - private abstract static class BaseEditSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction - { - protected BaseEditSchemaAction(Class commandClass) - { - super(commandClass); - } - - @Override - public void validateCommand(F form, Errors errors) - { - form.validate(errors); - } - - @Nullable - protected abstract T getCurrent(int externalSchemaId); - - @NotNull - protected T getDef(F form, boolean reshow) - { - T def; - Container defContainer; - - if (reshow) - { - def = form.getBean(); - T current = getCurrent(def.getExternalSchemaId()); - if (current == null) - throw new NotFoundException(); - - defContainer = current.lookupContainer(); - } - else - { - form.refreshFromDb(); - if (!form.isDataLoaded()) - throw new NotFoundException(); - - def = form.getBean(); - if (def == null) - throw new NotFoundException(); - - defContainer = def.lookupContainer(); - } - - if (!getContainer().equals(defContainer)) - throw new UnauthorizedException(); - - return def; - } - - @Override - public boolean handlePost(F form, BindException errors) throws Exception - { - T def = form.getBean(); - T fromDb = getCurrent(def.getExternalSchemaId()); - - // Unauthorized if def in the database reports a different container - if (!getContainer().equals(fromDb.lookupContainer())) - throw new UnauthorizedException(); - - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - form.doUpdate(); - auditSchemaAdminActivity(def, "updated", getContainer(), getUser()); - QueryManager.get().updateExternalSchemas(getContainer()); - t.commit(); - } - catch (RuntimeSQLException e) - { - if (e.isConstraintException()) - { - errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); - return false; - } - - throw e; - } - return true; - } - - @Override - public ActionURL getSuccessURL(F externalSchemaForm) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext()).addNavTrail(root); - root.addChild("Edit Schema", new ActionURL(getClass(), getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditLinkedSchemaAction extends BaseEditSchemaAction - { - public EditLinkedSchemaAction() - { - super(LinkedSchemaForm.class); - } - - @Nullable - @Override - protected LinkedSchemaDef getCurrent(int externalId) - { - return QueryManager.get().getLinkedSchemaDef(getContainer(), externalId); - } - - @Override - public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) - { - LinkedSchemaDef def = getDef(form, reshow); - - setHelpTopic("filterSchema"); - return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), def, false), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditExternalSchemaAction extends BaseEditSchemaAction - { - public EditExternalSchemaAction() - { - super(ExternalSchemaForm.class); - } - - @Nullable - @Override - protected ExternalSchemaDef getCurrent(int externalId) - { - return QueryManager.get().getExternalSchemaDef(getContainer(), externalId); - } - - @Override - public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) - { - ExternalSchemaDef def = getDef(form, reshow); - - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), def, false), errors); - } - } - - - public static class DataSourceInfo - { - public final String sourceName; - public final String displayName; - public final boolean editable; - - public DataSourceInfo(DbScope scope) - { - this(scope.getDataSourceName(), scope.getDisplayName(), scope.getSqlDialect().isEditable()); - } - - public DataSourceInfo(Container c) - { - this(c.getId(), c.getName(), false); - } - - public DataSourceInfo(String sourceName, String displayName, boolean editable) - { - this.sourceName = sourceName; - this.displayName = displayName; - this.editable = editable; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - DataSourceInfo that = (DataSourceInfo) o; - return sourceName != null ? sourceName.equals(that.sourceName) : that.sourceName == null; - } - - @Override - public int hashCode() - { - return sourceName != null ? sourceName.hashCode() : 0; - } - } - - public static abstract class BaseExternalSchemaBean - { - protected final Container _c; - protected final T _def; - protected final boolean _insert; - protected final Map _help = new HashMap<>(); - - public BaseExternalSchemaBean(Container c, T def, boolean insert) - { - _c = c; - _def = def; - _insert = insert; - - TableInfo ti = QueryManager.get().getTableInfoExternalSchema(); - - ti.getColumns() - .stream() - .filter(ci -> null != ci.getDescription()) - .forEach(ci -> _help.put(ci.getName(), ci.getDescription())); - } - - public abstract DataSourceInfo getInitialSource(); - - public T getSchemaDef() - { - return _def; - } - - public boolean isInsert() - { - return _insert; - } - - public ActionURL getReturnURL() - { - return new ActionURL(AdminAction.class, _c); - } - - public ActionURL getDeleteURL() - { - return new QueryUrlsImpl().urlDeleteSchema(_c, _def); - } - - public String getHelpHTML(String fieldName) - { - return _help.get(fieldName); - } - } - - public static class LinkedSchemaBean extends BaseExternalSchemaBean - { - public LinkedSchemaBean(Container c, LinkedSchemaDef def, boolean insert) - { - super(c, def, insert); - } - - @Override - public DataSourceInfo getInitialSource() - { - Container sourceContainer = getInitialContainer(); - return new DataSourceInfo(sourceContainer); - } - - private @NotNull Container getInitialContainer() - { - LinkedSchemaDef def = getSchemaDef(); - Container sourceContainer = def.lookupSourceContainer(); - if (sourceContainer == null) - sourceContainer = def.lookupContainer(); - if (sourceContainer == null) - sourceContainer = _c; - return sourceContainer; - } - } - - public static class ExternalSchemaBean extends BaseExternalSchemaBean - { - protected final Map> _sourcesAndSchemas = new LinkedHashMap<>(); - protected final Map> _sourcesAndSchemasIncludingSystem = new LinkedHashMap<>(); - - public ExternalSchemaBean(Container c, ExternalSchemaDef def, boolean insert) - { - super(c, def, insert); - initSources(); - } - - public Collection getSources() - { - return _sourcesAndSchemas.keySet(); - } - - public Collection getSchemaNames(DataSourceInfo source, boolean includeSystem) - { - if (includeSystem) - return _sourcesAndSchemasIncludingSystem.get(source); - else - return _sourcesAndSchemas.get(source); - } - - @Override - public DataSourceInfo getInitialSource() - { - ExternalSchemaDef def = getSchemaDef(); - DbScope scope = def.lookupDbScope(); - if (scope == null) - scope = DbScope.getLabKeyScope(); - return new DataSourceInfo(scope); - } - - protected void initSources() - { - ModuleLoader moduleLoader = ModuleLoader.getInstance(); - - for (DbScope scope : DbScope.getDbScopes()) - { - SqlDialect dialect = scope.getSqlDialect(); - - Collection schemaNames = new LinkedList<>(); - Collection schemaNamesIncludingSystem = new LinkedList<>(); - - for (String schemaName : scope.getSchemaNames()) - { - schemaNamesIncludingSystem.add(schemaName); - - if (dialect.isSystemSchema(schemaName)) - continue; - - if (null != moduleLoader.getModule(scope, schemaName)) - continue; - - schemaNames.add(schemaName); - } - - DataSourceInfo source = new DataSourceInfo(scope); - _sourcesAndSchemas.put(source, schemaNames); - _sourcesAndSchemasIncludingSystem.put(source, schemaNamesIncludingSystem); - } - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetTablesForm - { - private String _dataSource; - private String _schemaName; - private boolean _sorted; - - public String getDataSource() - { - return _dataSource; - } - - public void setDataSource(String dataSource) - { - _dataSource = dataSource; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isSorted() - { - return _sorted; - } - - public void setSorted(boolean sorted) - { - _sorted = sorted; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class GetTablesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(GetTablesForm form, BindException errors) - { - List> rows = new LinkedList<>(); - List tableNames = new ArrayList<>(); - - if (null != form.getSchemaName()) - { - DbScope scope = DbScope.getDbScope(form.getDataSource()); - if (null != scope) - { - DbSchema schema = scope.getSchema(form.getSchemaName(), DbSchemaType.Bare); - tableNames.addAll(schema.getTableNames()); - } - else - { - Container c = ContainerManager.getForId(form.getDataSource()); - if (null != c) - { - UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); - if (null != schema) - { - if (form.isSorted()) - for (TableInfo table : schema.getSortedTables()) - tableNames.add(table.getName()); - else - tableNames.addAll(schema.getTableAndQueryNames(true)); - } - } - } - } - - Collections.sort(tableNames); - - for (String tableName : tableNames) - { - Map row = new LinkedHashMap<>(); - row.put("table", tableName); - rows.add(row); - } - - Map properties = new HashMap<>(); - properties.put("rows", rows); - - return new ApiSimpleResponse(properties); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SchemaTemplateForm - { - private String _name; - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public static class SchemaTemplateAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SchemaTemplateForm form, BindException errors) - { - String name = form.getName(); - if (name == null) - throw new IllegalArgumentException("name required"); - - Container c = getContainer(); - TemplateSchemaType template = QueryServiceImpl.get().getSchemaTemplate(c, name); - if (template == null) - throw new NotFoundException("template not found"); - - JSONObject templateJson = QueryServiceImpl.get().schemaTemplateJson(name, template); - - return new ApiSimpleResponse("template", templateJson); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class SchemaTemplatesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object form, BindException errors) - { - Container c = getContainer(); - QueryServiceImpl svc = QueryServiceImpl.get(); - Map templates = svc.getSchemaTemplates(c); - - JSONArray ret = new JSONArray(); - for (String key : templates.keySet()) - { - TemplateSchemaType template = templates.get(key); - JSONObject templateJson = svc.schemaTemplateJson(key, template); - ret.put(templateJson); - } - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("templates", ret); - resp.put("success", true); - return resp; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ReloadExternalSchemaAction extends FormHandlerAction - { - private String _userSchemaName; - - @Override - public void validateCommand(SchemaForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SchemaForm form, BindException errors) - { - ExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), ExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - QueryManager.get().reloadExternalSchema(def); - _userSchemaName = def.getUserSchemaName(); - - return true; - } - - @Override - public ActionURL getSuccessURL(SchemaForm form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Schema " + _userSchemaName + " was reloaded successfully."); - } - } - - - @RequiresPermission(AdminPermission.class) - public static class ReloadAllUserSchemas extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - QueryManager.get().reloadAllExternalSchemas(getContainer()); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "All schemas in this folder were reloaded successfully."); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ReloadFailedConnectionsAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - DbScope.clearFailedDbScopes(); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Reconnection was attempted on all data sources that failed previous connection attempts."); - } - } - - @RequiresPermission(ReadPermission.class) - public static class TableInfoAction extends SimpleViewAction - { - @Override - public ModelAndView getView(TableInfoForm form, BindException errors) throws Exception - { - TablesDocument ret = TablesDocument.Factory.newInstance(); - TablesType tables = ret.addNewTables(); - - FieldKey[] fields = form.getFieldKeys(); - if (fields.length != 0) - { - TableInfo tinfo = QueryView.create(form, errors).getTable(); - Map columnMap = CustomViewImpl.getColumnInfos(tinfo, Arrays.asList(fields)); - TableXML.initTable(tables.addNewTable(), tinfo, null, columnMap.values()); - } - - for (FieldKey tableKey : form.getTableKeys()) - { - TableInfo tableInfo = form.getTableInfo(tableKey); - TableType xbTable = tables.addNewTable(); - TableXML.initTable(xbTable, tableInfo, tableKey); - } - getViewContext().getResponse().setContentType("text/xml"); - getViewContext().getResponse().getWriter().write(ret.toString()); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - // Issue 18870: Guest user can't revert unsaved custom view changes - // Permission will be checked inline (guests are allowed to delete their session custom views) - @RequiresNoPermission - @Action(ActionType.Configure.class) - public static class DeleteViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeleteViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - - if (getUser().isGuest()) - { - // Guests can only delete session custom views. - if (!view.isSession()) - throw new UnauthorizedException(); - } - else - { - // Logged in users must have read permission - if (!getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new UnauthorizedException(); - } - - if (view.isShared()) - { - if (!getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) - throw new UnauthorizedException(); - } - - view.delete(getUser(), getViewContext().getRequest()); - - // Delete the first shadowed custom view, if available. - if (form.isComplete()) - { - form.reset(); - CustomView shadowed = form.getCustomView(); - if (shadowed != null && shadowed.isEditable() && !(shadowed instanceof ModuleCustomView)) - { - if (!shadowed.isShared() || getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) - shadowed.delete(getUser(), getViewContext().getRequest()); - } - } - - // Try to get a custom view of the same name as the view we just deleted. - // The deleted view may have been a session view or a personal view masking shared view with the same name. - form.reset(); - view = form.getCustomView(); - String nextViewName = null; - if (view != null) - nextViewName = view.getName(); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("viewName", nextViewName); - return response; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SaveSessionViewForm extends QueryForm - { - private String newName; - private boolean inherit; - private boolean shared; - private boolean hidden; - private boolean replace; - private String containerPath; - - public String getNewName() - { - return newName; - } - - public void setNewName(String newName) - { - this.newName = newName; - } - - public boolean isInherit() - { - return inherit; - } - - public void setInherit(boolean inherit) - { - this.inherit = inherit; - } - - public boolean isShared() - { - return shared; - } - - public void setShared(boolean shared) - { - this.shared = shared; - } - - public String getContainerPath() - { - return containerPath; - } - - public void setContainerPath(String containerPath) - { - this.containerPath = containerPath; - } - - public boolean isHidden() - { - return hidden; - } - - public void setHidden(boolean hidden) - { - this.hidden = hidden; - } - - public boolean isReplace() - { - return replace; - } - - public void setReplace(boolean replace) - { - this.replace = replace; - } - } - - // Moves a session view into the database. - @RequiresPermission(ReadPermission.class) - public static class SaveSessionViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SaveSessionViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - if (!view.isSession()) - throw new IllegalArgumentException("This action only supports saving session views."); - - //if (!getContainer().getId().equals(view.getContainer().getId())) - // throw new IllegalArgumentException("View may only be saved from container it was created in."); - - assert !view.canInherit() && !view.isShared() && view.isEditable(): "Session view should never be inheritable or shared and always be editable"; - - // Users may save views to a location other than the current container - String containerPath = form.getContainerPath(); - Container container; - if (form.isInherit() && containerPath != null) - { - // Only respect this request if it's a view that is inheritable in subfolders - container = ContainerManager.getForPath(containerPath); - } - else - { - // Otherwise, save it in the current container - container = getContainer(); - } - - if (container == null) - throw new NotFoundException("No such container: " + containerPath); - - if (form.isShared() || form.isInherit()) - { - if (!container.hasPermission(getUser(), EditSharedViewPermission.class)) - throw new UnauthorizedException(); - } - - DbScope scope = QueryManager.get().getDbSchema().getScope(); - try (DbScope.Transaction tx = scope.ensureTransaction()) - { - // Delete the session view. The view will be restored if an exception is thrown. - view.delete(getUser(), getViewContext().getRequest()); - - // Get any previously existing non-session view. - // The session custom view and the view-to-be-saved may have different names. - // If they do have different names, we may need to delete an existing session view with that name. - // UNDONE: If the view has a different name, we will clobber it without asking. - CustomView existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); - if (existingView != null && existingView.isSession()) - { - // Delete any session view we are overwriting. - existingView.delete(getUser(), getViewContext().getRequest()); - existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); - } - - // save a new private view if shared is false but existing view is shared - if (existingView != null && !form.isShared() && existingView.getOwner() == null) - { - existingView = null; - } - - if (existingView != null && !form.isReplace() && !StringUtils.isEmpty(form.getNewName())) - throw new IllegalArgumentException("A saved view by the name \"" + form.getNewName() + "\" already exists. "); - - if (existingView == null || (existingView instanceof ModuleCustomView && existingView.isEditable())) - { - User owner = form.isShared() ? null : getUser(); - - CustomViewImpl viewCopy = new CustomViewImpl(form.getQueryDef(), owner, form.getNewName()); - viewCopy.setColumns(view.getColumns()); - viewCopy.setCanInherit(form.isInherit()); - viewCopy.setFilterAndSort(view.getFilterAndSort()); - viewCopy.setColumnProperties(view.getColumnProperties()); - viewCopy.setIsHidden(form.isHidden()); - if (form.isInherit()) - viewCopy.setContainer(container); - - viewCopy.save(getUser(), getViewContext().getRequest()); - } - else if (!existingView.isEditable()) - { - throw new IllegalArgumentException("Existing view '" + form.getNewName() + "' is not editable. You may save this view with a different name."); - } - else - { - // UNDONE: changing shared property of an existing view is unimplemented. Not sure if it makes sense from a usability point of view. - existingView.setColumns(view.getColumns()); - existingView.setFilterAndSort(view.getFilterAndSort()); - existingView.setColumnProperties(view.getColumnProperties()); - existingView.setCanInherit(form.isInherit()); - if (form.isInherit()) - ((CustomViewImpl)existingView).setContainer(container); - existingView.setIsHidden(form.isHidden()); - - existingView.save(getUser(), getViewContext().getRequest()); - } - - tx.commit(); - return new ApiSimpleResponse("success", true); - } - catch (Exception e) - { - // dirty the view then save the deleted session view back in session state - view.setName(view.getName()); - view.save(getUser(), getViewContext().getRequest()); - - throw e; - } - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class ManageViewsAction extends SimpleViewAction - { - @SuppressWarnings("UnusedDeclaration") - public ManageViewsAction() - { - } - - public ManageViewsAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return new JspView<>("/org/labkey/query/view/manageViews.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Views", QueryController.this.getViewContext().getActionURL()); - } - } - - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalDeleteView extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(InternalViewForm form, BindException errors) - { - return new JspView<>("/org/labkey/query/view/internalDeleteView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalViewForm form, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - QueryManager.get().delete(view); - return true; - } - - @Override - public void validateCommand(InternalViewForm internalViewForm, Errors errors) - { - } - - @Override - @NotNull - public ActionURL getSuccessURL(InternalViewForm internalViewForm) - { - return new ActionURL(ManageViewsAction.class, getContainer()); - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalSourceViewAction extends FormViewAction - { - @Override - public void validateCommand(InternalSourceViewForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(InternalSourceViewForm form, boolean reshow, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - form.ff_inherit = QueryManager.get().canInherit(view.getFlags()); - form.ff_hidden = QueryManager.get().isHidden(view.getFlags()); - form.ff_columnList = view.getColumns(); - form.ff_filter = view.getFilter(); - return new JspView<>("/org/labkey/query/view/internalSourceView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalSourceViewForm form, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - int flags = view.getFlags(); - flags = QueryManager.get().setCanInherit(flags, form.ff_inherit); - flags = QueryManager.get().setIsHidden(flags, form.ff_hidden); - view.setFlags(flags); - view.setColumns(form.ff_columnList); - view.setFilter(form.ff_filter); - QueryManager.get().update(getUser(), view); - return true; - } - - @Override - public ActionURL getSuccessURL(InternalSourceViewForm form) - { - return new ActionURL(ManageViewsAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new ManageViewsAction(getViewContext()).addNavTrail(root); - root.addChild("Edit source of Grid View"); - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalNewViewAction extends FormViewAction - { - int _customViewId = 0; - - @Override - public void validateCommand(InternalNewViewForm form, Errors errors) - { - if (StringUtils.trimToNull(form.ff_schemaName) == null) - { - errors.reject(ERROR_MSG, "Schema name cannot be blank."); - } - if (StringUtils.trimToNull(form.ff_queryName) == null) - { - errors.reject(ERROR_MSG, "Query name cannot be blank"); - } - } - - @Override - public ModelAndView getView(InternalNewViewForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/query/view/internalNewView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalNewViewForm form, BindException errors) - { - if (form.ff_share) - { - if (!getContainer().hasPermission(getUser(), AdminPermission.class)) - throw new UnauthorizedException(); - } - List existing = QueryManager.get().getCstmViews(getContainer(), form.ff_schemaName, form.ff_queryName, form.ff_viewName, form.ff_share ? null : getUser(), false, false); - CstmView view; - if (!existing.isEmpty()) - { - } - else - { - view = new CstmView(); - view.setSchema(form.ff_schemaName); - view.setQueryName(form.ff_queryName); - view.setName(form.ff_viewName); - view.setContainerId(getContainer().getId()); - if (form.ff_share) - { - view.setCustomViewOwner(null); - } - else - { - view.setCustomViewOwner(getUser().getUserId()); - } - if (form.ff_inherit) - { - view.setFlags(QueryManager.get().setCanInherit(view.getFlags(), form.ff_inherit)); - } - InternalViewForm.checkEdit(getViewContext(), view); - try - { - view = QueryManager.get().insert(getUser(), view); - } - catch (Exception e) - { - LogManager.getLogger(QueryController.class).error("Error", e); - errors.reject(ERROR_MSG, "An exception occurred: " + e); - return false; - } - _customViewId = view.getCustomViewId(); - } - return true; - } - - @Override - public ActionURL getSuccessURL(InternalNewViewForm form) - { - ActionURL forward = new ActionURL(InternalSourceViewAction.class, getContainer()); - forward.addParameter("customViewId", Integer.toString(_customViewId)); - return forward; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Create New Grid View"); - } - } - - - @ActionNames("clearSelected, selectNone") - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class SelectNoneAction extends MutatingApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) - { - errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - if (form.getQueryName() == null) - { - DataRegionSelection.clearAll(getViewContext(), form.getKey()); - return new DataRegionSelection.SelectionResponse(0); - } - - int count = DataRegionSelection.setSelectedFromForm(form); - return new DataRegionSelection.SelectionResponse(count); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SelectForm extends QueryForm - { - protected boolean clearSelected; - protected String key; - - public boolean isClearSelected() - { - return clearSelected; - } - - public void setClearSelected(boolean clearSelected) - { - this.clearSelected = clearSelected; - } - - public String getKey() - { - return key; - } - - public void setKey(String key) - { - this.key = key; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class SelectAllAction extends MutatingApiAction - { - @Override - public void validateForm(QueryForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() || form.getQueryName() == null) - { - errors.reject(ERROR_MSG, "schemaName and queryName required"); - } - } - - @Override - public ApiResponse execute(final QueryForm form, BindException errors) throws Exception - { - int count = DataRegionSelection.setSelectionForAll(form, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetSelectedAction extends ReadOnlyApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) - { - errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - getViewContext().getResponse().setHeader("Content-Type", CONTENT_TYPE_JSON); - Set selected; - - if (form.getQueryName() == null) - selected = DataRegionSelection.getSelected(getViewContext(), form.getKey(), form.isClearSelected()); - else - selected = DataRegionSelection.getSelected(form, form.isClearSelected()); - - return new ApiSimpleResponse("selected", selected); - } - } - - @ActionNames("setSelected, setCheck") - @RequiresPermission(ReadPermission.class) - public static class SetCheckAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) throws Exception - { - String[] ids = form.getId(getViewContext().getRequest()); - Set selection = new LinkedHashSet<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - int count; - if (form.getQueryName() != null && form.isValidateIds() && form.isChecked()) - { - selection = DataRegionSelection.getValidatedIds(selection, form); - } - - count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, form.isChecked()); - - return new DataRegionSelection.SelectionResponse(count); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SetCheckForm extends SelectForm - { - protected String[] ids; - protected boolean checked; - protected boolean validateIds; - - public String[] getId(HttpServletRequest request) - { - // 5025 : DataRegion checkbox names may contain comma - // Beehive parses a single parameter value with commas into an array - // which is not what we want. - String[] paramIds = request.getParameterValues("id"); - return paramIds == null ? ids: paramIds; - } - - public void setId(String[] ids) - { - this.ids = ids; - } - - public boolean isChecked() - { - return checked; - } - - public void setChecked(boolean checked) - { - this.checked = checked; - } - - public boolean isValidateIds() - { - return validateIds; - } - - public void setValidateIds(boolean validateIds) - { - this.validateIds = validateIds; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ReplaceSelectedAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) - { - String[] ids = form.getId(getViewContext().getRequest()); - List selection = new ArrayList<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - - DataRegionSelection.clearAll(getViewContext(), form.getKey()); - int count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class SetSnapshotSelectionAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) - { - String[] ids = form.getId(getViewContext().getRequest()); - List selection = new ArrayList<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - DataRegionSelection.clearAll(getViewContext(), form.getKey(), true); - int count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, true, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetSnapshotSelectionAction extends ReadOnlyApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (StringUtils.isEmpty(form.getKey())) - { - errors.reject(ERROR_MSG, "Selection key is required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - List selected = DataRegionSelection.getSnapshotSelected(getViewContext(), form.getKey()); - return new ApiSimpleResponse("selected", selected); - } - } - - public static String getMessage(SqlDialect d, SQLException x) - { - return x.getMessage(); - } - - - public static class GetSchemasForm - { - private boolean _includeHidden = true; - private SchemaKey _schemaName; - - public SchemaKey getSchemaName() - { - return _schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(SchemaKey schemaName) - { - _schemaName = schemaName; - } - - public boolean isIncludeHidden() - { - return _includeHidden; - } - - @SuppressWarnings("unused") - public void setIncludeHidden(boolean includeHidden) - { - _includeHidden = includeHidden; - } - } - - - @RequiresPermission(ReadPermission.class) - @ApiVersion(12.3) - public static class GetSchemasAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetSchemasForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetSchemasForm form, BindException errors) - { - final Container container = getContainer(); - final User user = getUser(); - - final boolean includeHidden = form.isIncludeHidden(); - if (getRequestedApiVersion() >= 9.3) - { - SimpleSchemaTreeVisitor visitor = new SimpleSchemaTreeVisitor<>(includeHidden) - { - @Override - public Void visitUserSchema(UserSchema schema, Path path, JSONObject json) - { - JSONObject schemaProps = new JSONObject(); - - schemaProps.put("schemaName", schema.getName()); - schemaProps.put("fullyQualifiedName", schema.getSchemaName()); - schemaProps.put("description", schema.getDescription()); - schemaProps.put("hidden", schema.isHidden()); - NavTree tree = schema.getSchemaBrowserLinks(user); - if (tree != null && tree.hasChildren()) - schemaProps.put("menu", tree.toJSON()); - - // Collect children schemas - JSONObject children = new JSONObject(); - visit(schema.getSchemas(_includeHidden), path, children); - if (!children.isEmpty()) - schemaProps.put("schemas", children); - - // Add node's schemaProps to the parent's json. - json.put(schema.getName(), schemaProps); - return null; - } - }; - - // By default, start from the root. - QuerySchema schema; - if (form.getSchemaName() != null) - schema = DefaultSchema.get(user, container, form.getSchemaName()); - else - schema = DefaultSchema.get(user, container); - - // Ensure consistent exception as other query actions - QueryForm.ensureSchemaNotNull(schema); - - // Create the JSON response by visiting the schema children. The parent schema information isn't included. - JSONObject ret = new JSONObject(); - visitor.visitTop(schema.getSchemas(includeHidden), ret); - - return new ApiSimpleResponse(ret); - } - else - { - return new ApiSimpleResponse("schemas", DefaultSchema.get(user, container).getUserSchemaPaths(includeHidden)); - } - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetQueriesForm - { - private String _schemaName; - private boolean _includeUserQueries = true; - private boolean _includeSystemQueries = true; - private boolean _includeColumns = true; - private boolean _includeViewDataUrl = true; - private boolean _includeTitle = true; - private boolean _queryDetailColumns = false; - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isIncludeUserQueries() - { - return _includeUserQueries; - } - - public void setIncludeUserQueries(boolean includeUserQueries) - { - _includeUserQueries = includeUserQueries; - } - - public boolean isIncludeSystemQueries() - { - return _includeSystemQueries; - } - - public void setIncludeSystemQueries(boolean includeSystemQueries) - { - _includeSystemQueries = includeSystemQueries; - } - - public boolean isIncludeColumns() - { - return _includeColumns; - } - - public void setIncludeColumns(boolean includeColumns) - { - _includeColumns = includeColumns; - } - - public boolean isQueryDetailColumns() - { - return _queryDetailColumns; - } - - public void setQueryDetailColumns(boolean queryDetailColumns) - { - _queryDetailColumns = queryDetailColumns; - } - - public boolean isIncludeViewDataUrl() - { - return _includeViewDataUrl; - } - - public void setIncludeViewDataUrl(boolean includeViewDataUrl) - { - _includeViewDataUrl = includeViewDataUrl; - } - - public boolean isIncludeTitle() - { - return _includeTitle; - } - - public void setIncludeTitle(boolean includeTitle) - { - _includeTitle = includeTitle; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class GetQueriesAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetQueriesForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetQueriesForm form, BindException errors) - { - if (null == StringUtils.trimToNull(form.getSchemaName())) - throw new IllegalArgumentException("You must supply a value for the 'schemaName' parameter!"); - - ApiSimpleResponse response = new ApiSimpleResponse(); - UserSchema uschema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); - if (null == uschema) - throw new NotFoundException("The schema name '" + form.getSchemaName() - + "' was not found within the folder '" + getContainer().getPath() + "'"); - - response.put("schemaName", form.getSchemaName()); - - List> qinfos = new ArrayList<>(); - - //user-defined queries - if (form.isIncludeUserQueries()) - { - for (QueryDefinition qdef : uschema.getQueryDefs().values()) - { - if (!qdef.isTemporary()) - { - ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; - qinfos.add(getQueryProps(qdef, viewDataUrl, true, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); - } - } - } - - //built-in tables - if (form.isIncludeSystemQueries()) - { - for (String qname : uschema.getVisibleTableNames()) - { - // Go direct against the UserSchema instead of calling into QueryService, which takes a schema and - // query name as strings and therefore has to create new instances - QueryDefinition qdef = uschema.getQueryDefForTable(qname); - if (qdef != null) - { - ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; - qinfos.add(getQueryProps(qdef, viewDataUrl, false, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); - } - } - } - response.put("queries", qinfos); - - return response; - } - - protected Map getQueryProps(QueryDefinition qdef, ActionURL viewDataUrl, boolean isUserDefined, UserSchema schema, boolean includeColumns, boolean useQueryDetailColumns, boolean includeTitle) - { - Map qinfo = new HashMap<>(); - qinfo.put("hidden", qdef.isHidden()); - qinfo.put("snapshot", qdef.isSnapshot()); - qinfo.put("inherit", qdef.canInherit()); - qinfo.put("isUserDefined", isUserDefined); - boolean canEdit = qdef.canEdit(getUser()); - qinfo.put("canEdit", canEdit); - qinfo.put("canEditSharedViews", getContainer().hasPermission(getUser(), EditSharedViewPermission.class)); - // CONSIDER: do we want to separate the 'canEditMetadata' property and 'isMetadataOverridable' properties to differentiate between capability and the permission check? - qinfo.put("isMetadataOverrideable", qdef.isMetadataEditable() && qdef.canEditMetadata(getUser())); - - if (isUserDefined) - qinfo.put("moduleName", qdef.getModuleName()); - boolean isInherited = qdef.canInherit() && !getContainer().equals(qdef.getDefinitionContainer()); - qinfo.put("isInherited", isInherited); - if (isInherited) - qinfo.put("containerPath", qdef.getDefinitionContainer().getPath()); - qinfo.put("isIncludedForLookups", qdef.isIncludedForLookups()); - - if (null != qdef.getDescription()) - qinfo.put("description", qdef.getDescription()); - if (viewDataUrl != null) - qinfo.put("viewDataUrl", viewDataUrl); - - String title = qdef.getName(); - String name = qdef.getName(); - try - { - // get the TableInfo if the user requested column info or title, otherwise skip (it can be expensive) - if (includeColumns || includeTitle) - { - TableInfo table = qdef.getTable(schema, null, true); - - if (null != table) - { - if (includeColumns) - { - Collection> columns; - - if (useQueryDetailColumns) - { - columns = JsonWriter - .getNativeColProps(table, Collections.emptyList(), null, false, false) - .values(); - } - else - { - columns = new ArrayList<>(); - for (ColumnInfo col : table.getColumns()) - { - Map cinfo = new HashMap<>(); - cinfo.put("name", col.getName()); - if (null != col.getLabel()) - cinfo.put("caption", col.getLabel()); - if (null != col.getShortLabel()) - cinfo.put("shortCaption", col.getShortLabel()); - if (null != col.getDescription()) - cinfo.put("description", col.getDescription()); - - columns.add(cinfo); - } - } - - if (!columns.isEmpty()) - qinfo.put("columns", columns); - } - - if (includeTitle) - { - name = table.getPublicName(); - title = table.getTitle(); - } - } - } - } - catch(Exception e) - { - //may happen due to query failing parse - } - - qinfo.put("title", title); - qinfo.put("name", name); - return qinfo; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetQueryViewsForm - { - private String _schemaName; - private String _queryName; - private String _viewName; - private boolean _metadata; - private boolean _excludeSessionView; - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - public String getViewName() - { - return _viewName; - } - - public void setViewName(String viewName) - { - _viewName = viewName; - } - - public boolean isMetadata() - { - return _metadata; - } - - public void setMetadata(boolean metadata) - { - _metadata = metadata; - } - - public boolean isExcludeSessionView() - { - return _excludeSessionView; - } - - public void setExcludeSessionView(boolean excludeSessionView) - { - _excludeSessionView = excludeSessionView; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class GetQueryViewsAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetQueryViewsForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetQueryViewsForm form, BindException errors) - { - if (null == StringUtils.trimToNull(form.getSchemaName())) - throw new IllegalArgumentException("You must pass a value for the 'schemaName' parameter!"); - if (null == StringUtils.trimToNull(form.getQueryName())) - throw new IllegalArgumentException("You must pass a value for the 'queryName' parameter!"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); - if (null == schema) - throw new NotFoundException("The schema name '" + form.getSchemaName() - + "' was not found within the folder '" + getContainer().getPath() + "'"); - - QueryDefinition querydef = QueryService.get().createQueryDefForTable(schema, form.getQueryName()); - if (null == querydef || querydef.getTable(null, true) == null) - throw new NotFoundException("The query '" + form.getQueryName() + "' was not found within the '" - + form.getSchemaName() + "' schema in the container '" - + getContainer().getPath() + "'!"); - - Map views = querydef.getCustomViews(getUser(), getViewContext().getRequest(), true, false, form.isExcludeSessionView()); - if (null == views) - views = Collections.emptyMap(); - - Map> columnMetadata = new HashMap<>(); - - List> viewInfos = Collections.emptyList(); - if (getViewContext().getBindPropertyValues().contains("viewName")) - { - // Get info for a named view or the default view (null) - String viewName = StringUtils.trimToNull(form.getViewName()); - CustomView view = views.get(viewName); - if (view != null) - { - viewInfos = Collections.singletonList(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); - } - else if (viewName == null) - { - // The default view was requested but it hasn't been customized yet. Create the 'default default' view. - viewInfos = Collections.singletonList(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); - } - } - else - { - boolean foundDefault = false; - viewInfos = new ArrayList<>(views.size()); - for (CustomView view : views.values()) - { - if (view.getName() == null) - foundDefault = true; - viewInfos.add(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); - } - - if (!foundDefault) - { - // The default view hasn't been customized yet. Create the 'default default' view. - viewInfos.add(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); - } - } - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("schemaName", form.getSchemaName()); - response.put("queryName", form.getQueryName()); - response.put("views", viewInfos); - - return response; - } - } - - @RequiresNoPermission - public static class GetServerDateAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - return new ApiSimpleResponse("date", new Date()); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - private static class SaveApiTestForm - { - private String _getUrl; - private String _postUrl; - private String _postData; - private String _response; - - public String getGetUrl() - { - return _getUrl; - } - - public void setGetUrl(String getUrl) - { - _getUrl = getUrl; - } - - public String getPostUrl() - { - return _postUrl; - } - - public void setPostUrl(String postUrl) - { - _postUrl = postUrl; - } - - public String getResponse() - { - return _response; - } - - public void setResponse(String response) - { - _response = response; - } - - public String getPostData() - { - return _postData; - } - - public void setPostData(String postData) - { - _postData = postData; - } - } - - - @RequiresPermission(ReadPermission.class) - public static class SaveApiTestAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SaveApiTestForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - ApiTestsDocument doc = ApiTestsDocument.Factory.newInstance(); - - TestCaseType test = doc.addNewApiTests().addNewTest(); - test.setName("recorded test case"); - ActionURL url = null; - - if (!StringUtils.isEmpty(form.getGetUrl())) - { - test.setType("get"); - url = new ActionURL(form.getGetUrl()); - } - else if (!StringUtils.isEmpty(form.getPostUrl())) - { - test.setType("post"); - test.setFormData(form.getPostData()); - url = new ActionURL(form.getPostUrl()); - } - - if (url != null) - { - String uri = url.getLocalURIString(); - if (uri.startsWith(url.getContextPath())) - uri = uri.substring(url.getContextPath().length() + 1); - - test.setUrl(uri); - } - test.setResponse(form.getResponse()); - - XmlOptions opts = new XmlOptions(); - opts.setSaveCDataEntityCountThreshold(0); - opts.setSaveCDataLengthThreshold(0); - opts.setSavePrettyPrint(); - opts.setUseDefaultNamespace(); - - response.put("xml", doc.xmlText(opts)); - - return response; - } - } - - - private abstract static class ParseAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - List qpe = new ArrayList<>(); - String expr = getViewContext().getRequest().getParameter("q"); - ArrayList html = new ArrayList<>(); - PageConfig config = getPageConfig(); - var inputId = config.makeId("submit_"); - config.addHandler(inputId, "click", "Ext.getBody().mask();"); - html.add("
\n" + - "" - ); - - QNode e = null; - if (null != expr) - { - try - { - e = _parse(expr,qpe); - } - catch (RuntimeException x) - { - qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); - } - } - - Tree tree = null; - if (null != expr) - { - try - { - tree = _tree(expr); - } catch (Exception x) - { - qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); - } - } - - for (Throwable x : qpe) - { - if (null != x.getCause() && x != x.getCause()) - x = x.getCause(); - html.add("
" + PageFlowUtil.filter(x.toString())); - LogManager.getLogger(QueryController.class).debug(expr,x); - } - if (null != e) - { - String prefix = SqlParser.toPrefixString(e); - html.add("
"); - html.add(PageFlowUtil.filter(prefix)); - } - if (null != tree) - { - String prefix = SqlParser.toPrefixString(tree); - html.add("
"); - html.add(PageFlowUtil.filter(prefix)); - } - html.add(""); - return HtmlView.unsafe(StringUtils.join(html,"")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - - abstract QNode _parse(String e, List errors); - abstract Tree _tree(String e) throws Exception; - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ParseExpressionAction extends ParseAction - { - @Override - QNode _parse(String s, List errors) - { - return new SqlParser().parseExpr(s, true, errors); - } - - @Override - Tree _tree(String e) - { - return null; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ParseQueryAction extends ParseAction - { - @Override - QNode _parse(String s, List errors) - { - return new SqlParser().parseQuery(s, errors, null); - } - - @Override - Tree _tree(String s) throws Exception - { - return new SqlParser().rawQuery(s); - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class ValidateQueryMetadataAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(QueryForm form, BindException errors) - { - UserSchema schema = form.getSchema(); - - if (null == schema) - { - errors.reject(ERROR_MSG, "could not resolve schema: " + form.getSchemaName()); - return null; - } - - List parseErrors = new ArrayList<>(); - List parseWarnings = new ArrayList<>(); - ApiSimpleResponse response = new ApiSimpleResponse(); - - try - { - TableInfo table = schema.getTable(form.getQueryName(), null); - - if (null == table) - { - errors.reject(ERROR_MSG, "could not resolve table: " + form.getQueryName()); - return null; - } - - if (!QueryManager.get().validateQuery(table, true, parseErrors, parseWarnings)) - { - for (QueryParseException e : parseErrors) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - return response; - } - - SchemaKey schemaKey = SchemaKey.fromString(form.getSchemaName()); - QueryManager.get().validateQueryMetadata(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); - QueryManager.get().validateQueryViews(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); - } - catch (QueryParseException e) - { - parseErrors.add(e); - } - - for (QueryParseException e : parseErrors) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - - for (QueryParseException e : parseWarnings) - { - errors.reject(ERROR_MSG, "WARNING: " + e.getMessage()); - } - - return response; - } - - @Override - protected ApiResponseWriter createResponseWriter() throws IOException - { - ApiResponseWriter result = super.createResponseWriter(); - // Issue 44875 - don't send a 400 or 500 response code when there's a bogus query or metadata - result.setErrorResponseStatus(HttpServletResponse.SC_OK); - return result; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class QueryExportAuditForm - { - private int rowId; - - public int getRowId() - { - return rowId; - } - - public void setRowId(int rowId) - { - this.rowId = rowId; - } - } - - /** - * Action used to redirect QueryAuditProvider [details] column to the exported table's grid view. - */ - @RequiresPermission(AdminPermission.class) - public static class QueryExportAuditRedirectAction extends SimpleRedirectAction - { - @Override - public URLHelper getRedirectURL(QueryExportAuditForm form) - { - if (form.getRowId() == 0) - throw new NotFoundException("Query export audit rowid required"); - - UserSchema auditSchema = QueryService.get().getUserSchema(getUser(), getContainer(), AbstractAuditTypeProvider.QUERY_SCHEMA_NAME); - TableInfo queryExportAuditTable = auditSchema.getTable(QueryExportAuditProvider.QUERY_AUDIT_EVENT, null); - if (null == queryExportAuditTable) - throw new NotFoundException(); - - TableSelector selector = new TableSelector(queryExportAuditTable, - PageFlowUtil.set( - QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME, - QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME, - QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL), - new SimpleFilter(FieldKey.fromParts(AbstractAuditTypeProvider.COLUMN_NAME_ROW_ID), form.getRowId()), null); - - Map result = selector.getMap(); - if (result == null) - throw new NotFoundException("Query export audit event not found for rowId"); - - String schemaName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME); - String queryName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME); - String detailsURL = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL); - - if (schemaName == null || queryName == null) - throw new NotFoundException("Query export audit event has not schemaName or queryName"); - - ActionURL url = new ActionURL(ExecuteQueryAction.class, getContainer()); - - // Apply the sorts and filters - if (detailsURL != null) - { - ActionURL sortFilterURL = new ActionURL(detailsURL); - url.setPropertyValues(sortFilterURL.getPropertyValues()); - } - - if (url.getParameter(QueryParam.schemaName) == null) - url.addParameter(QueryParam.schemaName, schemaName); - if (url.getParameter(QueryParam.queryName) == null && url.getParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName) == null) - url.addParameter(QueryParam.queryName, queryName); - - return url; - } - } - - @RequiresPermission(ReadPermission.class) - public static class AuditHistoryAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return QueryUpdateAuditProvider.createHistoryQueryView(getViewContext(), form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Audit History"); - } - } - - @RequiresPermission(ReadPermission.class) - public static class AuditDetailsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryDetailsForm form, BindException errors) - { - return QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Audit History"); - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class QueryDetailsForm extends QueryForm - { - String _keyValue; - - public String getKeyValue() - { - return _keyValue; - } - - public void setKeyValue(String keyValue) - { - _keyValue = keyValue; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportTablesAction extends FormViewAction - { - private ActionURL _successUrl; - - @Override - public void validateCommand(ExportTablesForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ExportTablesForm form, BindException errors) - { - HttpServletResponse httpResponse = getViewContext().getResponse(); - Container container = getContainer(); - QueryServiceImpl svc = (QueryServiceImpl)QueryService.get(); - - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream outputStream = new BufferedOutputStream(baos)) - { - try (ZipFile zip = new ZipFile(outputStream, true)) - { - svc.writeTables(container, getUser(), zip, form.getSchemas(), form.getHeaderType()); - } - - PageFlowUtil.streamFileBytes(httpResponse, FileUtil.makeFileNameWithTimestamp(container.getName(), "tables.zip"), baos.toByteArray(), false); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : e.getClass().getName()); - LOG.error("Errror exporting tables", e); - } - - if (errors.hasErrors()) - { - _successUrl = new ActionURL(ExportTablesAction.class, getContainer()); - } - - return !errors.hasErrors(); - } - - @Override - public ModelAndView getView(ExportTablesForm form, boolean reshow, BindException errors) - { - // When exporting the zip to the browser, the base action will attempt to reshow the view since we returned - // null as the success URL; returning null here causes the base action to stop pestering the action. - if (reshow && !errors.hasErrors()) - return null; - - return new JspView<>("/org/labkey/query/view/exportTables.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Export Tables"); - } - - @Override - public ActionURL getSuccessURL(ExportTablesForm form) - { - return _successUrl; - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportTablesForm implements HasBindParameters - { - ColumnHeaderType _headerType = ColumnHeaderType.DisplayFieldKey; - Map>> _schemas = new HashMap<>(); - - public ColumnHeaderType getHeaderType() - { - return _headerType; - } - - public void setHeaderType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public Map>> getSchemas() - { - return _schemas; - } - - public void setSchemas(Map>> schemas) - { - _schemas = schemas; - } - - @Override - public @NotNull BindException bindParameters(PropertyValues values) - { - BindException errors = new NullSafeBindException(this, "form"); - - PropertyValue schemasProperty = values.getPropertyValue("schemas"); - if (schemasProperty != null && schemasProperty.getValue() != null) - { - try - { - _schemas = JsonUtil.DEFAULT_MAPPER.readValue((String)schemasProperty.getValue(), _schemas.getClass()); - } - catch (IOException e) - { - errors.rejectValue("schemas", ERROR_MSG, e.getMessage()); - } - } - - PropertyValue headerTypeProperty = values.getPropertyValue("headerType"); - if (headerTypeProperty != null && headerTypeProperty.getValue() != null) - { - try - { - _headerType = ColumnHeaderType.valueOf(String.valueOf(headerTypeProperty.getValue())); - } - catch (IllegalArgumentException ex) - { - // ignore - } - } - - return errors; - } - } - - - @RequiresPermission(ReadPermission.class) - public static class SaveNamedSetAction extends MutatingApiAction - { - @Override - public Object execute(NamedSetForm namedSetForm, BindException errors) - { - QueryService.get().saveNamedSet(namedSetForm.getSetName(), namedSetForm.parseSetList()); - return new ApiSimpleResponse("success", true); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class NamedSetForm - { - String setName; - String[] setList; - - public String getSetName() - { - return setName; - } - - public void setSetName(String setName) - { - this.setName = setName; - } - - public String[] getSetList() - { - return setList; - } - - public void setSetList(String[] setList) - { - this.setList = setList; - } - - public List parseSetList() - { - return Arrays.asList(setList); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class DeleteNamedSetAction extends MutatingApiAction - { - - @Override - public Object execute(NamedSetForm namedSetForm, BindException errors) - { - QueryService.get().deleteNamedSet(namedSetForm.getSetName()); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(ReadPermission.class) - public static class AnalyzeQueriesAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - JSONObject ret = new JSONObject(); - - try - { - QueryService.QueryAnalysisService analysisService = QueryService.get().getQueryAnalysisService(); - if (analysisService != null) - { - DefaultSchema start = DefaultSchema.get(getUser(), getContainer()); - var deps = new HashSetValuedHashMap(); - - analysisService.analyzeFolder(start, deps); - ret.put("success", true); - - JSONObject objects = new JSONObject(); - for (var from : deps.keySet()) - { - objects.put(from.getKey(), from.toJSON()); - for (var to : deps.get(from)) - objects.put(to.getKey(), to.toJSON()); - } - ret.put("objects", objects); - - JSONArray dependants = new JSONArray(); - for (var from : deps.keySet()) - { - for (var to : deps.get(from)) - dependants.put(new String[] {from.getKey(), to.getKey()}); - } - ret.put("graph", dependants); - } - else - { - ret.put("success", false); - } - return ret; - } - catch (Throwable e) - { - LOG.error(e); - throw UnexpectedException.wrap(e); - } - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetQueryEditorMetadataAction extends ReadOnlyApiAction - { - @Override - protected ObjectMapper createRequestObjectMapper() - { - PropertyService propertyService = PropertyService.get(); - if (null != propertyService) - { - ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); - mapper.addMixIn(GWTPropertyDescriptor.class, MetadataTableJSONMixin.class); - return mapper; - } - else - { - throw new RuntimeException("Could not serialize request object"); - } - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return createRequestObjectMapper(); - } - - @Override - public Object execute(QueryForm queryForm, BindException errors) throws Exception - { - QueryDefinition queryDef = queryForm.getQueryDef(); - return MetadataTableJSON.getMetadata(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - public static class SaveQueryMetadataAction extends MutatingApiAction - { - @Override - protected ObjectMapper createRequestObjectMapper() - { - PropertyService propertyService = PropertyService.get(); - if (null != propertyService) - { - ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); - propertyService.configureObjectMapper(mapper, null); - return mapper; - } - else - { - throw new RuntimeException("Could not serialize request object"); - } - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return createRequestObjectMapper(); - } - - @Override - public Object execute(QueryMetadataApiForm queryMetadataApiForm, BindException errors) throws Exception - { - String schemaName = queryMetadataApiForm.getSchemaName(); - MetadataTableJSON domain = queryMetadataApiForm.getDomain(); - MetadataTableJSON.saveMetadata(schemaName, domain.getName(), null, domain.getFields(true), queryMetadataApiForm.isUserDefinedQuery(), false, getUser(), getContainer()); - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - resp.put("domain", MetadataTableJSON.getMetadata(schemaName, domain.getName(), getUser(), getContainer())); - return resp; - } - } - - @Marshal(Marshaller.Jackson) - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - public static class ResetQueryMetadataAction extends MutatingApiAction - { - @Override - public Object execute(QueryForm queryForm, BindException errors) throws Exception - { - QueryDefinition queryDef = queryForm.getQueryDef(); - return MetadataTableJSON.resetToDefault(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); - } - } - - private static class QueryMetadataApiForm - { - private MetadataTableJSON _domain; - private String _schemaName; - private boolean _userDefinedQuery; - - public MetadataTableJSON getDomain() - { - return _domain; - } - - @SuppressWarnings("unused") - public void setDomain(MetadataTableJSON domain) - { - _domain = domain; - } - - public String getSchemaName() - { - return _schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isUserDefinedQuery() - { - return _userDefinedQuery; - } - - @SuppressWarnings("unused") - public void setUserDefinedQuery(boolean userDefinedQuery) - { - _userDefinedQuery = userDefinedQuery; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetDefaultVisibleColumnsAction extends ReadOnlyApiAction - { - @Override - public Object execute(GetQueryDetailsAction.Form form, BindException errors) throws Exception - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - Container container = getContainer(); - User user = getUser(); - - if (StringUtils.isEmpty(form.getSchemaName())) - throw new NotFoundException("SchemaName not specified"); - - QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); - if (!(querySchema instanceof UserSchema schema)) - throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'"); - - QuerySettings settings = schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); - QueryDefinition queryDef = settings.getQueryDef(schema); - if (null == queryDef) - // Don't echo the provided query name, but schema name is legit since it was found. See #44528. - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'"); - - TableInfo tinfo = queryDef.getTable(null, true); - if (null == tinfo) - throw new NotFoundException("Could not find the specified query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - List fields = tinfo.getDefaultVisibleColumns(); - - List displayColumns = QueryService.get().getColumns(tinfo, fields) - .values() - .stream() - .filter(cinfo -> fields.contains(cinfo.getFieldKey())) - .map(cinfo -> cinfo.getDisplayColumnFactory().createRenderer(cinfo)) - .collect(Collectors.toList()); - - resp.put("columns", JsonWriter.getNativeColProps(displayColumns, null, false)); - - return resp; - } - } - - public static class ParseForm implements ApiJsonForm - { - String expression = ""; - Map columnMap = new HashMap<>(); - List phiColumns = new ArrayList<>(); - - Map getColumnMap() - { - return columnMap; - } - - public String getExpression() - { - return expression; - } - - public void setExpression(String expression) - { - this.expression = expression; - } - - public List getPhiColumns() - { - return phiColumns; - } - - public void setPhiColumns(List phiColumns) - { - this.phiColumns = phiColumns; - } - - @Override - public void bindJson(JSONObject json) - { - if (json.has("expression")) - setExpression(json.getString("expression")); - if (json.has("phiColumns")) - setPhiColumns(json.getJSONArray("phiColumns").toList().stream().map(s -> FieldKey.fromParts(s.toString())).collect(Collectors.toList())); - if (json.has("columnMap")) - { - JSONObject columnMap = json.getJSONObject("columnMap"); - for (String key : columnMap.keySet()) - { - try - { - getColumnMap().put(FieldKey.fromParts(key), JdbcType.valueOf(String.valueOf(columnMap.get(key)))); - } - catch (IllegalArgumentException iae) - { - getColumnMap().put(FieldKey.fromParts(key), JdbcType.OTHER); - } - } - } - } - } - - - /** - * Since this api purpose is to return parse errors, it does not generally return success:false. - *
- * The API expects JSON like this, note that column names should be in FieldKey.toString() encoded to match the response JSON format. - * 
-     *     { "expression": "A$ + B", "columnMap":{"A$D":"VARCHAR", "X":"VARCHAR"}}
-     *
- * and returns a response like this - * 
-     *     {
-     *       "jdbcType" : "OTHER",
-     *       "success" : true,
-     *       "columnMap" : {"A$D":"VARCHAR", "B":"OTHER"}
-     *       "errors" : [ { "msg" : "\"B\" not found.", "type" : "sql" } ]
-     *     }
-     *
- * The columnMap object keys are the names of columns found in the expression. Names are returned - * in FieldKey.toString() formatting e.g. dollar-sign encoded. The object structure - * is compatible with the columnMap input parameter, so it can be used as a template to make a second request - * with types filled in. If provided, the type will be copied from the input columnMap, otherwise it will be "OTHER". - *
- * Parse exceptions may contain a line (usually 1) and col location e.g. - * 
-     * {
-     *     "msg" : "Error on line 1: Syntax error near 'error', expected 'EOF'
-     *     "col" : 2,
-     *     "line" : 1,
-     *     "type" : "sql",
-     *     "errorStr" : "A error B"
-     *   }
-     *
- */ - @RequiresNoPermission - @CSRF(CSRF.Method.NONE) - public static class ParseCalculatedColumnAction extends ReadOnlyApiAction - { - @Override - public Object execute(ParseForm form, BindException errors) throws Exception - { - if (errors.hasErrors()) - return errors; - JSONObject result = new JSONObject(Map.of("success",true)); - var requiredColumns = new HashSet(); - JdbcType jdbcType = JdbcType.OTHER; - try - { - var schema = DefaultSchema.get(getViewContext().getUser(), getViewContext().getContainer()).getUserSchema("core"); - var table = new VirtualTable<>(schema.getDbSchema(), "EXPR", schema){}; - ColumnInfo calculatedCol = QueryServiceImpl.get().createQueryExpressionColumn(table, new FieldKey(null, "expr"), form.getExpression(), null); - Map columns = new HashMap<>(); - for (var entry : form.getColumnMap().entrySet()) - { - BaseColumnInfo entryCol = new BaseColumnInfo(entry.getKey(), entry.getValue()); - // bindQueryExpressionColumn has a check that restricts PHI columns from being used in expressions - // so we need to set the PHI level to something other than NotPHI on these fake BaseColumnInfo objects - if (form.getPhiColumns().contains(entry.getKey())) - entryCol.setPHI(PHI.PHI); - columns.put(entry.getKey(), entryCol); - table.addColumn(entryCol); - } - // TODO: calculating jdbcType still uses calculatedCol.getParentTable().getColumns() - QueryServiceImpl.get().bindQueryExpressionColumn(calculatedCol, columns, false, requiredColumns); - jdbcType = calculatedCol.getJdbcType(); - } - catch (QueryException x) - { - JSONArray parseErrors = new JSONArray(); - parseErrors.put(x.toJSON(form.getExpression())); - result.put("errors", parseErrors); - } - finally - { - if (!requiredColumns.isEmpty()) - { - JSONObject columnMap = new JSONObject(); - for (FieldKey fk : requiredColumns) - { - JdbcType type = Objects.requireNonNullElse(form.getColumnMap().get(fk), JdbcType.OTHER); - columnMap.put(fk.toString(), type); - } - result.put("columnMap", columnMap); - } - } - result.put("jdbcType", jdbcType.name()); - return result; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class QueryImportTemplateForm - { - private String schemaName; - private String queryName; - private String auditUserComment; - private List templateLabels; - private List templateUrls; - private Long _lastKnownModified; - - public void setQueryName(String queryName) - { - this.queryName = queryName; - } - - public List getTemplateLabels() - { - return templateLabels == null ? Collections.emptyList() : templateLabels; - } - - public void setTemplateLabels(List templateLabels) - { - this.templateLabels = templateLabels; - } - - public List getTemplateUrls() - { - return templateUrls == null ? Collections.emptyList() : templateUrls; - } - - public void setTemplateUrls(List templateUrls) - { - this.templateUrls = templateUrls; - } - - public String getSchemaName() - { - return schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(String schemaName) - { - this.schemaName = schemaName; - } - - public String getQueryName() - { - return queryName; - } - - public Long getLastKnownModified() - { - return _lastKnownModified; - } - - public void setLastKnownModified(Long lastKnownModified) - { - _lastKnownModified = lastKnownModified; - } - - public String getAuditUserComment() - { - return auditUserComment; - } - - public void setAuditUserComment(String auditUserComment) - { - this.auditUserComment = auditUserComment; - } - - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) //Real permissions will be enforced later on by the DomainKind - public static class UpdateQueryImportTemplateAction extends MutatingApiAction - { - private DomainKind _kind; - private UserSchema _schema; - private TableInfo _tInfo; - private QueryDefinition _queryDef; - private Domain _domain; - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return this.createRequestObjectMapper(); - } - - @Override - public void validateForm(QueryImportTemplateForm form, Errors errors) - { - User user = getUser(); - Container container = getContainer(); - String domainURI = PropertyService.get().getDomainURI(form.getSchemaName(), form.getQueryName(), container, user); - _kind = PropertyService.get().getDomainKind(domainURI); - _domain = PropertyService.get().getDomain(container, domainURI); - if (_domain == null) - throw new IllegalArgumentException("Domain '" + domainURI + "' not found."); - - if (!_kind.canEditDefinition(user, _domain)) - throw new UnauthorizedException("You don't have permission to update import templates for this domain."); - - QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); - if (!(querySchema instanceof UserSchema _schema)) - throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'."); - QuerySettings settings = _schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); - _queryDef = settings.getQueryDef(_schema); - if (null == _queryDef) - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); - if (!_queryDef.isMetadataEditable()) - throw new UnsupportedOperationException("Query metadata is not editable."); - _tInfo = _queryDef.getTable(_schema, new ArrayList<>(), true, true); - if (_tInfo == null) - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); - - } - - private Map getRowFiles() - { - Map rowFiles = new IntHashMap<>(); - if (getFileMap() != null) - { - for (Map.Entry fileEntry : getFileMap().entrySet()) - { - // allow for the fileMap key to include the row index for defining which row to attach this file to - // ex: "templateFile::0", "templateFile::1" - String fieldKey = fileEntry.getKey(); - int delimIndex = fieldKey.lastIndexOf("::"); - if (delimIndex > -1) - { - Integer fieldRowIndex = Integer.parseInt(fieldKey.substring(delimIndex + 2)); - SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); - rowFiles.put(fieldRowIndex, file.isEmpty() ? null : file); - } - } - } - return rowFiles; - } - - private List> getUploadedTemplates(QueryImportTemplateForm form, DomainKind kind) throws ValidationException, QueryUpdateServiceException, ExperimentException - { - FileContentService fcs = FileContentService.get(); - if (fcs == null) - throw new IllegalStateException("Unable to load file service."); - - User user = getUser(); - Container container = getContainer(); - - Map rowFiles = getRowFiles(); - List templateLabels = form.getTemplateLabels(); - Set labels = new HashSet<>(templateLabels); - if (labels.size() < templateLabels.size()) - throw new IllegalArgumentException("Duplicate template name is not allowed."); - - List templateUrls = form.getTemplateUrls(); - List> uploadedTemplates = new ArrayList<>(); - for (int rowIndex = 0; rowIndex < form.getTemplateLabels().size(); rowIndex++) - { - String templateLabel = templateLabels.get(rowIndex); - if (StringUtils.isBlank(templateLabel.trim())) - throw new IllegalArgumentException("Template name cannot be blank."); - String templateUrl = templateUrls.get(rowIndex); - Object file = rowFiles.get(rowIndex); - if (StringUtils.isEmpty(templateUrl) && file == null) - throw new IllegalArgumentException("Template file is not provided."); - - if (file instanceof MultipartFile || file instanceof SpringAttachmentFile) - { - String fileName; - if (file instanceof MultipartFile f) - fileName = f.getName(); - else - { - SpringAttachmentFile f = (SpringAttachmentFile) file; - fileName = f.getFilename(); - } - String fileNameValidation = FileUtil.validateFileName(fileName); - if (!StringUtils.isEmpty(fileNameValidation)) - throw new IllegalArgumentException(fileNameValidation); - - FileLike uploadDir = ensureUploadDirectory(container, kind.getDomainFileDirectory()); - uploadDir = uploadDir.resolveChild("_templates"); - Object savedFile = saveFile(user, container, "template file", file, uploadDir); - Path savedFilePath; - - if (savedFile instanceof File ioFile) - savedFilePath = ioFile.toPath(); - else if (savedFile instanceof FileLike fl) - savedFilePath = fl.toNioPathForRead(); - else - throw UnexpectedException.wrap(null,"Unable to upload template file."); - - templateUrl = fcs.getWebDavUrl(savedFilePath, container, FileContentService.PathType.serverRelative).toString(); - } - - uploadedTemplates.add(Pair.of(templateLabel, templateUrl)); - } - return uploadedTemplates; - } - - @Override - public Object execute(QueryImportTemplateForm form, BindException errors) throws ValidationException, QueryUpdateServiceException, SQLException, ExperimentException, MetadataUnavailableException - { - User user = getUser(); - Container container = getContainer(); - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - QueryDef queryDef = QueryManager.get().getQueryDef(container, schemaName, queryName, false); - if (queryDef != null && queryDef.getQueryDefId() != 0) - { - Long lastKnownModified = form.getLastKnownModified(); - if (lastKnownModified == null || lastKnownModified != queryDef.getModified().getTime()) - throw new ApiUsageException("Unable to save import templates. The templates appear out of date, reload the page and try again."); - } - - List> updatedTemplates = getUploadedTemplates(form, _kind); - - List> existingTemplates = _tInfo.getImportTemplates(getViewContext()); - List> existingCustomTemplates = new ArrayList<>(); - for (Pair template_ : existingTemplates) - { - if (!template_.second.toLowerCase().contains("exportexceltemplate")) - existingCustomTemplates.add(template_); - } - if (!updatedTemplates.equals(existingCustomTemplates)) - { - TablesDocument doc = null; - TableType xmlTable = null; - TableType.ImportTemplates xmlImportTemplates; - - if (queryDef != null) - { - try - { - doc = parseDocument(queryDef.getMetaData()); - } - catch (XmlException e) - { - throw new MetadataUnavailableException(e.getMessage()); - } - xmlTable = getTableType(form.getQueryName(), doc); - // when there is a queryDef but xmlTable is null it means the xmlMetaData contains tableName which does not - // match with actual queryName then reconstruct the xml table metadata : See Issue 43523 - if (xmlTable == null) - { - doc = null; - } - } - else - { - queryDef = new QueryDef(); - queryDef.setSchema(schemaName); - queryDef.setContainer(container.getId()); - queryDef.setName(queryName); - } - - if (doc == null) - { - doc = TablesDocument.Factory.newInstance(); - } - - if (xmlTable == null) - { - TablesType tables = doc.addNewTables(); - xmlTable = tables.addNewTable(); - xmlTable.setTableName(queryName); - } - - if (xmlTable.getTableDbType() == null) - { - xmlTable.setTableDbType("NOT_IN_DB"); - } - - // remove existing templates - if (xmlTable.isSetImportTemplates()) - xmlTable.unsetImportTemplates(); - xmlImportTemplates = xmlTable.addNewImportTemplates(); - - // set new templates - if (!updatedTemplates.isEmpty()) - { - for (Pair template_ : updatedTemplates) - { - ImportTemplateType importTemplateType = xmlImportTemplates.addNewTemplate(); - importTemplateType.setLabel(template_.first); - importTemplateType.setUrl(template_.second); - } - } - - XmlOptions xmlOptions = new XmlOptions(); - xmlOptions.setSavePrettyPrint(); - // Don't use an explicit namespace, making the XML much more readable - xmlOptions.setUseDefaultNamespace(); - queryDef.setMetaData(doc.xmlText(xmlOptions)); - if (queryDef.getQueryDefId() == 0) - { - QueryManager.get().insert(user, queryDef); - } - else - { - QueryManager.get().update(user, queryDef); - } - - DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "Import templates updated."); - event.setUserComment(form.getAuditUserComment()); - event.setDomainUri(_domain.getTypeURI()); - event.setDomainName(_domain.getName()); - AuditLogService.get().addEvent(user, event); - } - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - return resp; - } - } - - - public static class TestCase extends AbstractActionPermissionTest - { - @Override - public void testActionPermissions() - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - QueryController controller = new QueryController(); - - // @RequiresPermission(ReadPermission.class) - assertForReadPermission(user, false, - new BrowseAction(), - new BeginAction(), - controller.new SchemaAction(), - controller.new SourceQueryAction(), - controller.new ExecuteQueryAction(), - controller.new PrintRowsAction(), - new ExportScriptAction(), - new ExportRowsExcelAction(), - new ExportRowsXLSXAction(), - new ExportQueriesXLSXAction(), - new ExportExcelTemplateAction(), - new ExportRowsTsvAction(), - new ExcelWebQueryDefinitionAction(), - controller.new SaveQueryViewsAction(), - controller.new PropertiesQueryAction(), - controller.new SelectRowsAction(), - new GetDataAction(), - controller.new ExecuteSqlAction(), - controller.new SelectDistinctAction(), - controller.new GetColumnSummaryStatsAction(), - controller.new ImportAction(), - new ExportSqlAction(), - new UpdateRowsAction(), - new ImportRowsAction(), - new DeleteRowsAction(), - new TableInfoAction(), - new SaveSessionViewAction(), - new GetSchemasAction(), - new GetQueriesAction(), - new GetQueryViewsAction(), - new SaveApiTestAction(), - new ValidateQueryMetadataAction(), - new AuditHistoryAction(), - new AuditDetailsAction(), - new ExportTablesAction(), - new SaveNamedSetAction(), - new DeleteNamedSetAction(), - new ApiTestAction(), - new GetDefaultVisibleColumnsAction() - ); - - - // submitter should be allowed for InsertRows - assertForReadPermission(user, true, new InsertRowsAction()); - - // @RequiresPermission(DeletePermission.class) - assertForUpdateOrDeletePermission(user, - new DeleteQueryRowsAction() - ); - - // @RequiresPermission(AdminPermission.class) - assertForAdminPermission(user, - new DeleteQueryAction(), - controller.new MetadataQueryAction(), - controller.new NewQueryAction(), - new SaveSourceQueryAction(), - - new TruncateTableAction(), - new AdminAction(), - new ManageRemoteConnectionsAction(), - new ReloadExternalSchemaAction(), - new ReloadAllUserSchemas(), - controller.new ManageViewsAction(), - controller.new InternalDeleteView(), - controller.new InternalSourceViewAction(), - controller.new InternalNewViewAction(), - new QueryExportAuditRedirectAction() - ); - - // @RequiresPermission(AdminOperationsPermission.class) - assertForAdminOperationsPermission(user, - new EditRemoteConnectionAction(), - new DeleteRemoteConnectionAction(), - new TestRemoteConnectionAction(), - controller.new RawTableMetaDataAction(), - controller.new RawSchemaMetaDataAction(), - new InsertLinkedSchemaAction(), - new InsertExternalSchemaAction(), - new DeleteSchemaAction(), - new EditLinkedSchemaAction(), - new EditExternalSchemaAction(), - new GetTablesAction(), - new SchemaTemplateAction(), - new SchemaTemplatesAction(), - new ParseExpressionAction(), - new ParseQueryAction() - ); - - // @AdminConsoleAction - assertForAdminPermission(ContainerManager.getRoot(), user, - new DataSourceAdminAction() - ); - - // In addition to administrators (tested above), trusted analysts who are editors can create and edit queries - assertTrustedEditorPermission( - new DeleteQueryAction(), - controller.new MetadataQueryAction(), - controller.new NewQueryAction(), - new SaveSourceQueryAction() - ); - } - } - - public static class SaveRowsTestCase extends Assert - { - private static final String PROJECT_NAME1 = "SaveRowsTestProject1"; - private static final String PROJECT_NAME2 = "SaveRowsTestProject2"; - - private static final String USER_EMAIL = "saveRows@action.test"; - - private static final String LIST1 = "List1"; - private static final String LIST2 = "List2"; - - @Before - public void doSetup() throws Exception - { - doCleanup(); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME1, TestContext.get().getUser()); - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME2, TestContext.get().getUser()); - - //disable search so we dont get conflicts when deleting folder quickly - ContainerManager.updateSearchable(project1, false, TestContext.get().getUser()); - ContainerManager.updateSearchable(project2, false, TestContext.get().getUser()); - - ListDefinition ld1 = ListService.get().createList(project1, LIST1, ListDefinition.KeyType.Varchar); - ld1.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); - ld1.setKeyName("TextField"); - ld1.save(TestContext.get().getUser()); - - ListDefinition ld2 = ListService.get().createList(project2, LIST2, ListDefinition.KeyType.Varchar); - ld2.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); - ld2.setKeyName("TextField"); - ld2.save(TestContext.get().getUser()); - } - - @After - public void doCleanup() throws Exception - { - Container project = ContainerManager.getForPath(PROJECT_NAME1); - if (project != null) - { - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - if (project2 != null) - { - ContainerManager.deleteAll(project2, TestContext.get().getUser()); - } - - User u = UserManager.getUser(new ValidEmail(USER_EMAIL)); - if (u != null) - { - UserManager.deleteUser(u.getUserId()); - } - } - - private JSONObject getCommand(String val1, String val2) - { - JSONObject command1 = new JSONObject(); - command1.put("containerPath", ContainerManager.getForPath(PROJECT_NAME1).getPath()); - command1.put("command", "insert"); - command1.put("schemaName", "lists"); - command1.put("queryName", LIST1); - command1.put("rows", getTestRows(val1)); - - JSONObject command2 = new JSONObject(); - command2.put("containerPath", ContainerManager.getForPath(PROJECT_NAME2).getPath()); - command2.put("command", "insert"); - command2.put("schemaName", "lists"); - command2.put("queryName", LIST2); - command2.put("rows", getTestRows(val2)); - - JSONObject json = new JSONObject(); - json.put("commands", Arrays.asList(command1, command2)); - - return json; - } - - private MockHttpServletResponse makeRequest(JSONObject json, User user) throws Exception - { - Map headers = new HashMap<>(); - headers.put("Content-Type", "application/json"); - - HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), DetailsURL.fromString("/query/saveRows.view").copy(ContainerManager.getForPath(PROJECT_NAME1)).getActionURL(), user, headers, json.toString()); - return ViewServlet.mockDispatch(request, null); - } - - @Test - public void testCrossFolderSaveRows() throws Exception - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - JSONObject json = getCommand(PROJECT_NAME1, PROJECT_NAME2); - MockHttpServletResponse response = makeRequest(json, TestContext.get().getUser()); - if (response.getStatus() != HttpServletResponse.SC_OK) - { - JSONObject responseJson = new JSONObject(response.getContentAsString()); - throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); - } - - Container project1 = ContainerManager.getForPath(PROJECT_NAME1); - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - - TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); - TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); - - assertEquals("Incorrect row count, list1", 1L, new TableSelector(list1).getRowCount()); - assertEquals("Incorrect row count, list2", 1L, new TableSelector(list2).getRowCount()); - - assertEquals("Incorrect value", PROJECT_NAME1, new TableSelector(list1, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME1, String.class)); - assertEquals("Incorrect value", PROJECT_NAME2, new TableSelector(list2, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME2, String.class)); - - list1.getUpdateService().truncateRows(TestContext.get().getUser(), project1, null, null); - list2.getUpdateService().truncateRows(TestContext.get().getUser(), project2, null, null); - } - - @Test - public void testWithoutPermissions() throws Exception - { - // Now test failure without appropriate permissions: - User withoutPermissions = SecurityManager.addUser(new ValidEmail(USER_EMAIL), TestContext.get().getUser()).getUser(); - - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - Container project1 = ContainerManager.getForPath(PROJECT_NAME1); - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - - MutableSecurityPolicy securityPolicy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(project1)); - securityPolicy.addRoleAssignment(withoutPermissions, EditorRole.class); - SecurityPolicyManager.savePolicyForTests(securityPolicy, TestContext.get().getUser()); - - assertTrue("Should have insert permission", project1.hasPermission(withoutPermissions, InsertPermission.class)); - assertFalse("Should not have insert permission", project2.hasPermission(withoutPermissions, InsertPermission.class)); - - // repeat insert: - JSONObject json = getCommand("ShouldFail1", "ShouldFail2"); - MockHttpServletResponse response = makeRequest(json, withoutPermissions); - if (response.getStatus() != HttpServletResponse.SC_FORBIDDEN) - { - JSONObject responseJson = new JSONObject(response.getContentAsString()); - throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); - } - - TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); - TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); - - // The insert should have failed - assertEquals("Incorrect row count, list1", 0L, new TableSelector(list1).getRowCount()); - assertEquals("Incorrect row count, list2", 0L, new TableSelector(list2).getRowCount()); - } - - private JSONArray getTestRows(String val) - { - JSONArray rows = new JSONArray(); - rows.put(Map.of("TextField", val)); - - return rows; - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.query.controllers; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.xmlbeans.XmlError; +import org.apache.xmlbeans.XmlException; +import org.apache.xmlbeans.XmlOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONParserConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.action.Action; +import org.labkey.api.action.ActionType; +import org.labkey.api.action.ApiJsonForm; +import org.labkey.api.action.ApiJsonWriter; +import org.labkey.api.action.ApiQueryResponse; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiResponseWriter; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.ApiVersion; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.ExportException; +import org.labkey.api.action.ExtendedApiQueryResponse; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasBindParameters; +import org.labkey.api.action.JsonInputLimit; +import org.labkey.api.action.LabKeyError; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.NullSafeBindException; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReportingApiQueryResponse; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.collections.RowMapFactory; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.ActionButton; +import org.labkey.api.data.Aggregate; +import org.labkey.api.data.AnalyticsProviderItem; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.CachedResultSets; +import org.labkey.api.data.ColumnHeaderType; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.ExcelWriter; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.JdbcMetaDataSelector; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.JsonWriter; +import org.labkey.api.data.PHI; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.PropertyMap; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.QueryLogging; +import org.labkey.api.data.ResultSetView; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.ShowRows; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.VirtualTable; +import org.labkey.api.data.dialect.JdbcMetaDataLocator; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.ListofMapsDataIterator; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ProvenanceRecordingParams; +import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.query.AbstractQueryImportAction; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.CustomView; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.ExportScriptModel; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryAction; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QueryParam; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryUpdateForm; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.QueryUrls; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RuntimeValidationException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleSchemaTreeVisitor; +import org.labkey.api.query.TempQuerySettings; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.UserSchemaAction; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.report.ReportDescriptor; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.CSRF; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresAllOf; +import org.labkey.api.security.RequiresAnyOf; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AbstractActionPermissionTest; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.EditSharedViewPermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; +import org.labkey.api.stats.ColumnAnalyticsProvider; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.DOM; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.JavaScriptFragment; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.ReturnURLString; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.XmlBeansUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.DetailsView; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.InsertView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.UpdateView; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.writer.HtmlWriter; +import org.labkey.api.writer.ZipFile; +import org.labkey.data.xml.ColumnType; +import org.labkey.data.xml.ImportTemplateType; +import org.labkey.data.xml.TableType; +import org.labkey.data.xml.TablesDocument; +import org.labkey.data.xml.TablesType; +import org.labkey.data.xml.externalSchema.TemplateSchemaType; +import org.labkey.data.xml.queryCustomView.FilterType; +import org.labkey.query.AutoGeneratedDetailsCustomView; +import org.labkey.query.AutoGeneratedInsertCustomView; +import org.labkey.query.AutoGeneratedUpdateCustomView; +import org.labkey.query.CustomViewImpl; +import org.labkey.query.CustomViewUtil; +import org.labkey.query.EditQueriesPermission; +import org.labkey.query.EditableCustomView; +import org.labkey.query.LinkedTableInfo; +import org.labkey.query.MetadataTableJSON; +import org.labkey.query.ModuleCustomQueryDefinition; +import org.labkey.query.ModuleCustomView; +import org.labkey.query.QueryServiceImpl; +import org.labkey.query.TableXML; +import org.labkey.query.audit.QueryExportAuditProvider; +import org.labkey.query.audit.QueryUpdateAuditProvider; +import org.labkey.query.model.MetadataTableJSONMixin; +import org.labkey.query.persist.AbstractExternalSchemaDef; +import org.labkey.query.persist.CstmView; +import org.labkey.query.persist.ExternalSchemaDef; +import org.labkey.query.persist.ExternalSchemaDefCache; +import org.labkey.query.persist.LinkedSchemaDef; +import org.labkey.query.persist.QueryDef; +import org.labkey.query.persist.QueryManager; +import org.labkey.query.reports.ReportsController; +import org.labkey.query.reports.getdata.DataRequest; +import org.labkey.query.sql.QNode; +import org.labkey.query.sql.Query; +import org.labkey.query.sql.SqlParser; +import org.labkey.query.xml.ApiTestsDocument; +import org.labkey.query.xml.TestCaseType; +import org.labkey.remoteapi.RemoteConnections; +import org.labkey.remoteapi.SelectRowsStreamHack; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.vfs.FileLike; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.dao.DataAccessException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; +import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; +import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; +import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; +import static org.labkey.api.util.DOM.BR; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.FONT; +import static org.labkey.api.util.DOM.Renderable; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.query.MetadataTableJSON.getTableType; +import static org.labkey.query.MetadataTableJSON.parseDocument; + +@SuppressWarnings("DefaultAnnotationParam") + +public class QueryController extends SpringActionController +{ + private static final Logger LOG = LogManager.getLogger(QueryController.class); + private static final String ROW_ATTACHMENT_INDEX_DELIM = "::"; + + private static final Set RESERVED_VIEW_NAMES = CaseInsensitiveHashSet.of( + "Default", + AutoGeneratedDetailsCustomView.NAME, + AutoGeneratedInsertCustomView.NAME, + AutoGeneratedUpdateCustomView.NAME + ); + + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(QueryController.class, + ValidateQueryAction.class, + ValidateQueriesAction.class, + GetSchemaQueryTreeAction.class, + GetQueryDetailsAction.class, + ViewQuerySourceAction.class + ); + + public QueryController() + { + setActionResolver(_actionResolver); + } + + public static void registerAdminConsoleLinks() + { + AdminConsole.addLink(AdminConsole.SettingsLinkType.Diagnostics, "data sources", new ActionURL(DataSourceAdminAction.class, ContainerManager.getRoot())); + } + + public static class RemoteQueryConnectionUrls + { + public static ActionURL urlManageRemoteConnection(Container c) + { + return new ActionURL(ManageRemoteConnectionsAction.class, c); + } + + public static ActionURL urlCreateRemoteConnection(Container c) + { + return new ActionURL(EditRemoteConnectionAction.class, c); + } + + public static ActionURL urlEditRemoteConnection(Container c, String connectionName) + { + ActionURL url = new ActionURL(EditRemoteConnectionAction.class, c); + url.addParameter("connectionName", connectionName); + return url; + } + + public static ActionURL urlSaveRemoteConnection(Container c) + { + return new ActionURL(EditRemoteConnectionAction.class, c); + } + + public static ActionURL urlDeleteRemoteConnection(Container c, @Nullable String connectionName) + { + ActionURL url = new ActionURL(DeleteRemoteConnectionAction.class, c); + if (connectionName != null) + url.addParameter("connectionName", connectionName); + return url; + } + + public static ActionURL urlTestRemoteConnection(Container c, String connectionName) + { + ActionURL url = new ActionURL(TestRemoteConnectionAction.class, c); + url.addParameter("connectionName", connectionName); + return url; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); + if (!errors.hasErrors()) + { + String name = remoteConnectionForm.getConnectionName(); + // package the remote-connection properties into the remoteConnectionForm and pass them along + Map map1 = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); + remoteConnectionForm.setUrl(map1.get("URL")); + remoteConnectionForm.setUserEmail(map1.get("user")); + remoteConnectionForm.setPassword(map1.get("password")); + remoteConnectionForm.setFolderPath(map1.get("container")); + } + setHelpTopic("remoteConnection"); + return new JspView<>("/org/labkey/query/view/createRemoteConnection.jsp", remoteConnectionForm, errors); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + return RemoteConnections.createOrEditRemoteConnection(remoteConnectionForm, getContainer(), errors); + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Create/Edit Remote Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class DeleteRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/query/view/confirmDeleteConnection.jsp", remoteConnectionForm, errors); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); + return RemoteConnections.deleteRemoteConnection(remoteConnectionForm, getContainer()); + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Confirm Delete Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + String name = remoteConnectionForm.getConnectionName(); + String schemaName = "core"; // test Schema Name + String queryName = "Users"; // test Query Name + + // Extract the username, password, and container from the secure property store + Map singleConnectionMap = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); + if (singleConnectionMap.isEmpty()) + throw new NotFoundException(); + String url = singleConnectionMap.get(RemoteConnections.FIELD_URL); + String user = singleConnectionMap.get(RemoteConnections.FIELD_USER); + String password = singleConnectionMap.get(RemoteConnections.FIELD_PASSWORD); + String container = singleConnectionMap.get(RemoteConnections.FIELD_CONTAINER); + + // connect to the remote server and retrieve an input stream + org.labkey.remoteapi.Connection cn = new org.labkey.remoteapi.Connection(url, user, password); + final SelectRowsCommand cmd = new SelectRowsCommand(schemaName, queryName); + try + { + DataIteratorBuilder source = SelectRowsStreamHack.go(cn, container, cmd, getContainer()); + // immediately close the source after opening it, this is a test. + source.getDataIterator(new DataIteratorContext()).close(); + } + catch (Exception e) + { + errors.addError(new LabKeyError("The listed credentials for this remote connection failed to connect.")); + return new JspView<>("/org/labkey/query/view/testRemoteConnectionsFailure.jsp", remoteConnectionForm); + } + + return new JspView<>("/org/labkey/query/view/testRemoteConnectionsSuccess.jsp", remoteConnectionForm); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + return true; + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + public static class QueryUrlsImpl implements QueryUrls + { + @Override + public ActionURL urlSchemaBrowser(Container c) + { + return new ActionURL(BeginAction.class, c); + } + + @Override + public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName) + { + ActionURL ret = urlSchemaBrowser(c); + if (schemaName != null) + { + ret.addParameter(QueryParam.schemaName.toString(), schemaName); + } + return ret; + } + + @Override + public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName, @Nullable String queryName) + { + if (StringUtils.isEmpty(queryName)) + return urlSchemaBrowser(c, schemaName); + ActionURL ret = urlSchemaBrowser(c); + ret.addParameter(QueryParam.schemaName.toString(), trimToEmpty(schemaName)); + ret.addParameter(QueryParam.queryName.toString(), trimToEmpty(queryName)); + return ret; + } + + public ActionURL urlExternalSchemaAdmin(Container c) + { + return urlExternalSchemaAdmin(c, null); + } + + public ActionURL urlExternalSchemaAdmin(Container c, @Nullable String message) + { + ActionURL url = new ActionURL(AdminAction.class, c); + + if (null != message) + url.addParameter("message", message); + + return url; + } + + public ActionURL urlInsertExternalSchema(Container c) + { + return new ActionURL(InsertExternalSchemaAction.class, c); + } + + public ActionURL urlNewQuery(Container c) + { + return new ActionURL(NewQueryAction.class, c); + } + + public ActionURL urlUpdateExternalSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(EditExternalSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + public ActionURL urlReloadExternalSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(ReloadExternalSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + public ActionURL urlDeleteSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(DeleteSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + @Override + public ActionURL urlStartBackgroundRReport(@NotNull ActionURL baseURL, String reportId) + { + ActionURL result = baseURL.clone(); + result.setAction(ReportsController.StartBackgroundRReportAction.class); + result.replaceParameter(ReportDescriptor.Prop.reportId, reportId); + return result; + } + + @Override + public ActionURL urlExecuteQuery(@NotNull ActionURL baseURL) + { + ActionURL result = baseURL.clone(); + result.setAction(ExecuteQueryAction.class); + return result; + } + + @Override + public ActionURL urlExecuteQuery(Container c, String schemaName, String queryName) + { + return new ActionURL(ExecuteQueryAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter(QueryParam.queryName, queryName); + } + + @Override + public @NotNull ActionURL urlCreateExcelTemplate(Container c, String schemaName, String queryName) + { + return new ActionURL(ExportExcelTemplateAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter("query.queryName", queryName); + } + + @Override + public ActionURL urlMetadataQuery(Container c, String schemaName, String queryName) + { + return new ActionURL(MetadataQueryAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter(QueryParam.queryName, queryName); + } + } + + @Override + public PageConfig defaultPageConfig() + { + // set default help topic for query controller + PageConfig config = super.defaultPageConfig(); + config.setHelpTopic("querySchemaBrowser"); + return config; + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class DataSourceAdminAction extends SimpleViewAction + { + public DataSourceAdminAction() + { + } + + public DataSourceAdminAction(ViewContext viewContext) + { + setViewContext(viewContext); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Site Admin or Troubleshooter? Troubleshooters can see all the information but can't test data sources. + boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); + List allDefs = QueryManager.get().getExternalSchemaDefs(null); + + MultiValuedMap byDataSourceName = new ArrayListValuedHashMap<>(); + + for (ExternalSchemaDef def : allDefs) + byDataSourceName.put(def.getDataSource(), def); + + MutableInt row = new MutableInt(); + + Renderable r = DOM.DIV( + DIV("This page lists all the data sources defined in your " + AppProps.getInstance().getWebappConfigurationFilename() + " file that were available when first referenced and the external schemas defined in each."), + BR(), + TABLE(cl("labkey-data-region"), + TR(cl("labkey-show-borders"), + hasAdminOpsPerms ? TD(cl("labkey-column-header"), "Test") : null, + TD(cl("labkey-column-header"), "Data Source"), + TD(cl("labkey-column-header"), "Current Status"), + TD(cl("labkey-column-header"), "URL"), + TD(cl("labkey-column-header"), "Database Name"), + TD(cl("labkey-column-header"), "Product Name"), + TD(cl("labkey-column-header"), "Product Version"), + TD(cl("labkey-column-header"), "Max Connections"), + TD(cl("labkey-column-header"), "Active Connections"), + TD(cl("labkey-column-header"), "Idle Connections"), + TD(cl("labkey-column-header"), "Max Wait (ms)") + ), + DbScope.getDbScopes().stream() + .flatMap(scope -> { + String rowStyle = row.getAndIncrement() % 2 == 0 ? "labkey-alternate-row labkey-show-borders" : "labkey-row labkey-show-borders"; + Object status; + boolean connected = false; + try (Connection ignore = scope.getConnection()) + { + status = "connected"; + connected = true; + } + catch (Exception e) + { + status = FONT(cl("labkey-error"), "disconnected"); + } + + return Stream.of( + TR( + cl(rowStyle), + hasAdminOpsPerms ? TD(connected ? new ButtonBuilder("Test").href(new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", scope.getDataSourceName())) : "") : null, + TD(HtmlString.NBSP, scope.getDisplayName()), + TD(status), + TD(scope.getDatabaseUrl()), + TD(scope.getDatabaseName()), + TD(scope.getDatabaseProductName()), + TD(scope.getDatabaseProductVersion()), + TD(scope.getDataSourceProperties().getMaxTotal()), + TD(scope.getDataSourceProperties().getNumActive()), + TD(scope.getDataSourceProperties().getNumIdle()), + TD(scope.getDataSourceProperties().getMaxWaitMillis()) + ), + TR( + cl(rowStyle), + TD(HtmlString.NBSP), + TD(at(DOM.Attribute.colspan, 10), getDataSourceTable(byDataSourceName.get(scope.getDataSourceName()))) + ) + ); + }) + ) + ); + + return new HtmlView(r); + } + + private Renderable getDataSourceTable(Collection dsDefs) + { + if (dsDefs.isEmpty()) + return TABLE(TR(TD(HtmlString.NBSP))); + + MultiValuedMap byContainerPath = new ArrayListValuedHashMap<>(); + + for (ExternalSchemaDef def : dsDefs) + byContainerPath.put(def.getContainerPath(), def); + + TreeSet paths = new TreeSet<>(byContainerPath.keySet()); + + return TABLE(paths.stream() + .map(path -> TR(TD(at(DOM.Attribute.colspan, 4), getDataSourcePath(path, byContainerPath.get(path))))) + ); + } + + private Renderable getDataSourcePath(String path, Collection unsorted) + { + List defs = new ArrayList<>(unsorted); + defs.sort(Comparator.comparing(AbstractExternalSchemaDef::getUserSchemaName, String.CASE_INSENSITIVE_ORDER)); + Container c = ContainerManager.getForPath(path); + + if (null == c) + return TD(); + + boolean hasRead = c.hasPermission(getUser(), ReadPermission.class); + QueryUrlsImpl urls = new QueryUrlsImpl(); + + return + TD(TABLE( + TR(TD( + at(DOM.Attribute.colspan, 3), + hasRead ? LinkBuilder.simpleLink(path, urls.urlExternalSchemaAdmin(c)) : path + )), + TR(TD(TABLE( + defs.stream() + .map(def -> TR(TD( + at(DOM.Attribute.style, "padding-left:20px"), + hasRead ? LinkBuilder.simpleLink(def.getUserSchemaName() + + (!Strings.CS.equals(def.getSourceSchemaName(), def.getUserSchemaName()) ? " (" + def.getSourceSchemaName() + ")" : ""), urls.urlUpdateExternalSchema(c, def)) + : def.getUserSchemaName() + ))) + ))) + )); + } + + @Override + public void addNavTrail(NavTree root) + { + urlProvider(AdminUrls.class).addAdminNavTrail(root, "Data Source Administration", getClass(), getContainer()); + } + } + + public static class TestDataSourceForm + { + private String _dataSource; + + public String getDataSource() + { + return _dataSource; + } + + @SuppressWarnings("unused") + public void setDataSource(String dataSource) + { + _dataSource = dataSource; + } + } + + public static class TestDataSourceConfirmForm extends TestDataSourceForm + { + private String _excludeSchemas; + private String _excludeTables; + + public String getExcludeSchemas() + { + return _excludeSchemas; + } + + @SuppressWarnings("unused") + public void setExcludeSchemas(String excludeSchemas) + { + _excludeSchemas = excludeSchemas; + } + + public String getExcludeTables() + { + return _excludeTables; + } + + @SuppressWarnings("unused") + public void setExcludeTables(String excludeTables) + { + _excludeTables = excludeTables; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestDataSourceConfirmAction extends FormViewAction + { + private DbScope _scope; + + @Override + public ModelAndView getView(TestDataSourceConfirmForm form, boolean reshow, BindException errors) throws Exception + { + validateCommand(form, errors); + return new JspView<>("/org/labkey/query/view/testDataSourceConfirm.jsp", _scope); + } + + @Override + public void validateCommand(TestDataSourceConfirmForm form, Errors errors) + { + _scope = DbScope.getDbScope(form.getDataSource()); + + if (null == _scope) + throw new NotFoundException("Could not resolve data source " + form.getDataSource()); + } + + @Override + public boolean handlePost(TestDataSourceConfirmForm form, BindException errors) throws Exception + { + saveTestDataSourceProperties(form); + return true; + } + + @Override + public URLHelper getSuccessURL(TestDataSourceConfirmForm form) + { + return new ActionURL(TestDataSourceAction.class, getContainer()).addParameter("dataSource", _scope.getDataSourceName()); + } + + @Override + public void addNavTrail(NavTree root) + { + new DataSourceAdminAction(getViewContext()).addNavTrail(root); + root.addChild("Prepare Test of " + _scope.getDataSourceName()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestDataSourceAction extends SimpleViewAction + { + private DbScope _scope; + + @Override + public ModelAndView getView(TestDataSourceForm form, BindException errors) + { + _scope = DbScope.getDbScope(form.getDataSource()); + + if (null == _scope) + throw new NotFoundException("Could not resolve data source " + form.getDataSource()); + + return new JspView<>("/org/labkey/query/view/testDataSource.jsp", _scope); + } + + @Override + public void addNavTrail(NavTree root) + { + new DataSourceAdminAction(getViewContext()).addNavTrail(root); + root.addChild("Test " + _scope.getDataSourceName()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ResetDataSourcePropertiesAction extends FormHandlerAction + { + @Override + public void validateCommand(TestDataSourceForm target, Errors errors) + { + } + + @Override + public boolean handlePost(TestDataSourceForm form, BindException errors) throws Exception + { + WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), false); + if (map != null) + map.delete(); + return true; + } + + @Override + public URLHelper getSuccessURL(TestDataSourceForm form) + { + return new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", form.getDataSource()) ; + } + } + + private static final String TEST_DATA_SOURCE_CATEGORY = "testDataSourceProperties"; + private static final String TEST_DATA_SOURCE_SCHEMAS_PROPERTY = "excludeSchemas"; + private static final String TEST_DATA_SOURCE_TABLES_PROPERTY = "excludeTables"; + + private static String getCategory(String dataSourceName) + { + return TEST_DATA_SOURCE_CATEGORY + "|" + dataSourceName; + } + + public static void saveTestDataSourceProperties(TestDataSourceConfirmForm form) + { + WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), true); + // Save empty entries as empty string to distinguish from null (which results in default values) + map.put(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, StringUtils.trimToEmpty(form.getExcludeSchemas())); + map.put(TEST_DATA_SOURCE_TABLES_PROPERTY, StringUtils.trimToEmpty(form.getExcludeTables())); + map.save(); + } + + public static TestDataSourceConfirmForm getTestDataSourceProperties(DbScope scope) + { + TestDataSourceConfirmForm form = new TestDataSourceConfirmForm(); + PropertyMap map = PropertyManager.getProperties(getCategory(scope.getDataSourceName())); + form.setExcludeSchemas(map.getOrDefault(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, scope.getSqlDialect().getDefaultSchemasToExcludeFromTesting())); + form.setExcludeTables(map.getOrDefault(TEST_DATA_SOURCE_TABLES_PROPERTY, scope.getSqlDialect().getDefaultTablesToExcludeFromTesting())); + + return form; + } + + @RequiresPermission(ReadPermission.class) + public static class BrowseAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/query/view/browse.jsp", null); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Schema Browser"); + } + } + + @RequiresPermission(ReadPermission.class) + public static class BeginAction extends QueryViewAction + { + @SuppressWarnings("UnusedDeclaration") + public BeginAction() + { + } + + public BeginAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + JspView view = new JspView<>("/org/labkey/query/view/browse.jsp", form); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Query Schema Browser", new QueryUrlsImpl().urlSchemaBrowser(getContainer())); + } + } + + @RequiresPermission(ReadPermission.class) + public class SchemaAction extends QueryViewAction + { + public SchemaAction() {} + + SchemaAction(QueryForm form) + { + _form = form; + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + _form = form; + return new JspView<>("/org/labkey/query/view/browse.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + if (_form != null && _form.getSchema() != null) + addSchemaActionNavTrail(root, _form.getSchema().getSchemaPath(), _form.getQueryName()); + } + } + + + void addSchemaActionNavTrail(NavTree root, SchemaKey schemaKey, String queryName) + { + if (getContainer().hasOneOf(getUser(), AdminPermission.class, PlatformDeveloperPermission.class)) + { + // Don't show the full query nav trail to non-admin/non-developer users as they almost certainly don't + // want it + try + { + String schemaName = schemaKey.toDisplayString(); + ActionURL url = new ActionURL(BeginAction.class, getContainer()); + url.addParameter("schemaName", schemaKey.toString()); + url.addParameter("queryName", queryName); + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild(schemaName + " Schema", url); + } + catch (NullPointerException e) + { + LOG.error("NullPointerException in addNavTrail", e); + } + } + } + + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.SelectData.class) + public class NewQueryAction extends FormViewAction + { + private NewQueryForm _form; + private ActionURL _successUrl; + + @Override + public void validateCommand(NewQueryForm target, org.springframework.validation.Errors errors) + { + target.ff_newQueryName = StringUtils.trimToNull(target.ff_newQueryName); + if (null == target.ff_newQueryName) + errors.reject(ERROR_MSG, "QueryName is required"); + } + + @Override + public ModelAndView getView(NewQueryForm form, boolean reshow, BindException errors) + { + form.ensureSchemaExists(); + + if (!form.getSchema().canCreate()) + { + throw new UnauthorizedException(); + } + + getPageConfig().setFocusId("ff_newQueryName"); + _form = form; + setHelpTopic("sqlTutorial"); + return new JspView<>("/org/labkey/query/view/newQuery.jsp", form, errors); + } + + @Override + public boolean handlePost(NewQueryForm form, BindException errors) + { + form.ensureSchemaExists(); + + if (!form.getSchema().canCreate()) + { + throw new UnauthorizedException(); + } + + try + { + if (StringUtils.isEmpty(form.ff_baseTableName)) + { + errors.reject(ERROR_MSG, "You must select a base table or query name."); + return false; + } + + UserSchema schema = form.getSchema(); + String newQueryName = form.ff_newQueryName; + QueryDef existing = QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), newQueryName, true); + if (existing != null) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); + return false; + } + TableInfo existingTable = form.getSchema().getTable(newQueryName, null); + if (existingTable != null) + { + errors.reject(ERROR_MSG, "A table with the name '" + newQueryName + "' already exists."); + return false; + } + // bug 6095 -- conflicting query and dataset names + if (form.getSchema().getTableNames().contains(newQueryName)) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists as a table"); + return false; + } + QueryDefinition newDef = QueryService.get().createQueryDef(getUser(), getContainer(), form.getSchemaKey(), form.ff_newQueryName); + Query query = new Query(schema); + query.setRootTable(FieldKey.fromParts(form.ff_baseTableName)); + String sql = query.getQueryText(); + if (null == sql) + sql = "SELECT * FROM \"" + form.ff_baseTableName + "\""; + newDef.setSql(sql); + + try + { + newDef.save(getUser(), getContainer()); + } + catch (SQLException x) + { + if (RuntimeSQLException.isConstraintException(x)) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); + return false; + } + else + { + throw x; + } + } + + _successUrl = newDef.urlFor(form.ff_redirect); + return true; + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); + errors.reject(ERROR_MSG, Objects.toString(e.getMessage(), e.toString())); + return false; + } + } + + @Override + public ActionURL getSuccessURL(NewQueryForm newQueryForm) + { + return _successUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + root.addChild("New Query", new QueryUrlsImpl().urlNewQuery(getContainer())); + } + } + + // CONSIDER : deleting this action after the SQL editor UI changes are finalized, keep in mind that built-in views + // use this view as well via the edit metadata page. + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) // Note: This action deals with just meta data; it AJAXes data into place using GetWebPartAction + public class SourceQueryAction extends SimpleViewAction + { + public SourceForm _form; + public UserSchema _schema; + public QueryDefinition _queryDef; + + + @Override + public void validate(SourceForm target, BindException errors) + { + _form = target; + if (StringUtils.isEmpty(target.getSchemaName())) + throw new NotFoundException("schema name not specified"); + if (StringUtils.isEmpty(target.getQueryName())) + throw new NotFoundException("query name not specified"); + + QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); + if (null == querySchema) + throw new NotFoundException("schema not found: " + _form.getSchemaKey().toDisplayString()); + if (!(querySchema instanceof UserSchema)) + throw new NotFoundException("Could not find the schema '" + _form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); + _schema = (UserSchema)querySchema; + } + + + @Override + public ModelAndView getView(SourceForm form, BindException errors) + { + _queryDef = _schema.getQueryDef(form.getQueryName()); + if (null == _queryDef) + _queryDef = _schema.getQueryDefForTable(form.getQueryName()); + if (null == _queryDef) + throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + try + { + if (form.ff_queryText == null) + { + form.ff_queryText = _queryDef.getSql(); + form.ff_metadataText = _queryDef.getMetadataXml(); + if (null == form.ff_metadataText) + form.ff_metadataText = form.getDefaultMetadataText(); + } + + for (QueryException qpe : _queryDef.getParseErrors(_schema)) + { + errors.reject(ERROR_MSG, Objects.toString(qpe.getMessage(), qpe.toString())); + } + } + catch (Exception e) + { + try + { + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); + } + catch (Throwable t) + { + // + } + errors.reject("ERROR_MSG", e.toString()); + LOG.error("Error", e); + } + + Renderable moduleWarning = null; + if (_queryDef instanceof ModuleCustomQueryDefinition mcqd && _queryDef.canEdit(getUser())) + { + moduleWarning = DIV(cl("labkey-warning-messages"), + "This SQL query is defined in the '" + mcqd.getModuleName() + "' module in directory '" + mcqd.getSqlFile().getParent() + "'.", + BR(), + "Changes to this query will be reflected in all usages across different folders on the server." + ); + } + + var sourceQueryView = new JspView<>("/org/labkey/query/view/sourceQuery.jsp", this, errors); + WebPartView ret = sourceQueryView; + if (null != moduleWarning) + ret = new VBox(new HtmlView(moduleWarning), sourceQueryView); + return ret; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("useSqlEditor"); + + addSchemaActionNavTrail(root, _form.getSchemaKey(), _form.getQueryName()); + + root.addChild("Edit " + _form.getQueryName()); + } + } + + + /** + * Ajax action to save a query. If the save is successful the request will return successfully. A query + * with SQL syntax errors can still be saved successfully. + * + * If the SQL contains parse errors, a parseErrors object will be returned which contains an array of + * JSON serialized error information. + */ + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.Configure.class) + public static class SaveSourceQueryAction extends MutatingApiAction + { + private UserSchema _schema; + + @Override + public void validateForm(SourceForm form, Errors errors) + { + if (StringUtils.isEmpty(form.getSchemaName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + if (StringUtils.isEmpty(form.getQueryName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + + QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), form.getSchemaKey()); + if (null == querySchema) + throw new NotFoundException("schema not found: " + form.getSchemaKey().toDisplayString()); + if (!(querySchema instanceof UserSchema)) + throw new NotFoundException("Could not find the schema '" + form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); + _schema = (UserSchema)querySchema; + + XmlOptions options = XmlBeansUtil.getDefaultParseOptions(); + List xmlErrors = new ArrayList<>(); + options.setErrorListener(xmlErrors); + try + { + // had a couple of real-world failures due to null pointers in this code, so it's time to be paranoid + if (form.ff_metadataText != null) + { + TablesDocument tablesDoc = TablesDocument.Factory.parse(form.ff_metadataText, options); + if (tablesDoc != null) + { + tablesDoc.validate(options); + TablesType tablesType = tablesDoc.getTables(); + if (tablesType != null) + { + for (TableType tableType : tablesType.getTableArray()) + { + if (null != tableType) + { + if (!Objects.equals(tableType.getTableName(), form.getQueryName())) + { + errors.reject(ERROR_MSG, "Table name in the XML metadata must match the table/query name: " + form.getQueryName()); + } + + TableType.Columns tableColumns = tableType.getColumns(); + if (null != tableColumns) + { + ColumnType[] tableColumnArray = tableColumns.getColumnArray(); + for (ColumnType column : tableColumnArray) + { + if (column.isSetPhi() || column.isSetProtected()) + { + throw new IllegalArgumentException("PHI/protected metadata must not be set here."); + } + + ColumnType.Fk fk = column.getFk(); + if (null != fk) + { + try + { + validateForeignKey(fk, column, errors); + validateLookupFilter(AbstractTableInfo.parseXMLLookupFilters(fk.getFilters()), errors); + } + catch (ValidationException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + } + } + } + } + } + } + } + } + } + catch (XmlException e) + { + throw new RuntimeValidationException(e); + } + + for (XmlError xmle : xmlErrors) + { + errors.reject(ERROR_MSG, XmlBeansUtil.getErrorMessage(xmle)); + } + } + + private void validateForeignKey(ColumnType.Fk fk, ColumnType column, Errors errors) + { + if (fk.isSetFkMultiValued()) + { + // issue 51695 : don't let users create unsupported MVFK types + String type = fk.getFkMultiValued(); + if (!AbstractTableInfo.MultiValuedFkType.junction.name().equals(type)) + { + errors.reject(ERROR_MSG, String.format("Column : \"%s\" has an invalid fkMultiValued value : \"%s\" is not supported.", column.getColumnName(), type)); + } + } + } + + private void validateLookupFilter(Map> filterMap, Errors errors) + { + filterMap.forEach((operation, filters) -> { + + String displayStr = "Filter for operation : " + operation.name(); + for (FilterType filter : filters) + { + if (isBlank(filter.getColumn())) + errors.reject(ERROR_MSG, displayStr + " requires columnName"); + + if (null == filter.getOperator()) + { + errors.reject(ERROR_MSG, displayStr + " requires operator"); + } + else + { + CompareType compareType = CompareType.getByURLKey(filter.getOperator().toString()); + if (null == compareType) + { + errors.reject(ERROR_MSG, displayStr + " operator is invalid"); + } + else + { + if (compareType.isDataValueRequired() && null == filter.getValue()) + errors.reject(ERROR_MSG, displayStr + " requires a value but none is specified"); + } + } + } + + try + { + // attempt to convert to something we can query against + SimpleFilter.fromXml(filters.toArray(new FilterType[0])); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + }); + } + + @Override + public ApiResponse execute(SourceForm form, BindException errors) + { + var queryDef = _schema.getQueryDef(form.getQueryName()); + if (null == queryDef) + queryDef = _schema.getQueryDefForTable(form.getQueryName()); + if (null == queryDef) + throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + if (form.ff_queryText != null) + { + if (!queryDef.isSqlEditable()) + throw new UnauthorizedException("Query SQL is not editable."); + + if (!queryDef.canEdit(getUser())) + throw new UnauthorizedException("Edit permissions are required."); + + queryDef.setSql(form.ff_queryText); + } + + String metadataText = StringUtils.trimToNull(form.ff_metadataText); + if (!Objects.equals(metadataText, queryDef.getMetadataXml())) + { + if (queryDef.isMetadataEditable()) + { + if (!queryDef.canEditMetadata(getUser())) + throw new UnauthorizedException("Edit metadata permissions are required."); + + if (!getUser().isTrustedBrowserDev()) + { + JavaScriptFragment.ensureXMLMetadataNoJavaScript(metadataText); + } + + queryDef.setMetadataXml(metadataText); + } + else + { + if (metadataText != null) + throw new UnsupportedOperationException("Query metadata is not editable."); + } + } + + queryDef.save(getUser(), getContainer()); + + // the query was successfully saved, validate the query but return any errors in the success response + List parseErrors = new ArrayList<>(); + List parseWarnings = new ArrayList<>(); + queryDef.validateQuery(_schema, parseErrors, parseWarnings); + if (!parseErrors.isEmpty()) + { + JSONArray errorArray = new JSONArray(); + + for (QueryException e : parseErrors) + { + errorArray.put(e.toJSON(form.ff_queryText)); + } + response.put("parseErrors", errorArray); + } + else if (!parseWarnings.isEmpty()) + { + JSONArray errorArray = new JSONArray(); + + for (QueryException e : parseWarnings) + { + errorArray.put(e.toJSON(form.ff_queryText)); + } + response.put("parseWarnings", errorArray); + } + } + catch (SQLException e) + { + errors.reject(ERROR_MSG, "An exception occurred: " + e); + LOG.error("Error", e); + } + catch (RuntimeException e) + { + errors.reject(ERROR_MSG, "An exception occurred: " + e.getMessage()); + LOG.error("Error", e); + } + + if (errors.hasErrors()) + return null; + + //if we got here, the query is OK + response.put("success", true); + return response; + } + + } + + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, DeletePermission.class}) + @Action(ActionType.Configure.class) + public static class DeleteQueryAction extends ConfirmAction + { + public SourceForm _form; + public QuerySchema _baseSchema; + public QueryDefinition _queryDef; + + + @Override + public void validateCommand(SourceForm target, Errors errors) + { + _form = target; + if (StringUtils.isEmpty(target.getSchemaName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + if (StringUtils.isEmpty(target.getQueryName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + + _baseSchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); + if (null == _baseSchema) + throw new NotFoundException("Schema not found: " + _form.getSchemaKey().toDisplayString()); + } + + + @Override + public ModelAndView getConfirmView(SourceForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Query"); + _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); + + if (null == _queryDef) + throw new NotFoundException("Query not found: " + form.getQueryName()); + + if (!_queryDef.canDelete(getUser())) + { + errors.reject(ERROR_MSG, "Sorry, this query can not be deleted"); + } + + return new JspView<>("/org/labkey/query/view/deleteQuery.jsp", this, errors); + } + + + @Override + public boolean handlePost(SourceForm form, BindException errors) throws Exception + { + _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); + + if (null == _queryDef) + return false; + try + { + _queryDef.delete(getUser()); + } + catch (OptimisticConflictException x) + { + /* reshow will throw NotFound, so just ignore */ + } + return true; + } + + @Override + @NotNull + public ActionURL getSuccessURL(SourceForm queryForm) + { + return ((UserSchema)_baseSchema).urlFor(QueryAction.schema); + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public class ExecuteQueryAction extends QueryViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _form = form; + + if (errors.hasErrors()) + return new SimpleErrorView(errors, true); + + QueryView queryView = Objects.requireNonNull(form.getQueryView()); + + var t = queryView.getTable(); + if (null != t && !t.allowRobotsIndex()) + { + getPageConfig().setRobotsNone(); + } + + if (isPrint()) + { + queryView.setPrintView(true); + getPageConfig().setTemplate(PageConfig.Template.Print); + getPageConfig().setShowPrintDialog(true); + } + + queryView.setShadeAlternatingRows(true); + queryView.setShowBorders(true); + setHelpTopic("customSQL"); + _queryView = queryView; + return queryView; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + TableInfo ti = null; + try + { + if (null != _queryView) + ti = _queryView.getTable(); + } + catch (QueryParseException x) + { + /* */ + } + String display = ti == null ? _form.getQueryName() : ti.getTitle(); + root.addChild(display); + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class RawTableMetaDataAction extends QueryViewAction + { + private String _dbSchemaName; + private String _dbTableName; + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _form = form; + + QueryView queryView = form.getQueryView(); + String userSchemaName = queryView.getSchema().getName(); + TableInfo ti = queryView.getTable(); + if (null == ti) + throw new NotFoundException(); + + DbScope scope = ti.getSchema().getScope(); + + // Test for provisioned table + if (ti.getDomain() != null) + { + Domain domain = ti.getDomain(); + if (domain.getStorageTableName() != null) + { + // Use the real table and schema names for getting the metadata + _dbTableName = domain.getStorageTableName(); + _dbSchemaName = domain.getDomainKind().getStorageSchemaName(); + } + } + + // No domain or domain with non-provisioned storage (e.g., core.Users) + if (null == _dbSchemaName || null == _dbTableName) + { + DbSchema dbSchema = ti.getSchema(); + _dbSchemaName = dbSchema.getName(); + + // Try to get the underlying schema table and use the meta data name, #12015 + if (ti instanceof FilteredTable fti) + ti = fti.getRealTable(); + + if (ti instanceof SchemaTableInfo) + _dbTableName = ti.getMetaDataIdentifier().getId(); + else if (ti instanceof LinkedTableInfo) + _dbTableName = ti.getName(); + + if (null == _dbTableName) + { + TableInfo tableInfo = dbSchema.getTable(ti.getName()); + if (null != tableInfo) + _dbTableName = tableInfo.getMetaDataIdentifier().getId(); + } + } + + if (null != _dbTableName) + { + VBox result = new VBox(); + + ActionURL url = null; + QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(userSchemaName); + if (qs != null) + { + url = new ActionURL(RawSchemaMetaDataAction.class, getContainer()); + url.addParameter("schemaName", userSchemaName); + } + + SqlDialect dialect = scope.getSqlDialect(); + ScopeView scopeInfo = new ScopeView("Scope and Schema Information", scope, _dbSchemaName, url, _dbTableName); + + result.addView(scopeInfo); + + try (JdbcMetaDataLocator locator = dialect.getTableResolver().getSingleTableLocator(scope, _dbSchemaName, _dbTableName)) + { + JdbcMetaDataSelector columnSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getColumns(l.getCatalogName(), l.getSchemaNamePattern(), l.getTableNamePattern(), null)); + result.addView(new ResultSetView(CachedResultSets.create(columnSelector.getResultSet(), true, Table.ALL_ROWS), "Table Meta Data")); + + JdbcMetaDataSelector pkSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getPrimaryKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(pkSelector.getResultSet(), true, Table.ALL_ROWS), "Primary Key Meta Data")); + + if (dialect.canCheckIndices(ti)) + { + JdbcMetaDataSelector indexSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getIndexInfo(l.getCatalogName(), l.getSchemaName(), l.getTableName(), false, false)); + result.addView(new ResultSetView(CachedResultSets.create(indexSelector.getResultSet(), true, Table.ALL_ROWS), "Other Index Meta Data")); + } + + JdbcMetaDataSelector ikSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getImportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(ikSelector.getResultSet(), true, Table.ALL_ROWS), "Imported Keys Meta Data")); + + JdbcMetaDataSelector ekSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getExportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(ekSelector.getResultSet(), true, Table.ALL_ROWS), "Exported Keys Meta Data")); + } + return result; + } + else + { + errors.reject(ERROR_MSG, "Raw metadata not accessible for table " + ti.getName()); + return new SimpleErrorView(errors); + } + } + + @Override + public void addNavTrail(NavTree root) + { + (new SchemaAction(_form)).addNavTrail(root); + if (null != _dbTableName) + root.addChild("JDBC Meta Data For Table \"" + _dbSchemaName + "." + _dbTableName + "\""); + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class RawSchemaMetaDataAction extends SimpleViewAction + { + private String _schemaName; + + @Override + public ModelAndView getView(Object form, BindException errors) throws Exception + { + _schemaName = getViewContext().getActionURL().getParameter("schemaName"); + if (null == _schemaName) + throw new NotFoundException(); + QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(_schemaName); + if (null == qs) + throw new NotFoundException(_schemaName); + DbSchema schema = qs.getDbSchema(); + String dbSchemaName = schema.getName(); + DbScope scope = schema.getScope(); + SqlDialect dialect = scope.getSqlDialect(); + + HttpView scopeInfo = new ScopeView("Scope Information", scope); + + ModelAndView tablesView; + + try (JdbcMetaDataLocator locator = dialect.getTableResolver().getAllTablesLocator(scope, dbSchemaName)) + { + JdbcMetaDataSelector selector = new JdbcMetaDataSelector(locator, + (dbmd, locator1) -> dbmd.getTables(locator1.getCatalogName(), locator1.getSchemaNamePattern(), locator1.getTableNamePattern(), null)); + Set tableNames = Sets.newCaseInsensitiveHashSet(qs.getTableNames()); + + ActionURL url = new ActionURL(RawTableMetaDataAction.class, getContainer()) + .addParameter("schemaName", _schemaName) + .addParameter("query.queryName", null); + tablesView = new ResultSetView(CachedResultSets.create(selector.getResultSet(), true, Table.ALL_ROWS), "Tables", "TABLE_NAME", url) + { + @Override + protected boolean shouldLink(ResultSet rs) throws SQLException + { + // Only link to tables and views (not indexes or sequences). And only if they're defined in the query schema. + String name = rs.getString("TABLE_NAME"); + String type = rs.getString("TABLE_TYPE"); + return ("TABLE".equalsIgnoreCase(type) || "VIEW".equalsIgnoreCase(type)) && tableNames.contains(name); + } + }; + } + + return new VBox(scopeInfo, tablesView); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("JDBC Meta Data For Schema \"" + _schemaName + "\""); + } + } + + + public static class ScopeView extends WebPartView + { + private final DbScope _scope; + private final String _schemaName; + private final String _tableName; + private final ActionURL _url; + + private ScopeView(String title, DbScope scope) + { + this(title, scope, null, null, null); + } + + private ScopeView(String title, DbScope scope, String schemaName, ActionURL url, String tableName) + { + super(title); + _scope = scope; + _schemaName = schemaName; + _tableName = tableName; + _url = url; + } + + @Override + protected void renderView(Object model, HtmlWriter out) + { + TABLE( + null != _schemaName ? getLabelAndContents("Schema", _url == null ? _schemaName : LinkBuilder.simpleLink(_schemaName, _url)) : null, + null != _tableName ? getLabelAndContents("Table", _tableName) : null, + getLabelAndContents("Scope", _scope.getDisplayName()), + getLabelAndContents("Dialect", _scope.getSqlDialect().getClass().getSimpleName()), + getLabelAndContents("URL", _scope.getDatabaseUrl()) + ).appendTo(out); + } + + // Return a single row (TR) with styled label and contents in separate TDs + private Renderable getLabelAndContents(String label, Object contents) + { + return TR( + TD( + cl("labkey-form-label"), + label + ), + TD( + contents + ) + ); + } + } + + // for backwards compat same as _executeQuery.view ?_print=1 + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public class PrintRowsAction extends ExecuteQueryAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _print = true; + ModelAndView result = super.getView(form, errors); + String title = form.getQueryName(); + if (StringUtils.isEmpty(title)) + title = form.getSchemaName(); + getPageConfig().setTitle(title, true); + return result; + } + } + + + abstract static class _ExportQuery extends SimpleViewAction + { + @Override + public ModelAndView getView(K form, BindException errors) throws Exception + { + QueryView view = form.getQueryView(); + getPageConfig().setTemplate(PageConfig.Template.None); + HttpServletResponse response = getViewContext().getResponse(); + response.setHeader("X-Robots-Tag", "noindex"); + try + { + _export(form, view); + return null; + } + catch (QueryService.NamedParameterNotProvided | QueryParseException x) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + throw x; + } + } + + abstract void _export(K form, QueryView view) throws Exception; + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportScriptForm extends QueryForm + { + private String _type; + + public String getScriptType() + { + return _type; + } + + public void setScriptType(String type) + { + _type = type; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) // This is called "export" but it doesn't export any data + @CSRF(CSRF.Method.ALL) + public static class ExportScriptAction extends SimpleViewAction + { + @Override + public void validate(ExportScriptForm form, BindException errors) + { + // calling form.getQueryView() as a validation check as it will throw if schema/query missing + form.getQueryView(); + + if (StringUtils.isEmpty(form.getScriptType())) + throw new NotFoundException("Missing required parameter: scriptType."); + } + + @Override + public ModelAndView getView(ExportScriptForm form, BindException errors) + { + return ExportScriptModel.getExportScriptView(QueryView.create(form, errors), form.getScriptType(), getPageConfig(), getViewContext().getResponse()); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsExcelAction extends _ExportQuery + { + @Override + void _export(ExportQueryForm form, QueryView view) throws Exception + { + view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xls, form.getRenameColumnMap()); + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsXLSXAction extends _ExportQuery + { + @Override + void _export(ExportQueryForm form, QueryView view) throws Exception + { + view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xlsx, form.getRenameColumnMap()); + } + } + + public static class ExportQueriesForm extends ExportQueryForm implements ApiJsonForm + { + private String filename; + private List queryForms; + + public void setFilename(String filename) + { + this.filename = filename; + } + + public String getFilename() + { + return filename; + } + + public void setQueryForms(List queryForms) + { + this.queryForms = queryForms; + } + + public List getQueryForms() + { + return queryForms; + } + + /** + * Map JSON to Spring PropertyValue objects. + * @param json the properties + */ + private MutablePropertyValues getPropertyValues(JSONObject json) + { + // Collecting mapped properties as a list because adding them to an existing MutablePropertyValues object replaces existing values + List properties = new ArrayList<>(); + + for (String key : json.keySet()) + { + Object value = json.get(key); + if (value instanceof JSONArray val) + { + // Split arrays into individual pairs to be bound (Issue #45452) + for (int i = 0; i < val.length(); i++) + { + properties.add(new PropertyValue(key, val.get(i).toString())); + } + } + else + { + properties.add(new PropertyValue(key, value)); + } + } + + return new MutablePropertyValues(properties); + } + + @Override + public void bindJson(JSONObject json) + { + setFilename(json.get("filename").toString()); + List forms = new ArrayList<>(); + + JSONArray models = json.optJSONArray("queryForms"); + if (models == null) + { + QueryController.LOG.error("No models to export; Form's `queryForms` property was null"); + throw new RuntimeValidationException("No queries to export; Form's `queryForms` property was null"); + } + + for (JSONObject queryModel : JsonUtil.toJSONObjectList(models)) + { + ExportQueryForm qf = new ExportQueryForm(); + qf.setViewContext(getViewContext()); + + qf.bindParameters(getPropertyValues(queryModel)); + forms.add(qf); + } + + setQueryForms(forms); + } + } + + /** + * Export multiple query forms + */ + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportQueriesXLSXAction extends ReadOnlyApiAction + { + @Override + public Object execute(ExportQueriesForm form, BindException errors) throws Exception + { + getPageConfig().setTemplate(PageConfig.Template.None); + HttpServletResponse response = getViewContext().getResponse(); + response.setHeader("X-Robots-Tag", "noindex"); + ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment); + ViewContext viewContext = getViewContext(); + + Map> nameFormMap = new CaseInsensitiveHashMap<>(); + Map sheetNames = new HashMap<>(); + form.getQueryForms().forEach(qf -> { + String sheetName = qf.getSheetName(); + QueryView qv = qf.getQueryView(); + // use the given sheet name if provided, otherwise try the query definition name + String name = StringUtils.isNotBlank(sheetName) ? sheetName : qv.getQueryDef().getName(); + // if there is no sheet name or queryDefinition name, use a data region name if provided. Otherwise, use "Data" + name = StringUtils.isNotBlank(name) ? name : StringUtils.isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; + // clean it to remove undesirable characters and make it of an acceptable length + name = ExcelWriter.cleanSheetName(name); + nameFormMap.computeIfAbsent(name, k -> new ArrayList<>()).add(qf); + }); + // Issue 53722: Need to assure unique names for the sheets in the presence of really long names + for (Map.Entry> entry : nameFormMap.entrySet()) { + String name = entry.getKey(); + if (entry.getValue().size() > 1) + { + List queryForms = entry.getValue(); + int countLength = String.valueOf(queryForms.size()).length() + 2; + if (countLength > name.length()) + throw new IllegalArgumentException("Cannot create sheet names from overlapping query names."); + for (int i = 0; i < queryForms.size(); i++) + { + sheetNames.put(entry.getValue().get(i), name.substring(0, name.length() - countLength) + "(" + i + ")"); + } + } + else + { + sheetNames.put(entry.getValue().get(0), name); + } + } + ExcelWriter writer = new ExcelWriter(ExcelWriter.ExcelDocumentType.xlsx) { + @Override + protected void renderSheets(Workbook workbook) + { + for (ExportQueryForm qf : form.getQueryForms()) + { + qf.setViewContext(viewContext); + qf.getSchema(); + + QueryView qv = qf.getQueryView(); + QueryView.ExcelExportConfig config = new QueryView.ExcelExportConfig(response, qf.getHeaderType()) + .setExcludeColumns(qf.getExcludeColumns()) + .setRenamedColumns(qf.getRenameColumnMap()); + qv.configureExcelWriter(this, config); + setSheetName(sheetNames.get(qf)); + setAutoSize(true); + renderNewSheet(workbook); + qv.logAuditEvent("Exported to Excel", getDataRowCount()); + } + + workbook.setActiveSheet(0); + } + }; + writer.setFilenamePrefix(form.getFilename()); + writer.renderWorkbook(response); + return null; //Returning anything here will cause error as excel writer will close the response stream + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class TemplateForm extends ExportQueryForm + { + boolean insertColumnsOnly = true; + String filenamePrefix; + FieldKey[] includeColumn; + String fileType; + + public TemplateForm() + { + _headerType = ColumnHeaderType.Caption; + } + + // "captionType" field backwards compatibility + public void setCaptionType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public ColumnHeaderType getCaptionType() + { + return _headerType; + } + + public List getIncludeColumns() + { + if (includeColumn == null || includeColumn.length == 0) + return Collections.emptyList(); + return Arrays.asList(includeColumn); + } + + public FieldKey[] getIncludeColumn() + { + return includeColumn; + } + + public void setIncludeColumn(FieldKey[] includeColumn) + { + this.includeColumn = includeColumn; + } + + @NotNull + public String getFilenamePrefix() + { + return filenamePrefix == null ? getQueryName() : filenamePrefix; + } + + public void setFilenamePrefix(String prefix) + { + filenamePrefix = prefix; + } + + public String getFileType() + { + return fileType; + } + + public void setFileType(String fileType) + { + this.fileType = fileType; + } + } + + + /** + * Can be used to generate an Excel template for import into a table. Supported URL params include: + *
+ *
filenamePrefix
+ *
the prefix of the excel file that is generated, defaults to '_data'
+ * + *
query.viewName
+ *
if provided, the resulting excel file will use the fields present in this view. + * Non-usereditable columns will be skipped. + * Non-existent columns (like a lookup) unless includeMissingColumns is true. + * Any required columns missing from this view will be appended to the end of the query. + *
+ * + *
includeColumn
+ *
List of column names to include, even if the column doesn't exist or is non-userEditable. + * For example, this can be used to add a fake column that is only supported during the import process. + *
+ * + *
excludeColumn
+ *
List of column names to exclude. + *
+ * + *
exportAlias.columns
+ *
Use alternative column name in excel: exportAlias.originalColumnName=aliasColumnName + *
+ * + *
captionType
+ *
determines which column property is used in the header, either Label or Name
+ *
+ */ + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportExcelTemplateAction extends _ExportQuery + { + public ExportExcelTemplateAction() + { + setCommandClass(TemplateForm.class); + } + + @Override + void _export(TemplateForm form, QueryView view) throws Exception + { + boolean respectView = form.getViewName() != null; + ExcelWriter.ExcelDocumentType fileType = ExcelWriter.ExcelDocumentType.xlsx; + if (form.getFileType() != null) + { + try + { + fileType = ExcelWriter.ExcelDocumentType.valueOf(form.getFileType().toLowerCase()); + } + catch (IllegalArgumentException ignored) {} + } + view.exportToExcel( new QueryView.ExcelExportConfig(getViewContext().getResponse(), form.getHeaderType()) + .setTemplateOnly(true) + .setInsertColumnsOnly(form.insertColumnsOnly) + .setDocType(fileType) + .setRespectView(respectView) + .setIncludeColumns(form.getIncludeColumns()) + .setExcludeColumns(form.getExcludeColumns()) + .setRenamedColumns(form.getRenameColumnMap()) + .setPrefix((StringUtils.isEmpty(form.getFilenamePrefix()) ? "Import" : form.getFilenamePrefix()) + "_Template") // Issue 48028: Change template file names + ); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportQueryForm extends QueryForm + { + protected ColumnHeaderType _headerType = null; // QueryView will provide a default header type if the user doesn't select one + FieldKey[] excludeColumn; + Map renameColumns = null; + private String sheetName; + + public void setSheetName(String sheetName) + { + this.sheetName = sheetName; + } + + public String getSheetName() + { + return sheetName; + } + + public ColumnHeaderType getHeaderType() + { + return _headerType; + } + + public void setHeaderType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public List getExcludeColumns() + { + if (excludeColumn == null || excludeColumn.length == 0) + return Collections.emptyList(); + return Arrays.asList(excludeColumn); + } + + public void setExcludeColumn(FieldKey[] excludeColumn) + { + this.excludeColumn = excludeColumn; + } + + public Map getRenameColumnMap() + { + if (renameColumns != null) + return renameColumns; + + renameColumns = new CaseInsensitiveHashMap<>(); + final String renameParamPrefix = "exportAlias."; + PropertyValue[] pvs = getInitParameters().getPropertyValues(); + for (PropertyValue pv : pvs) + { + String paramName = pv.getName(); + if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) + continue; + + renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); + } + + return renameColumns; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportRowsTsvForm extends ExportQueryForm + { + private TSVWriter.DELIM _delim = TSVWriter.DELIM.TAB; + private TSVWriter.QUOTE _quote = TSVWriter.QUOTE.DOUBLE; + + public TSVWriter.DELIM getDelim() + { + return _delim; + } + + public void setDelim(TSVWriter.DELIM delim) + { + _delim = delim; + } + + public TSVWriter.QUOTE getQuote() + { + return _quote; + } + + public void setQuote(TSVWriter.QUOTE quote) + { + _quote = quote; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsTsvAction extends _ExportQuery + { + public ExportRowsTsvAction() + { + setCommandClass(ExportRowsTsvForm.class); + } + + @Override + void _export(ExportRowsTsvForm form, QueryView view) throws Exception + { + view.exportToTsv(getViewContext().getResponse(), form.getDelim(), form.getQuote(), form.getHeaderType(), form.getRenameColumnMap()); + } + } + + + @RequiresNoPermission + @IgnoresTermsOfUse + @Action(ActionType.Export.class) + public static class ExcelWebQueryAction extends ExportRowsTsvAction + { + @Override + public ModelAndView getView(ExportRowsTsvForm form, BindException errors) throws Exception + { + if (!getContainer().hasPermission(getUser(), ReadPermission.class)) + { + if (!getUser().isGuest()) + { + throw new UnauthorizedException(); + } + getViewContext().getResponse().setHeader("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); + getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); + return null; + } + + // Bug 5610. Excel web queries don't work over SSL if caching is disabled, + // so we need to allow caching so that Excel can read from IE on Windows. + HttpServletResponse response = getViewContext().getResponse(); + // Set the headers to allow the client to cache, but not proxies + ResponseHelper.setPrivate(response); + + QueryView view = form.getQueryView(); + getPageConfig().setTemplate(PageConfig.Template.None); + view.exportToExcelWebQuery(getViewContext().getResponse()); + return null; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExcelWebQueryDefinitionAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + getPageConfig().setTemplate(PageConfig.Template.None); + form.getQueryView(); + String queryViewActionURL = form.getQueryViewActionURL(); + ActionURL url; + if (queryViewActionURL != null) + { + url = new ActionURL(queryViewActionURL); + } + else + { + url = getViewContext().cloneActionURL(); + url.setAction(ExcelWebQueryAction.class); + } + getViewContext().getResponse().setContentType("text/x-ms-iqy"); + String filename = FileUtil.makeFileNameWithTimestamp(form.getQueryName(), "iqy"); + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, filename); + PrintWriter writer = getViewContext().getResponse().getWriter(); + writer.println("WEB"); + writer.println("1"); + writer.println(url.getURIString()); + + QueryService.get().addAuditEvent(getUser(), getContainer(), form.getSchemaName(), form.getQueryName(), url, "Exported to Excel Web Query definition", null); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.SelectMetaData.class) + public class MetadataQueryAction extends SimpleViewAction + { + QueryForm _form = null; + + @Override + public ModelAndView getView(QueryForm queryForm, BindException errors) throws Exception + { + String schemaName = queryForm.getSchemaName(); + String queryName = queryForm.getQueryName(); + + _form = queryForm; + + if (schemaName.isEmpty() && (null == queryName || queryName.isEmpty())) + { + throw new NotFoundException("Must provide schemaName and queryName."); + } + + if (schemaName.isEmpty()) + { + throw new NotFoundException("Must provide schemaName."); + } + + if (null == queryName || queryName.isEmpty()) + { + throw new NotFoundException("Must provide queryName."); + } + + if (!queryForm.getQueryDef().isMetadataEditable()) + throw new UnauthorizedException("Query metadata is not editable"); + + if (!queryForm.canEditMetadata()) + throw new UnauthorizedException("You do not have permission to edit the query metadata"); + + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("queryMetadataEditor")); + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + var metadataQuery = _form.getQueryDef().getName(); + if (null != metadataQuery) + root.addChild("Edit Metadata: " + _form.getQueryName(), metadataQuery); + else + root.addChild("Edit Metadata: " + _form.getQueryName()); + } + } + + // Uck. Supports the old and new view designer. + protected JSONObject saveCustomView(Container container, QueryDefinition queryDef, + String regionName, String viewName, boolean replaceExisting, + boolean share, boolean inherit, + boolean session, boolean saveFilter, + boolean hidden, JSONObject jsonView, + ActionURL returnUrl, + BindException errors) + { + User owner = getUser(); + boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); + if (share && canSaveForAllUsers && !session) + { + owner = null; + } + String name = StringUtils.trimToNull(viewName); + + if (name != null && RESERVED_VIEW_NAMES.contains(name.toLowerCase())) + errors.reject(ERROR_MSG, "The grid view name '" + name + "' is not allowed."); + + boolean isHidden = hidden; + CustomView view; + if (owner == null) + view = queryDef.getSharedCustomView(name); + else + view = queryDef.getCustomView(owner, getViewContext().getRequest(), name); + + if (view != null && !replaceExisting && !StringUtils.isEmpty(name)) + errors.reject(ERROR_MSG, "A saved view by the name \"" + viewName + "\" already exists. "); + + // 11179: Allow editing the view if we're saving to session. + // NOTE: Check for session flag first otherwise the call to canEdit() will add errors to the errors collection. + boolean canEdit = view == null || session || view.canEdit(container, errors); + if (errors.hasErrors()) + return null; + + if (canEdit) + { + // Issue 13594: Disallow setting of the customview inherit bit for query views + // that have no available container filter types. Unfortunately, the only way + // to get the container filters is from the QueryView. Ideally, the query def + // would know if it was container filterable or not instead of using the QueryView. + if (inherit && canSaveForAllUsers && !session) + { + UserSchema schema = queryDef.getSchema(); + QueryView queryView = schema.createView(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, queryDef.getName(), errors); + if (queryView != null) + { + Set allowableContainerFilterTypes = queryView.getAllowableContainerFilterTypes(); + if (allowableContainerFilterTypes.size() <= 1) + { + errors.reject(ERROR_MSG, "QueryView doesn't support inherited custom views"); + return null; + } + } + } + + // Create a new view if none exists or the current view is a shared view + // and the user wants to override the shared view with a personal view. + if (view == null || (owner != null && view.isShared())) + { + if (owner == null) + view = queryDef.createSharedCustomView(name); + else + view = queryDef.createCustomView(owner, name); + + if (owner != null && session) + ((CustomViewImpl) view).isSession(true); + view.setIsHidden(hidden); + } + else if (session != view.isSession()) + { + if (session) + { + assert !view.isSession(); + if (owner == null) + { + errors.reject(ERROR_MSG, "Session views can't be saved for all users"); + return null; + } + + // The form is saving to session but the view is in the database. + // Make a copy in case it's a read-only version from an XML file + view = queryDef.createCustomView(owner, name); + ((CustomViewImpl) view).isSession(true); + } + else + { + // Remove the session view and call saveCustomView again to either create a new view or update an existing view. + assert view.isSession(); + boolean success = false; + try + { + view.delete(getUser(), getViewContext().getRequest()); + JSONObject ret = saveCustomView(container, queryDef, regionName, viewName, replaceExisting, share, inherit, session, saveFilter, hidden, jsonView, returnUrl, errors); + success = !errors.hasErrors() && ret != null; + return success ? ret : null; + } + finally + { + if (!success) + { + // dirty the view then save the deleted session view back in session state + view.setName(view.getName()); + view.save(getUser(), getViewContext().getRequest()); + } + } + } + } + + // NOTE: Updating, saving, and deleting the view may throw an exception + CustomViewImpl cview = null; + if (view instanceof EditableCustomView && view.isOverridable()) + { + cview = ((EditableCustomView)view).getEditableViewInfo(owner, session); + } + if (null == cview) + { + throw new IllegalArgumentException("View cannot be edited"); + } + + cview.update(jsonView, saveFilter); + if (canSaveForAllUsers && !session) + { + cview.setCanInherit(inherit); + } + isHidden = view.isHidden(); + cview.setContainer(container); + cview.save(getUser(), getViewContext().getRequest()); + if (owner == null) + { + // New view is shared so delete any previous custom view owned by the user with the same name. + CustomView personalView = queryDef.getCustomView(getUser(), getViewContext().getRequest(), name); + if (personalView != null && !personalView.isShared()) + { + personalView.delete(getUser(), getViewContext().getRequest()); + } + } + } + + if (null == returnUrl) + { + returnUrl = getViewContext().cloneActionURL().setAction(ExecuteQueryAction.class); + } + else + { + returnUrl = returnUrl.clone(); + if (name == null || !canEdit) + { + returnUrl.deleteParameter(regionName + "." + QueryParam.viewName); + } + else if (!isHidden) + { + returnUrl.replaceParameter(regionName + "." + QueryParam.viewName, name); + } + returnUrl.deleteParameter(regionName + "." + QueryParam.ignoreFilter); + if (saveFilter) + { + for (String key : returnUrl.getKeysByPrefix(regionName + ".")) + { + if (isFilterOrSort(regionName, key)) + returnUrl.deleteFilterParameters(key); + } + } + } + + JSONObject ret = new JSONObject(); + ret.put("redirect", returnUrl); + Map viewAsMap = CustomViewUtil.toMap(view, getUser(), true); + try + { + ret.put("view", new JSONObject(viewAsMap, new JSONParserConfiguration().withMaxNestingDepth(10))); + } + catch (JSONException e) + { + LOG.error("Failed to save view: {}", jsonView, e); + } + return ret; + } + + private boolean isFilterOrSort(String dataRegionName, String param) + { + assert param.startsWith(dataRegionName + "."); + String check = param.substring(dataRegionName.length() + 1); + if (check.contains("~")) + return true; + if ("sort".equals(check)) + return true; + if (check.equals("containerFilterName")) + return true; + return false; + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Configure.class) + @JsonInputLimit(100_000) + public class SaveQueryViewsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SimpleApiJsonForm form, BindException errors) + { + JSONObject json = form.getJsonObject(); + if (json == null) + throw new NotFoundException("Empty request"); + + String schemaName = json.optString(QueryParam.schemaName.toString(), null); + String queryName = json.optString(QueryParam.queryName.toString(), null); + if (schemaName == null || queryName == null) + throw new NotFoundException("schemaName and queryName are required"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + if (schema == null) + throw new NotFoundException("schema not found"); + + QueryDefinition queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), schemaName, queryName); + if (queryDef == null) + queryDef = schema.getQueryDefForTable(queryName); + if (queryDef == null) + throw new NotFoundException("query not found"); + + JSONObject response = new JSONObject(); + response.put(QueryParam.schemaName.toString(), schemaName); + response.put(QueryParam.queryName.toString(), queryName); + JSONArray views = new JSONArray(); + response.put("views", views); + + ActionURL redirect = null; + JSONArray jsonViews = json.getJSONArray("views"); + for (int i = 0; i < jsonViews.length(); i++) + { + final JSONObject jsonView = jsonViews.getJSONObject(i); + String viewName = jsonView.optString("name", null); + if (viewName == null) + throw new NotFoundException("'name' is required all views'"); + + boolean shared = jsonView.optBoolean("shared", false); + boolean replace = jsonView.optBoolean("replace", true); // "replace" was the default before the flag is introduced + boolean inherit = jsonView.optBoolean("inherit", false); + boolean session = jsonView.optBoolean("session", false); + boolean hidden = jsonView.optBoolean("hidden", false); + // Users may save views to a location other than the current container + String containerPath = jsonView.optString("containerPath", getContainer().getPath()); + Container container; + if (inherit) + { + // Only respect this request if it's a view that is inheritable in subfolders + container = ContainerManager.getForPath(containerPath); + } + else + { + // Otherwise, save it in the current container + container = getContainer().getContainerFor(ContainerType.DataType.customQueryViews); + } + + if (container == null) + { + throw new NotFoundException("No such container: " + containerPath); + } + + JSONObject savedView = saveCustomView( + container, queryDef, QueryView.DATAREGIONNAME_DEFAULT, viewName, replace, + shared, inherit, session, true, hidden, jsonView, null, errors); + + if (savedView != null) + { + if (redirect == null) + redirect = (ActionURL)savedView.get("redirect"); + views.put(savedView.getJSONObject("view")); + } + } + + if (redirect != null) + response.put("redirect", redirect); + + if (errors.hasErrors()) + return null; + else + return new ApiSimpleResponse(response); + } + } + + public static class RenameQueryViewForm extends QueryForm + { + private String newName; + + public String getNewName() + { + return newName; + } + + public void setNewName(String newName) + { + this.newName = newName; + } + } + + @RequiresPermission(ReadPermission.class) + public class RenameQueryViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RenameQueryViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + + Container container = getContainer(); + User user = getUser(); + + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + if (schemaName == null || queryName == null) + throw new NotFoundException("schemaName and queryName are required"); + + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); + if (schema == null) + throw new NotFoundException("schema not found"); + + QueryDefinition queryDef = QueryService.get().getQueryDef(user, container, schemaName, queryName); + if (queryDef == null) + queryDef = schema.getQueryDefForTable(queryName); + if (queryDef == null) + throw new NotFoundException("query not found"); + + renameCustomView(container, queryDef, view, form.getNewName(), errors); + + if (errors.hasErrors()) + return null; + else + return new ApiSimpleResponse("success", true); + } + } + + protected void renameCustomView(Container container, QueryDefinition queryDef, CustomView fromView, String newViewName, BindException errors) + { + if (newViewName != null && RESERVED_VIEW_NAMES.contains(newViewName.toLowerCase())) + errors.reject(ERROR_MSG, "The grid view name '" + newViewName + "' is not allowed."); + + String newName = StringUtils.trimToNull(newViewName); + if (StringUtils.isEmpty(newName)) + errors.reject(ERROR_MSG, "View name cannot be blank."); + + if (errors.hasErrors()) + return; + + User owner = getUser(); + boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); + + if (!fromView.canEdit(container, errors)) + return; + + if (fromView.isSession()) + { + errors.reject(ERROR_MSG, "Cannot rename a session view."); + return; + } + + CustomView duplicateView = queryDef.getCustomView(owner, getViewContext().getRequest(), newName); + if (duplicateView == null && canSaveForAllUsers) + duplicateView = queryDef.getSharedCustomView(newName); + if (duplicateView != null) + { + // only allow duplicate view name if creating a new private view to shadow an existing shared view + if (!(!fromView.isShared() && duplicateView.isShared())) + { + errors.reject(ERROR_MSG, "Another saved view by the name \"" + newName + "\" already exists. "); + return; + } + } + + fromView.setName(newViewName); + fromView.save(getUser(), getViewContext().getRequest()); + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Configure.class) + public class PropertiesQueryAction extends FormViewAction + { + PropertiesForm _form = null; + private String _queryName; + + @Override + public void validateCommand(PropertiesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(PropertiesForm form, boolean reshow, BindException errors) + { + // assertQueryExists requires that it be well-formed + // assertQueryExists(form); + QueryDefinition queryDef = form.getQueryDef(); + _form = form; + _form.setDescription(queryDef.getDescription()); + _form.setInheritable(queryDef.canInherit()); + _form.setHidden(queryDef.isHidden()); + setHelpTopic("editQueryProperties"); + _queryName = form.getQueryName(); + + return new JspView<>("/org/labkey/query/view/propertiesQuery.jsp", form, errors); + } + + @Override + public boolean handlePost(PropertiesForm form, BindException errors) throws Exception + { + // assertQueryExists requires that it be well-formed + // assertQueryExists(form); + if (!form.canEdit()) + { + throw new UnauthorizedException(); + } + QueryDefinition queryDef = form.getQueryDef(); + _queryName = form.getQueryName(); + if (!queryDef.getDefinitionContainer().getId().equals(getContainer().getId())) + throw new NotFoundException("Query not found"); + + _form = form; + + if (!StringUtils.isEmpty(form.rename) && !form.rename.equalsIgnoreCase(queryDef.getName())) + { + // issue 17766: check if query or table exist with this name + if (null != QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), form.rename, true) + || null != form.getSchema().getTable(form.rename,null)) + { + errors.reject(ERROR_MSG, "A query or table with the name \"" + form.rename + "\" already exists."); + return false; + } + + // Issue 40895: update queryName in xml metadata + updateXmlMetadata(queryDef); + queryDef.setName(form.rename); + // update form so getSuccessURL() works + _form = new PropertiesForm(form.getSchemaName(), form.rename); + _form.setViewContext(form.getViewContext()); + _queryName = form.rename; + } + + queryDef.setDescription(form.description); + queryDef.setCanInherit(form.inheritable); + queryDef.setIsHidden(form.hidden); + queryDef.save(getUser(), getContainer()); + return true; + } + + private void updateXmlMetadata(QueryDefinition queryDef) throws XmlException + { + if (null != queryDef.getMetadataXml()) + { + TablesDocument doc = TablesDocument.Factory.parse(queryDef.getMetadataXml()); + if (null != doc) + { + for (TableType tableType : doc.getTables().getTableArray()) + { + if (tableType.getTableName().equalsIgnoreCase(queryDef.getName())) + { + // update tableName in xml + tableType.setTableName(_form.rename); + } + } + XmlOptions xmlOptions = new XmlOptions(); + xmlOptions.setSavePrettyPrint(); + // Don't use an explicit namespace, making the XML much more readable + xmlOptions.setUseDefaultNamespace(); + queryDef.setMetadataXml(doc.xmlText(xmlOptions)); + } + } + } + + @Override + public ActionURL getSuccessURL(PropertiesForm propertiesForm) + { + ActionURL url = new ActionURL(BeginAction.class, propertiesForm.getViewContext().getContainer()); + url.addParameter("schemaName", propertiesForm.getSchemaName()); + if (null != _queryName) + url.addParameter("queryName", _queryName); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + root.addChild("Edit query properties"); + } + } + + @ActionNames("truncateTable") + @RequiresPermission(AdminPermission.class) + public static class TruncateTableAction extends MutatingApiAction + { + UserSchema schema; + TableInfo table; + + @Override + public void validateForm(QueryForm form, Errors errors) + { + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + + if (isBlank(schemaName) || isBlank(queryName)) + throw new NotFoundException("schemaName and queryName are required"); + + schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + if (null == schema) + throw new NotFoundException("The schema '" + schemaName + "' does not exist."); + + table = schema.getTable(queryName, null); + if (null == table) + throw new NotFoundException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); + } + + @Override + public ApiResponse execute(QueryForm form, BindException errors) throws Exception + { + int deletedRows; + QueryUpdateService qus = table.getUpdateService(); + + if (null == qus) + throw new IllegalArgumentException("The query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "' is not truncatable."); + + try (DbScope.Transaction transaction = table.getSchema().getScope().ensureTransaction()) + { + deletedRows = qus.truncateRows(getUser(), getContainer(), null, null); + transaction.commit(); + } + + ApiSimpleResponse response = new ApiSimpleResponse(); + + response.put("success", true); + response.put(BaseSaveRowsAction.PROP_SCHEMA_NAME, form.getSchemaName()); + response.put(BaseSaveRowsAction.PROP_QUERY_NAME, form.getQueryName()); + response.put("deletedRows", deletedRows); + + return response; + } + } + + + @RequiresPermission(DeletePermission.class) + public static class DeleteQueryRowsAction extends FormHandlerAction + { + @Override + public void validateCommand(QueryForm target, Errors errors) + { + } + + @Override + public boolean handlePost(QueryForm form, BindException errors) + { + TableInfo table = form.getQueryView().getTable(); + + if (!table.hasPermission(getUser(), DeletePermission.class)) + { + throw new UnauthorizedException(); + } + + QueryUpdateService updateService = table.getUpdateService(); + if (updateService == null) + throw new UnsupportedOperationException("Unable to delete - no QueryUpdateService registered for " + form.getSchemaName() + "." + form.getQueryName()); + + Set ids = DataRegionSelection.getSelected(form.getViewContext(), null, true); + List pks = table.getPkColumns(); + int numPks = pks.size(); + + //normalize the pks to arrays of correctly-typed objects + List> keyValues = new ArrayList<>(ids.size()); + for (String id : ids) + { + String[] stringValues; + if (numPks > 1) + { + stringValues = id.split(","); + if (stringValues.length != numPks) + throw new IllegalStateException("This table has " + numPks + " primary-key columns, but " + stringValues.length + " primary-key values were provided!"); + } + else + stringValues = new String[]{id}; + + Map rowKeyValues = new CaseInsensitiveHashMap<>(); + for (int idx = 0; idx < numPks; ++idx) + { + ColumnInfo keyColumn = pks.get(idx); + Object keyValue = keyColumn.getJavaClass() == String.class ? stringValues[idx] : ConvertUtils.convert(stringValues[idx], keyColumn.getJavaClass()); + rowKeyValues.put(keyColumn.getName(), keyValue); + } + keyValues.add(rowKeyValues); + } + + DbSchema dbSchema = table.getSchema(); + try + { + dbSchema.getScope().executeWithRetry(tx -> + { + try + { + updateService.deleteRows(getUser(), getContainer(), keyValues, null, null); + } + catch (SQLException x) + { + if (!RuntimeSQLException.isConstraintException(x)) + throw new RuntimeSQLException(x); + errors.reject(ERROR_MSG, getMessage(table.getSchema().getSqlDialect(), x)); + } + catch (DataIntegrityViolationException | OptimisticConflictException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + catch (BatchValidationException x) + { + x.addToErrors(errors); + } + catch (Exception x) + { + errors.reject(ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); + } + // need to throw here to avoid committing tx + if (errors.hasErrors()) + throw new DbScope.RetryPassthroughException(errors); + return true; + }); + } + catch (DbScope.RetryPassthroughException x) + { + if (x.getCause() != errors) + x.throwRuntimeException(); + } + return !errors.hasErrors(); + } + + @Override + public ActionURL getSuccessURL(QueryForm form) + { + return form.getReturnActionURL(); + } + } + + @RequiresPermission(ReadPermission.class) + public static class DetailsQueryRowAction extends UserSchemaAction + { + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + + if (_schema != null && _table != null) + { + if (_table.hasPermission(getUser(), UpdatePermission.class)) + { + StringExpression updateExpr = _form.getQueryDef().urlExpr(QueryAction.updateQueryRow, _schema.getContainer()); + if (updateExpr != null) + { + String url = updateExpr.eval(tableForm.getTypedValues()); + if (url != null) + { + ActionURL updateUrl = new ActionURL(url); + ActionButton editButton = new ActionButton("Edit", updateUrl); + bb.add(editButton); + } + } + } + + + ActionURL gridUrl; + if (_form.getReturnActionURL() != null) + { + // If we have a specific return URL requested, use that + gridUrl = _form.getReturnActionURL(); + } + else + { + // Otherwise go back to the default grid view + gridUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); + } + if (gridUrl != null) + { + ActionButton gridButton = new ActionButton("Show Grid", gridUrl); + bb.add(gridButton); + } + } + + DetailsView detailsView = new DetailsView(tableForm); + detailsView.setFrame(WebPartView.FrameType.PORTAL); + detailsView.getDataRegion().setButtonBar(bb); + + VBox view = new VBox(detailsView); + + DetailsURL detailsURL = QueryService.get().getAuditDetailsURL(getUser(), getContainer(), _table); + + if (detailsURL != null) + { + String url = detailsURL.eval(tableForm.getTypedValues()); + if (url != null) + { + ActionURL auditURL = new ActionURL(url); + + QueryView historyView = QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), + auditURL.getParameter(QueryParam.schemaName), + auditURL.getParameter(QueryParam.queryName), + auditURL.getParameter("keyValue"), errors); + + if (null != historyView) + { + historyView.setFrame(WebPartView.FrameType.PORTAL); + historyView.setTitle("History"); + + view.addView(historyView); + } + } + } + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + return false; + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Details"); + } + } + + @RequiresPermission(InsertPermission.class) + public static class InsertQueryRowAction extends UserSchemaAction + { + @Override + public BindException bindParameters(PropertyValues m) throws Exception + { + BindException bind = super.bindParameters(m); + + // what is going on with UserSchemaAction and form binding? Why doesn't successUrl bind? + QueryUpdateForm form = (QueryUpdateForm)bind.getTarget(); + if (null == form.getSuccessUrl() && null != m.getPropertyValue(ActionURL.Param.successUrl.name())) + form.setSuccessUrl(new ReturnURLString(m.getPropertyValue(ActionURL.Param.successUrl.name()).getValue().toString())); + return bind; + } + + Map insertedRow = null; + + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Insert Row"); + + InsertView view = new InsertView(tableForm, errors); + view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + List> list = doInsertUpdate(tableForm, errors, true); + if (null != list && list.size() == 1) + insertedRow = list.get(0); + return 0 == errors.getErrorCount(); + } + + /** + * NOTE: UserSchemaAction.addNavTrail() uses this method getSuccessURL() for the nav trail link (form==null). + * It is used for where to go on success, and also as a "back" link in the nav trail + * If there is a setSuccessUrl specified, we will use that for successful submit + */ + @Override + public ActionURL getSuccessURL(QueryUpdateForm form) + { + if (null == form) + return super.getSuccessURL(null); + + String str = null; + if (form.getSuccessUrl() != null) + str = form.getSuccessUrl().toString(); + if (isBlank(str)) + str = form.getReturnUrl(); + + if ("details.view".equals(str)) + { + if (null == insertedRow) + return super.getSuccessURL(form); + StringExpression se = form.getTable().getDetailsURL(null, getContainer()); + if (null == se) + return super.getSuccessURL(form); + str = se.eval(insertedRow); + } + try + { + if (!isBlank(str)) + return new ActionURL(str); + } + catch (IllegalArgumentException x) + { + // pass + } + return super.getSuccessURL(form); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Insert " + _table.getName()); + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateQueryRowAction extends UserSchemaAction + { + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + ButtonBar bb = createSubmitCancelButtonBar(tableForm); + UpdateView view = new UpdateView(tableForm, errors); + view.getDataRegion().setButtonBar(bb); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception + { + doInsertUpdate(tableForm, errors, false); + return 0 == errors.getErrorCount(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Edit " + _table.getName()); + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateQueryRowsAction extends UpdateQueryRowAction + { + @Override + public ModelAndView handleRequest(QueryUpdateForm tableForm, BindException errors) throws Exception + { + tableForm.setBulkUpdate(true); + return super.handleRequest(tableForm, errors); + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception + { + boolean ret; + + if (tableForm.isDataSubmit()) + { + ret = super.handlePost(tableForm, errors); + if (ret) + DataRegionSelection.clearAll(getViewContext(), null); // in case we altered primary keys, see issue #35055 + return ret; + } + + return false; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Edit Multiple " + _table.getName()); + } + } + + // alias + public static class DeleteAction extends DeleteQueryRowsAction + { + } + + public abstract static class QueryViewAction extends SimpleViewAction + { + QueryForm _form; + QueryView _queryView; + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class APIQueryForm extends ContainerFilterQueryForm + { + private Integer _start; + private Integer _limit; + private boolean _includeDetailsColumn = false; + private boolean _includeUpdateColumn = false; + private boolean _includeTotalCount = true; + private boolean _includeStyle = false; + private boolean _includeDisplayValues = false; + private boolean _minimalColumns = true; + private boolean _includeMetadata = true; + + public Integer getStart() + { + return _start; + } + + public void setStart(Integer start) + { + _start = start; + } + + public Integer getLimit() + { + return _limit; + } + + public void setLimit(Integer limit) + { + _limit = limit; + } + + public boolean isIncludeTotalCount() + { + return _includeTotalCount; + } + + public void setIncludeTotalCount(boolean includeTotalCount) + { + _includeTotalCount = includeTotalCount; + } + + public boolean isIncludeStyle() + { + return _includeStyle; + } + + public void setIncludeStyle(boolean includeStyle) + { + _includeStyle = includeStyle; + } + + public boolean isIncludeDetailsColumn() + { + return _includeDetailsColumn; + } + + public void setIncludeDetailsColumn(boolean includeDetailsColumn) + { + _includeDetailsColumn = includeDetailsColumn; + } + + public boolean isIncludeUpdateColumn() + { + return _includeUpdateColumn; + } + + public void setIncludeUpdateColumn(boolean includeUpdateColumn) + { + _includeUpdateColumn = includeUpdateColumn; + } + + public boolean isIncludeDisplayValues() + { + return _includeDisplayValues; + } + + public void setIncludeDisplayValues(boolean includeDisplayValues) + { + _includeDisplayValues = includeDisplayValues; + } + + public boolean isMinimalColumns() + { + return _minimalColumns; + } + + public void setMinimalColumns(boolean minimalColumns) + { + _minimalColumns = minimalColumns; + } + + public boolean isIncludeMetadata() + { + return _includeMetadata; + } + + public void setIncludeMetadata(boolean includeMetadata) + { + _includeMetadata = includeMetadata; + } + + @Override + protected QuerySettings createQuerySettings(UserSchema schema) + { + QuerySettings results = super.createQuerySettings(schema); + + // See dataintegration/202: The java client api / remote ETL calls selectRows with showRows=all. We need to test _initParameters to properly read this + boolean missingShowRows = null == getViewContext().getRequest().getParameter(getDataRegionName() + "." + QueryParam.showRows) && null == _initParameters.getPropertyValue(getDataRegionName() + "." + QueryParam.showRows); + if (null == getLimit() && !results.isMaxRowsSet() && missingShowRows) + { + results.setShowRows(ShowRows.PAGINATED); + results.setMaxRows(DEFAULT_API_MAX_ROWS); + } + + if (getLimit() != null) + { + results.setShowRows(ShowRows.PAGINATED); + results.setMaxRows(getLimit()); + } + if (getStart() != null) + results.setOffset(getStart()); + + return results; + } + } + + public static final int DEFAULT_API_MAX_ROWS = 100000; + + @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 + @ActionNames("selectRows, getQuery") + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.1) + @Action(ActionType.SelectData.class) + public class SelectRowsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(APIQueryForm form, BindException errors) + { + // Issue 12233: add implicit maxRows=100k when using client API + QueryView view = form.getQueryView(); + + view.setShowPagination(form.isIncludeTotalCount()); + + //if viewName was specified, ensure that it was actually found and used + //QueryView.create() will happily ignore an invalid view name and just return the default view + if (null != StringUtils.trimToNull(form.getViewName()) && + null == view.getQueryDef().getCustomView(getUser(), getViewContext().getRequest(), form.getViewName())) + { + throw new NotFoundException("The requested view '" + form.getViewName() + "' does not exist for this user."); + } + + TableInfo t = view.getTable(); + if (null == t) + { + List qpes = view.getParseErrors(); + if (!qpes.isEmpty()) + throw qpes.get(0); + throw new NotFoundException(form.getQueryName()); + } + + boolean isEditable = isQueryEditable(view.getTable()); + boolean metaDataOnly = form.getQuerySettings().getMaxRows() == 0; + boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; + boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; + + ApiQueryResponse response; + + // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. + if (getRequestedApiVersion() >= 13.2) + { + ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, true, view.getQueryDef().getName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); + fancyResponse.includeFormattedValue(includeFormattedValue); + response = fancyResponse; + } + //if requested version is >= 9.1, use the extended api query response + else if (getRequestedApiVersion() >= 9.1) + { + response = new ExtendedApiQueryResponse(view, isEditable, true, + form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + } + else + { + response = new ApiQueryResponse(view, isEditable, true, + form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), + form.isIncludeDisplayValues(), form.isIncludeMetadata()); + } + response.includeStyle(form.isIncludeStyle()); + + // Issues 29515 and 32269 - force key and other non-requested columns to be sent back, but only if the client has + // requested minimal columns, as we now do for ExtJS stores + if (form.isMinimalColumns()) + { + // Be sure to use the settings from the view, as it may have swapped it out with a customized version. + // See issue 38747. + response.setColumnFilter(view.getSettings().getFieldKeys()); + } + + return response; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class GetDataAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + ObjectMapper mapper = JsonUtil.createDefaultMapper(); + mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + JSONObject object = form.getJsonObject(); + if (object == null) + { + object = new JSONObject(); + } + DataRequest builder = mapper.readValue(object.toString(), DataRequest.class); + + return builder.render(getViewContext(), errors); + } + } + + protected boolean isQueryEditable(TableInfo table) + { + if (!getContainer().hasPermission("isQueryEditable", getUser(), DeletePermission.class)) + return false; + QueryUpdateService updateService = null; + try + { + updateService = table.getUpdateService(); + } + catch(Exception ignore) {} + return null != table && null != updateService; + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExecuteSqlForm extends APIQueryForm + { + private String _sql; + private Integer _maxRows; + private Integer _offset; + private boolean _saveInSession; + + public String getSql() + { + return _sql; + } + + public void setSql(String sql) + { + _sql = PageFlowUtil.wafDecode(StringUtils.trim(sql)); + } + + public Integer getMaxRows() + { + return _maxRows; + } + + public void setMaxRows(Integer maxRows) + { + _maxRows = maxRows; + } + + public Integer getOffset() + { + return _offset; + } + + public void setOffset(Integer offset) + { + _offset = offset; + } + + @Override + public void setLimit(Integer limit) + { + _maxRows = limit; + } + + @Override + public void setStart(Integer start) + { + _offset = start; + } + + public boolean isSaveInSession() + { + return _saveInSession; + } + + public void setSaveInSession(boolean saveInSession) + { + _saveInSession = saveInSession; + } + + @Override + public String getQueryName() + { + // ExecuteSqlAction doesn't allow setting query name parameter. + return null; + } + + @Override + public void setQueryName(String name) + { + // ExecuteSqlAction doesn't allow setting query name parameter. + } + } + + @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.1) + @Action(ActionType.SelectData.class) + public class ExecuteSqlAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(ExecuteSqlForm form, BindException errors) + { + form.ensureSchemaExists(); + + String schemaName = StringUtils.trimToNull(form.getQuerySettings().getSchemaName()); + if (null == schemaName) + throw new IllegalArgumentException("No value was supplied for the required parameter 'schemaName'."); + String sql = form.getSql(); + if (StringUtils.isBlank(sql)) + throw new IllegalArgumentException("No value was supplied for the required parameter 'sql'."); + + //create a temp query settings object initialized with the posted LabKey SQL + //this will provide a temporary QueryDefinition to Query + QuerySettings settings = form.getQuerySettings(); + if (form.isSaveInSession()) + { + HttpSession session = getViewContext().getSession(); + if (session == null) + throw new IllegalStateException("Session required"); + + QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), schemaName, sql); + settings.setDataRegionName("executeSql"); + settings.setQueryName(def.getName()); + } + else + { + settings = new TempQuerySettings(getViewContext(), sql, settings); + } + + //need to explicitly turn off various UI options that will try to refer to the + //current URL and query string + settings.setAllowChooseView(false); + settings.setAllowCustomizeView(false); + + // Issue 12233: add implicit maxRows=100k when using client API + settings.setShowRows(ShowRows.PAGINATED); + settings.setMaxRows(DEFAULT_API_MAX_ROWS); + + // 16961: ExecuteSql API without maxRows parameter defaults to returning 100 rows + //apply optional settings (maxRows, offset) + boolean metaDataOnly = false; + if (null != form.getMaxRows() && (form.getMaxRows() >= 0 || form.getMaxRows() == Table.ALL_ROWS)) + { + settings.setMaxRows(form.getMaxRows()); + metaDataOnly = Table.NO_ROWS == form.getMaxRows(); + } + + int offset = 0; + if (null != form.getOffset()) + { + settings.setOffset(form.getOffset().longValue()); + offset = form.getOffset(); + } + + //build a query view using the schema and settings + QueryView view = new QueryView(form.getSchema(), settings, errors); + view.setShowRecordSelectors(false); + view.setShowExportButtons(false); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + view.setShowPagination(form.isIncludeTotalCount()); + + TableInfo t = view.getTable(); + boolean isEditable = null != t && isQueryEditable(view.getTable()); + boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; + boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; + + ApiQueryResponse response; + + // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. + if (getRequestedApiVersion() >= 13.2) + { + ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, false, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); + fancyResponse.includeFormattedValue(includeFormattedValue); + response = fancyResponse; + } + else if (getRequestedApiVersion() >= 9.1) + { + response = new ExtendedApiQueryResponse(view, isEditable, + false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + } + else + { + response = new ApiQueryResponse(view, isEditable, + false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), + form.isIncludeDisplayValues()); + } + response.includeStyle(form.isIncludeStyle()); + + return response; + } + } + + public static class ContainerFilterQueryForm extends QueryForm + { + private String _containerFilter; + + public String getContainerFilter() + { + return _containerFilter; + } + + public void setContainerFilter(String containerFilter) + { + _containerFilter = containerFilter; + } + + @Override + protected QuerySettings createQuerySettings(UserSchema schema) + { + var result = super.createQuerySettings(schema); + if (getContainerFilter() != null) + { + // If the user specified an incorrect filter, throw an IllegalArgumentException + try + { + ContainerFilter.Type containerFilterType = ContainerFilter.Type.valueOf(getContainerFilter()); + result.setContainerFilterName(containerFilterType.name()); + } + catch (IllegalArgumentException e) + { + // Remove bogus value from error message, Issue 45567 + throw new IllegalArgumentException("'containerFilter' parameter is not valid"); + } + } + return result; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public class SelectDistinctAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(ContainerFilterQueryForm form, BindException errors) throws Exception + { + TableInfo table = form.getQueryView().getTable(); + if (null == table) + throw new NotFoundException(); + SqlSelector sqlSelector = getDistinctSql(table, form, errors); + + if (errors.hasErrors() || null == sqlSelector) + return null; + + ApiResponseWriter writer = new ApiJsonWriter(getViewContext().getResponse()); + + try (ResultSet rs = sqlSelector.getResultSet()) + { + writer.startResponse(); + writer.writeProperty("schemaName", form.getSchemaName()); + writer.writeProperty("queryName", form.getQueryName()); + writer.startList("values"); + + while (rs.next()) + { + writer.writeListEntry(rs.getObject(1)); + } + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + catch (DataAccessException x) // Spring error translator can return various subclasses of this + { + throw new RuntimeException(x); + } + writer.endList(); + writer.endResponse(); + + return null; + } + + @Nullable + private SqlSelector getDistinctSql(TableInfo table, ContainerFilterQueryForm form, BindException errors) + { + QuerySettings settings = form.getQuerySettings(); + QueryService service = QueryService.get(); + + if (null == getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())) + { + settings.setMaxRows(DEFAULT_API_MAX_ROWS); + } + else + { + try + { + int maxRows = Integer.parseInt(getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())); + settings.setMaxRows(maxRows); + } + catch (NumberFormatException e) + { + // Standard exception message, Issue 45567 + QuerySettings.throwParameterParseException(QueryParam.maxRows); + } + } + + List fieldKeys = settings.getFieldKeys(); + if (null == fieldKeys || fieldKeys.size() != 1) + { + errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); + return null; + } + Map columns = service.getColumns(table, fieldKeys); + if (columns.size() != 1) + { + errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); + return null; + } + + ColumnInfo col = columns.get(settings.getFieldKeys().get(0)); + if (col == null) + { + errors.reject(ERROR_MSG, "\"" + settings.getFieldKeys().get(0).getName() + "\" is not a valid column."); + return null; + } + + try + { + SimpleFilter filter = getFilterFromQueryForm(form); + + // Strip out filters on columns that don't exist - issue 21669 + service.ensureRequiredColumns(table, columns.values(), filter, null, new HashSet<>()); + QueryLogging queryLogging = new QueryLogging(); + QueryService.SelectBuilder builder = service.getSelectBuilder(table) + .columns(columns.values()) + .filter(filter) + .queryLogging(queryLogging) + .distinct(true); + SQLFragment selectSql = builder.buildSqlFragment(); + + // TODO: queryLogging.isShouldAudit() is always false at this point. + // The only place that seems to set this is ComplianceQueryLoggingProfileListener.queryInvoked() + if (queryLogging.isShouldAudit() && null != queryLogging.getExceptionToThrowIfLoggingIsEnabled()) + { + // this is probably a more helpful message + errors.reject(ERROR_MSG, "Cannot choose values from a column that requires logging."); + return null; + } + + // Regenerate the column since the alias may have changed after call to getSelectSQL() + columns = service.getColumns(table, settings.getFieldKeys()); + var colGetAgain = columns.get(settings.getFieldKeys().get(0)); + // I don't believe the above comment, so here's an assert + assert(colGetAgain.getAlias().equals(col.getAlias())); + + SQLFragment sql = new SQLFragment("SELECT ").appendIdentifier(col.getAlias()).append(" AS value FROM ("); + sql.append(selectSql); + sql.append(") S ORDER BY value"); + + sql = table.getSqlDialect().limitRows(sql, settings.getMaxRows()); + + // 18875: Support Parameterized queries in Select Distinct + Map _namedParameters = settings.getQueryParameters(); + + service.bindNamedParameters(sql, _namedParameters); + service.validateNamedParameters(sql); + + return new SqlSelector(table.getSchema().getScope(), sql, queryLogging); + } + catch (ConversionException | QueryService.NamedParameterNotProvided e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return null; + } + } + } + + private SimpleFilter getFilterFromQueryForm(QueryForm form) + { + QuerySettings settings = form.getQuerySettings(); + SimpleFilter filter = null; + + // 21032: Respect 'ignoreFilter' + if (settings != null && !settings.getIgnoreUserFilter()) + { + // Attach any URL-based filters. This would apply to 'filterArray' from the JavaScript API. + filter = new SimpleFilter(settings.getBaseFilter()); + + String dataRegionName = form.getDataRegionName(); + if (StringUtils.trimToNull(dataRegionName) == null) + dataRegionName = QueryView.DATAREGIONNAME_DEFAULT; + + // Support for 'viewName' + CustomView view = settings.getCustomView(getViewContext(), form.getQueryDef()); + if (null != view && view.hasFilterOrSort() && !settings.getIgnoreViewFilter()) + { + ActionURL url = new ActionURL(SelectDistinctAction.class, getContainer()); + view.applyFilterAndSortToURL(url, dataRegionName); + filter.addAllClauses(new SimpleFilter(url, dataRegionName)); + } + + filter.addUrlFilters(settings.getSortFilterURL(), dataRegionName, Collections.emptyList(), getUser(), getContainer()); + } + + return filter; + } + + @RequiresPermission(ReadPermission.class) + public class GetColumnSummaryStatsAction extends ReadOnlyApiAction + { + private FieldKey _colFieldKey; + + @Override + public void validateForm(QueryForm form, Errors errors) + { + QuerySettings settings = form.getQuerySettings(); + List fieldKeys = settings != null ? settings.getFieldKeys() : null; + if (null == fieldKeys || fieldKeys.size() != 1) + errors.reject(ERROR_MSG, "GetColumnSummaryStats requires that only one column be requested."); + else + _colFieldKey = fieldKeys.get(0); + } + + @Override + public ApiResponse execute(QueryForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + QueryView view = form.getQueryView(); + DisplayColumn displayColumn = null; + + for (DisplayColumn dc : view.getDisplayColumns()) + { + if (dc.getColumnInfo() != null && _colFieldKey.equals(dc.getColumnInfo().getFieldKey())) + { + displayColumn = dc; + break; + } + } + + if (displayColumn != null && displayColumn.getColumnInfo() != null) + { + // get the map of the analytics providers to their relevant aggregates and add the information to the response + Map> analyticsProviders = new LinkedHashMap<>(); + Set colAggregates = new HashSet<>(); + for (ColumnAnalyticsProvider analyticsProvider : displayColumn.getAnalyticsProviders()) + { + if (analyticsProvider instanceof BaseAggregatesAnalyticsProvider baseAggProvider) + { + Map props = new HashMap<>(); + props.put("label", baseAggProvider.getLabel()); + + List aggregateNames = new ArrayList<>(); + for (Aggregate aggregate : AnalyticsProviderItem.createAggregates(baseAggProvider, _colFieldKey, null)) + { + aggregateNames.add(aggregate.getType().getName()); + colAggregates.add(aggregate); + } + props.put("aggregates", aggregateNames); + + analyticsProviders.put(baseAggProvider.getName(), props); + } + } + + // get the filter set from the queryform and verify that they resolve + SimpleFilter filter = getFilterFromQueryForm(form); + if (filter != null) + { + Map resolvedCols = QueryService.get().getColumns(view.getTable(), filter.getAllFieldKeys()); + for (FieldKey filterFieldKey : filter.getAllFieldKeys()) + { + if (!resolvedCols.containsKey(filterFieldKey)) + filter.deleteConditions(filterFieldKey); + } + } + + // query the table/view for the aggregate results + Collection columns = Collections.singleton(displayColumn.getColumnInfo()); + TableSelector selector = new TableSelector(view.getTable(), columns, filter, null).setNamedParameters(form.getQuerySettings().getQueryParameters()); + Map> aggResults = selector.getAggregates(new ArrayList<>(colAggregates)); + + // create a response object mapping the analytics providers to their relevant aggregate results + Map> aggregateResults = new HashMap<>(); + if (aggResults.containsKey(_colFieldKey.toString())) + { + for (Aggregate.Result r : aggResults.get(_colFieldKey.toString())) + { + Map props = new HashMap<>(); + Aggregate.Type type = r.getAggregate().getType(); + props.put("label", type.getFullLabel()); + props.put("description", type.getDescription()); + props.put("value", r.getFormattedValue(displayColumn, getContainer()).value()); + aggregateResults.put(type.getName(), props); + } + + response.put("success", true); + response.put("analyticsProviders", analyticsProviders); + response.put("aggregateResults", aggregateResults); + } + else + { + response.put("success", false); + response.put("message", "Unable to get aggregate results for " + _colFieldKey); + } + } + else + { + response.put("success", false); + response.put("message", "Unable to find ColumnInfo for " + _colFieldKey); + } + + return response; + } + } + + @RequiresPermission(ReadPermission.class) + public class ImportAction extends AbstractQueryImportAction + { + private QueryForm _form; + + @Override + protected void initRequest(QueryForm form) throws ServletException + { + _form = form; + + _insertOption = form.getInsertOption(); + QueryDefinition query = form.getQueryDef(); + List qpe = new ArrayList<>(); + TableInfo t = query.getTable(form.getSchema(), qpe, true); + if (!qpe.isEmpty()) + throw qpe.get(0); + if (null != t) + setTarget(t); + _auditBehaviorType = form.getAuditBehavior(); + _auditUserComment = form.getAuditUserComment(); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + initRequest(form); + return super.getDefaultImportView(form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + var executeQuery = _form.urlFor(QueryAction.executeQuery); + if (null == executeQuery) + root.addChild(_form.getQueryName()); + else + root.addChild(_form.getQueryName(), executeQuery); + root.addChild("Import Data"); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportSqlForm + { + private String _sql; + private String _schemaName; + private String _containerFilter; + private String _format = "excel"; + + public String getSql() + { + return _sql; + } + + public void setSql(String sql) + { + _sql = PageFlowUtil.wafDecode(sql); + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getContainerFilter() + { + return _containerFilter; + } + + public void setContainerFilter(String containerFilter) + { + _containerFilter = containerFilter; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.2) + @Action(ActionType.Export.class) + public static class ExportSqlAction extends ExportAction + { + @Override + public void export(ExportSqlForm form, HttpServletResponse response, BindException errors) throws IOException, ExportException + { + String schemaName = StringUtils.trimToNull(form.getSchemaName()); + if (null == schemaName) + throw new NotFoundException("No value was supplied for the required parameter 'schemaName'"); + String sql = StringUtils.trimToNull(form.getSql()); + if (null == sql) + throw new NotFoundException("No value was supplied for the required parameter 'sql'"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + + if (null == schema) + throw new NotFoundException("Schema '" + schemaName + "' not found in this folder"); + + //create a temp query settings object initialized with the posted LabKey SQL + //this will provide a temporary QueryDefinition to Query + TempQuerySettings settings = new TempQuerySettings(getViewContext(), sql); + + //need to explicitly turn off various UI options that will try to refer to the + //current URL and query string + settings.setAllowChooseView(false); + settings.setAllowCustomizeView(false); + + //return all rows + settings.setShowRows(ShowRows.ALL); + + //add container filter if supplied + if (form.getContainerFilter() != null && !form.getContainerFilter().isEmpty()) + { + ContainerFilter.Type containerFilterType = + ContainerFilter.Type.valueOf(form.getContainerFilter()); + settings.setContainerFilterName(containerFilterType.name()); + } + + //build a query view using the schema and settings + QueryView view = new QueryView(schema, settings, errors); + view.setShowRecordSelectors(false); + view.setShowExportButtons(false); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + + //export it + ResponseHelper.setPrivate(response); + response.setHeader("X-Robots-Tag", "noindex"); + + if ("excel".equalsIgnoreCase(form.getFormat())) + view.exportToExcel(response); + else if ("tsv".equalsIgnoreCase(form.getFormat())) + view.exportToTsv(response); + else + errors.reject(null, "Invalid format specified; must be 'excel' or 'tsv'"); + + for (QueryException qe : view.getParseErrors()) + errors.reject(null, qe.getMessage()); + + if (errors.hasErrors()) + throw new ExportException(new SimpleErrorView(errors, false)); + } + } + + public static class ApiSaveRowsForm extends SimpleApiJsonForm + { + } + + private enum CommandType + { + insert(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException + { + BatchValidationException errors = new BatchValidationException(); + List> insertedRows = qus.insertRows(user, container, rows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + // Issue 42519: Submitter role not able to insert + // as per the definition of submitter, should allow insert without read + if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) + { + return qus.getRows(user, container, insertedRows); + } + else + { + return insertedRows; + } + } + }, + insertWithKeys(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException + { + List> newRows = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + for (Map row : rows) + { + //issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null + CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); + newRows.add(newMap); + + CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); + oldKeys.add(oldMap); + } + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.insertRows(user, container, newRows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + // Issue 42519: Submitter role not able to insert + // as per the definition of submitter, should allow insert without read + if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) + { + updatedRows = qus.getRows(user, container, updatedRows); + } + List> results = new ArrayList<>(); + for (int i = 0; i < updatedRows.size(); i++) + { + Map result = new HashMap<>(); + result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); + result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); + results.add(result); + } + return results; + } + }, + importRows(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + DataIteratorBuilder it = new ListofMapsDataIterator.Builder(rows.get(0).keySet(), rows); + qus.importRows(user, container, it, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return Collections.emptyList(); + } + }, + moveRows(MoveEntitiesPermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + + Container targetContainer = (Container) configParameters.get(QueryUpdateService.ConfigParameters.TargetContainer); + Map updatedCounts = qus.moveRows(user, container, targetContainer, rows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return Collections.singletonList(updatedCounts); + } + }, + update(UpdatePermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.updateRows(user, container, rows, null, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return shouldReselect(configParameters) ? qus.getRows(user, container, updatedRows) : updatedRows; + } + }, + updateChangingKeys(UpdatePermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + List> newRows = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + for (Map row : rows) + { + // issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null. + // this should never happen on an update, but we will let it fail later with a better error message instead of the NPE here + CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); + newRows.add(newMap); + + CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); + oldKeys.add(oldMap); + } + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.updateRows(user, container, newRows, oldKeys, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + if (shouldReselect(configParameters)) + updatedRows = qus.getRows(user, container, updatedRows); + List> results = new ArrayList<>(); + for (int i = 0; i < updatedRows.size(); i++) + { + Map result = new HashMap<>(); + result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); + result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); + results.add(result); + } + return results; + } + }, + delete(DeletePermission.class, QueryService.AuditAction.DELETE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + return qus.deleteRows(user, container, rows, configParameters, extraContext); + } + }; + + private final Class _permission; + private final QueryService.AuditAction _auditAction; + + CommandType(Class permission, QueryService.AuditAction auditAction) + { + _permission = permission; + _auditAction = auditAction; + } + + public Class getPermission() + { + return _permission; + } + + public QueryService.AuditAction getAuditAction() + { + return _auditAction; + } + + public static boolean shouldReselect(Map configParameters) + { + if (configParameters == null || !configParameters.containsKey(QueryUpdateService.ConfigParameters.SkipReselectRows)) + return true; + + return Boolean.TRUE != configParameters.get(QueryUpdateService.ConfigParameters.SkipReselectRows); + } + + public abstract List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException; + } + + /** + * Base action class for insert/update/delete actions + */ + protected abstract static class BaseSaveRowsAction
extends MutatingApiAction + { + public static final String PROP_SCHEMA_NAME = "schemaName"; + public static final String PROP_QUERY_NAME = "queryName"; + public static final String PROP_CONTAINER_PATH = "containerPath"; + public static final String PROP_TARGET_CONTAINER_PATH = "targetContainerPath"; + public static final String PROP_COMMAND = "command"; + public static final String PROP_ROWS = "rows"; + + private JSONObject _json; + + @Override + public void validateForm(FORM apiSaveRowsForm, Errors errors) + { + _json = apiSaveRowsForm.getJsonObject(); + + // if the POST was done using FormData, the apiSaveRowsForm would not have bound the json data, so + // we'll instead look for that data in the request param directly + if (_json == null && getViewContext().getRequest() != null && getViewContext().getRequest().getParameter("json") != null) + _json = new JSONObject(getViewContext().getRequest().getParameter("json")); + } + + protected JSONObject getJsonObject() + { + return _json; + } + + protected Container getContainerForCommand(JSONObject json) + { + return getContainerForCommand(json, PROP_CONTAINER_PATH, getContainer()); + } + + protected Container getContainerForCommand(JSONObject json, String containerPathProp, @Nullable Container defaultContainer) + { + Container container; + String containerPath = StringUtils.trimToNull(json.optString(containerPathProp)); + if (containerPath == null) + { + if (defaultContainer != null) + container = defaultContainer; + else + throw new IllegalArgumentException(containerPathProp + " is required but was not provided."); + } + else + { + container = ContainerManager.getForPath(containerPath); + if (container == null) + { + throw new IllegalArgumentException("Unknown container: " + containerPath); + } + } + + // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more downstream + if (!container.hasPermission(getUser(), ReadPermission.class) && + !container.hasPermission(getUser(), DeletePermission.class) && + !container.hasPermission(getUser(), InsertPermission.class) && + !container.hasPermission(getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + return container; + } + + protected String getTargetContainerProp() + { + JSONObject json = getJsonObject(); + return json.optString(PROP_TARGET_CONTAINER_PATH, null); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors) throws Exception + { + return executeJson(json, commandType, allowTransaction, errors, false); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction) throws Exception + { + return executeJson(json, commandType, allowTransaction, errors, isNestedTransaction, null); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction, @Nullable Integer commandIndex) throws Exception + { + JSONObject response = new JSONObject(); + Container container = getContainerForCommand(json); + User user = getUser(); + + if (json == null) + throw new ValidationException("Empty request"); + + JSONArray rows; + try + { + rows = json.getJSONArray(PROP_ROWS); + if (rows.isEmpty()) + throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); + } + catch (JSONException x) + { + throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); + } + + String schemaName = json.getString(PROP_SCHEMA_NAME); + String queryName = json.getString(PROP_QUERY_NAME); + TableInfo table = getTableInfo(container, user, schemaName, queryName); + + if (!table.hasPermission(user, commandType.getPermission())) + throw new UnauthorizedException(); + + if (commandType != CommandType.insert && table.getPkColumns().isEmpty()) + throw new IllegalArgumentException("The table '" + table.getPublicSchemaName() + "." + + table.getPublicName() + "' cannot be updated because it has no primary key defined!"); + + QueryUpdateService qus = table.getUpdateService(); + if (null == qus) + throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + + "' is not updatable via the HTTP-based APIs."); + + int rowsAffected = 0; + + List> rowsToProcess = new ArrayList<>(); + + // NOTE RowMapFactory is faster, but for update it's important to preserve missing v explicit NULL values + // Do we need to support some sort of UNDEFINED and NULL instance of MvFieldWrapper? + RowMapFactory f = null; + if (commandType == CommandType.insert || commandType == CommandType.insertWithKeys || commandType == CommandType.delete) + f = new RowMapFactory<>(); + CaseInsensitiveHashMap referenceCasing = new CaseInsensitiveHashMap<>(); + + for (int idx = 0; idx < rows.length(); ++idx) + { + JSONObject jsonObj; + try + { + jsonObj = rows.getJSONObject(idx); + } + catch (JSONException x) + { + throw new IllegalArgumentException("rows[" + idx + "] is not an object."); + } + if (null != jsonObj) + { + Map rowMap = null == f ? new CaseInsensitiveHashMap<>(new HashMap<>(), referenceCasing) : f.getRowMap(); + // Use shallow copy since jsonObj.toMap() will translate contained JSONObjects into Maps, which we don't want + boolean conflictingCasing = JsonUtil.fillMapShallow(jsonObj, rowMap); + if (conflictingCasing) + { + // Issue 52616 + LOG.error("Row contained conflicting casing for key names in the incoming row: {}", jsonObj); + } + if (allowRowAttachments()) + addRowAttachments(table, rowMap, idx, commandIndex); + + rowsToProcess.add(rowMap); + rowsAffected++; + } + } + + Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); + + Map configParameters = new HashMap<>(); + + // Check first if the audit behavior has been defined for the table either in code or through XML. + // If not defined there, check for the audit behavior defined in the action form (json). + AuditBehaviorType behaviorType = table.getEffectiveAuditBehavior(json.optString("auditBehavior", null)); + if (behaviorType != null) + { + configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, behaviorType); + String auditComment = json.optString("auditUserComment", null); + if (!StringUtils.isEmpty(auditComment)) + configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment, auditComment); + } + + boolean skipReselectRows = json.optBoolean("skipReselectRows", false); + if (skipReselectRows) + configParameters.put(QueryUpdateService.ConfigParameters.SkipReselectRows, true); + + if (getTargetContainerProp() != null) + { + Container targetContainer = getContainerForCommand(json, PROP_TARGET_CONTAINER_PATH, null); + configParameters.put(QueryUpdateService.ConfigParameters.TargetContainer, targetContainer); + } + + //set up the response, providing the schema name, query name, and operation + //so that the client can sort out which request this response belongs to + //(clients often submit these async) + response.put(PROP_SCHEMA_NAME, schemaName); + response.put(PROP_QUERY_NAME, queryName); + response.put("command", commandType.name()); + response.put("containerPath", container.getPath()); + + //we will transact operations by default, but the user may + //override this by sending a "transacted" property set to false + // 11741: A transaction may already be active if we're trying to + // insert/update/delete from within a transformation/validation script. + boolean transacted = allowTransaction && json.optBoolean("transacted", true); + TransactionAuditProvider.TransactionAuditEvent auditEvent = null; + try (DbScope.Transaction transaction = transacted ? table.getSchema().getScope().ensureTransaction() : NO_OP_TRANSACTION) + { + if (behaviorType != null && behaviorType != AuditBehaviorType.NONE) + { + DbScope.Transaction auditTransaction = !transacted && isNestedTransaction ? table.getSchema().getScope().getCurrentTransaction() : transaction; + if (auditTransaction == null) + auditTransaction = NO_OP_TRANSACTION; + + if (auditTransaction.getAuditEvent() != null) + { + auditEvent = auditTransaction.getAuditEvent(); + // detect trigger event? + } + else + { + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), getTransactionAuditDetails()); + AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); + } + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.QueryCommand, commandType.name()); + } + + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); + List> responseRows = + commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); + if (auditEvent != null) + { + auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); + if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.BatchAction))) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true); + } + + if (commandType == CommandType.moveRows) + { + // moveRows returns a single map of updateCounts + response.put("updateCounts", responseRows.get(0)); + } + else if (commandType != CommandType.importRows) + { + response.put("rows", responseRows.stream() + .map(JsonUtil::toMapPreserveNonFinite) + .map(JsonUtil::toJsonPreserveNulls) + .collect(LabKeyCollectors.toJSONArray())); + } + + // if there is any provenance information, save it here + ProvenanceService svc = ProvenanceService.get(); + if (json.has("provenance")) + { + JSONObject provenanceJSON = json.getJSONObject("provenance"); + ProvenanceRecordingParams params = svc.createRecordingParams(getViewContext(), provenanceJSON, ProvenanceService.ADD_RECORDING); + RecordedAction action = svc.createRecordedAction(getViewContext(), params); + if (action != null && params.getRecordingId() != null) + { + // check for any row level provenance information + if (json.has("rows")) + { + Object rowObject = json.get("rows"); + if (rowObject instanceof JSONArray jsonArray) + { + // we need to match any provenance object inputs to the object outputs from the response rows, this typically would + // be the row lsid but it configurable in the provenance recording params + // + List> provenanceMap = svc.createProvenanceMapFromRows(getViewContext(), params, jsonArray, responseRows); + if (!provenanceMap.isEmpty()) + { + action.getProvenanceMap().addAll(provenanceMap); + } + svc.addRecordingStep(getViewContext().getRequest(), params.getRecordingId(), action); + } + else + { + errors.reject(SpringActionController.ERROR_MSG, "Unable to process provenance information, the rows object was not an array"); + } + } + } + } + transaction.commit(); + } + catch (OptimisticConflictException e) + { + //issue 13967: provide better message for OptimisticConflictException + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + catch (QueryUpdateServiceException | ConversionException | DuplicateKeyException | DataIntegrityViolationException e) + { + //Issue 14294: improve handling of ConversionException (and DuplicateKeyException (Issue 28037), and DataIntegrity (uniqueness) (Issue 22779) + errors.reject(SpringActionController.ERROR_MSG, e.getMessage() == null ? e.toString() : e.getMessage()); + } + catch (BatchValidationException e) + { + if (isSuccessOnValidationError()) + { + response.put("errors", createResponseWriter().toJSON(e)); + } + else + { + ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + throw e; + } + } + if (auditEvent != null) + { + response.put("transactionAuditId", auditEvent.getRowId()); + response.put("reselectRowCount", auditEvent.hasMultiActions()); + } + + response.put("rowsAffected", rowsAffected); + + return response; + } + + protected boolean allowRowAttachments() + { + return false; + } + + private void addRowAttachments(TableInfo tableInfo, Map rowMap, int rowIndex, @Nullable Integer commandIndex) + { + if (getFileMap() != null) + { + for (Map.Entry fileEntry : getFileMap().entrySet()) + { + // Allow for the fileMap key to include the row index, and optionally command index, for defining + // which row to attach this file to + String fullKey = fileEntry.getKey(); + String fieldKey = fullKey; + // Issue 52827: Cannot attach a file if the field name contains :: + // use lastIndexOf instead of split to get the proper parts + int lastDelimIndex = fullKey.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); + if (lastDelimIndex > -1) + { + String fieldKeyExcludeIndex = fullKey.substring(0, lastDelimIndex); + String fieldRowIndex = fullKey.substring(lastDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); + if (!fieldRowIndex.equals(rowIndex+"")) continue; + + if (commandIndex == null) + { + // Single command, so we're parsing file names in the format of: FileField::0 + fieldKey = fieldKeyExcludeIndex; + } + else + { + // Multi-command, so we're parsing file names in the format of: FileField::0::1 + int subDelimIndex = fieldKeyExcludeIndex.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); + if (subDelimIndex > -1) + { + fieldKey = fieldKeyExcludeIndex.substring(0, subDelimIndex); + String fieldCommandIndex = fieldKeyExcludeIndex.substring(subDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); + if (!fieldCommandIndex.equals(commandIndex+"")) + continue; + } + else + continue; + } + } + + SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); + rowMap.put(fieldKey, file.isEmpty() ? null : file); + } + } + + for (ColumnInfo col : tableInfo.getColumns()) + DataIteratorUtil.MatchType.multiPartFormData.updateRowMap(col, rowMap); + } + + protected boolean isSuccessOnValidationError() + { + return getRequestedApiVersion() >= 13.2; + } + + @NotNull + protected TableInfo getTableInfo(Container container, User user, String schemaName, String queryName) + { + if (null == schemaName || null == queryName) + throw new IllegalArgumentException("You must supply a schemaName and queryName!"); + + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); + if (null == schema) + throw new IllegalArgumentException("The schema '" + schemaName + "' does not exist."); + + TableInfo table = schema.getTableForInsert(queryName); + if (table == null) + throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); + return table; + } + } + + // Issue: 20522 - require read access to the action but executeJson will check for update privileges from the table + // + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class UpdateRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.update, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + } + + @RequiresAnyOf({ReadPermission.class, InsertPermission.class}) //will check below + @ApiVersion(8.3) + public static class InsertRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.insert, true, errors); + if (response == null || errors.hasErrors()) + return null; + + return new ApiSimpleResponse(response); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + } + + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class ImportRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.importRows, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + } + + @ActionNames("deleteRows, delRows") + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class DeleteRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.delete, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + } + + @RequiresPermission(ReadPermission.class) //will check below + public static class MoveRowsAction extends BaseSaveRowsAction + { + private Container _targetContainer; + + @Override + public void validateForm(MoveRowsForm form, Errors errors) + { + super.validateForm(form, errors); + + JSONObject json = getJsonObject(); + if (json == null) + { + errors.reject(ERROR_GENERIC, "Empty request"); + } + else + { + // Since we are moving between containers, we know we have product folders enabled + if (getContainer().getProject().getAuditCommentsRequired() && StringUtils.isBlank(json.optString("auditUserComment"))) + errors.reject(ERROR_GENERIC, "A reason for the move of data is required."); + else + { + String queryName = json.optString(PROP_QUERY_NAME, null); + _targetContainer = ContainerManager.getMoveTargetContainer(queryName, getContainer(), getUser(), getTargetContainerProp(), errors); + } + } + } + + @Override + public ApiResponse execute(MoveRowsForm form, BindException errors) throws Exception + { + // if JSON does not have rows array, see if they were provided via selectionKey + if (!getJsonObject().has(PROP_ROWS)) + setRowsFromSelectionKey(form); + + JSONObject response = executeJson(getJsonObject(), CommandType.moveRows, true, errors); + if (response == null || errors.hasErrors()) + return null; + + updateSelections(form); + + response.put("success", true); + response.put("containerPath", _targetContainer.getPath()); + return new ApiSimpleResponse(response); + } + + private void updateSelections(MoveRowsForm form) + { + String selectionKey = form.getDataRegionSelectionKey(); + if (selectionKey != null) + { + Set rowIds = form.getIds(getViewContext(), false) + .stream().map(Object::toString).collect(Collectors.toSet()); + DataRegionSelection.setSelected(getViewContext(), selectionKey, rowIds, false); + + // if moving entities from a type, the selections from other selectionKeys in that container will + // possibly be holding onto invalid keys after the move, so clear them based on the containerPath and selectionKey suffix + String[] keyParts = selectionKey.split("|"); + if (keyParts.length > 1) + DataRegionSelection.clearRelatedByContainerPath(getViewContext(), keyParts[keyParts.length - 1]); + } + } + + private void setRowsFromSelectionKey(MoveRowsForm form) + { + Set rowIds = form.getIds(getViewContext(), false); // handle clear of selectionKey after move complete + + // convert rowIds to a JSONArray of JSONObjects with a single property "RowId" + JSONArray rows = new JSONArray(); + for (Long rowId : rowIds) + { + JSONObject row = new JSONObject(); + row.put("RowId", rowId); + rows.put(row); + } + getJsonObject().put(PROP_ROWS, rows); + } + } + + public static class MoveRowsForm extends ApiSaveRowsForm + { + private String _dataRegionSelectionKey; + private boolean _useSnapshotSelection; + + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + public void setDataRegionSelectionKey(String dataRegionSelectionKey) + { + _dataRegionSelectionKey = dataRegionSelectionKey; + } + + public boolean isUseSnapshotSelection() + { + return _useSnapshotSelection; + } + + public void setUseSnapshotSelection(boolean useSnapshotSelection) + { + _useSnapshotSelection = useSnapshotSelection; + } + + @Override + public void bindJson(JSONObject json) + { + super.bindJson(json); + _dataRegionSelectionKey = json.optString("dataRegionSelectionKey", null); + _useSnapshotSelection = json.optBoolean("useSnapshotSelection", false); + } + + public Set getIds(ViewContext context, boolean clear) + { + if (_useSnapshotSelection) + return new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(context, getDataRegionSelectionKey())); + else + return DataRegionSelection.getSelectedIntegers(context, getDataRegionSelectionKey(), clear); + } + } + + @RequiresNoPermission //will check below + public static class SaveRowsAction extends BaseSaveRowsAction + { + public static final String PROP_VALUES = "values"; + public static final String PROP_OLD_KEYS = "oldKeys"; + + @Override + protected boolean isFailure(BindException errors) + { + return !isSuccessOnValidationError() && super.isFailure(errors); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more + // specific permissions later once we've figured out exactly what they're trying to do. This helps us + // give a better HTTP response code when they're trying to access a resource that's not available to guests + if (!getContainer().hasPermission(getUser(), ReadPermission.class) && + !getContainer().hasPermission(getUser(), DeletePermission.class) && + !getContainer().hasPermission(getUser(), InsertPermission.class) && + !getContainer().hasPermission(getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + JSONObject json = getJsonObject(); + if (json == null) + throw new IllegalArgumentException("Empty request"); + + JSONArray commands = json.optJSONArray("commands"); + if (commands == null || commands.isEmpty()) + { + throw new NotFoundException("Empty request"); + } + + boolean validateOnly = json.optBoolean("validateOnly", false); + // If we are going to validate and not commit, we need to be sure we're transacted as well. Otherwise, + // respect the client's request. + boolean transacted = validateOnly || json.optBoolean("transacted", true); + + // Keep track of whether we end up committing or not + boolean committed = false; + + DbScope scope = null; + if (transacted) + { + for (int i = 0; i < commands.length(); i++) + { + JSONObject commandJSON = commands.getJSONObject(i); + String schemaName = commandJSON.getString(PROP_SCHEMA_NAME); + String queryName = commandJSON.getString(PROP_QUERY_NAME); + Container container = getContainerForCommand(commandJSON); + TableInfo tableInfo = getTableInfo(container, getUser(), schemaName, queryName); + if (scope == null) + { + scope = tableInfo.getSchema().getScope(); + } + else if (scope != tableInfo.getSchema().getScope()) + { + throw new IllegalArgumentException("All queries must be from the same source database"); + } + } + assert scope != null; + } + + JSONArray resultArray = new JSONArray(); + JSONObject extraContext = json.optJSONObject("extraContext"); + + int startingErrorIndex = 0; + int errorCount = 0; + // 11741: A transaction may already be active if we're trying to + // insert/update/delete from within a transformation/validation script. + + try (DbScope.Transaction transaction = transacted ? scope.ensureTransaction() : NO_OP_TRANSACTION) + { + for (int i = 0; i < commands.length(); i++) + { + JSONObject commandObject = commands.getJSONObject(i); + String commandName = commandObject.getString(PROP_COMMAND); + if (commandName == null) + { + throw new ApiUsageException(PROP_COMMAND + " is required but was missing"); + } + CommandType command = CommandType.valueOf(commandName); + + // Copy the top-level 'extraContext' and merge in the command-level extraContext. + Map commandExtraContext = new HashMap<>(); + if (extraContext != null) + commandExtraContext.putAll(extraContext.toMap()); + if (commandObject.has("extraContext")) + { + commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); + } + commandObject.put("extraContext", commandExtraContext); + + JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); + // Bail out immediately if we're going to return a failure-type response message + if (commandResponse == null || (errors.hasErrors() && !isSuccessOnValidationError())) + return null; + + //this would be populated in executeJson when a BatchValidationException is thrown + if (commandResponse.has("errors")) + { + errorCount += commandResponse.getJSONObject("errors").getInt("errorCount"); + } + + // If we encountered errors with this particular command and the client requested that don't treat + // the whole request as a failure (non-200 HTTP status code), stash the errors for this particular + // command in its response section. + // NOTE: executeJson should handle and serialize BatchValidationException + // these errors upstream + if (errors.getErrorCount() > startingErrorIndex && isSuccessOnValidationError()) + { + commandResponse.put("errors", ApiResponseWriter.convertToJSON(errors, startingErrorIndex).getValue()); + startingErrorIndex = errors.getErrorCount(); + } + + resultArray.put(commandResponse); + } + + // Don't commit if we had errors or if the client requested that we only validate (and not commit) + if (!errors.hasErrors() && !validateOnly && errorCount == 0) + { + transaction.commit(); + committed = true; + } + } + + errorCount += errors.getErrorCount(); + JSONObject result = new JSONObject(); + result.put("result", resultArray); + result.put("committed", committed); + result.put("errorCount", errorCount); + + return new ApiSimpleResponse(result); + } + } + + @RequiresPermission(ReadPermission.class) + public static class ApiTestAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/query/view/apitest.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("API Test"); + } + } + + + @RequiresPermission(AdminPermission.class) + public static class AdminAction extends SimpleViewAction + { + @SuppressWarnings("UnusedDeclaration") + public AdminAction() + { + } + + public AdminAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/admin.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Schema Administration", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ResetRemoteConnectionsForm + { + private boolean _reset; + + public boolean isReset() + { + return _reset; + } + + public void setReset(boolean reset) + { + _reset = reset; + } + } + + + @RequiresPermission(AdminPermission.class) + public static class ManageRemoteConnectionsAction extends FormViewAction + { + @Override + public void validateCommand(ResetRemoteConnectionsForm target, Errors errors) {} + + @Override + public boolean handlePost(ResetRemoteConnectionsForm form, BindException errors) + { + if (form.isReset()) + { + PropertyManager.getEncryptedStore().deletePropertySet(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); + } + return true; + } + + @Override + public URLHelper getSuccessURL(ResetRemoteConnectionsForm queryForm) + { + return new ActionURL(ManageRemoteConnectionsAction.class, getContainer()); + } + + @Override + public ModelAndView getView(ResetRemoteConnectionsForm queryForm, boolean reshow, BindException errors) + { + Map connectionMap; + try + { + // if the encrypted property store is configured but no values have yet been set, and empty map is returned + connectionMap = PropertyManager.getEncryptedStore().getProperties(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); + } + catch (Exception e) + { + connectionMap = null; // render the failure page + } + setHelpTopic("remoteConnection"); + return new JspView<>("/org/labkey/query/view/manageRemoteConnections.jsp", connectionMap, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + private abstract static class BaseInsertExternalSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction + { + protected BaseInsertExternalSchemaAction(Class commandClass) + { + super(commandClass); + } + + @Override + public void validateCommand(F form, Errors errors) + { + form.validate(errors); + } + + @Override + public boolean handlePost(F form, BindException errors) throws Exception + { + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + form.doInsert(); + auditSchemaAdminActivity(form.getBean(), "created", getContainer(), getUser()); + QueryManager.get().updateExternalSchemas(getContainer()); + + t.commit(); + } + catch (RuntimeSQLException e) + { + if (e.isConstraintException()) + { + errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); + return false; + } + + throw e; + } + + return true; + } + + @Override + public ActionURL getSuccessURL(F form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext()).addNavTrail(root); + root.addChild("Define Schema", new ActionURL(getClass(), getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class InsertLinkedSchemaAction extends BaseInsertExternalSchemaAction + { + public InsertLinkedSchemaAction() + { + super(LinkedSchemaForm.class); + } + + @Override + public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) + { + setHelpTopic("filterSchema"); + return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), form.getBean(), true), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class InsertExternalSchemaAction extends BaseInsertExternalSchemaAction + { + public InsertExternalSchemaAction() + { + super(ExternalSchemaForm.class); + } + + @Override + public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) + { + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), form.getBean(), true), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class DeleteSchemaAction extends ConfirmAction + { + @Override + public String getConfirmText() + { + return "Delete"; + } + + @Override + public ModelAndView getConfirmView(SchemaForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Schema"); + + AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + String schemaName = isBlank(def.getUserSchemaName()) ? "this schema" : "the schema '" + def.getUserSchemaName() + "'"; + return new HtmlView(HtmlString.of("Are you sure you want to delete " + schemaName + "? The tables and queries defined in this schema will no longer be accessible.")); + } + + @Override + public boolean handlePost(SchemaForm form, BindException errors) + { + AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + auditSchemaAdminActivity(def, "deleted", getContainer(), getUser()); + QueryManager.get().delete(def); + t.commit(); + } + return true; + } + + @Override + public void validateCommand(SchemaForm form, Errors errors) + { + } + + @Override + @NotNull + public ActionURL getSuccessURL(SchemaForm form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + } + + private static void auditSchemaAdminActivity(AbstractExternalSchemaDef def, String action, Container container, User user) + { + String comment = StringUtils.capitalize(def.getSchemaType().toString()) + " schema '" + def.getUserSchemaName() + "' " + action; + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, comment); + AuditLogService.get().addEvent(user, event); + } + + + private abstract static class BaseEditSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction + { + protected BaseEditSchemaAction(Class commandClass) + { + super(commandClass); + } + + @Override + public void validateCommand(F form, Errors errors) + { + form.validate(errors); + } + + @Nullable + protected abstract T getCurrent(int externalSchemaId); + + @NotNull + protected T getDef(F form, boolean reshow) + { + T def; + Container defContainer; + + if (reshow) + { + def = form.getBean(); + T current = getCurrent(def.getExternalSchemaId()); + if (current == null) + throw new NotFoundException(); + + defContainer = current.lookupContainer(); + } + else + { + form.refreshFromDb(); + if (!form.isDataLoaded()) + throw new NotFoundException(); + + def = form.getBean(); + if (def == null) + throw new NotFoundException(); + + defContainer = def.lookupContainer(); + } + + if (!getContainer().equals(defContainer)) + throw new UnauthorizedException(); + + return def; + } + + @Override + public boolean handlePost(F form, BindException errors) throws Exception + { + T def = form.getBean(); + T fromDb = getCurrent(def.getExternalSchemaId()); + + // Unauthorized if def in the database reports a different container + if (!getContainer().equals(fromDb.lookupContainer())) + throw new UnauthorizedException(); + + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + form.doUpdate(); + auditSchemaAdminActivity(def, "updated", getContainer(), getUser()); + QueryManager.get().updateExternalSchemas(getContainer()); + t.commit(); + } + catch (RuntimeSQLException e) + { + if (e.isConstraintException()) + { + errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); + return false; + } + + throw e; + } + return true; + } + + @Override + public ActionURL getSuccessURL(F externalSchemaForm) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext()).addNavTrail(root); + root.addChild("Edit Schema", new ActionURL(getClass(), getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditLinkedSchemaAction extends BaseEditSchemaAction + { + public EditLinkedSchemaAction() + { + super(LinkedSchemaForm.class); + } + + @Nullable + @Override + protected LinkedSchemaDef getCurrent(int externalId) + { + return QueryManager.get().getLinkedSchemaDef(getContainer(), externalId); + } + + @Override + public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) + { + LinkedSchemaDef def = getDef(form, reshow); + + setHelpTopic("filterSchema"); + return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), def, false), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditExternalSchemaAction extends BaseEditSchemaAction + { + public EditExternalSchemaAction() + { + super(ExternalSchemaForm.class); + } + + @Nullable + @Override + protected ExternalSchemaDef getCurrent(int externalId) + { + return QueryManager.get().getExternalSchemaDef(getContainer(), externalId); + } + + @Override + public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) + { + ExternalSchemaDef def = getDef(form, reshow); + + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), def, false), errors); + } + } + + + public static class DataSourceInfo + { + public final String sourceName; + public final String displayName; + public final boolean editable; + + public DataSourceInfo(DbScope scope) + { + this(scope.getDataSourceName(), scope.getDisplayName(), scope.getSqlDialect().isEditable()); + } + + public DataSourceInfo(Container c) + { + this(c.getId(), c.getName(), false); + } + + public DataSourceInfo(String sourceName, String displayName, boolean editable) + { + this.sourceName = sourceName; + this.displayName = displayName; + this.editable = editable; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DataSourceInfo that = (DataSourceInfo) o; + return sourceName != null ? sourceName.equals(that.sourceName) : that.sourceName == null; + } + + @Override + public int hashCode() + { + return sourceName != null ? sourceName.hashCode() : 0; + } + } + + public static abstract class BaseExternalSchemaBean + { + protected final Container _c; + protected final T _def; + protected final boolean _insert; + protected final Map _help = new HashMap<>(); + + public BaseExternalSchemaBean(Container c, T def, boolean insert) + { + _c = c; + _def = def; + _insert = insert; + + TableInfo ti = QueryManager.get().getTableInfoExternalSchema(); + + ti.getColumns() + .stream() + .filter(ci -> null != ci.getDescription()) + .forEach(ci -> _help.put(ci.getName(), ci.getDescription())); + } + + public abstract DataSourceInfo getInitialSource(); + + public T getSchemaDef() + { + return _def; + } + + public boolean isInsert() + { + return _insert; + } + + public ActionURL getReturnURL() + { + return new ActionURL(AdminAction.class, _c); + } + + public ActionURL getDeleteURL() + { + return new QueryUrlsImpl().urlDeleteSchema(_c, _def); + } + + public String getHelpHTML(String fieldName) + { + return _help.get(fieldName); + } + } + + public static class LinkedSchemaBean extends BaseExternalSchemaBean + { + public LinkedSchemaBean(Container c, LinkedSchemaDef def, boolean insert) + { + super(c, def, insert); + } + + @Override + public DataSourceInfo getInitialSource() + { + Container sourceContainer = getInitialContainer(); + return new DataSourceInfo(sourceContainer); + } + + private @NotNull Container getInitialContainer() + { + LinkedSchemaDef def = getSchemaDef(); + Container sourceContainer = def.lookupSourceContainer(); + if (sourceContainer == null) + sourceContainer = def.lookupContainer(); + if (sourceContainer == null) + sourceContainer = _c; + return sourceContainer; + } + } + + public static class ExternalSchemaBean extends BaseExternalSchemaBean + { + protected final Map> _sourcesAndSchemas = new LinkedHashMap<>(); + protected final Map> _sourcesAndSchemasIncludingSystem = new LinkedHashMap<>(); + + public ExternalSchemaBean(Container c, ExternalSchemaDef def, boolean insert) + { + super(c, def, insert); + initSources(); + } + + public Collection getSources() + { + return _sourcesAndSchemas.keySet(); + } + + public Collection getSchemaNames(DataSourceInfo source, boolean includeSystem) + { + if (includeSystem) + return _sourcesAndSchemasIncludingSystem.get(source); + else + return _sourcesAndSchemas.get(source); + } + + @Override + public DataSourceInfo getInitialSource() + { + ExternalSchemaDef def = getSchemaDef(); + DbScope scope = def.lookupDbScope(); + if (scope == null) + scope = DbScope.getLabKeyScope(); + return new DataSourceInfo(scope); + } + + protected void initSources() + { + ModuleLoader moduleLoader = ModuleLoader.getInstance(); + + for (DbScope scope : DbScope.getDbScopes()) + { + SqlDialect dialect = scope.getSqlDialect(); + + Collection schemaNames = new LinkedList<>(); + Collection schemaNamesIncludingSystem = new LinkedList<>(); + + for (String schemaName : scope.getSchemaNames()) + { + schemaNamesIncludingSystem.add(schemaName); + + if (dialect.isSystemSchema(schemaName)) + continue; + + if (null != moduleLoader.getModule(scope, schemaName)) + continue; + + schemaNames.add(schemaName); + } + + DataSourceInfo source = new DataSourceInfo(scope); + _sourcesAndSchemas.put(source, schemaNames); + _sourcesAndSchemasIncludingSystem.put(source, schemaNamesIncludingSystem); + } + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetTablesForm + { + private String _dataSource; + private String _schemaName; + private boolean _sorted; + + public String getDataSource() + { + return _dataSource; + } + + public void setDataSource(String dataSource) + { + _dataSource = dataSource; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isSorted() + { + return _sorted; + } + + public void setSorted(boolean sorted) + { + _sorted = sorted; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class GetTablesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(GetTablesForm form, BindException errors) + { + List> rows = new LinkedList<>(); + List tableNames = new ArrayList<>(); + + if (null != form.getSchemaName()) + { + DbScope scope = DbScope.getDbScope(form.getDataSource()); + if (null != scope) + { + DbSchema schema = scope.getSchema(form.getSchemaName(), DbSchemaType.Bare); + tableNames.addAll(schema.getTableNames()); + } + else + { + Container c = ContainerManager.getForId(form.getDataSource()); + if (null != c) + { + UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); + if (null != schema) + { + if (form.isSorted()) + for (TableInfo table : schema.getSortedTables()) + tableNames.add(table.getName()); + else + tableNames.addAll(schema.getTableAndQueryNames(true)); + } + } + } + } + + Collections.sort(tableNames); + + for (String tableName : tableNames) + { + Map row = new LinkedHashMap<>(); + row.put("table", tableName); + rows.add(row); + } + + Map properties = new HashMap<>(); + properties.put("rows", rows); + + return new ApiSimpleResponse(properties); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SchemaTemplateForm + { + private String _name; + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public static class SchemaTemplateAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SchemaTemplateForm form, BindException errors) + { + String name = form.getName(); + if (name == null) + throw new IllegalArgumentException("name required"); + + Container c = getContainer(); + TemplateSchemaType template = QueryServiceImpl.get().getSchemaTemplate(c, name); + if (template == null) + throw new NotFoundException("template not found"); + + JSONObject templateJson = QueryServiceImpl.get().schemaTemplateJson(name, template); + + return new ApiSimpleResponse("template", templateJson); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class SchemaTemplatesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) + { + Container c = getContainer(); + QueryServiceImpl svc = QueryServiceImpl.get(); + Map templates = svc.getSchemaTemplates(c); + + JSONArray ret = new JSONArray(); + for (String key : templates.keySet()) + { + TemplateSchemaType template = templates.get(key); + JSONObject templateJson = svc.schemaTemplateJson(key, template); + ret.put(templateJson); + } + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("templates", ret); + resp.put("success", true); + return resp; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ReloadExternalSchemaAction extends FormHandlerAction + { + private String _userSchemaName; + + @Override + public void validateCommand(SchemaForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SchemaForm form, BindException errors) + { + ExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), ExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + QueryManager.get().reloadExternalSchema(def); + _userSchemaName = def.getUserSchemaName(); + + return true; + } + + @Override + public ActionURL getSuccessURL(SchemaForm form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Schema " + _userSchemaName + " was reloaded successfully."); + } + } + + + @RequiresPermission(AdminPermission.class) + public static class ReloadAllUserSchemas extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + QueryManager.get().reloadAllExternalSchemas(getContainer()); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "All schemas in this folder were reloaded successfully."); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ReloadFailedConnectionsAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + DbScope.clearFailedDbScopes(); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Reconnection was attempted on all data sources that failed previous connection attempts."); + } + } + + @RequiresPermission(ReadPermission.class) + public static class TableInfoAction extends SimpleViewAction + { + @Override + public ModelAndView getView(TableInfoForm form, BindException errors) throws Exception + { + TablesDocument ret = TablesDocument.Factory.newInstance(); + TablesType tables = ret.addNewTables(); + + FieldKey[] fields = form.getFieldKeys(); + if (fields.length != 0) + { + TableInfo tinfo = QueryView.create(form, errors).getTable(); + Map columnMap = CustomViewImpl.getColumnInfos(tinfo, Arrays.asList(fields)); + TableXML.initTable(tables.addNewTable(), tinfo, null, columnMap.values()); + } + + for (FieldKey tableKey : form.getTableKeys()) + { + TableInfo tableInfo = form.getTableInfo(tableKey); + TableType xbTable = tables.addNewTable(); + TableXML.initTable(xbTable, tableInfo, tableKey); + } + getViewContext().getResponse().setContentType("text/xml"); + getViewContext().getResponse().getWriter().write(ret.toString()); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + // Issue 18870: Guest user can't revert unsaved custom view changes + // Permission will be checked inline (guests are allowed to delete their session custom views) + @RequiresNoPermission + @Action(ActionType.Configure.class) + public static class DeleteViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeleteViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + + if (getUser().isGuest()) + { + // Guests can only delete session custom views. + if (!view.isSession()) + throw new UnauthorizedException(); + } + else + { + // Logged in users must have read permission + if (!getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new UnauthorizedException(); + } + + if (view.isShared()) + { + if (!getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) + throw new UnauthorizedException(); + } + + view.delete(getUser(), getViewContext().getRequest()); + + // Delete the first shadowed custom view, if available. + if (form.isComplete()) + { + form.reset(); + CustomView shadowed = form.getCustomView(); + if (shadowed != null && shadowed.isEditable() && !(shadowed instanceof ModuleCustomView)) + { + if (!shadowed.isShared() || getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) + shadowed.delete(getUser(), getViewContext().getRequest()); + } + } + + // Try to get a custom view of the same name as the view we just deleted. + // The deleted view may have been a session view or a personal view masking shared view with the same name. + form.reset(); + view = form.getCustomView(); + String nextViewName = null; + if (view != null) + nextViewName = view.getName(); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("viewName", nextViewName); + return response; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SaveSessionViewForm extends QueryForm + { + private String newName; + private boolean inherit; + private boolean shared; + private boolean hidden; + private boolean replace; + private String containerPath; + + public String getNewName() + { + return newName; + } + + public void setNewName(String newName) + { + this.newName = newName; + } + + public boolean isInherit() + { + return inherit; + } + + public void setInherit(boolean inherit) + { + this.inherit = inherit; + } + + public boolean isShared() + { + return shared; + } + + public void setShared(boolean shared) + { + this.shared = shared; + } + + public String getContainerPath() + { + return containerPath; + } + + public void setContainerPath(String containerPath) + { + this.containerPath = containerPath; + } + + public boolean isHidden() + { + return hidden; + } + + public void setHidden(boolean hidden) + { + this.hidden = hidden; + } + + public boolean isReplace() + { + return replace; + } + + public void setReplace(boolean replace) + { + this.replace = replace; + } + } + + // Moves a session view into the database. + @RequiresPermission(ReadPermission.class) + public static class SaveSessionViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SaveSessionViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + if (!view.isSession()) + throw new IllegalArgumentException("This action only supports saving session views."); + + //if (!getContainer().getId().equals(view.getContainer().getId())) + // throw new IllegalArgumentException("View may only be saved from container it was created in."); + + assert !view.canInherit() && !view.isShared() && view.isEditable(): "Session view should never be inheritable or shared and always be editable"; + + // Users may save views to a location other than the current container + String containerPath = form.getContainerPath(); + Container container; + if (form.isInherit() && containerPath != null) + { + // Only respect this request if it's a view that is inheritable in subfolders + container = ContainerManager.getForPath(containerPath); + } + else + { + // Otherwise, save it in the current container + container = getContainer(); + } + + if (container == null) + throw new NotFoundException("No such container: " + containerPath); + + if (form.isShared() || form.isInherit()) + { + if (!container.hasPermission(getUser(), EditSharedViewPermission.class)) + throw new UnauthorizedException(); + } + + DbScope scope = QueryManager.get().getDbSchema().getScope(); + try (DbScope.Transaction tx = scope.ensureTransaction()) + { + // Delete the session view. The view will be restored if an exception is thrown. + view.delete(getUser(), getViewContext().getRequest()); + + // Get any previously existing non-session view. + // The session custom view and the view-to-be-saved may have different names. + // If they do have different names, we may need to delete an existing session view with that name. + // UNDONE: If the view has a different name, we will clobber it without asking. + CustomView existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); + if (existingView != null && existingView.isSession()) + { + // Delete any session view we are overwriting. + existingView.delete(getUser(), getViewContext().getRequest()); + existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); + } + + // save a new private view if shared is false but existing view is shared + if (existingView != null && !form.isShared() && existingView.getOwner() == null) + { + existingView = null; + } + + if (existingView != null && !form.isReplace() && !StringUtils.isEmpty(form.getNewName())) + throw new IllegalArgumentException("A saved view by the name \"" + form.getNewName() + "\" already exists. "); + + if (existingView == null || (existingView instanceof ModuleCustomView && existingView.isEditable())) + { + User owner = form.isShared() ? null : getUser(); + + CustomViewImpl viewCopy = new CustomViewImpl(form.getQueryDef(), owner, form.getNewName()); + viewCopy.setColumns(view.getColumns()); + viewCopy.setCanInherit(form.isInherit()); + viewCopy.setFilterAndSort(view.getFilterAndSort()); + viewCopy.setColumnProperties(view.getColumnProperties()); + viewCopy.setIsHidden(form.isHidden()); + if (form.isInherit()) + viewCopy.setContainer(container); + + viewCopy.save(getUser(), getViewContext().getRequest()); + } + else if (!existingView.isEditable()) + { + throw new IllegalArgumentException("Existing view '" + form.getNewName() + "' is not editable. You may save this view with a different name."); + } + else + { + // UNDONE: changing shared property of an existing view is unimplemented. Not sure if it makes sense from a usability point of view. + existingView.setColumns(view.getColumns()); + existingView.setFilterAndSort(view.getFilterAndSort()); + existingView.setColumnProperties(view.getColumnProperties()); + existingView.setCanInherit(form.isInherit()); + if (form.isInherit()) + ((CustomViewImpl)existingView).setContainer(container); + existingView.setIsHidden(form.isHidden()); + + existingView.save(getUser(), getViewContext().getRequest()); + } + + tx.commit(); + return new ApiSimpleResponse("success", true); + } + catch (Exception e) + { + // dirty the view then save the deleted session view back in session state + view.setName(view.getName()); + view.save(getUser(), getViewContext().getRequest()); + + throw e; + } + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class ManageViewsAction extends SimpleViewAction + { + @SuppressWarnings("UnusedDeclaration") + public ManageViewsAction() + { + } + + public ManageViewsAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return new JspView<>("/org/labkey/query/view/manageViews.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Views", QueryController.this.getViewContext().getActionURL()); + } + } + + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalDeleteView extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(InternalViewForm form, BindException errors) + { + return new JspView<>("/org/labkey/query/view/internalDeleteView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalViewForm form, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + QueryManager.get().delete(view); + return true; + } + + @Override + public void validateCommand(InternalViewForm internalViewForm, Errors errors) + { + } + + @Override + @NotNull + public ActionURL getSuccessURL(InternalViewForm internalViewForm) + { + return new ActionURL(ManageViewsAction.class, getContainer()); + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalSourceViewAction extends FormViewAction + { + @Override + public void validateCommand(InternalSourceViewForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(InternalSourceViewForm form, boolean reshow, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + form.ff_inherit = QueryManager.get().canInherit(view.getFlags()); + form.ff_hidden = QueryManager.get().isHidden(view.getFlags()); + form.ff_columnList = view.getColumns(); + form.ff_filter = view.getFilter(); + return new JspView<>("/org/labkey/query/view/internalSourceView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalSourceViewForm form, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + int flags = view.getFlags(); + flags = QueryManager.get().setCanInherit(flags, form.ff_inherit); + flags = QueryManager.get().setIsHidden(flags, form.ff_hidden); + view.setFlags(flags); + view.setColumns(form.ff_columnList); + view.setFilter(form.ff_filter); + QueryManager.get().update(getUser(), view); + return true; + } + + @Override + public ActionURL getSuccessURL(InternalSourceViewForm form) + { + return new ActionURL(ManageViewsAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new ManageViewsAction(getViewContext()).addNavTrail(root); + root.addChild("Edit source of Grid View"); + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalNewViewAction extends FormViewAction + { + int _customViewId = 0; + + @Override + public void validateCommand(InternalNewViewForm form, Errors errors) + { + if (StringUtils.trimToNull(form.ff_schemaName) == null) + { + errors.reject(ERROR_MSG, "Schema name cannot be blank."); + } + if (StringUtils.trimToNull(form.ff_queryName) == null) + { + errors.reject(ERROR_MSG, "Query name cannot be blank"); + } + } + + @Override + public ModelAndView getView(InternalNewViewForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/query/view/internalNewView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalNewViewForm form, BindException errors) + { + if (form.ff_share) + { + if (!getContainer().hasPermission(getUser(), AdminPermission.class)) + throw new UnauthorizedException(); + } + List existing = QueryManager.get().getCstmViews(getContainer(), form.ff_schemaName, form.ff_queryName, form.ff_viewName, form.ff_share ? null : getUser(), false, false); + CstmView view; + if (!existing.isEmpty()) + { + } + else + { + view = new CstmView(); + view.setSchema(form.ff_schemaName); + view.setQueryName(form.ff_queryName); + view.setName(form.ff_viewName); + view.setContainerId(getContainer().getId()); + if (form.ff_share) + { + view.setCustomViewOwner(null); + } + else + { + view.setCustomViewOwner(getUser().getUserId()); + } + if (form.ff_inherit) + { + view.setFlags(QueryManager.get().setCanInherit(view.getFlags(), form.ff_inherit)); + } + InternalViewForm.checkEdit(getViewContext(), view); + try + { + view = QueryManager.get().insert(getUser(), view); + } + catch (Exception e) + { + LogManager.getLogger(QueryController.class).error("Error", e); + errors.reject(ERROR_MSG, "An exception occurred: " + e); + return false; + } + _customViewId = view.getCustomViewId(); + } + return true; + } + + @Override + public ActionURL getSuccessURL(InternalNewViewForm form) + { + ActionURL forward = new ActionURL(InternalSourceViewAction.class, getContainer()); + forward.addParameter("customViewId", Integer.toString(_customViewId)); + return forward; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Create New Grid View"); + } + } + + + @ActionNames("clearSelected, selectNone") + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class SelectNoneAction extends MutatingApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) + { + errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + if (form.getQueryName() == null) + { + DataRegionSelection.clearAll(getViewContext(), form.getKey()); + return new DataRegionSelection.SelectionResponse(0); + } + + int count = DataRegionSelection.setSelectedFromForm(form); + return new DataRegionSelection.SelectionResponse(count); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SelectForm extends QueryForm + { + protected boolean clearSelected; + protected String key; + + public boolean isClearSelected() + { + return clearSelected; + } + + public void setClearSelected(boolean clearSelected) + { + this.clearSelected = clearSelected; + } + + public String getKey() + { + return key; + } + + public void setKey(String key) + { + this.key = key; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class SelectAllAction extends MutatingApiAction + { + @Override + public void validateForm(QueryForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() || form.getQueryName() == null) + { + errors.reject(ERROR_MSG, "schemaName and queryName required"); + } + } + + @Override + public ApiResponse execute(final QueryForm form, BindException errors) throws Exception + { + int count = DataRegionSelection.setSelectionForAll(form, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetSelectedAction extends ReadOnlyApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) + { + errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + getViewContext().getResponse().setHeader("Content-Type", CONTENT_TYPE_JSON); + Set selected; + + if (form.getQueryName() == null) + selected = DataRegionSelection.getSelected(getViewContext(), form.getKey(), form.isClearSelected()); + else + selected = DataRegionSelection.getSelected(form, form.isClearSelected()); + + return new ApiSimpleResponse("selected", selected); + } + } + + @ActionNames("setSelected, setCheck") + @RequiresPermission(ReadPermission.class) + public static class SetCheckAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) throws Exception + { + String[] ids = form.getId(getViewContext().getRequest()); + Set selection = new LinkedHashSet<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + int count; + if (form.getQueryName() != null && form.isValidateIds() && form.isChecked()) + { + selection = DataRegionSelection.getValidatedIds(selection, form); + } + + count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, form.isChecked()); + + return new DataRegionSelection.SelectionResponse(count); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SetCheckForm extends SelectForm + { + protected String[] ids; + protected boolean checked; + protected boolean validateIds; + + public String[] getId(HttpServletRequest request) + { + // 5025 : DataRegion checkbox names may contain comma + // Beehive parses a single parameter value with commas into an array + // which is not what we want. + String[] paramIds = request.getParameterValues("id"); + return paramIds == null ? ids: paramIds; + } + + public void setId(String[] ids) + { + this.ids = ids; + } + + public boolean isChecked() + { + return checked; + } + + public void setChecked(boolean checked) + { + this.checked = checked; + } + + public boolean isValidateIds() + { + return validateIds; + } + + public void setValidateIds(boolean validateIds) + { + this.validateIds = validateIds; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ReplaceSelectedAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) + { + String[] ids = form.getId(getViewContext().getRequest()); + List selection = new ArrayList<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + + DataRegionSelection.clearAll(getViewContext(), form.getKey()); + int count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class SetSnapshotSelectionAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) + { + String[] ids = form.getId(getViewContext().getRequest()); + List selection = new ArrayList<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + DataRegionSelection.clearAll(getViewContext(), form.getKey(), true); + int count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, true, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetSnapshotSelectionAction extends ReadOnlyApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (StringUtils.isEmpty(form.getKey())) + { + errors.reject(ERROR_MSG, "Selection key is required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + List selected = DataRegionSelection.getSnapshotSelected(getViewContext(), form.getKey()); + return new ApiSimpleResponse("selected", selected); + } + } + + public static String getMessage(SqlDialect d, SQLException x) + { + return x.getMessage(); + } + + + public static class GetSchemasForm + { + private boolean _includeHidden = true; + private SchemaKey _schemaName; + + public SchemaKey getSchemaName() + { + return _schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(SchemaKey schemaName) + { + _schemaName = schemaName; + } + + public boolean isIncludeHidden() + { + return _includeHidden; + } + + @SuppressWarnings("unused") + public void setIncludeHidden(boolean includeHidden) + { + _includeHidden = includeHidden; + } + } + + + @RequiresPermission(ReadPermission.class) + @ApiVersion(12.3) + public static class GetSchemasAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetSchemasForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetSchemasForm form, BindException errors) + { + final Container container = getContainer(); + final User user = getUser(); + + final boolean includeHidden = form.isIncludeHidden(); + if (getRequestedApiVersion() >= 9.3) + { + SimpleSchemaTreeVisitor visitor = new SimpleSchemaTreeVisitor<>(includeHidden) + { + @Override + public Void visitUserSchema(UserSchema schema, Path path, JSONObject json) + { + JSONObject schemaProps = new JSONObject(); + + schemaProps.put("schemaName", schema.getName()); + schemaProps.put("fullyQualifiedName", schema.getSchemaName()); + schemaProps.put("description", schema.getDescription()); + schemaProps.put("hidden", schema.isHidden()); + NavTree tree = schema.getSchemaBrowserLinks(user); + if (tree != null && tree.hasChildren()) + schemaProps.put("menu", tree.toJSON()); + + // Collect children schemas + JSONObject children = new JSONObject(); + visit(schema.getSchemas(_includeHidden), path, children); + if (!children.isEmpty()) + schemaProps.put("schemas", children); + + // Add node's schemaProps to the parent's json. + json.put(schema.getName(), schemaProps); + return null; + } + }; + + // By default, start from the root. + QuerySchema schema; + if (form.getSchemaName() != null) + schema = DefaultSchema.get(user, container, form.getSchemaName()); + else + schema = DefaultSchema.get(user, container); + + // Ensure consistent exception as other query actions + QueryForm.ensureSchemaNotNull(schema); + + // Create the JSON response by visiting the schema children. The parent schema information isn't included. + JSONObject ret = new JSONObject(); + visitor.visitTop(schema.getSchemas(includeHidden), ret); + + return new ApiSimpleResponse(ret); + } + else + { + return new ApiSimpleResponse("schemas", DefaultSchema.get(user, container).getUserSchemaPaths(includeHidden)); + } + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetQueriesForm + { + private String _schemaName; + private boolean _includeUserQueries = true; + private boolean _includeSystemQueries = true; + private boolean _includeColumns = true; + private boolean _includeViewDataUrl = true; + private boolean _includeTitle = true; + private boolean _queryDetailColumns = false; + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isIncludeUserQueries() + { + return _includeUserQueries; + } + + public void setIncludeUserQueries(boolean includeUserQueries) + { + _includeUserQueries = includeUserQueries; + } + + public boolean isIncludeSystemQueries() + { + return _includeSystemQueries; + } + + public void setIncludeSystemQueries(boolean includeSystemQueries) + { + _includeSystemQueries = includeSystemQueries; + } + + public boolean isIncludeColumns() + { + return _includeColumns; + } + + public void setIncludeColumns(boolean includeColumns) + { + _includeColumns = includeColumns; + } + + public boolean isQueryDetailColumns() + { + return _queryDetailColumns; + } + + public void setQueryDetailColumns(boolean queryDetailColumns) + { + _queryDetailColumns = queryDetailColumns; + } + + public boolean isIncludeViewDataUrl() + { + return _includeViewDataUrl; + } + + public void setIncludeViewDataUrl(boolean includeViewDataUrl) + { + _includeViewDataUrl = includeViewDataUrl; + } + + public boolean isIncludeTitle() + { + return _includeTitle; + } + + public void setIncludeTitle(boolean includeTitle) + { + _includeTitle = includeTitle; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class GetQueriesAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetQueriesForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetQueriesForm form, BindException errors) + { + if (null == StringUtils.trimToNull(form.getSchemaName())) + throw new IllegalArgumentException("You must supply a value for the 'schemaName' parameter!"); + + ApiSimpleResponse response = new ApiSimpleResponse(); + UserSchema uschema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); + if (null == uschema) + throw new NotFoundException("The schema name '" + form.getSchemaName() + + "' was not found within the folder '" + getContainer().getPath() + "'"); + + response.put("schemaName", form.getSchemaName()); + + List> qinfos = new ArrayList<>(); + + //user-defined queries + if (form.isIncludeUserQueries()) + { + for (QueryDefinition qdef : uschema.getQueryDefs().values()) + { + if (!qdef.isTemporary()) + { + ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; + qinfos.add(getQueryProps(qdef, viewDataUrl, true, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); + } + } + } + + //built-in tables + if (form.isIncludeSystemQueries()) + { + for (String qname : uschema.getVisibleTableNames()) + { + // Go direct against the UserSchema instead of calling into QueryService, which takes a schema and + // query name as strings and therefore has to create new instances + QueryDefinition qdef = uschema.getQueryDefForTable(qname); + if (qdef != null) + { + ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; + qinfos.add(getQueryProps(qdef, viewDataUrl, false, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); + } + } + } + response.put("queries", qinfos); + + return response; + } + + protected Map getQueryProps(QueryDefinition qdef, ActionURL viewDataUrl, boolean isUserDefined, UserSchema schema, boolean includeColumns, boolean useQueryDetailColumns, boolean includeTitle) + { + Map qinfo = new HashMap<>(); + qinfo.put("hidden", qdef.isHidden()); + qinfo.put("snapshot", qdef.isSnapshot()); + qinfo.put("inherit", qdef.canInherit()); + qinfo.put("isUserDefined", isUserDefined); + boolean canEdit = qdef.canEdit(getUser()); + qinfo.put("canEdit", canEdit); + qinfo.put("canEditSharedViews", getContainer().hasPermission(getUser(), EditSharedViewPermission.class)); + // CONSIDER: do we want to separate the 'canEditMetadata' property and 'isMetadataOverridable' properties to differentiate between capability and the permission check? + qinfo.put("isMetadataOverrideable", qdef.isMetadataEditable() && qdef.canEditMetadata(getUser())); + + if (isUserDefined) + qinfo.put("moduleName", qdef.getModuleName()); + boolean isInherited = qdef.canInherit() && !getContainer().equals(qdef.getDefinitionContainer()); + qinfo.put("isInherited", isInherited); + if (isInherited) + qinfo.put("containerPath", qdef.getDefinitionContainer().getPath()); + qinfo.put("isIncludedForLookups", qdef.isIncludedForLookups()); + + if (null != qdef.getDescription()) + qinfo.put("description", qdef.getDescription()); + if (viewDataUrl != null) + qinfo.put("viewDataUrl", viewDataUrl); + + String title = qdef.getName(); + String name = qdef.getName(); + try + { + // get the TableInfo if the user requested column info or title, otherwise skip (it can be expensive) + if (includeColumns || includeTitle) + { + TableInfo table = qdef.getTable(schema, null, true); + + if (null != table) + { + if (includeColumns) + { + Collection> columns; + + if (useQueryDetailColumns) + { + columns = JsonWriter + .getNativeColProps(table, Collections.emptyList(), null, false, false) + .values(); + } + else + { + columns = new ArrayList<>(); + for (ColumnInfo col : table.getColumns()) + { + Map cinfo = new HashMap<>(); + cinfo.put("name", col.getName()); + if (null != col.getLabel()) + cinfo.put("caption", col.getLabel()); + if (null != col.getShortLabel()) + cinfo.put("shortCaption", col.getShortLabel()); + if (null != col.getDescription()) + cinfo.put("description", col.getDescription()); + + columns.add(cinfo); + } + } + + if (!columns.isEmpty()) + qinfo.put("columns", columns); + } + + if (includeTitle) + { + name = table.getPublicName(); + title = table.getTitle(); + } + } + } + } + catch(Exception e) + { + //may happen due to query failing parse + } + + qinfo.put("title", title); + qinfo.put("name", name); + return qinfo; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetQueryViewsForm + { + private String _schemaName; + private String _queryName; + private String _viewName; + private boolean _metadata; + private boolean _excludeSessionView; + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + public String getViewName() + { + return _viewName; + } + + public void setViewName(String viewName) + { + _viewName = viewName; + } + + public boolean isMetadata() + { + return _metadata; + } + + public void setMetadata(boolean metadata) + { + _metadata = metadata; + } + + public boolean isExcludeSessionView() + { + return _excludeSessionView; + } + + public void setExcludeSessionView(boolean excludeSessionView) + { + _excludeSessionView = excludeSessionView; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class GetQueryViewsAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetQueryViewsForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetQueryViewsForm form, BindException errors) + { + if (null == StringUtils.trimToNull(form.getSchemaName())) + throw new IllegalArgumentException("You must pass a value for the 'schemaName' parameter!"); + if (null == StringUtils.trimToNull(form.getQueryName())) + throw new IllegalArgumentException("You must pass a value for the 'queryName' parameter!"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); + if (null == schema) + throw new NotFoundException("The schema name '" + form.getSchemaName() + + "' was not found within the folder '" + getContainer().getPath() + "'"); + + QueryDefinition querydef = QueryService.get().createQueryDefForTable(schema, form.getQueryName()); + if (null == querydef || querydef.getTable(null, true) == null) + throw new NotFoundException("The query '" + form.getQueryName() + "' was not found within the '" + + form.getSchemaName() + "' schema in the container '" + + getContainer().getPath() + "'!"); + + Map views = querydef.getCustomViews(getUser(), getViewContext().getRequest(), true, false, form.isExcludeSessionView()); + if (null == views) + views = Collections.emptyMap(); + + Map> columnMetadata = new HashMap<>(); + + List> viewInfos = Collections.emptyList(); + if (getViewContext().getBindPropertyValues().contains("viewName")) + { + // Get info for a named view or the default view (null) + String viewName = StringUtils.trimToNull(form.getViewName()); + CustomView view = views.get(viewName); + if (view != null) + { + viewInfos = Collections.singletonList(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); + } + else if (viewName == null) + { + // The default view was requested but it hasn't been customized yet. Create the 'default default' view. + viewInfos = Collections.singletonList(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); + } + } + else + { + boolean foundDefault = false; + viewInfos = new ArrayList<>(views.size()); + for (CustomView view : views.values()) + { + if (view.getName() == null) + foundDefault = true; + viewInfos.add(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); + } + + if (!foundDefault) + { + // The default view hasn't been customized yet. Create the 'default default' view. + viewInfos.add(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); + } + } + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("schemaName", form.getSchemaName()); + response.put("queryName", form.getQueryName()); + response.put("views", viewInfos); + + return response; + } + } + + @RequiresNoPermission + public static class GetServerDateAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + return new ApiSimpleResponse("date", new Date()); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + private static class SaveApiTestForm + { + private String _getUrl; + private String _postUrl; + private String _postData; + private String _response; + + public String getGetUrl() + { + return _getUrl; + } + + public void setGetUrl(String getUrl) + { + _getUrl = getUrl; + } + + public String getPostUrl() + { + return _postUrl; + } + + public void setPostUrl(String postUrl) + { + _postUrl = postUrl; + } + + public String getResponse() + { + return _response; + } + + public void setResponse(String response) + { + _response = response; + } + + public String getPostData() + { + return _postData; + } + + public void setPostData(String postData) + { + _postData = postData; + } + } + + + @RequiresPermission(ReadPermission.class) + public static class SaveApiTestAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SaveApiTestForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + ApiTestsDocument doc = ApiTestsDocument.Factory.newInstance(); + + TestCaseType test = doc.addNewApiTests().addNewTest(); + test.setName("recorded test case"); + ActionURL url = null; + + if (!StringUtils.isEmpty(form.getGetUrl())) + { + test.setType("get"); + url = new ActionURL(form.getGetUrl()); + } + else if (!StringUtils.isEmpty(form.getPostUrl())) + { + test.setType("post"); + test.setFormData(form.getPostData()); + url = new ActionURL(form.getPostUrl()); + } + + if (url != null) + { + String uri = url.getLocalURIString(); + if (uri.startsWith(url.getContextPath())) + uri = uri.substring(url.getContextPath().length() + 1); + + test.setUrl(uri); + } + test.setResponse(form.getResponse()); + + XmlOptions opts = new XmlOptions(); + opts.setSaveCDataEntityCountThreshold(0); + opts.setSaveCDataLengthThreshold(0); + opts.setSavePrettyPrint(); + opts.setUseDefaultNamespace(); + + response.put("xml", doc.xmlText(opts)); + + return response; + } + } + + + private abstract static class ParseAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + List qpe = new ArrayList<>(); + String expr = getViewContext().getRequest().getParameter("q"); + ArrayList html = new ArrayList<>(); + PageConfig config = getPageConfig(); + var inputId = config.makeId("submit_"); + config.addHandler(inputId, "click", "Ext.getBody().mask();"); + html.add("
\n" + + "" + ); + + QNode e = null; + if (null != expr) + { + try + { + e = _parse(expr,qpe); + } + catch (RuntimeException x) + { + qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); + } + } + + Tree tree = null; + if (null != expr) + { + try + { + tree = _tree(expr); + } catch (Exception x) + { + qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); + } + } + + for (Throwable x : qpe) + { + if (null != x.getCause() && x != x.getCause()) + x = x.getCause(); + html.add("
" + PageFlowUtil.filter(x.toString())); + LogManager.getLogger(QueryController.class).debug(expr,x); + } + if (null != e) + { + String prefix = SqlParser.toPrefixString(e); + html.add("
"); + html.add(PageFlowUtil.filter(prefix)); + } + if (null != tree) + { + String prefix = SqlParser.toPrefixString(tree); + html.add("
"); + html.add(PageFlowUtil.filter(prefix)); + } + html.add(""); + return HtmlView.unsafe(StringUtils.join(html,"")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + + abstract QNode _parse(String e, List errors); + abstract Tree _tree(String e) throws Exception; + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ParseExpressionAction extends ParseAction + { + @Override + QNode _parse(String s, List errors) + { + return new SqlParser().parseExpr(s, true, errors); + } + + @Override + Tree _tree(String e) + { + return null; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ParseQueryAction extends ParseAction + { + @Override + QNode _parse(String s, List errors) + { + return new SqlParser().parseQuery(s, errors, null); + } + + @Override + Tree _tree(String s) throws Exception + { + return new SqlParser().rawQuery(s); + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class ValidateQueryMetadataAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(QueryForm form, BindException errors) + { + UserSchema schema = form.getSchema(); + + if (null == schema) + { + errors.reject(ERROR_MSG, "could not resolve schema: " + form.getSchemaName()); + return null; + } + + List parseErrors = new ArrayList<>(); + List parseWarnings = new ArrayList<>(); + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo table = schema.getTable(form.getQueryName(), null); + + if (null == table) + { + errors.reject(ERROR_MSG, "could not resolve table: " + form.getQueryName()); + return null; + } + + if (!QueryManager.get().validateQuery(table, true, parseErrors, parseWarnings)) + { + for (QueryParseException e : parseErrors) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + return response; + } + + SchemaKey schemaKey = SchemaKey.fromString(form.getSchemaName()); + QueryManager.get().validateQueryMetadata(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); + QueryManager.get().validateQueryViews(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); + } + catch (QueryParseException e) + { + parseErrors.add(e); + } + + for (QueryParseException e : parseErrors) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + + for (QueryParseException e : parseWarnings) + { + errors.reject(ERROR_MSG, "WARNING: " + e.getMessage()); + } + + return response; + } + + @Override + protected ApiResponseWriter createResponseWriter() throws IOException + { + ApiResponseWriter result = super.createResponseWriter(); + // Issue 44875 - don't send a 400 or 500 response code when there's a bogus query or metadata + result.setErrorResponseStatus(HttpServletResponse.SC_OK); + return result; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class QueryExportAuditForm + { + private int rowId; + + public int getRowId() + { + return rowId; + } + + public void setRowId(int rowId) + { + this.rowId = rowId; + } + } + + /** + * Action used to redirect QueryAuditProvider [details] column to the exported table's grid view. + */ + @RequiresPermission(AdminPermission.class) + public static class QueryExportAuditRedirectAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(QueryExportAuditForm form) + { + if (form.getRowId() == 0) + throw new NotFoundException("Query export audit rowid required"); + + UserSchema auditSchema = QueryService.get().getUserSchema(getUser(), getContainer(), AbstractAuditTypeProvider.QUERY_SCHEMA_NAME); + TableInfo queryExportAuditTable = auditSchema.getTable(QueryExportAuditProvider.QUERY_AUDIT_EVENT, null); + if (null == queryExportAuditTable) + throw new NotFoundException(); + + TableSelector selector = new TableSelector(queryExportAuditTable, + PageFlowUtil.set( + QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME, + QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME, + QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL), + new SimpleFilter(FieldKey.fromParts(AbstractAuditTypeProvider.COLUMN_NAME_ROW_ID), form.getRowId()), null); + + Map result = selector.getMap(); + if (result == null) + throw new NotFoundException("Query export audit event not found for rowId"); + + String schemaName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME); + String queryName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME); + String detailsURL = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL); + + if (schemaName == null || queryName == null) + throw new NotFoundException("Query export audit event has not schemaName or queryName"); + + ActionURL url = new ActionURL(ExecuteQueryAction.class, getContainer()); + + // Apply the sorts and filters + if (detailsURL != null) + { + ActionURL sortFilterURL = new ActionURL(detailsURL); + url.setPropertyValues(sortFilterURL.getPropertyValues()); + } + + if (url.getParameter(QueryParam.schemaName) == null) + url.addParameter(QueryParam.schemaName, schemaName); + if (url.getParameter(QueryParam.queryName) == null && url.getParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName) == null) + url.addParameter(QueryParam.queryName, queryName); + + return url; + } + } + + @RequiresPermission(ReadPermission.class) + public static class AuditHistoryAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return QueryUpdateAuditProvider.createHistoryQueryView(getViewContext(), form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Audit History"); + } + } + + @RequiresPermission(ReadPermission.class) + public static class AuditDetailsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryDetailsForm form, BindException errors) + { + return QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Audit History"); + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class QueryDetailsForm extends QueryForm + { + String _keyValue; + + public String getKeyValue() + { + return _keyValue; + } + + public void setKeyValue(String keyValue) + { + _keyValue = keyValue; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportTablesAction extends FormViewAction + { + private ActionURL _successUrl; + + @Override + public void validateCommand(ExportTablesForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ExportTablesForm form, BindException errors) + { + HttpServletResponse httpResponse = getViewContext().getResponse(); + Container container = getContainer(); + QueryServiceImpl svc = (QueryServiceImpl)QueryService.get(); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream outputStream = new BufferedOutputStream(baos)) + { + try (ZipFile zip = new ZipFile(outputStream, true)) + { + svc.writeTables(container, getUser(), zip, form.getSchemas(), form.getHeaderType()); + } + + PageFlowUtil.streamFileBytes(httpResponse, FileUtil.makeFileNameWithTimestamp(container.getName(), "tables.zip"), baos.toByteArray(), false); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : e.getClass().getName()); + LOG.error("Errror exporting tables", e); + } + + if (errors.hasErrors()) + { + _successUrl = new ActionURL(ExportTablesAction.class, getContainer()); + } + + return !errors.hasErrors(); + } + + @Override + public ModelAndView getView(ExportTablesForm form, boolean reshow, BindException errors) + { + // When exporting the zip to the browser, the base action will attempt to reshow the view since we returned + // null as the success URL; returning null here causes the base action to stop pestering the action. + if (reshow && !errors.hasErrors()) + return null; + + return new JspView<>("/org/labkey/query/view/exportTables.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Export Tables"); + } + + @Override + public ActionURL getSuccessURL(ExportTablesForm form) + { + return _successUrl; + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportTablesForm implements HasBindParameters + { + ColumnHeaderType _headerType = ColumnHeaderType.DisplayFieldKey; + Map>> _schemas = new HashMap<>(); + + public ColumnHeaderType getHeaderType() + { + return _headerType; + } + + public void setHeaderType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public Map>> getSchemas() + { + return _schemas; + } + + public void setSchemas(Map>> schemas) + { + _schemas = schemas; + } + + @Override + public @NotNull BindException bindParameters(PropertyValues values) + { + BindException errors = new NullSafeBindException(this, "form"); + + PropertyValue schemasProperty = values.getPropertyValue("schemas"); + if (schemasProperty != null && schemasProperty.getValue() != null) + { + try + { + _schemas = JsonUtil.DEFAULT_MAPPER.readValue((String)schemasProperty.getValue(), _schemas.getClass()); + } + catch (IOException e) + { + errors.rejectValue("schemas", ERROR_MSG, e.getMessage()); + } + } + + PropertyValue headerTypeProperty = values.getPropertyValue("headerType"); + if (headerTypeProperty != null && headerTypeProperty.getValue() != null) + { + try + { + _headerType = ColumnHeaderType.valueOf(String.valueOf(headerTypeProperty.getValue())); + } + catch (IllegalArgumentException ex) + { + // ignore + } + } + + return errors; + } + } + + + @RequiresPermission(ReadPermission.class) + public static class SaveNamedSetAction extends MutatingApiAction + { + @Override + public Object execute(NamedSetForm namedSetForm, BindException errors) + { + QueryService.get().saveNamedSet(namedSetForm.getSetName(), namedSetForm.parseSetList()); + return new ApiSimpleResponse("success", true); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class NamedSetForm + { + String setName; + String[] setList; + + public String getSetName() + { + return setName; + } + + public void setSetName(String setName) + { + this.setName = setName; + } + + public String[] getSetList() + { + return setList; + } + + public void setSetList(String[] setList) + { + this.setList = setList; + } + + public List parseSetList() + { + return Arrays.asList(setList); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class DeleteNamedSetAction extends MutatingApiAction + { + + @Override + public Object execute(NamedSetForm namedSetForm, BindException errors) + { + QueryService.get().deleteNamedSet(namedSetForm.getSetName()); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(ReadPermission.class) + public static class AnalyzeQueriesAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + JSONObject ret = new JSONObject(); + + try + { + QueryService.QueryAnalysisService analysisService = QueryService.get().getQueryAnalysisService(); + if (analysisService != null) + { + DefaultSchema start = DefaultSchema.get(getUser(), getContainer()); + var deps = new HashSetValuedHashMap(); + + analysisService.analyzeFolder(start, deps); + ret.put("success", true); + + JSONObject objects = new JSONObject(); + for (var from : deps.keySet()) + { + objects.put(from.getKey(), from.toJSON()); + for (var to : deps.get(from)) + objects.put(to.getKey(), to.toJSON()); + } + ret.put("objects", objects); + + JSONArray dependants = new JSONArray(); + for (var from : deps.keySet()) + { + for (var to : deps.get(from)) + dependants.put(new String[] {from.getKey(), to.getKey()}); + } + ret.put("graph", dependants); + } + else + { + ret.put("success", false); + } + return ret; + } + catch (Throwable e) + { + LOG.error(e); + throw UnexpectedException.wrap(e); + } + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetQueryEditorMetadataAction extends ReadOnlyApiAction + { + @Override + protected ObjectMapper createRequestObjectMapper() + { + PropertyService propertyService = PropertyService.get(); + if (null != propertyService) + { + ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); + mapper.addMixIn(GWTPropertyDescriptor.class, MetadataTableJSONMixin.class); + return mapper; + } + else + { + throw new RuntimeException("Could not serialize request object"); + } + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return createRequestObjectMapper(); + } + + @Override + public Object execute(QueryForm queryForm, BindException errors) throws Exception + { + QueryDefinition queryDef = queryForm.getQueryDef(); + return MetadataTableJSON.getMetadata(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + public static class SaveQueryMetadataAction extends MutatingApiAction + { + @Override + protected ObjectMapper createRequestObjectMapper() + { + PropertyService propertyService = PropertyService.get(); + if (null != propertyService) + { + ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); + propertyService.configureObjectMapper(mapper, null); + return mapper; + } + else + { + throw new RuntimeException("Could not serialize request object"); + } + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return createRequestObjectMapper(); + } + + @Override + public Object execute(QueryMetadataApiForm queryMetadataApiForm, BindException errors) throws Exception + { + String schemaName = queryMetadataApiForm.getSchemaName(); + MetadataTableJSON domain = queryMetadataApiForm.getDomain(); + MetadataTableJSON.saveMetadata(schemaName, domain.getName(), null, domain.getFields(true), queryMetadataApiForm.isUserDefinedQuery(), false, getUser(), getContainer()); + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + resp.put("domain", MetadataTableJSON.getMetadata(schemaName, domain.getName(), getUser(), getContainer())); + return resp; + } + } + + @Marshal(Marshaller.Jackson) + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + public static class ResetQueryMetadataAction extends MutatingApiAction + { + @Override + public Object execute(QueryForm queryForm, BindException errors) throws Exception + { + QueryDefinition queryDef = queryForm.getQueryDef(); + return MetadataTableJSON.resetToDefault(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); + } + } + + private static class QueryMetadataApiForm + { + private MetadataTableJSON _domain; + private String _schemaName; + private boolean _userDefinedQuery; + + public MetadataTableJSON getDomain() + { + return _domain; + } + + @SuppressWarnings("unused") + public void setDomain(MetadataTableJSON domain) + { + _domain = domain; + } + + public String getSchemaName() + { + return _schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isUserDefinedQuery() + { + return _userDefinedQuery; + } + + @SuppressWarnings("unused") + public void setUserDefinedQuery(boolean userDefinedQuery) + { + _userDefinedQuery = userDefinedQuery; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetDefaultVisibleColumnsAction extends ReadOnlyApiAction + { + @Override + public Object execute(GetQueryDetailsAction.Form form, BindException errors) throws Exception + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + + Container container = getContainer(); + User user = getUser(); + + if (StringUtils.isEmpty(form.getSchemaName())) + throw new NotFoundException("SchemaName not specified"); + + QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); + if (!(querySchema instanceof UserSchema schema)) + throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'"); + + QuerySettings settings = schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); + QueryDefinition queryDef = settings.getQueryDef(schema); + if (null == queryDef) + // Don't echo the provided query name, but schema name is legit since it was found. See #44528. + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'"); + + TableInfo tinfo = queryDef.getTable(null, true); + if (null == tinfo) + throw new NotFoundException("Could not find the specified query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + List fields = tinfo.getDefaultVisibleColumns(); + + List displayColumns = QueryService.get().getColumns(tinfo, fields) + .values() + .stream() + .filter(cinfo -> fields.contains(cinfo.getFieldKey())) + .map(cinfo -> cinfo.getDisplayColumnFactory().createRenderer(cinfo)) + .collect(Collectors.toList()); + + resp.put("columns", JsonWriter.getNativeColProps(displayColumns, null, false)); + + return resp; + } + } + + public static class ParseForm implements ApiJsonForm + { + String expression = ""; + Map columnMap = new HashMap<>(); + List phiColumns = new ArrayList<>(); + + Map getColumnMap() + { + return columnMap; + } + + public String getExpression() + { + return expression; + } + + public void setExpression(String expression) + { + this.expression = expression; + } + + public List getPhiColumns() + { + return phiColumns; + } + + public void setPhiColumns(List phiColumns) + { + this.phiColumns = phiColumns; + } + + @Override + public void bindJson(JSONObject json) + { + if (json.has("expression")) + setExpression(json.getString("expression")); + if (json.has("phiColumns")) + setPhiColumns(json.getJSONArray("phiColumns").toList().stream().map(s -> FieldKey.fromParts(s.toString())).collect(Collectors.toList())); + if (json.has("columnMap")) + { + JSONObject columnMap = json.getJSONObject("columnMap"); + for (String key : columnMap.keySet()) + { + try + { + getColumnMap().put(FieldKey.fromParts(key), JdbcType.valueOf(String.valueOf(columnMap.get(key)))); + } + catch (IllegalArgumentException iae) + { + getColumnMap().put(FieldKey.fromParts(key), JdbcType.OTHER); + } + } + } + } + } + + + /** + * Since this api purpose is to return parse errors, it does not generally return success:false. + *
+ * The API expects JSON like this, note that column names should be in FieldKey.toString() encoded to match the response JSON format. + * 
+     *     { "expression": "A$ + B", "columnMap":{"A$D":"VARCHAR", "X":"VARCHAR"}}
+     *
+ * and returns a response like this + * 
+     *     {
+     *       "jdbcType" : "OTHER",
+     *       "success" : true,
+     *       "columnMap" : {"A$D":"VARCHAR", "B":"OTHER"}
+     *       "errors" : [ { "msg" : "\"B\" not found.", "type" : "sql" } ]
+     *     }
+     *
+ * The columnMap object keys are the names of columns found in the expression. Names are returned + * in FieldKey.toString() formatting e.g. dollar-sign encoded. The object structure + * is compatible with the columnMap input parameter, so it can be used as a template to make a second request + * with types filled in. If provided, the type will be copied from the input columnMap, otherwise it will be "OTHER". + *
+ * Parse exceptions may contain a line (usually 1) and col location e.g. + * 
+     * {
+     *     "msg" : "Error on line 1: Syntax error near 'error', expected 'EOF'
+     *     "col" : 2,
+     *     "line" : 1,
+     *     "type" : "sql",
+     *     "errorStr" : "A error B"
+     *   }
+     *
+ */ + @RequiresNoPermission + @CSRF(CSRF.Method.NONE) + public static class ParseCalculatedColumnAction extends ReadOnlyApiAction + { + @Override + public Object execute(ParseForm form, BindException errors) throws Exception + { + if (errors.hasErrors()) + return errors; + JSONObject result = new JSONObject(Map.of("success",true)); + var requiredColumns = new HashSet(); + JdbcType jdbcType = JdbcType.OTHER; + try + { + var schema = DefaultSchema.get(getViewContext().getUser(), getViewContext().getContainer()).getUserSchema("core"); + var table = new VirtualTable<>(schema.getDbSchema(), "EXPR", schema){}; + ColumnInfo calculatedCol = QueryServiceImpl.get().createQueryExpressionColumn(table, new FieldKey(null, "expr"), form.getExpression(), null); + Map columns = new HashMap<>(); + for (var entry : form.getColumnMap().entrySet()) + { + BaseColumnInfo entryCol = new BaseColumnInfo(entry.getKey(), entry.getValue()); + // bindQueryExpressionColumn has a check that restricts PHI columns from being used in expressions + // so we need to set the PHI level to something other than NotPHI on these fake BaseColumnInfo objects + if (form.getPhiColumns().contains(entry.getKey())) + entryCol.setPHI(PHI.PHI); + columns.put(entry.getKey(), entryCol); + table.addColumn(entryCol); + } + // TODO: calculating jdbcType still uses calculatedCol.getParentTable().getColumns() + QueryServiceImpl.get().bindQueryExpressionColumn(calculatedCol, columns, false, requiredColumns); + jdbcType = calculatedCol.getJdbcType(); + } + catch (QueryException x) + { + JSONArray parseErrors = new JSONArray(); + parseErrors.put(x.toJSON(form.getExpression())); + result.put("errors", parseErrors); + } + finally + { + if (!requiredColumns.isEmpty()) + { + JSONObject columnMap = new JSONObject(); + for (FieldKey fk : requiredColumns) + { + JdbcType type = Objects.requireNonNullElse(form.getColumnMap().get(fk), JdbcType.OTHER); + columnMap.put(fk.toString(), type); + } + result.put("columnMap", columnMap); + } + } + result.put("jdbcType", jdbcType.name()); + return result; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class QueryImportTemplateForm + { + private String schemaName; + private String queryName; + private String auditUserComment; + private List templateLabels; + private List templateUrls; + private Long _lastKnownModified; + + public void setQueryName(String queryName) + { + this.queryName = queryName; + } + + public List getTemplateLabels() + { + return templateLabels == null ? Collections.emptyList() : templateLabels; + } + + public void setTemplateLabels(List templateLabels) + { + this.templateLabels = templateLabels; + } + + public List getTemplateUrls() + { + return templateUrls == null ? Collections.emptyList() : templateUrls; + } + + public void setTemplateUrls(List templateUrls) + { + this.templateUrls = templateUrls; + } + + public String getSchemaName() + { + return schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(String schemaName) + { + this.schemaName = schemaName; + } + + public String getQueryName() + { + return queryName; + } + + public Long getLastKnownModified() + { + return _lastKnownModified; + } + + public void setLastKnownModified(Long lastKnownModified) + { + _lastKnownModified = lastKnownModified; + } + + public String getAuditUserComment() + { + return auditUserComment; + } + + public void setAuditUserComment(String auditUserComment) + { + this.auditUserComment = auditUserComment; + } + + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) //Real permissions will be enforced later on by the DomainKind + public static class UpdateQueryImportTemplateAction extends MutatingApiAction + { + private DomainKind _kind; + private UserSchema _schema; + private TableInfo _tInfo; + private QueryDefinition _queryDef; + private Domain _domain; + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return this.createRequestObjectMapper(); + } + + @Override + public void validateForm(QueryImportTemplateForm form, Errors errors) + { + User user = getUser(); + Container container = getContainer(); + String domainURI = PropertyService.get().getDomainURI(form.getSchemaName(), form.getQueryName(), container, user); + _kind = PropertyService.get().getDomainKind(domainURI); + _domain = PropertyService.get().getDomain(container, domainURI); + if (_domain == null) + throw new IllegalArgumentException("Domain '" + domainURI + "' not found."); + + if (!_kind.canEditDefinition(user, _domain)) + throw new UnauthorizedException("You don't have permission to update import templates for this domain."); + + QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); + if (!(querySchema instanceof UserSchema _schema)) + throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'."); + QuerySettings settings = _schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); + _queryDef = settings.getQueryDef(_schema); + if (null == _queryDef) + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); + if (!_queryDef.isMetadataEditable()) + throw new UnsupportedOperationException("Query metadata is not editable."); + _tInfo = _queryDef.getTable(_schema, new ArrayList<>(), true, true); + if (_tInfo == null) + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); + + } + + private Map getRowFiles() + { + Map rowFiles = new IntHashMap<>(); + if (getFileMap() != null) + { + for (Map.Entry fileEntry : getFileMap().entrySet()) + { + // allow for the fileMap key to include the row index for defining which row to attach this file to + // ex: "templateFile::0", "templateFile::1" + String fieldKey = fileEntry.getKey(); + int delimIndex = fieldKey.lastIndexOf("::"); + if (delimIndex > -1) + { + Integer fieldRowIndex = Integer.parseInt(fieldKey.substring(delimIndex + 2)); + SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); + rowFiles.put(fieldRowIndex, file.isEmpty() ? null : file); + } + } + } + return rowFiles; + } + + private List> getUploadedTemplates(QueryImportTemplateForm form, DomainKind kind) throws ValidationException, QueryUpdateServiceException, ExperimentException + { + FileContentService fcs = FileContentService.get(); + if (fcs == null) + throw new IllegalStateException("Unable to load file service."); + + User user = getUser(); + Container container = getContainer(); + + Map rowFiles = getRowFiles(); + List templateLabels = form.getTemplateLabels(); + Set labels = new HashSet<>(templateLabels); + if (labels.size() < templateLabels.size()) + throw new IllegalArgumentException("Duplicate template name is not allowed."); + + List templateUrls = form.getTemplateUrls(); + List> uploadedTemplates = new ArrayList<>(); + for (int rowIndex = 0; rowIndex < form.getTemplateLabels().size(); rowIndex++) + { + String templateLabel = templateLabels.get(rowIndex); + if (StringUtils.isBlank(templateLabel.trim())) + throw new IllegalArgumentException("Template name cannot be blank."); + String templateUrl = templateUrls.get(rowIndex); + Object file = rowFiles.get(rowIndex); + if (StringUtils.isEmpty(templateUrl) && file == null) + throw new IllegalArgumentException("Template file is not provided."); + + if (file instanceof MultipartFile || file instanceof SpringAttachmentFile) + { + String fileName; + if (file instanceof MultipartFile f) + fileName = f.getName(); + else + { + SpringAttachmentFile f = (SpringAttachmentFile) file; + fileName = f.getFilename(); + } + String fileNameValidation = FileUtil.validateFileName(fileName); + if (!StringUtils.isEmpty(fileNameValidation)) + throw new IllegalArgumentException(fileNameValidation); + + FileLike uploadDir = ensureUploadDirectory(container, kind.getDomainFileDirectory()); + uploadDir = uploadDir.resolveChild("_templates"); + Object savedFile = saveFile(user, container, "template file", file, uploadDir); + Path savedFilePath; + + if (savedFile instanceof File ioFile) + savedFilePath = ioFile.toPath(); + else if (savedFile instanceof FileLike fl) + savedFilePath = fl.toNioPathForRead(); + else + throw UnexpectedException.wrap(null,"Unable to upload template file."); + + templateUrl = fcs.getWebDavUrl(savedFilePath, container, FileContentService.PathType.serverRelative).toString(); + } + + uploadedTemplates.add(Pair.of(templateLabel, templateUrl)); + } + return uploadedTemplates; + } + + @Override + public Object execute(QueryImportTemplateForm form, BindException errors) throws ValidationException, QueryUpdateServiceException, SQLException, ExperimentException, MetadataUnavailableException + { + User user = getUser(); + Container container = getContainer(); + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + QueryDef queryDef = QueryManager.get().getQueryDef(container, schemaName, queryName, false); + if (queryDef != null && queryDef.getQueryDefId() != 0) + { + Long lastKnownModified = form.getLastKnownModified(); + if (lastKnownModified == null || lastKnownModified != queryDef.getModified().getTime()) + throw new ApiUsageException("Unable to save import templates. The templates appear out of date, reload the page and try again."); + } + + List> updatedTemplates = getUploadedTemplates(form, _kind); + + List> existingTemplates = _tInfo.getImportTemplates(getViewContext()); + List> existingCustomTemplates = new ArrayList<>(); + for (Pair template_ : existingTemplates) + { + if (!template_.second.toLowerCase().contains("exportexceltemplate")) + existingCustomTemplates.add(template_); + } + if (!updatedTemplates.equals(existingCustomTemplates)) + { + TablesDocument doc = null; + TableType xmlTable = null; + TableType.ImportTemplates xmlImportTemplates; + + if (queryDef != null) + { + try + { + doc = parseDocument(queryDef.getMetaData()); + } + catch (XmlException e) + { + throw new MetadataUnavailableException(e.getMessage()); + } + xmlTable = getTableType(form.getQueryName(), doc); + // when there is a queryDef but xmlTable is null it means the xmlMetaData contains tableName which does not + // match with actual queryName then reconstruct the xml table metadata : See Issue 43523 + if (xmlTable == null) + { + doc = null; + } + } + else + { + queryDef = new QueryDef(); + queryDef.setSchema(schemaName); + queryDef.setContainer(container.getId()); + queryDef.setName(queryName); + } + + if (doc == null) + { + doc = TablesDocument.Factory.newInstance(); + } + + if (xmlTable == null) + { + TablesType tables = doc.addNewTables(); + xmlTable = tables.addNewTable(); + xmlTable.setTableName(queryName); + } + + if (xmlTable.getTableDbType() == null) + { + xmlTable.setTableDbType("NOT_IN_DB"); + } + + // remove existing templates + if (xmlTable.isSetImportTemplates()) + xmlTable.unsetImportTemplates(); + xmlImportTemplates = xmlTable.addNewImportTemplates(); + + // set new templates + if (!updatedTemplates.isEmpty()) + { + for (Pair template_ : updatedTemplates) + { + ImportTemplateType importTemplateType = xmlImportTemplates.addNewTemplate(); + importTemplateType.setLabel(template_.first); + importTemplateType.setUrl(template_.second); + } + } + + XmlOptions xmlOptions = new XmlOptions(); + xmlOptions.setSavePrettyPrint(); + // Don't use an explicit namespace, making the XML much more readable + xmlOptions.setUseDefaultNamespace(); + queryDef.setMetaData(doc.xmlText(xmlOptions)); + if (queryDef.getQueryDefId() == 0) + { + QueryManager.get().insert(user, queryDef); + } + else + { + QueryManager.get().update(user, queryDef); + } + + DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "Import templates updated."); + event.setUserComment(form.getAuditUserComment()); + event.setDomainUri(_domain.getTypeURI()); + event.setDomainName(_domain.getName()); + AuditLogService.get().addEvent(user, event); + } + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + return resp; + } + } + + + public static class TestCase extends AbstractActionPermissionTest + { + @Override + public void testActionPermissions() + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + QueryController controller = new QueryController(); + + // @RequiresPermission(ReadPermission.class) + assertForReadPermission(user, false, + new BrowseAction(), + new BeginAction(), + controller.new SchemaAction(), + controller.new SourceQueryAction(), + controller.new ExecuteQueryAction(), + controller.new PrintRowsAction(), + new ExportScriptAction(), + new ExportRowsExcelAction(), + new ExportRowsXLSXAction(), + new ExportQueriesXLSXAction(), + new ExportExcelTemplateAction(), + new ExportRowsTsvAction(), + new ExcelWebQueryDefinitionAction(), + controller.new SaveQueryViewsAction(), + controller.new PropertiesQueryAction(), + controller.new SelectRowsAction(), + new GetDataAction(), + controller.new ExecuteSqlAction(), + controller.new SelectDistinctAction(), + controller.new GetColumnSummaryStatsAction(), + controller.new ImportAction(), + new ExportSqlAction(), + new UpdateRowsAction(), + new ImportRowsAction(), + new DeleteRowsAction(), + new TableInfoAction(), + new SaveSessionViewAction(), + new GetSchemasAction(), + new GetQueriesAction(), + new GetQueryViewsAction(), + new SaveApiTestAction(), + new ValidateQueryMetadataAction(), + new AuditHistoryAction(), + new AuditDetailsAction(), + new ExportTablesAction(), + new SaveNamedSetAction(), + new DeleteNamedSetAction(), + new ApiTestAction(), + new GetDefaultVisibleColumnsAction() + ); + + + // submitter should be allowed for InsertRows + assertForReadPermission(user, true, new InsertRowsAction()); + + // @RequiresPermission(DeletePermission.class) + assertForUpdateOrDeletePermission(user, + new DeleteQueryRowsAction() + ); + + // @RequiresPermission(AdminPermission.class) + assertForAdminPermission(user, + new DeleteQueryAction(), + controller.new MetadataQueryAction(), + controller.new NewQueryAction(), + new SaveSourceQueryAction(), + + new TruncateTableAction(), + new AdminAction(), + new ManageRemoteConnectionsAction(), + new ReloadExternalSchemaAction(), + new ReloadAllUserSchemas(), + controller.new ManageViewsAction(), + controller.new InternalDeleteView(), + controller.new InternalSourceViewAction(), + controller.new InternalNewViewAction(), + new QueryExportAuditRedirectAction() + ); + + // @RequiresPermission(AdminOperationsPermission.class) + assertForAdminOperationsPermission(user, + new EditRemoteConnectionAction(), + new DeleteRemoteConnectionAction(), + new TestRemoteConnectionAction(), + controller.new RawTableMetaDataAction(), + controller.new RawSchemaMetaDataAction(), + new InsertLinkedSchemaAction(), + new InsertExternalSchemaAction(), + new DeleteSchemaAction(), + new EditLinkedSchemaAction(), + new EditExternalSchemaAction(), + new GetTablesAction(), + new SchemaTemplateAction(), + new SchemaTemplatesAction(), + new ParseExpressionAction(), + new ParseQueryAction() + ); + + // @AdminConsoleAction + assertForAdminPermission(ContainerManager.getRoot(), user, + new DataSourceAdminAction() + ); + + // In addition to administrators (tested above), trusted analysts who are editors can create and edit queries + assertTrustedEditorPermission( + new DeleteQueryAction(), + controller.new MetadataQueryAction(), + controller.new NewQueryAction(), + new SaveSourceQueryAction() + ); + } + } + + public static class SaveRowsTestCase extends Assert + { + private static final String PROJECT_NAME1 = "SaveRowsTestProject1"; + private static final String PROJECT_NAME2 = "SaveRowsTestProject2"; + + private static final String USER_EMAIL = "saveRows@action.test"; + + private static final String LIST1 = "List1"; + private static final String LIST2 = "List2"; + + @Before + public void doSetup() throws Exception + { + doCleanup(); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME1, TestContext.get().getUser()); + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME2, TestContext.get().getUser()); + + //disable search so we dont get conflicts when deleting folder quickly + ContainerManager.updateSearchable(project1, false, TestContext.get().getUser()); + ContainerManager.updateSearchable(project2, false, TestContext.get().getUser()); + + ListDefinition ld1 = ListService.get().createList(project1, LIST1, ListDefinition.KeyType.Varchar); + ld1.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); + ld1.setKeyName("TextField"); + ld1.save(TestContext.get().getUser()); + + ListDefinition ld2 = ListService.get().createList(project2, LIST2, ListDefinition.KeyType.Varchar); + ld2.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); + ld2.setKeyName("TextField"); + ld2.save(TestContext.get().getUser()); + } + + @After + public void doCleanup() throws Exception + { + Container project = ContainerManager.getForPath(PROJECT_NAME1); + if (project != null) + { + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + if (project2 != null) + { + ContainerManager.deleteAll(project2, TestContext.get().getUser()); + } + + User u = UserManager.getUser(new ValidEmail(USER_EMAIL)); + if (u != null) + { + UserManager.deleteUser(u.getUserId()); + } + } + + private JSONObject getCommand(String val1, String val2) + { + JSONObject command1 = new JSONObject(); + command1.put("containerPath", ContainerManager.getForPath(PROJECT_NAME1).getPath()); + command1.put("command", "insert"); + command1.put("schemaName", "lists"); + command1.put("queryName", LIST1); + command1.put("rows", getTestRows(val1)); + + JSONObject command2 = new JSONObject(); + command2.put("containerPath", ContainerManager.getForPath(PROJECT_NAME2).getPath()); + command2.put("command", "insert"); + command2.put("schemaName", "lists"); + command2.put("queryName", LIST2); + command2.put("rows", getTestRows(val2)); + + JSONObject json = new JSONObject(); + json.put("commands", Arrays.asList(command1, command2)); + + return json; + } + + private MockHttpServletResponse makeRequest(JSONObject json, User user) throws Exception + { + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/json"); + + HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), DetailsURL.fromString("/query/saveRows.view").copy(ContainerManager.getForPath(PROJECT_NAME1)).getActionURL(), user, headers, json.toString()); + return ViewServlet.mockDispatch(request, null); + } + + @Test + public void testCrossFolderSaveRows() throws Exception + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + JSONObject json = getCommand(PROJECT_NAME1, PROJECT_NAME2); + MockHttpServletResponse response = makeRequest(json, TestContext.get().getUser()); + if (response.getStatus() != HttpServletResponse.SC_OK) + { + JSONObject responseJson = new JSONObject(response.getContentAsString()); + throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); + } + + Container project1 = ContainerManager.getForPath(PROJECT_NAME1); + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + + TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); + TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); + + assertEquals("Incorrect row count, list1", 1L, new TableSelector(list1).getRowCount()); + assertEquals("Incorrect row count, list2", 1L, new TableSelector(list2).getRowCount()); + + assertEquals("Incorrect value", PROJECT_NAME1, new TableSelector(list1, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME1, String.class)); + assertEquals("Incorrect value", PROJECT_NAME2, new TableSelector(list2, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME2, String.class)); + + list1.getUpdateService().truncateRows(TestContext.get().getUser(), project1, null, null); + list2.getUpdateService().truncateRows(TestContext.get().getUser(), project2, null, null); + } + + @Test + public void testWithoutPermissions() throws Exception + { + // Now test failure without appropriate permissions: + User withoutPermissions = SecurityManager.addUser(new ValidEmail(USER_EMAIL), TestContext.get().getUser()).getUser(); + + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + Container project1 = ContainerManager.getForPath(PROJECT_NAME1); + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + + MutableSecurityPolicy securityPolicy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(project1)); + securityPolicy.addRoleAssignment(withoutPermissions, EditorRole.class); + SecurityPolicyManager.savePolicyForTests(securityPolicy, TestContext.get().getUser()); + + assertTrue("Should have insert permission", project1.hasPermission(withoutPermissions, InsertPermission.class)); + assertFalse("Should not have insert permission", project2.hasPermission(withoutPermissions, InsertPermission.class)); + + // repeat insert: + JSONObject json = getCommand("ShouldFail1", "ShouldFail2"); + MockHttpServletResponse response = makeRequest(json, withoutPermissions); + if (response.getStatus() != HttpServletResponse.SC_FORBIDDEN) + { + JSONObject responseJson = new JSONObject(response.getContentAsString()); + throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); + } + + TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); + TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); + + // The insert should have failed + assertEquals("Incorrect row count, list1", 0L, new TableSelector(list1).getRowCount()); + assertEquals("Incorrect row count, list2", 0L, new TableSelector(list2).getRowCount()); + } + + private JSONArray getTestRows(String val) + { + JSONArray rows = new JSONArray(); + rows.put(Map.of("TextField", val)); + + return rows; + } + } +} diff --git a/study/src/org/labkey/study/query/DatasetUpdateService.java b/study/src/org/labkey/study/query/DatasetUpdateService.java index 9a28eb8b9e1..d64019bd1a3 100644 --- a/study/src/org/labkey/study/query/DatasetUpdateService.java +++ b/study/src/org/labkey/study/query/DatasetUpdateService.java @@ -1,1136 +1,1144 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.study.query; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.assay.AssayFileWriter; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.ResultSetRowMapFactory; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.MvFieldWrapper; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.TimepointType; -import org.labkey.api.study.security.StudySecurityEscalator; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.TestContext; -import org.labkey.study.model.DatasetDataIteratorBuilder; -import org.labkey.study.dataset.DatasetAuditProvider; -import org.labkey.study.model.DatasetDefinition; -import org.labkey.study.model.DatasetDomainKind; -import org.labkey.study.model.DatasetLsidImportHelper; -import org.labkey.study.model.ParticipantIdImportHelper; -import org.labkey.study.model.ParticipantSeqNumImportHelper; -import org.labkey.study.model.QCStateImportHelper; -import org.labkey.study.model.SecurityType; -import org.labkey.study.model.SequenceNumImportHelper; -import org.labkey.study.model.StudyImpl; -import org.labkey.study.model.StudyManager; -import org.labkey.study.visitmanager.PurgeParticipantsJob.ParticipantPurger; - -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import static org.labkey.api.util.IntegerUtils.asInteger; -import static org.labkey.api.gwt.client.AuditBehaviorType.DETAILED; -import static org.labkey.api.gwt.client.AuditBehaviorType.NONE; - -/* -* User: Dave -* Date: Jun 13, 2008 -* Time: 4:15:51 PM -*/ - -/** - * QueryUpdateService implementation for Study datasets. - *

- * Since datasets are of an unpredictable shape, this class just implements - * the QueryUpdateService directly, working with Map<String,Object> - * collections for the row data. - */ -public class DatasetUpdateService extends DefaultQueryUpdateService -{ - // These are that can be passed into DatasetUpdateService via DataIteratorContext.configParameters. - // These used to be passed to DatasetDataIterator via - // DatasetDefinition.importDatasetData()->DatasetDefinition.insertData(). - // Moving these options into DataInteratorContext allows for even more consistency and code sharing - // also see QueryUpdateService.ConfigParameters.Logger - public enum Config - { - CheckForDuplicates, // expected: enum CheckForDuplicates - DefaultQCState, // expected: class QCState - SkipResyncStudy, // expected: Boolean - - // NOTE: There really has to be better way to handle the functionality of StudyImportContext.getTableIdMap() - // NOTE: Could this be handled by a method on StudySchema or something??? - // see StudyImportContext.getTableIdMapMap() - StudyImportMaps, // expected: Map> - - KeyList, // expected: List - AllowImportManagedKey // expected: Boolean - } - - private static final Logger LOG = LogManager.getLogger(DatasetUpdateService.class); - - private final DatasetDefinition _dataset; - private final Set _potentiallyNewParticipants = new HashSet<>(); - private final Set _potentiallyDeletedParticipants = new HashSet<>(); - private boolean _participantVisitResyncRequired = false; - - private final boolean _skipAuditLogging = false; - - public DatasetUpdateService(DatasetTableImpl table) - { - super(table, table.getDatasetDefinition().getStorageTableInfo(false), createMVMapping(table.getDatasetDefinition().getDomain())); - _dataset = table.getDatasetDefinition(); - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) - { - if (StudySecurityEscalator.isEscalated()) { - return true; - } - else { - return super.hasPermission(user, acl); - } - } - - - @Override - protected Map getRow(User user, Container container, Map keys) - throws InvalidKeyException - { - String lsid = keyFromMap(keys); - SimpleFilter filter = new SimpleFilter() - .addCondition(new FieldKey(null,"container"), container.getId()) - .addCondition(new FieldKey(null,"lsid"),lsid); - - // NOTE getQueryTable().getColumns() returns a bunch of columns that getDatasetRow() did not such as: - // Container, Dataset, DatasetId, Datasets, Folder, Modified, ModifiedBy, MouseVisit, ParticipantSequenceNum, VisitDay, VisitRowId - // Mostly this is harmless, but there is some noise. - HashSet nameset = new HashSet<>(getQueryTable().getColumnNameSet()); - List.of("Container","Datasets","DatasetId","Dataset","Folder").forEach(nameset::remove); - List columns = new ArrayList<>(getQueryTable().getColumns(nameset.toArray(new String[0]))); - - // filter out calculated columns which can be expensive to reselect - columns.removeIf(ColumnInfo::isCalculated); - - // This is a general version of DatasetDefinition.canonicalizeDatasetRow() - // The caller needs to make sure names are unique. Not suitable for use w/ lookups etc where there can be name collisions. - // CONSIDER: might be nice to make this a TableSelector method. - var map = new CaseInsensitiveHashMap<>(); - try (var str = new TableSelector(getQueryTable(), columns, filter, null).uncachedResultSetStream()) - { - str.forEach(rs -> { - try - { - for (int i = 0; i < columns.size(); i++) - { - Object o = rs.getObject(i + 1); - o = ResultSetRowMapFactory.translateResultSetObject(o, false); - map.put(columns.get(i).getName(), o); - } - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - }); - } - return map.isEmpty() ? null : map; - } - - - /* TODO for performance, NOTE need to return rows in order of input list - @Override - public List> getRows(User user, Container container, List> keys) throws InvalidKeyException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - ArrayList lsids = new ArrayList<>(keys.size()); - for (var m : keys) - lsids.add(keyFromMap(m)); - var result = (List)(new TableSelector(getQueryTable(), - TableSelector.ALL_COLUMNS, - new SimpleFilter(new FieldKey(null,"lsid"), lsids, CompareType.IN), - null)) - .getArrayList(Map.class); - return (List>)result; - } - */ - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - { - int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - if (count > 0) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container, null, null, true); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - } - return count; - } - - @Override - public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - int count = _importRowsUsingDIB(user, container, rows, null, context, extraScriptContext); - if (count > 0 && !Boolean.TRUE.equals(context.getConfigParameterBoolean(Config.SkipResyncStudy))) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container, null, null, true); - } - catch (ValidationException e) - { - context.getErrors().addRowError(e); - } - } - return count; - } - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) - { - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - - return loadRows(user, container, rows, context, extraScriptContext); - } - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws QueryUpdateServiceException - { - for (Map row : rows) - { - aliasColumns(_columnMapping, row); - } - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - if (_skipAuditLogging) - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE); - else if (!isBulkLoad()) - { - // default to DETAILED unless there is a metadata XML override - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, - getQueryTable().getXmlAuditBehaviorType() != null ? getQueryTable().getXmlAuditBehaviorType() : DETAILED); - } - - List> result = super._insertRowsUsingDIB(user, container, rows, context, extraScriptContext); - - if (null != result && result.size() > 0) - { - for (Map row : result) - { - String participantID = getParticipant(row, user, container); - _potentiallyNewParticipants.add(participantID); - } - - _participantVisitResyncRequired = true; // 13717 : Study failing to resync() on dataset insert - if (configParameters == null || !Boolean.TRUE.equals(configParameters.get(DatasetUpdateService.Config.SkipResyncStudy))) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - } - } - return result; - } - - @Override - protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) - { - // If we're using a managed GUID as a key, wire it up here so that it's available to trigger scripts - if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER && - _dataset.getKeyManagementType() == Dataset.KeyManagementType.GUID && - _dataset.getKeyPropertyName() != null) - { - return new DataIteratorBuilder() - { - @Override - public DataIterator getDataIterator(DataIteratorContext context) - { - DataIterator input = in.getDataIterator(context); - if (null == input) - return null; // Can happen if context has errors - - final SimpleTranslator result = new SimpleTranslator(input, context); - - boolean foundKeyCol = false; - for (int c = 1; c <= input.getColumnCount(); c++) - { - ColumnInfo col = input.getColumnInfo(c); - - // Incoming data has a matching field - if (col.getName().equalsIgnoreCase(_dataset.getKeyPropertyName())) - { - // make sure guid is not null (12884) - result.addCoalesceColumn(col.getName(), c, new SimpleTranslator.GuidColumn()); - foundKeyCol = true; - } - else - { - // Pass it through as-is - result.addColumn(c); - } - } - - if (!foundKeyCol) - { - // Inject a column with a new GUID - ColumnInfo key = getQueryTable().getColumn(_dataset.getKeyPropertyName()); - result.addColumn(new BaseColumnInfo(key), new SimpleTranslator.GuidColumn()); - } - - return result; - } - }; - } - return in; - } - - @Override - public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) - { - if (null == context.getConfigParameter(Config.DefaultQCState)) - { - context.putConfigParameter(Config.DefaultQCState, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); - } - - if (null == context.getConfigParameter(Config.CheckForDuplicates)) - { - DatasetDefinition.CheckForDuplicates dupePolicy; - if (isBulkLoad()) - dupePolicy = DatasetDefinition.CheckForDuplicates.never; - else if (context.getInsertOption().mergeRows) - dupePolicy = DatasetDefinition.CheckForDuplicates.sourceOnly; - else - dupePolicy = DatasetDefinition.CheckForDuplicates.sourceAndDestination; - context.putConfigParameter(Config.CheckForDuplicates, dupePolicy); - } - - // NOTE: This was done to help coalesce some old code paths. However, this is a little weird, because - // the DI is over the DatasetSchemaTableInfo not the DatasetTableImpl you'd expect. This all still works - // because of property URI matching in StatementDataIterator. - return _dataset.getInsertDataIterator(user, container, data, context); - } - - - @Override - protected int _pump(DataIteratorBuilder etl, final ArrayList> rows, DataIteratorContext context) - { - try - { - boolean hasRowId = _dataset.getKeyManagementType() == Dataset.KeyManagementType.RowId; - - if (null != rows) - { - // TODO: consider creating DataIterator metadata to mark "internal" cols (not to be returned via API) - DataIterator it = etl.getDataIterator(context); - DataIteratorBuilder cleanMap = new MapDataIterator.MapDataIteratorImpl(it, true, CaseInsensitiveHashSet.of( - it.getColumnInfo(0).getName() - )); - etl = cleanMap; - } - - if (!hasRowId) - { - return super._pump(etl, rows, context); - } - - synchronized (_dataset.getManagedKeyLock()) - { - return super._pump(etl, rows, context); - } - } - catch (RuntimeSQLException e) - { - String translated = _dataset.translateSQLException(e); - if (translated != null) - { - context.getErrors().addRowError(new ValidationException(translated)); - return 0; - } - throw e; - } - } - - - @NotNull String getParticipant(Map row, User user, Container container) throws QueryUpdateServiceException - { - String columnName = _dataset.getStudy().getSubjectColumnName(); - Object participant = row.get(columnName); - if (participant == null) - { - participant = row.get("ParticipantId"); - } - if (participant == null) - { - try - { - // This may be an update or delete where the user specified the LSID as the key, but didn't bother - // sending the participant, so look it up - Map originalRow = getRow(user, container, row); - participant = originalRow == null ? null : originalRow.get(columnName); - if (participant == null) - { - participant = originalRow.get("ParticipantId"); - } - } - catch (InvalidKeyException e) - { - throw new QueryUpdateServiceException(e); - } - } - if (participant == null) - { - throw new QueryUpdateServiceException("All dataset rows must include a value for " + columnName); - } - return participant.toString(); - } - - static class PurgeParticipantCommitTask implements Runnable - { - private final Container _container; - private final Set _potentiallyDeletedParticipants; - - PurgeParticipantCommitTask(Container container, Set potentiallyDeletedParticipants) - { - _container = container; - _potentiallyDeletedParticipants = new HashSet<>(potentiallyDeletedParticipants); - } - - @Override - public void run() - { - new ParticipantPurger(_container, _potentiallyDeletedParticipants, LOG::info, LOG::error).purgeParticipants(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - PurgeParticipantCommitTask that = (PurgeParticipantCommitTask) o; - - if (!Objects.equals(_container, that._container)) return false; - - return true; - } - - @Override - public int hashCode() - { - return _container != null ? _container.hashCode() : 0; - } - } - - - @Override - public List> updateRows(User user, final Container container, List> rows, List> oldKeys, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - List> result = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (null != extraScriptContext && Boolean.TRUE.equals(extraScriptContext.get("synchronousParticipantPurge"))) - { - PurgeParticipantCommitTask addObj = new PurgeParticipantCommitTask(container, _potentiallyDeletedParticipants); - PurgeParticipantCommitTask setObj = getQueryTable().getSchema().getScope().addCommitTask(addObj, DbScope.CommitTaskOption.POSTCOMMIT); - setObj._potentiallyDeletedParticipants.addAll(addObj._potentiallyDeletedParticipants); - } - - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - return result; - } - - private void resyncStudy(User user, Container container) throws ValidationException - { - resyncStudy(user, container, _potentiallyNewParticipants, _potentiallyDeletedParticipants, _participantVisitResyncRequired); - - _participantVisitResyncRequired = false; - _potentiallyNewParticipants.clear(); - _potentiallyDeletedParticipants.clear(); - } - - /** - * Resyncs the study : updates the participant, visit, and (optionally) participant visit tables. Also updates automatic cohort assignments. - * - * @param potentiallyAddedParticipants optionally, the specific participants that may have been added to the study. - * If null, all the changedDatasets and specimens will be checked to see if they contain new participants - * @param potentiallyDeletedParticipants optionally, the specific participants that may have been removed from the - * study. If null, all participants will be checked to see if they are still in the study. - * @param participantVisitResyncRequired If true, will force an update of the ParticipantVisit mapping for this study - */ - private void resyncStudy(User user, Container container, @Nullable Set potentiallyAddedParticipants, - @Nullable Set potentiallyDeletedParticipants, - boolean participantVisitResyncRequired) throws ValidationException - { - StudyImpl study = StudyManager.getInstance().getStudy(container); - Study sharedStudy = StudyManager.getInstance().getSharedStudy(study); - - ValidationException errors = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singletonList(_dataset), - potentiallyAddedParticipants, potentiallyDeletedParticipants, participantVisitResyncRequired, - sharedStudy != null ? sharedStudy.isFailForUndefinedTimepoints() : study.isFailForUndefinedTimepoints(), null); - - if (errors.hasErrors()) - throw errors; - } - - @Override - protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException - { - // Issue 53320 : ensure a valid file link path - if (fileLinkDirPath == null) - fileLinkDirPath = AssayFileWriter.getUploadDirectoryPath(c, "datasetdata").toNioPathForWrite(); - - super.convertTypes(user, c, row, t, fileLinkDirPath); - } - - @Override - protected Map _update(User user, Container container, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException - { - try (DbScope.Transaction transaction = StudyService.get().getDatasetSchema().getScope().ensureTransaction()) - { - String lsid = keyFromMap(oldRow); - checkDuplicateUpdate(lsid); - // Make sure we've found the original participant before doing the update - String oldParticipant = getParticipant(oldRow, user, container); - String newLsid; - - Long rowId = (Long)oldRow.get(DatasetDomainKind.DSROWID); - Map oldData = _dataset.getDatasetRow(user, lsid); - - if (oldData == null) - { - // No old record found, so we can't update - ValidationException error = new ValidationException(); - error.addError(new SimpleValidationError("Record not found with lsid: " + lsid)); - throw error; - } - - // values that are always recalculated - getComputedValues(user, row, oldRow); - - newLsid = (String)row.get(DatasetDomainKind.LSID); - Table.update(user, getDbTable(), row, rowId); - - if (!isBulkLoad()) - { - DatasetTableImpl target = (DatasetTableImpl)_dataset.getTableInfo(user); - new DatasetDefinition.DatasetAuditHandler(_dataset).addAuditEvent(user, container, target, AuditBehaviorType.DETAILED, null, QueryService.AuditAction.UPDATE, - List.of(row), List.of(oldData)); - } - - // Successfully updated - transaction.commit(); - - // return updated row - var returnRow = getRow(user, container, Map.of(DatasetDomainKind.LSID, newLsid)); - - String newParticipant = getParticipant(returnRow, user, container); - if (!oldParticipant.equals(newParticipant)) - { - // Participant has changed - might be a reference to a new participant, or removal of the last reference to - // the old participant - _potentiallyNewParticipants.add(newParticipant); - _potentiallyDeletedParticipants.add(oldParticipant); - - // Need to resync the ParticipantVisit table too - _participantVisitResyncRequired = true; - } - // Check if the timepoint may have changed, but only if we don't already know we need to resync - else if (!_participantVisitResyncRequired) - { - String columnName = StudyManager.getInstance().getStudy(container).getTimepointType().isVisitBased() ? - "SequenceNum" : "Date"; - Object oldTimepoint = oldRow.get(columnName); - Object newTimepoint = returnRow.get(columnName); - if (!Objects.equals(oldTimepoint, newTimepoint)) - { - _participantVisitResyncRequired = true; - } - } - - return returnRow; - } - catch (QueryUpdateServiceException | InvalidKeyException e) - { - throw new ValidationException(e.getMessage()); - } - } - - private void getComputedValues(User user, Map row, Map oldRow) throws ValidationException - { - String subjectColumnName = _dataset.getStudy().getSubjectColumnName(); - TableInfo table = _dataset.getTableInfo(user); - ColumnInfo subjectColumn = table.getColumn(subjectColumnName); - ColumnInfo sequenceNumColumn = table.getColumn(DatasetDomainKind.SEQUENCENUM); - ColumnInfo dateColumn = table.getColumn(DatasetDomainKind.DATE); - String managedKey = null; - if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER) - managedKey = _dataset.getKeyPropertyName(); - ColumnInfo managedKeyColumn = managedKey != null ? table.getColumn(managedKey) : null; - - Object inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(row, subjectColumn); - if (inputSubjectId == null) - inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(oldRow, subjectColumn); - Object inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(row, sequenceNumColumn); - if (inputSeqNum == null) - inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(oldRow, sequenceNumColumn); - Date inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(row, dateColumn); - if (inputDate == null) - inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(oldRow, dateColumn); - Object inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(row, managedKeyColumn); - if (inputManagedKey == null) - inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(oldRow, managedKeyColumn); - Integer inputQCState = asInteger(DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_ID_COLNAME))); - - SequenceNumImportHelper snih = new SequenceNumImportHelper(_dataset.getStudy(), _dataset); - Double sequenceNum = snih.translateSequenceNum(inputSeqNum, inputDate); - - ParticipantIdImportHelper helper = new ParticipantIdImportHelper(_dataset.getStudy(), user, _dataset); - String subjectId = helper.translateParticipantId(inputSubjectId); - - // generate participant sequence number - String participantSeqNum = ParticipantSeqNumImportHelper.translateParticipantSeqNum(subjectId, sequenceNum); - - // re-generate a new lsid - DatasetLsidImportHelper dlih = new DatasetLsidImportHelper(_dataset); - String lsid = dlih.translateLsid(subjectId, sequenceNum, inputDate, inputManagedKey, null); - - // handle default QC states - if (inputQCState == null) - { - String inputQCText = (String)DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_LABEL_COLNAME)); - QCStateImportHelper qcih = new QCStateImportHelper(user, _dataset, true, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); - Long qcState = qcih.translateQCState(inputQCText); - if (qcState != null) - row.put(DatasetTableImpl.QCSTATE_ID_COLNAME, qcState); - } - row.put(DatasetDomainKind.LSID, lsid); - row.put(DatasetDomainKind.PARTICIPANTSEQUENCENUM, participantSeqNum); - row.put(DatasetDomainKind.PARTICIPANTID, subjectId); - } - - @Override - public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - List> result = super.deleteRows(user, container, keys, configParameters, extraScriptContext); - try - { - resyncStudy(user, container); - } - catch (ValidationException e) - { - throw new BatchValidationException(e); - } - return result; - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRow) - throws InvalidKeyException, QueryUpdateServiceException - { - // Make sure we've found the original participant before doing the delete - String participant = getParticipant(oldRow, user, container); - _dataset.deleteDatasetRows(user, Collections.singleton(keyFromMap(oldRow)), isBulkLoad()); - _potentiallyDeletedParticipants.add(participant); - _participantVisitResyncRequired = true; - return oldRow; - } - - @Override - protected int truncateRows(User user, Container container) - { - return _dataset.deleteRows((Date) null); - } - - @Override - public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws BatchValidationException, QueryUpdateServiceException, SQLException - { - Map updatedParams = configParameters; - if (updatedParams == null) - updatedParams = new HashMap<>(); - updatedParams.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.SUMMARY); - return super.truncateRows(user, container, updatedParams, extraScriptContext); - } - - public String keyFromMap(Map map) throws InvalidKeyException - { - Object lsid = map.get("lsid"); - if (lsid != null) - return lsid.toString(); - lsid = map.get("LSID"); - if (lsid != null) - return lsid.toString(); - - boolean isDemographic = _dataset.isDemographicData(); - - // if there was no explicit lsid and KeyManagementType == None, there is no non-lsid key that is unique by itself. - // Unless of course it is a demographic table. - if (!isDemographic && _dataset.getKeyManagementType() == DatasetDefinition.KeyManagementType.None) - { - throw new InvalidKeyException("No lsid, and no KeyManagement"); - } - - String keyPropertyName = isDemographic ? _dataset.getStudy().getSubjectColumnName() : _dataset.getKeyPropertyName(); - Object id = map.get(keyPropertyName); - - if (null == id) - { - id = map.get("Key"); - } - - // if there was no other type of key, this query is invalid - if (null == id) - { - throw new InvalidKeyException(String.format("key needs one of 'lsid', '%s' or 'Key', none of which were found in %s", keyPropertyName, map)); - } - - // now look up lsid - // if one is found, return that - // if 0, it's legal to return null - // if > 1, there is an integrity problem that should raise alarm - String[] lsids = new TableSelector(getQueryTable().getColumn("LSID"), new SimpleFilter(keyPropertyName, id), null).getArray(String.class); - - if (lsids.length == 0) - { - return null; - } - else if (lsids.length > 1) - { - throw new IllegalStateException("More than one row matched for key '" + id + "' in column " + - _dataset.getKeyPropertyName() + " in dataset " + - _dataset.getName() + " in folder " + - _dataset.getContainer().getPath()); - } - else return lsids[0]; - } - - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert - { - private static final String SUBJECT_COLUMN_NAME = "SubjectID"; - private static final String DATASET_NAME = "DS1"; - TestContext _context = null; - User _user = null; - Container _container = null; - StudyImpl _junitStudy = null; - StudyManager _manager = StudyManager.getInstance(); - String longName = "this is a very long name (with punctuation) that raises many questions \"?\" about your database design choices"; - - private void createDataset() throws Exception - { - if (DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME) != null) - { - return; - } - - var dsd = new DatasetDefinition(_junitStudy, 1001, DATASET_NAME, DATASET_NAME, null, null, null); - _manager.createDatasetDefinition(_user, dsd); - dsd = _manager.getDatasetDefinition(_junitStudy, 1001); - dsd.refreshDomain(); - { - var domain = dsd.getDomain(true); - DomainProperty p; - - p = domain.addProperty(); - p.setName("Field1"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName("SELECT"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName(longName); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName("Value1"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - p = domain.addProperty(); - p.setName("Value2"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - p = domain.addProperty(); - p.setName("Value3"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - domain.save(_user); - } - } - - private long getDatasetAuditRowCount() - { - return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT)).getRowCount(); - } - - private String getLatestAuditMessage() - { - return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT), PageFlowUtil.set("Comment"), null, new Sort("-rowId")).setMaxRows(1).getObject(String.class); - } - - @Test - public void testAuditing() throws Exception - { - createDataset(); - TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); - t.getUpdateService().truncateRows(_user, _container, null, null); - - final QueryUpdateService qus = t.getUpdateService(); - BatchValidationException errors = new BatchValidationException(); - - long actualAuditRows = getDatasetAuditRowCount(); - long expectedAuditRows; - - List> insertedRows = qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S1", - "SequenceNum", "1.2345", - longName, "NA"), - Map.of( - SUBJECT_COLUMN_NAME, "S2", - "SequenceNum", "1.2345", - longName, "WithoutBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A new dataset record was inserted", getLatestAuditMessage()); - - qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S3", - "SequenceNum", "1.2345", - longName, "WithoutBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 1; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - // Now update: - insertedRows.get(0).put(longName, "NewValue"); - insertedRows.get(1).put(longName, "NewValue"); - List> oldKeys = Arrays.asList( - Map.of("lsid", insertedRows.get(0).get("lsid")), - Map.of("lsid", insertedRows.get(1).get("lsid")) - ); - qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A dataset record was modified", getLatestAuditMessage()); - - qus.deleteRows(_user, _container, oldKeys, null, null); - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A dataset record was deleted", getLatestAuditMessage()); - - // Repeat using bulkLoad=true: - qus.setBulkLoad(true); - - insertedRows = qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S4", - "SequenceNum", "1.2345", - longName, "WithBulkLoad"), - Map.of( - SUBJECT_COLUMN_NAME, "S5", - "SequenceNum", "1.2345", - longName, "WithBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - // Now update: - insertedRows.get(0).put(longName, "NewValue"); - insertedRows.get(1).put(longName, "NewValue"); - oldKeys = Arrays.asList( - Map.of("lsid", insertedRows.get(0).get("lsid")), - Map.of("lsid", insertedRows.get(1).get("lsid")) - ); - qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - qus.deleteRows(_user, _container, oldKeys, null, null); - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - } - - @Test - public void updateRowTest() throws Exception - { - createDataset(); - - TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); - assertNotNull(t); - assertTrue("Field1".equalsIgnoreCase(t.getColumn("Field1").getAlias().getId())); - assertFalse("SELECT".equalsIgnoreCase(t.getColumn("SELECT").getAlias().getId())); - assertFalse(longName.equalsIgnoreCase(t.getColumn(longName).getAlias().getId())); - var up = t.getUpdateService(); - assertNotNull(up); - var errors = new BatchValidationException(); - - var result = up.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, " S1 \t", - "SequenceNum", "1.2345", - "Field1", "f", - "SELECT", "s", - longName, "l", - "value1", "1.0", - "value2", "NA", - "VALUE3", "NA")), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertFalse(errors.hasErrors()); - assertNotNull(result); - assertEquals(1, result.size()); - var map = result.get(0); - assertEquals("S1", map.get(SUBJECT_COLUMN_NAME)); - assertEquals("f", map.get("Field1")); - assertEquals("s", map.get("SELECT")); - assertEquals("l", map.get(longName)); - assertEquals( 1.0d, map.get("value1")); // 1.0 - var v2 = map.get("value2"); // NA - assertTrue(v2 instanceof MvFieldWrapper); - assertEquals("NA", ((MvFieldWrapper)v2).getMvIndicator()); - var v3 = map.get("value3"); // NA - assertTrue(v3 instanceof MvFieldWrapper); - assertEquals("NA", ((MvFieldWrapper)v3).getMvIndicator()); - assertNotNull(map.get("lsid")); - assertTrue(((String)map.get("lsid")).endsWith(":1001.S1.1.2345")); - String lsid = (String)map.get("lsid"); - - // update subjectid column - result = up.updateRows(_user, _container, - List.of(Map.of(SUBJECT_COLUMN_NAME, "\tS2 ")), - List.of(Map.of("lsid", lsid)), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertNotNull(result); - assertEquals(1, result.size()); - map = result.get(0); - assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); - // All other columns are preserved - assertEquals("f", map.get("Field1")); - assertEquals("s", map.get("SELECT")); - assertEquals("l", map.get(longName)); - assertEquals( 1.0d, map.get("value1")); // 1.0 - // DIFFERENCE - updateRows() does not return MvFieldWrapper - assertNull(map.get("value2")); // NA - assertEquals("NA", map.get("value2mvindicator")); - assertNull(map.get("VALUE3")); // NA - assertEquals("NA", map.get("value3MvIndicator")); - // LSID is updated - assertNotNull(map.get("lsid")); - assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); - lsid = (String)map.get("lsid"); - - // update other columns - result = up.updateRows(_user, _container, - List.of(Map.of( - "Field1", "fUpdated", - "SELECT", "sUpdated", - longName, "lUpdated", - "value1", "NA", // 1.0 -> NA - "value2", "2.0", // NA -> 2.0 - "value3", "QA") // NA -> QA - ), - List.of(Map.of("lsid", lsid)), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertNotNull(result); - assertEquals(1, result.size()); - map = result.get(0); - assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); - assertEquals("fUpdated", map.get("Field1")); - assertEquals("sUpdated", map.get("SELECT")); - assertEquals("lUpdated", map.get(longName)); - assertNull(map.get("value1")); // NA - assertEquals("NA", map.get("Value1MVIndicator")); - assertEquals(2.0d, map.get("value2")); // 2.0 - assertNull(map.get("Value2MVIndicator")); - assertNull(map.get("value3")); // QA - assertEquals("QA", map.get("value3mVindicator")); - assertNotNull(map.get("lsid")); - // unchanged - assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); - map.get("lsid"); - } - - @Before - public void createStudy() - { - _context = TestContext.get(); - Container junit = JunitUtil.getTestContainer(); - String name = GUID.makeHash(); - Container c = ContainerManager.createContainer(junit, name, _context.getUser()); - MvUtil.assignMvIndicators(c, new String[] {"NA","QA"}, new String[] {"NA","QA"}); - StudyImpl s = new StudyImpl(c, "Junit Study"); - s.setTimepointType(TimepointType.VISIT); - s.setStartDate(new Date(DateUtil.parseDateTime(c, "2014-01-01"))); - s.setSubjectColumnName(SUBJECT_COLUMN_NAME); - s.setSubjectNounPlural("Subjects"); - s.setSubjectNounSingular("Subject"); - s.setSecurityType(SecurityType.BASIC_WRITE); - _junitStudy = StudyManager.getInstance().createStudy(_context.getUser(), s); - _user = _context.getUser(); - _container = _junitStudy.getContainer(); - } - - @After - public void tearDown() - { - if (null != _junitStudy) - { - assertTrue(ContainerManager.delete(_junitStudy.getContainer(), _context.getUser())); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.study.query; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.ResultSetRowMapFactory; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.MvFieldWrapper; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.TimepointType; +import org.labkey.api.study.security.StudySecurityEscalator; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.TestContext; +import org.labkey.study.model.DatasetDataIteratorBuilder; +import org.labkey.study.dataset.DatasetAuditProvider; +import org.labkey.study.model.DatasetDefinition; +import org.labkey.study.model.DatasetDomainKind; +import org.labkey.study.model.DatasetLsidImportHelper; +import org.labkey.study.model.ParticipantIdImportHelper; +import org.labkey.study.model.ParticipantSeqNumImportHelper; +import org.labkey.study.model.QCStateImportHelper; +import org.labkey.study.model.SecurityType; +import org.labkey.study.model.SequenceNumImportHelper; +import org.labkey.study.model.StudyImpl; +import org.labkey.study.model.StudyManager; +import org.labkey.study.visitmanager.PurgeParticipantsJob.ParticipantPurger; + +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.labkey.api.util.IntegerUtils.asInteger; +import static org.labkey.api.gwt.client.AuditBehaviorType.DETAILED; +import static org.labkey.api.gwt.client.AuditBehaviorType.NONE; + +/* +* User: Dave +* Date: Jun 13, 2008 +* Time: 4:15:51 PM +*/ + +/** + * QueryUpdateService implementation for Study datasets. + *

+ * Since datasets are of an unpredictable shape, this class just implements + * the QueryUpdateService directly, working with Map<String,Object> + * collections for the row data. + */ +public class DatasetUpdateService extends DefaultQueryUpdateService +{ + // These are that can be passed into DatasetUpdateService via DataIteratorContext.configParameters. + // These used to be passed to DatasetDataIterator via + // DatasetDefinition.importDatasetData()->DatasetDefinition.insertData(). + // Moving these options into DataInteratorContext allows for even more consistency and code sharing + // also see QueryUpdateService.ConfigParameters.Logger + public enum Config + { + CheckForDuplicates, // expected: enum CheckForDuplicates + DefaultQCState, // expected: class QCState + SkipResyncStudy, // expected: Boolean + + // NOTE: There really has to be better way to handle the functionality of StudyImportContext.getTableIdMap() + // NOTE: Could this be handled by a method on StudySchema or something??? + // see StudyImportContext.getTableIdMapMap() + StudyImportMaps, // expected: Map> + + KeyList, // expected: List + AllowImportManagedKey // expected: Boolean + } + + private static final Logger LOG = LogManager.getLogger(DatasetUpdateService.class); + + private final DatasetDefinition _dataset; + private final Set _potentiallyNewParticipants = new HashSet<>(); + private final Set _potentiallyDeletedParticipants = new HashSet<>(); + private boolean _participantVisitResyncRequired = false; + + private final boolean _skipAuditLogging = false; + + public DatasetUpdateService(DatasetTableImpl table) + { + super(table, table.getDatasetDefinition().getStorageTableInfo(false), createMVMapping(table.getDatasetDefinition().getDomain())); + _dataset = table.getDatasetDefinition(); + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) + { + if (StudySecurityEscalator.isEscalated()) { + return true; + } + else { + return super.hasPermission(user, acl); + } + } + + + @Override + protected Map getRow(User user, Container container, Map keys) + throws InvalidKeyException + { + String lsid = keyFromMap(keys); + SimpleFilter filter = new SimpleFilter() + .addCondition(new FieldKey(null,"container"), container.getId()) + .addCondition(new FieldKey(null,"lsid"),lsid); + + // NOTE getQueryTable().getColumns() returns a bunch of columns that getDatasetRow() did not such as: + // Container, Dataset, DatasetId, Datasets, Folder, Modified, ModifiedBy, MouseVisit, ParticipantSequenceNum, VisitDay, VisitRowId + // Mostly this is harmless, but there is some noise. + HashSet nameset = new HashSet<>(getQueryTable().getColumnNameSet()); + List.of("Container","Datasets","DatasetId","Dataset","Folder").forEach(nameset::remove); + List columns = new ArrayList<>(getQueryTable().getColumns(nameset.toArray(new String[0]))); + + // filter out calculated columns which can be expensive to reselect + columns.removeIf(ColumnInfo::isCalculated); + + // This is a general version of DatasetDefinition.canonicalizeDatasetRow() + // The caller needs to make sure names are unique. Not suitable for use w/ lookups etc where there can be name collisions. + // CONSIDER: might be nice to make this a TableSelector method. + var map = new CaseInsensitiveHashMap<>(); + try (var str = new TableSelector(getQueryTable(), columns, filter, null).uncachedResultSetStream()) + { + str.forEach(rs -> { + try + { + for (int i = 0; i < columns.size(); i++) + { + Object o = rs.getObject(i + 1); + o = ResultSetRowMapFactory.translateResultSetObject(o, false); + map.put(columns.get(i).getName(), o); + } + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + }); + } + return map.isEmpty() ? null : map; + } + + + /* TODO for performance, NOTE need to return rows in order of input list + @Override + public List> getRows(User user, Container container, List> keys) throws InvalidKeyException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + ArrayList lsids = new ArrayList<>(keys.size()); + for (var m : keys) + lsids.add(keyFromMap(m)); + var result = (List)(new TableSelector(getQueryTable(), + TableSelector.ALL_COLUMNS, + new SimpleFilter(new FieldKey(null,"lsid"), lsids, CompareType.IN), + null)) + .getArrayList(Map.class); + return (List>)result; + } + */ + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + if (count > 0) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container, null, null, true); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + } + return count; + } + + @Override + public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + int count = _importRowsUsingDIB(user, container, rows, null, context, extraScriptContext); + if (count > 0 && !Boolean.TRUE.equals(context.getConfigParameterBoolean(Config.SkipResyncStudy))) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container, null, null, true); + } + catch (ValidationException e) + { + context.getErrors().addRowError(e); + } + } + return count; + } + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) + { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + + return loadRows(user, container, rows, context, extraScriptContext); + } + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws QueryUpdateServiceException + { + for (Map row : rows) + { + aliasColumns(_columnMapping, row); + } + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + if (_skipAuditLogging) + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE); + else if (!isBulkLoad()) + { + // default to DETAILED unless there is a metadata XML override + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, + getQueryTable().getXmlAuditBehaviorType() != null ? getQueryTable().getXmlAuditBehaviorType() : DETAILED); + } + + List> result = super._insertRowsUsingDIB(user, container, rows, context, extraScriptContext); + + if (null != result && result.size() > 0) + { + for (Map row : result) + { + String participantID = getParticipant(row, user, container); + _potentiallyNewParticipants.add(participantID); + } + + _participantVisitResyncRequired = true; // 13717 : Study failing to resync() on dataset insert + if (configParameters == null || !Boolean.TRUE.equals(configParameters.get(DatasetUpdateService.Config.SkipResyncStudy))) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + } + } + return result; + } + + @Override + protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) + { + // If we're using a managed GUID as a key, wire it up here so that it's available to trigger scripts + if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER && + _dataset.getKeyManagementType() == Dataset.KeyManagementType.GUID && + _dataset.getKeyPropertyName() != null) + { + return new DataIteratorBuilder() + { + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator input = in.getDataIterator(context); + if (null == input) + return null; // Can happen if context has errors + + final SimpleTranslator result = new SimpleTranslator(input, context); + + boolean foundKeyCol = false; + for (int c = 1; c <= input.getColumnCount(); c++) + { + ColumnInfo col = input.getColumnInfo(c); + + // Incoming data has a matching field + if (col.getName().equalsIgnoreCase(_dataset.getKeyPropertyName())) + { + // make sure guid is not null (12884) + result.addCoalesceColumn(col.getName(), c, new SimpleTranslator.GuidColumn()); + foundKeyCol = true; + } + else + { + // Pass it through as-is + result.addColumn(c); + } + } + + if (!foundKeyCol) + { + // Inject a column with a new GUID + ColumnInfo key = getQueryTable().getColumn(_dataset.getKeyPropertyName()); + result.addColumn(new BaseColumnInfo(key), new SimpleTranslator.GuidColumn()); + } + + return result; + } + }; + } + return in; + } + + @Override + public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) + { + if (null == context.getConfigParameter(Config.DefaultQCState)) + { + context.putConfigParameter(Config.DefaultQCState, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); + } + + if (null == context.getConfigParameter(Config.CheckForDuplicates)) + { + DatasetDefinition.CheckForDuplicates dupePolicy; + if (isBulkLoad()) + dupePolicy = DatasetDefinition.CheckForDuplicates.never; + else if (context.getInsertOption().mergeRows) + dupePolicy = DatasetDefinition.CheckForDuplicates.sourceOnly; + else + dupePolicy = DatasetDefinition.CheckForDuplicates.sourceAndDestination; + context.putConfigParameter(Config.CheckForDuplicates, dupePolicy); + } + + // NOTE: This was done to help coalesce some old code paths. However, this is a little weird, because + // the DI is over the DatasetSchemaTableInfo not the DatasetTableImpl you'd expect. This all still works + // because of property URI matching in StatementDataIterator. + return _dataset.getInsertDataIterator(user, container, data, context); + } + + + @Override + protected int _pump(DataIteratorBuilder etl, final ArrayList> rows, DataIteratorContext context) + { + try + { + boolean hasRowId = _dataset.getKeyManagementType() == Dataset.KeyManagementType.RowId; + + if (null != rows) + { + // TODO: consider creating DataIterator metadata to mark "internal" cols (not to be returned via API) + DataIterator it = etl.getDataIterator(context); + DataIteratorBuilder cleanMap = new MapDataIterator.MapDataIteratorImpl(it, true, CaseInsensitiveHashSet.of( + it.getColumnInfo(0).getName() + )); + etl = cleanMap; + } + + if (!hasRowId) + { + return super._pump(etl, rows, context); + } + + synchronized (_dataset.getManagedKeyLock()) + { + return super._pump(etl, rows, context); + } + } + catch (RuntimeSQLException e) + { + String translated = _dataset.translateSQLException(e); + if (translated != null) + { + context.getErrors().addRowError(new ValidationException(translated)); + return 0; + } + throw e; + } + } + + + @NotNull String getParticipant(Map row, User user, Container container) throws QueryUpdateServiceException + { + String columnName = _dataset.getStudy().getSubjectColumnName(); + Object participant = row.get(columnName); + if (participant == null) + { + participant = row.get("ParticipantId"); + } + if (participant == null) + { + try + { + // This may be an update or delete where the user specified the LSID as the key, but didn't bother + // sending the participant, so look it up + Map originalRow = getRow(user, container, row); + participant = originalRow == null ? null : originalRow.get(columnName); + if (participant == null) + { + participant = originalRow.get("ParticipantId"); + } + } + catch (InvalidKeyException e) + { + throw new QueryUpdateServiceException(e); + } + } + if (participant == null) + { + throw new QueryUpdateServiceException("All dataset rows must include a value for " + columnName); + } + return participant.toString(); + } + + static class PurgeParticipantCommitTask implements Runnable + { + private final Container _container; + private final Set _potentiallyDeletedParticipants; + + PurgeParticipantCommitTask(Container container, Set potentiallyDeletedParticipants) + { + _container = container; + _potentiallyDeletedParticipants = new HashSet<>(potentiallyDeletedParticipants); + } + + @Override + public void run() + { + new ParticipantPurger(_container, _potentiallyDeletedParticipants, LOG::info, LOG::error).purgeParticipants(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PurgeParticipantCommitTask that = (PurgeParticipantCommitTask) o; + + if (!Objects.equals(_container, that._container)) return false; + + return true; + } + + @Override + public int hashCode() + { + return _container != null ? _container.hashCode() : 0; + } + } + + + @Override + public List> updateRows(User user, final Container container, List> rows, List> oldKeys, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + List> result = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (null != extraScriptContext && Boolean.TRUE.equals(extraScriptContext.get("synchronousParticipantPurge"))) + { + PurgeParticipantCommitTask addObj = new PurgeParticipantCommitTask(container, _potentiallyDeletedParticipants); + PurgeParticipantCommitTask setObj = getQueryTable().getSchema().getScope().addCommitTask(addObj, DbScope.CommitTaskOption.POSTCOMMIT); + setObj._potentiallyDeletedParticipants.addAll(addObj._potentiallyDeletedParticipants); + } + + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + return result; + } + + private void resyncStudy(User user, Container container) throws ValidationException + { + resyncStudy(user, container, _potentiallyNewParticipants, _potentiallyDeletedParticipants, _participantVisitResyncRequired); + + _participantVisitResyncRequired = false; + _potentiallyNewParticipants.clear(); + _potentiallyDeletedParticipants.clear(); + } + + /** + * Resyncs the study : updates the participant, visit, and (optionally) participant visit tables. Also updates automatic cohort assignments. + * + * @param potentiallyAddedParticipants optionally, the specific participants that may have been added to the study. + * If null, all the changedDatasets and specimens will be checked to see if they contain new participants + * @param potentiallyDeletedParticipants optionally, the specific participants that may have been removed from the + * study. If null, all participants will be checked to see if they are still in the study. + * @param participantVisitResyncRequired If true, will force an update of the ParticipantVisit mapping for this study + */ + private void resyncStudy(User user, Container container, @Nullable Set potentiallyAddedParticipants, + @Nullable Set potentiallyDeletedParticipants, + boolean participantVisitResyncRequired) throws ValidationException + { + StudyImpl study = StudyManager.getInstance().getStudy(container); + Study sharedStudy = StudyManager.getInstance().getSharedStudy(study); + + ValidationException errors = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singletonList(_dataset), + potentiallyAddedParticipants, potentiallyDeletedParticipants, participantVisitResyncRequired, + sharedStudy != null ? sharedStudy.isFailForUndefinedTimepoints() : study.isFailForUndefinedTimepoints(), null); + + if (errors.hasErrors()) + throw errors; + } + + @Override + protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException + { + // Issue 53320 : ensure a valid file link path + if (fileLinkDirPath == null) + fileLinkDirPath = AssayFileWriter.getUploadDirectoryPath(c, "datasetdata").toNioPathForWrite(); + + super.convertTypes(user, c, row, t, fileLinkDirPath); + } + + @Override + protected Map _update(User user, Container container, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException + { + try (DbScope.Transaction transaction = StudyService.get().getDatasetSchema().getScope().ensureTransaction()) + { + String lsid = keyFromMap(oldRow); + checkDuplicateUpdate(lsid); + // Make sure we've found the original participant before doing the update + String oldParticipant = getParticipant(oldRow, user, container); + String newLsid; + + Long rowId = (Long)oldRow.get(DatasetDomainKind.DSROWID); + Map oldData = _dataset.getDatasetRow(user, lsid); + + if (oldData == null) + { + // No old record found, so we can't update + ValidationException error = new ValidationException(); + error.addError(new SimpleValidationError("Record not found with lsid: " + lsid)); + throw error; + } + + // values that are always recalculated + getComputedValues(user, row, oldRow); + + newLsid = (String)row.get(DatasetDomainKind.LSID); + Table.update(user, getDbTable(), row, rowId); + + if (!isBulkLoad()) + { + DatasetTableImpl target = (DatasetTableImpl)_dataset.getTableInfo(user); + new DatasetDefinition.DatasetAuditHandler(_dataset).addAuditEvent(user, container, target, AuditBehaviorType.DETAILED, null, QueryService.AuditAction.UPDATE, + List.of(row), List.of(oldData)); + } + + // Successfully updated + transaction.commit(); + + // return updated row + var returnRow = getRow(user, container, Map.of(DatasetDomainKind.LSID, newLsid)); + + String newParticipant = getParticipant(returnRow, user, container); + if (!oldParticipant.equals(newParticipant)) + { + // Participant has changed - might be a reference to a new participant, or removal of the last reference to + // the old participant + _potentiallyNewParticipants.add(newParticipant); + _potentiallyDeletedParticipants.add(oldParticipant); + + // Need to resync the ParticipantVisit table too + _participantVisitResyncRequired = true; + } + // Check if the timepoint may have changed, but only if we don't already know we need to resync + else if (!_participantVisitResyncRequired) + { + String columnName = StudyManager.getInstance().getStudy(container).getTimepointType().isVisitBased() ? + "SequenceNum" : "Date"; + Object oldTimepoint = oldRow.get(columnName); + Object newTimepoint = returnRow.get(columnName); + if (!Objects.equals(oldTimepoint, newTimepoint)) + { + _participantVisitResyncRequired = true; + } + } + + return returnRow; + } + catch (QueryUpdateServiceException | InvalidKeyException e) + { + throw new ValidationException(e.getMessage()); + } + } + + private void getComputedValues(User user, Map row, Map oldRow) throws ValidationException + { + String subjectColumnName = _dataset.getStudy().getSubjectColumnName(); + TableInfo table = _dataset.getTableInfo(user); + ColumnInfo subjectColumn = table.getColumn(subjectColumnName); + ColumnInfo sequenceNumColumn = table.getColumn(DatasetDomainKind.SEQUENCENUM); + ColumnInfo dateColumn = table.getColumn(DatasetDomainKind.DATE); + String managedKey = null; + if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER) + managedKey = _dataset.getKeyPropertyName(); + ColumnInfo managedKeyColumn = managedKey != null ? table.getColumn(managedKey) : null; + + Object inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(row, subjectColumn); + if (inputSubjectId == null) + inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(oldRow, subjectColumn); + Object inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(row, sequenceNumColumn); + if (inputSeqNum == null) + inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(oldRow, sequenceNumColumn); + Date inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(row, dateColumn); + if (inputDate == null) + inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(oldRow, dateColumn); + Object inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(row, managedKeyColumn); + if (inputManagedKey == null) + inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(oldRow, managedKeyColumn); + Integer inputQCState = asInteger(DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_ID_COLNAME))); + + SequenceNumImportHelper snih = new SequenceNumImportHelper(_dataset.getStudy(), _dataset); + Double sequenceNum = snih.translateSequenceNum(inputSeqNum, inputDate); + + ParticipantIdImportHelper helper = new ParticipantIdImportHelper(_dataset.getStudy(), user, _dataset); + String subjectId = helper.translateParticipantId(inputSubjectId); + + // generate participant sequence number + String participantSeqNum = ParticipantSeqNumImportHelper.translateParticipantSeqNum(subjectId, sequenceNum); + + // re-generate a new lsid + DatasetLsidImportHelper dlih = new DatasetLsidImportHelper(_dataset); + String lsid = dlih.translateLsid(subjectId, sequenceNum, inputDate, inputManagedKey, null); + + // handle default QC states + if (inputQCState == null) + { + String inputQCText = (String)DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_LABEL_COLNAME)); + QCStateImportHelper qcih = new QCStateImportHelper(user, _dataset, true, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); + Long qcState = qcih.translateQCState(inputQCText); + if (qcState != null) + row.put(DatasetTableImpl.QCSTATE_ID_COLNAME, qcState); + } + row.put(DatasetDomainKind.LSID, lsid); + row.put(DatasetDomainKind.PARTICIPANTSEQUENCENUM, participantSeqNum); + row.put(DatasetDomainKind.PARTICIPANTID, subjectId); + } + + @Override + public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + List> result = super.deleteRows(user, container, keys, configParameters, extraScriptContext); + try + { + resyncStudy(user, container); + } + catch (ValidationException e) + { + throw new BatchValidationException(e); + } + return result; + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRow) + throws InvalidKeyException, QueryUpdateServiceException + { + // Make sure we've found the original participant before doing the delete + String participant = getParticipant(oldRow, user, container); + _dataset.deleteDatasetRows(user, Collections.singleton(keyFromMap(oldRow)), isBulkLoad()); + _potentiallyDeletedParticipants.add(participant); + _participantVisitResyncRequired = true; + return oldRow; + } + + @Override + protected int truncateRows(User user, Container container) + { + return _dataset.deleteRows((Date) null); + } + + @Override + public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws BatchValidationException, QueryUpdateServiceException, SQLException + { + Map updatedParams = configParameters; + if (updatedParams == null) + updatedParams = new HashMap<>(); + updatedParams.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.SUMMARY); + return super.truncateRows(user, container, updatedParams, extraScriptContext); + } + + public String keyFromMap(Map map) throws InvalidKeyException + { + Object lsid = map.get("lsid"); + if (lsid != null) + return lsid.toString(); + lsid = map.get("LSID"); + if (lsid != null) + return lsid.toString(); + + boolean isDemographic = _dataset.isDemographicData(); + + // if there was no explicit lsid and KeyManagementType == None, there is no non-lsid key that is unique by itself. + // Unless of course it is a demographic table. + if (!isDemographic && _dataset.getKeyManagementType() == DatasetDefinition.KeyManagementType.None) + { + throw new InvalidKeyException("No lsid, and no KeyManagement"); + } + + String keyPropertyName = isDemographic ? _dataset.getStudy().getSubjectColumnName() : _dataset.getKeyPropertyName(); + Object id = map.get(keyPropertyName); + + if (null == id) + { + id = map.get("Key"); + } + + // if there was no other type of key, this query is invalid + if (null == id) + { + throw new InvalidKeyException(String.format("key needs one of 'lsid', '%s' or 'Key', none of which were found in %s", keyPropertyName, map)); + } + + // now look up lsid + // if one is found, return that + // if 0, it's legal to return null + // if > 1, there is an integrity problem that should raise alarm + String[] lsids = new TableSelector(getQueryTable().getColumn("LSID"), new SimpleFilter(keyPropertyName, id), null).getArray(String.class); + + if (lsids.length == 0) + { + return null; + } + else if (lsids.length > 1) + { + throw new IllegalStateException("More than one row matched for key '" + id + "' in column " + + _dataset.getKeyPropertyName() + " in dataset " + + _dataset.getName() + " in folder " + + _dataset.getContainer().getPath()); + } + else return lsids[0]; + } + + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert + { + private static final String SUBJECT_COLUMN_NAME = "SubjectID"; + private static final String DATASET_NAME = "DS1"; + TestContext _context = null; + User _user = null; + Container _container = null; + StudyImpl _junitStudy = null; + StudyManager _manager = StudyManager.getInstance(); + String longName = "this is a very long name (with punctuation) that raises many questions \"?\" about your database design choices"; + + private void createDataset() throws Exception + { + if (DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME) != null) + { + return; + } + + var dsd = new DatasetDefinition(_junitStudy, 1001, DATASET_NAME, DATASET_NAME, null, null, null); + _manager.createDatasetDefinition(_user, dsd); + dsd = _manager.getDatasetDefinition(_junitStudy, 1001); + dsd.refreshDomain(); + { + var domain = dsd.getDomain(true); + DomainProperty p; + + p = domain.addProperty(); + p.setName("Field1"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName("SELECT"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName(longName); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName("Value1"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + p = domain.addProperty(); + p.setName("Value2"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + p = domain.addProperty(); + p.setName("Value3"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + domain.save(_user); + } + } + + private long getDatasetAuditRowCount() + { + return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT)).getRowCount(); + } + + private String getLatestAuditMessage() + { + return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT), PageFlowUtil.set("Comment"), null, new Sort("-rowId")).setMaxRows(1).getObject(String.class); + } + + @Test + public void testAuditing() throws Exception + { + createDataset(); + TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); + t.getUpdateService().truncateRows(_user, _container, null, null); + + final QueryUpdateService qus = t.getUpdateService(); + BatchValidationException errors = new BatchValidationException(); + + long actualAuditRows = getDatasetAuditRowCount(); + long expectedAuditRows; + + List> insertedRows = qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S1", + "SequenceNum", "1.2345", + longName, "NA"), + Map.of( + SUBJECT_COLUMN_NAME, "S2", + "SequenceNum", "1.2345", + longName, "WithoutBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A new dataset record was inserted", getLatestAuditMessage()); + + qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S3", + "SequenceNum", "1.2345", + longName, "WithoutBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 1; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + // Now update: + insertedRows.get(0).put(longName, "NewValue"); + insertedRows.get(1).put(longName, "NewValue"); + List> oldKeys = Arrays.asList( + Map.of("lsid", insertedRows.get(0).get("lsid")), + Map.of("lsid", insertedRows.get(1).get("lsid")) + ); + qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A dataset record was modified", getLatestAuditMessage()); + + qus.deleteRows(_user, _container, oldKeys, null, null); + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A dataset record was deleted", getLatestAuditMessage()); + + // Repeat using bulkLoad=true: + qus.setBulkLoad(true); + + insertedRows = qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S4", + "SequenceNum", "1.2345", + longName, "WithBulkLoad"), + Map.of( + SUBJECT_COLUMN_NAME, "S5", + "SequenceNum", "1.2345", + longName, "WithBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + // Now update: + insertedRows.get(0).put(longName, "NewValue"); + insertedRows.get(1).put(longName, "NewValue"); + oldKeys = Arrays.asList( + Map.of("lsid", insertedRows.get(0).get("lsid")), + Map.of("lsid", insertedRows.get(1).get("lsid")) + ); + qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + qus.deleteRows(_user, _container, oldKeys, null, null); + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + } + + @Test + public void updateRowTest() throws Exception + { + createDataset(); + + TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); + assertNotNull(t); + assertTrue("Field1".equalsIgnoreCase(t.getColumn("Field1").getAlias().getId())); + assertFalse("SELECT".equalsIgnoreCase(t.getColumn("SELECT").getAlias().getId())); + assertFalse(longName.equalsIgnoreCase(t.getColumn(longName).getAlias().getId())); + var up = t.getUpdateService(); + assertNotNull(up); + var errors = new BatchValidationException(); + + var result = up.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, " S1 \t", + "SequenceNum", "1.2345", + "Field1", "f", + "SELECT", "s", + longName, "l", + "value1", "1.0", + "value2", "NA", + "VALUE3", "NA")), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertFalse(errors.hasErrors()); + assertNotNull(result); + assertEquals(1, result.size()); + var map = result.get(0); + assertEquals("S1", map.get(SUBJECT_COLUMN_NAME)); + assertEquals("f", map.get("Field1")); + assertEquals("s", map.get("SELECT")); + assertEquals("l", map.get(longName)); + assertEquals( 1.0d, map.get("value1")); // 1.0 + var v2 = map.get("value2"); // NA + assertTrue(v2 instanceof MvFieldWrapper); + assertEquals("NA", ((MvFieldWrapper)v2).getMvIndicator()); + var v3 = map.get("value3"); // NA + assertTrue(v3 instanceof MvFieldWrapper); + assertEquals("NA", ((MvFieldWrapper)v3).getMvIndicator()); + assertNotNull(map.get("lsid")); + assertTrue(((String)map.get("lsid")).endsWith(":1001.S1.1.2345")); + String lsid = (String)map.get("lsid"); + + // update subjectid column + result = up.updateRows(_user, _container, + List.of(Map.of(SUBJECT_COLUMN_NAME, "\tS2 ")), + List.of(Map.of("lsid", lsid)), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertNotNull(result); + assertEquals(1, result.size()); + map = result.get(0); + assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); + // All other columns are preserved + assertEquals("f", map.get("Field1")); + assertEquals("s", map.get("SELECT")); + assertEquals("l", map.get(longName)); + assertEquals( 1.0d, map.get("value1")); // 1.0 + // DIFFERENCE - updateRows() does not return MvFieldWrapper + assertNull(map.get("value2")); // NA + assertEquals("NA", map.get("value2mvindicator")); + assertNull(map.get("VALUE3")); // NA + assertEquals("NA", map.get("value3MvIndicator")); + // LSID is updated + assertNotNull(map.get("lsid")); + assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); + lsid = (String)map.get("lsid"); + + // update other columns + result = up.updateRows(_user, _container, + List.of(Map.of( + "Field1", "fUpdated", + "SELECT", "sUpdated", + longName, "lUpdated", + "value1", "NA", // 1.0 -> NA + "value2", "2.0", // NA -> 2.0 + "value3", "QA") // NA -> QA + ), + List.of(Map.of("lsid", lsid)), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertNotNull(result); + assertEquals(1, result.size()); + map = result.get(0); + assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); + assertEquals("fUpdated", map.get("Field1")); + assertEquals("sUpdated", map.get("SELECT")); + assertEquals("lUpdated", map.get(longName)); + assertNull(map.get("value1")); // NA + assertEquals("NA", map.get("Value1MVIndicator")); + assertEquals(2.0d, map.get("value2")); // 2.0 + assertNull(map.get("Value2MVIndicator")); + assertNull(map.get("value3")); // QA + assertEquals("QA", map.get("value3mVindicator")); + assertNotNull(map.get("lsid")); + // unchanged + assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); + map.get("lsid"); + } + + @Before + public void createStudy() + { + _context = TestContext.get(); + Container junit = JunitUtil.getTestContainer(); + String name = GUID.makeHash(); + Container c = ContainerManager.createContainer(junit, name, _context.getUser()); + MvUtil.assignMvIndicators(c, new String[] {"NA","QA"}, new String[] {"NA","QA"}); + StudyImpl s = new StudyImpl(c, "Junit Study"); + s.setTimepointType(TimepointType.VISIT); + s.setStartDate(new Date(DateUtil.parseDateTime(c, "2014-01-01"))); + s.setSubjectColumnName(SUBJECT_COLUMN_NAME); + s.setSubjectNounPlural("Subjects"); + s.setSubjectNounSingular("Subject"); + s.setSecurityType(SecurityType.BASIC_WRITE); + _junitStudy = StudyManager.getInstance().createStudy(_context.getUser(), s); + _user = _context.getUser(); + _container = _junitStudy.getContainer(); + } + + @After + public void tearDown() + { + if (null != _junitStudy) + { + assertTrue(ContainerManager.delete(_junitStudy.getContainer(), _context.getUser())); + } + } + } +} From 484b30fca1335db6b95249d5e10b81ffee231d9c Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 23 Oct 2025 22:03:53 -0700 Subject: [PATCH 02/13] Add auditing of what method was used for data imports, updates, deletes --- .../org/labkey/api/action/BaseViewAction.java | 16 ++++-- .../pipeline/AssayUploadPipelineJob.java | 2 +- .../api/audit/TransactionAuditProvider.java | 52 +++++++++++++------ api/src/org/labkey/api/util/HttpUtil.java | 24 ++++++++- .../assay/pipeline/AssayImportRunTask.java | 2 +- .../query/controllers/QueryController.java | 6 ++- 6 files changed, 77 insertions(+), 25 deletions(-) diff --git a/api/src/org/labkey/api/action/BaseViewAction.java b/api/src/org/labkey/api/action/BaseViewAction.java index 0ea6e665da5..9b7fc011d5b 100644 --- a/api/src/org/labkey/api/action/BaseViewAction.java +++ b/api/src/org/labkey/api/action/BaseViewAction.java @@ -547,13 +547,21 @@ public Map getTransactionAud public static Map getTransactionAuditDetails(ViewContext viewContext) { Map map = new HashMap<>(); - map.put(TransactionAuditProvider.TransactionDetail.APIAction, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); - String productName = HttpUtil.getProductName(viewContext.getRequest()); - if (null != productName) - map.put(TransactionAuditProvider.TransactionDetail.Product, productName); + map.put(TransactionAuditProvider.TransactionDetail.Action, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); String clientLibrary = HttpUtil.getClientLibrary(viewContext.getRequest()); if (null != clientLibrary) map.put(TransactionAuditProvider.TransactionDetail.ClientLibrary, clientLibrary); + else + { + String productName = HttpUtil.getProductNameFromReferer(viewContext.getRequest()); // app + if (null != productName) + map.put(TransactionAuditProvider.TransactionDetail.Product, productName); + else // LKS + { + String refererRelativeURL = HttpUtil.getRefererRelativeURL(viewContext.getRequest()); + map.put(TransactionAuditProvider.TransactionDetail.Product, refererRelativeURL); + } + } return map; } diff --git a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java index 57fb8ba2823..283c1f7025d 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java @@ -160,7 +160,7 @@ public void doWork() Map transactionDetails = new HashMap<>(); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, _primaryFile.getName()); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); - transactionDetails.put(TransactionAuditProvider.TransactionDetail.APIAction, "AssayUploadPipelineJob"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayUploadPipelineJob"); ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails/*TODO*/); setStatus(TaskStatus.complete); getLogger().info("Finished assay upload"); diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 94b764323d1..d3585a8a6f1 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -1,6 +1,7 @@ package org.labkey.api.audit; import com.fasterxml.jackson.core.JsonProcessingException; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; @@ -126,29 +127,46 @@ public static TransactionAuditEvent getCurrentTransactionAuditEvent() public enum TransactionDetail { - Operation(false), - AuditEvents(true), - ImportFileName(true), - //BackgroundImport(false), - ClientLibrary(false), - Product(false), - // CrossFolderImport(false), - // CrossTypeImport(false), - // AllowCreateStorage(false), - // UseTransactionAuditCache(false), - APIVersion(false), - APIAction(false), - QueryCommand(true), - BatchAction(false), - ImportOptions(true) - ; + AuditEvents(true, "The types of audit events generated during the transaction"), + ImportFileName(true, "The input filenames used for the import action"), + ClientLibrary(false, "The client library (R, Python, etc) used to perform the action"), + Product(false, "The product (Sample Manager, etc) this action originated from"), + Action(false, "The controller-action for this request"), + QueryCommand(true, "The query commands (insert.update) executed during the transaction"), + BatchAction(false, "If data iterator was used for insert/update"), + ImportOptions(true, "Various import parameters (CrossType, CrossFolder, etc) used during the import action"), + EditMethod(false, "The method used to insert/update data from the app (e.g., 'DetailEdit', 'GridEdit', etc)"), + RequestSource(false, "The URL where the request originated from"); private final boolean multiValue; - TransactionDetail(boolean multiValue) + TransactionDetail(boolean multiValue, String description) { this.multiValue = multiValue; } + public static TransactionDetail fromString(String key) + { + for (TransactionDetail detail : values()) + { + if (detail.name().equalsIgnoreCase(key)) + return detail; + } + return null; + } + + public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull Map auditDetails) + { + if (!auditDetails.isEmpty()) + { + for (Map.Entry entry : auditDetails.entrySet()) + { + TransactionAuditProvider.TransactionDetail detail = TransactionAuditProvider.TransactionDetail.fromString(entry.getKey()); + if (detail != null) + detail.add(transactionDetails, entry.getValue()); + } + } + } + public void add(Map detailMap, Object value) { if (value == null) diff --git a/api/src/org/labkey/api/util/HttpUtil.java b/api/src/org/labkey/api/util/HttpUtil.java index 9094f3a8f71..7657e2ba5f2 100644 --- a/api/src/org/labkey/api/util/HttpUtil.java +++ b/api/src/org/labkey/api/util/HttpUtil.java @@ -301,7 +301,7 @@ else if (userAgent.startsWith("LabKey SAS API")) return request.getRemoteAddr(); } - public static @Nullable String getProductName(HttpServletRequest request) + public static @Nullable String getProductNameFromReferer(HttpServletRequest request) { if (!isBrowser(request)) return null; @@ -324,4 +324,26 @@ else if (userAgent.startsWith("LabKey SAS API")) return null; } + + public static @Nullable String getRefererRelativeURL(HttpServletRequest request) + { + if (!isBrowser(request)) + return null; + + String referer = request.getHeader("Referer"); + if (referer != null) + { + try + { + ActionURL url = new ActionURL(referer); + return url.toContainerRelativeURL(); + } + catch (Exception e) + { + return null; + } + } + + return null; + } } diff --git a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java index b33271297d4..6a48f825d5e 100644 --- a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java +++ b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java @@ -841,7 +841,7 @@ public RecordedActionSet run() throws PipelineJobException Map transactionDetails = new HashMap<>(); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, uploadedData.getName()); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); - transactionDetails.put(TransactionAuditProvider.TransactionDetail.APIAction, "AssayImportRunTask"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayImportRunTask"); Pair pair = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, false, transactionDetails); ExpRun run = pair.second; diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 0a168d9aa21..7b9e9c71908 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -4645,6 +4645,8 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); + Map auditDetails = json.has("auditDetails") ? json.getJSONObject("auditDetails").toMap() : new CaseInsensitiveHashMap<>(); + Map configParameters = new HashMap<>(); // Check first if the audit behavior has been defined for the table either in code or through XML. @@ -4697,7 +4699,9 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole } else { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), getTransactionAuditDetails()); + Map transactionDetails = getTransactionAuditDetails(); + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetails); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); } auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.QueryCommand, commandType.name()); From dffa1d61b9aa7861634b2631c1b2b59343638ed5 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 23 Oct 2025 22:16:43 -0700 Subject: [PATCH 03/13] crlf --- .../org/labkey/api/action/BaseViewAction.java | 1568 +- .../api/assay/DefaultAssayRunCreator.java | 2542 +-- .../org/labkey/api/audit/AuditTypeEvent.java | 534 +- .../provider/FileSystemAuditProvider.java | 468 +- .../api/query/AbstractQueryUpdateService.java | 3040 +-- .../labkey/api/query/UserSchemaAction.java | 670 +- .../experiment/SampleTypeAuditProvider.java | 440 +- .../experiment/api/SampleTypeServiceImpl.java | 4740 ++--- .../labkey/list/model/ListAuditProvider.java | 546 +- .../list/model/ListQueryUpdateService.java | 1780 +- .../query/audit/QueryUpdateAuditProvider.java | 608 +- .../query/controllers/QueryController.java | 17564 ++++++++-------- .../study/query/DatasetUpdateService.java | 2288 +- 13 files changed, 18394 insertions(+), 18394 deletions(-) diff --git a/api/src/org/labkey/api/action/BaseViewAction.java b/api/src/org/labkey/api/action/BaseViewAction.java index 9b7fc011d5b..be9a16e452b 100644 --- a/api/src/org/labkey/api/action/BaseViewAction.java +++ b/api/src/org/labkey/api/action/BaseViewAction.java @@ -1,784 +1,784 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.action; - -import jakarta.servlet.ServletRequest; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.beanutils.DynaBean; -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.data.Container; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.security.User; -import org.labkey.api.util.HelpTopic; -import org.labkey.api.util.HttpUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.writer.ContainerUser; -import org.springframework.beans.AbstractPropertyAccessor; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.BeanWrapper; -import org.springframework.beans.BeansException; -import org.springframework.beans.InvalidPropertyException; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.NotReadablePropertyException; -import org.springframework.beans.NotWritablePropertyException; -import org.springframework.beans.PropertyAccessException; -import org.springframework.beans.PropertyValue; -import org.springframework.beans.PropertyValues; -import org.springframework.beans.TypeMismatchException; -import org.springframework.core.MethodParameter; -import org.springframework.core.convert.TypeDescriptor; -import org.springframework.lang.Nullable; -import org.springframework.validation.BeanPropertyBindingResult; -import org.springframework.validation.BindException; -import org.springframework.validation.BindingErrorProcessor; -import org.springframework.validation.BindingResult; -import org.springframework.validation.FieldError; -import org.springframework.validation.ObjectError; -import org.springframework.validation.Validator; -import org.springframework.web.bind.ServletRequestDataBinder; -import org.springframework.web.bind.ServletRequestParameterPropertyValues; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.multipart.MultipartHttpServletRequest; -import org.springframework.web.servlet.ModelAndView; - -import java.beans.PropertyDescriptor; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Predicate; - -public abstract class BaseViewAction

extends PermissionCheckableAction implements Validator, HasPageConfig, ContainerUser -{ - protected static final Logger logger = LogHelper.getLogger(BaseViewAction.class, "BaseViewAction"); - - private PageConfig _pageConfig = null; - private PropertyValues _pvs; - private boolean _robot = false; // Is this request from GoogleBot or some other crawler? - private boolean _debug = false; - - protected boolean _print = false; - protected Class _commandClass; - protected String _commandName = "form"; - - protected BaseViewAction() - { - String methodName = getCommandClassMethodName(); - - if (null == methodName) - return; - - // inspect the action's *public* methods to determine form class - Class typeBest = null; - for (Method m : this.getClass().getMethods()) - { - if (methodName.equals(m.getName())) - { - Class[] types = m.getParameterTypes(); - if (types.length < 1) - continue; - Class typeCurrent = types[0]; - assert null == _commandClass || typeCurrent.equals(_commandClass); - - // Using templated classes to extend a base action can lead to multiple - // versions of a method with acceptable types, so take the most extended - // type we can find. - if (typeBest == null || typeBest.isAssignableFrom(typeCurrent)) - typeBest = typeCurrent; - } - } - if (typeBest != null) - setCommandClass(typeBest); - } - - - protected abstract String getCommandClassMethodName(); - - - protected BaseViewAction(@NotNull Class commandClass) - { - setCommandClass(commandClass); - } - - - public void setProperties(PropertyValues pvs) - { - _pvs = pvs; - } - - - public void setProperties(Map m) - { - _pvs = new MutablePropertyValues(m); - } - - - /* Doesn't guarantee non-null, non-empty */ - public Object getProperty(String key, String d) - { - PropertyValue pv = _pvs.getPropertyValue(key); - return pv == null ? d : pv.getValue(); - } - - - public Object getProperty(Enum key) - { - PropertyValue pv = _pvs.getPropertyValue(key.name()); - return pv == null ? null : pv.getValue(); - } - - - public Object getProperty(String key) - { - PropertyValue pv = _pvs.getPropertyValue(key); - return pv == null ? null : pv.getValue(); - } - - public PropertyValues getPropertyValues() - { - return _pvs; - } - - - public static PropertyValues getPropertyValuesForFormBinding(PropertyValues pvs, @NotNull Predicate allowBind) - { - if (null == pvs) - return null; - MutablePropertyValues ret = new MutablePropertyValues(); - for (PropertyValue pv : pvs.getPropertyValues()) - { - if (allowBind.test(pv.getName())) - ret.addPropertyValue(pv); - } - return ret; - } - - static final String FORM_DATE_ENCODED_PARAM = "formDataEncoded"; - - /** - * When a double quote is encountered in a multipart/form-data context, it is encoded as %22 using URL-encoding by browsers. - * This process replaces the double quote with its hexadecimal equivalent in a URL-safe format, preventing it from being misinterpreted as the end of a value or a boundary. - * The consequence of such encoding is we can't distinguish '"' from the actual '%22' in parameter name. - * As a workaround, a client-side util `encodeFormDataQuote` is used to convert %22 to %2522 and " to %22 explicitly, while passing in an additional param formDataEncoded=true. - * This class converts those encoded param names back to its decoded form during PropertyValues binding. - * See Issue 52827, 52925 and 52119 for more information. - */ - static public class ViewActionParameterPropertyValues extends ServletRequestParameterPropertyValues - { - - public ViewActionParameterPropertyValues(ServletRequest request) { - this(request, null, null); - } - - public ViewActionParameterPropertyValues(ServletRequest request, @Nullable String prefix, @Nullable String prefixSeparator) - { - super(request, prefix, prefixSeparator); - if (isFormDataEncoded()) - { - for (int i = 0; i < getPropertyValues().length; i++) - { - PropertyValue formDataPropValue = getPropertyValues()[i]; - String propValueName = formDataPropValue.getName(); - String decoded = PageFlowUtil.decodeQuoteEncodedFormDataKey(propValueName); - if (!propValueName.equals(decoded)) - setPropertyValueAt(new PropertyValue(decoded, formDataPropValue.getValue()), i); - } - } - } - - private boolean isFormDataEncoded() - { - PropertyValue formDataPropValue = getPropertyValue(FORM_DATE_ENCODED_PARAM); - if (formDataPropValue != null) - { - Object v = formDataPropValue.getValue(); - String formDataPropValueStr = v == null ? null : String.valueOf(v); - if (StringUtils.isNotBlank(formDataPropValueStr)) - return (Boolean) ConvertUtils.convert(formDataPropValueStr, Boolean.class); - } - - return false; - } - } - - @Override - public ModelAndView handleRequest(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) throws Exception - { - if (null == getPropertyValues()) - setProperties(new ViewActionParameterPropertyValues(request)); - getViewContext().setBindPropertyValues(getPropertyValues()); - handleSpecialProperties(); - - return handleRequest(); - } - - - private void handleSpecialProperties() - { - _robot = PageFlowUtil.isRobotUserAgent(getViewContext().getRequest().getHeader("User-Agent")); - - // Special flag puts actions in "debug" mode, during which they should log extra information that would be - // helpful for testing or debugging problems - if (!_robot && hasStringValue("_debug")) - { - _debug = true; - } - - // SpringActionController.defaultPageConfig() has logic for isPrint, don't need to duplicate here - _print = PageConfig.Template.Print == HttpView.currentPageConfig().getTemplate(); - } - - private boolean hasStringValue(String propertyName) - { - Object o = getProperty(propertyName); - if (o == null) - { - return false; - } - if (o instanceof String s) - { - return null != StringUtils.trimToNull(s); - } - if (o instanceof String[] strings) - { - for (String s : strings) - { - if (null != StringUtils.trimToNull(s)) - { - return true; - } - } - } - return false; - } - - public abstract ModelAndView handleRequest() throws Exception; - - - @Override - public void setPageConfig(PageConfig page) - { - _pageConfig = page; - } - - - @Override - public Container getContainer() - { - return getViewContext().getContainer(); - } - - - @Override - public User getUser() - { - return getViewContext().getUser(); - } - - - @Override - public PageConfig getPageConfig() - { - return _pageConfig; - } - - - public void setTitle(String title) - { - assert null != getPageConfig() : "action not initialized property"; - getPageConfig().setTitle(title); - } - - - public void setHelpTopic(String topicName) - { - setHelpTopic(new HelpTopic(topicName)); - } - - - public void setHelpTopic(HelpTopic topic) - { - assert null != getPageConfig() : "action not initialized properly"; - getPageConfig().setHelpTopic(topic); - } - - - protected Object newInstance(Class c) - { - try - { - return c == null ? null : c.getConstructor().newInstance(); - } - catch (Exception x) - { - if (x instanceof RuntimeException) - throw ((RuntimeException)x); - else - throw new RuntimeException(x); - } - } - - - protected @NotNull FORM getCommand(HttpServletRequest request) throws Exception - { - FORM command = (FORM) createCommand(); - - if (command instanceof HasViewContext) - ((HasViewContext)command).setViewContext(getViewContext()); - - return command; - } - - - protected @NotNull FORM getCommand() throws Exception - { - return getCommand(getViewContext().getRequest()); - } - - - // - // PARAMETER BINDING - // - // don't assume parameters always come from a request, use PropertyValues interface - // - - public @NotNull BindException defaultBindParameters(FORM form, PropertyValues params) - { - return defaultBindParameters(form, getCommandName(), params); - } - - - public static @NotNull BindException defaultBindParameters(Object form, String commandName, PropertyValues params) - { - /* check for do-it-myself forms */ - if (form instanceof HasBindParameters) - { - return ((HasBindParameters)form).bindParameters(params); - } - - if (form instanceof DynaBean) - { - return simpleBindParameters(form, commandName, params); - } - else - { - return springBindParameters(form, commandName, params); - } - } - - public static @NotNull BindException springBindParameters(Object command, String commandName, PropertyValues params) - { - Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); - ServletRequestDataBinder binder = new ServletRequestDataBinder(command, commandName); - - String[] fields = binder.getDisallowedFields(); - List fieldList = new ArrayList<>(fields != null ? Arrays.asList(fields) : Collections.emptyList()); - fieldList.addAll(Arrays.asList("class.*", "Class.*", "*.class.*", "*.Class.*")); - binder.setDisallowedFields(fieldList.toArray(new String[] {})); - - ConvertHelper.getPropertyEditorRegistrar().registerCustomEditors(binder); - BindingErrorProcessor defaultBEP = binder.getBindingErrorProcessor(); - binder.setBindingErrorProcessor(getBindingErrorProcessor(defaultBEP)); - binder.setFieldMarkerPrefix(SpringActionController.FIELD_MARKER); - try - { - // most paths probably called getPropertyValuesForFormBinding() already, but this is a public static method, so call it again - binder.bind(getPropertyValuesForFormBinding(params, allow)); - BindException errors = new NullSafeBindException(binder.getBindingResult()); - return errors; - } - catch (InvalidPropertyException x) - { - // Maybe we should propagate exception and return SC_BAD_REQUEST (in ExceptionUtil.handleException()) - // most POST handlers check errors.hasErrors(), but not all GET handlers do - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding property: " + x.getPropertyName()); - return errors; - } - catch (NumberFormatException x) - { - // Malformed array parameter throws this exception, unfortunately. Just reject the request. #21931 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; invalid array index (" + x.getMessage() + ")"); - return errors; - } - catch (NegativeArraySizeException x) - { - // Another malformed array parameter throws this exception. #23929 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; negative array size (" + x.getMessage() + ")"); - return errors; - } - catch (IllegalArgumentException x) - { - // General bean binding problem. #23929 - BindException errors = new BindException(command, commandName); - errors.reject(SpringActionController.ERROR_MSG, "Error binding property; (" + x.getMessage() + ")"); - return errors; - } - } - - - static BindingErrorProcessor getBindingErrorProcessor(final BindingErrorProcessor defaultBEP) - { - return new BindingErrorProcessor() - { - @Override - public void processMissingFieldError(String missingField, BindingResult bindingResult) - { - defaultBEP.processMissingFieldError(missingField, bindingResult); - } - - @Override - public void processPropertyAccessException(PropertyAccessException ex, BindingResult bindingResult) - { - Object newValue = ex.getPropertyChangeEvent().getNewValue(); - if (newValue instanceof String) - newValue = StringUtils.trimToNull((String)newValue); - - // convert NULL conversion errors to required errors - if (null == newValue) - defaultBEP.processMissingFieldError(ex.getPropertyChangeEvent().getPropertyName(), bindingResult); - else - defaultBEP.processPropertyAccessException(ex, bindingResult); - } - }; - } - - - /* - * This binder doesn't have much to offer over the standard spring data binding except that it will - * handle DynaBeans. - */ - public static @NotNull BindException simpleBindParameters(Object command, String commandName, PropertyValues params) - { - Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); - - BindException errors = new NullSafeBindException(command, "Form"); - - // unfortunately ObjectFactory and BeanObjectFactory are not good about reporting errors - // do this by hand - for (PropertyValue pv : params.getPropertyValues()) - { - String propertyName = pv.getName(); - Object value = pv.getValue(); - if (!allow.test(propertyName)) - continue; - - try - { - Object converted = value; - Class propClass = PropertyUtils.getPropertyType(command, propertyName); - if (null == propClass) - continue; - if (value == null) - { - /* */ - } - else if (propClass.isPrimitive()) - { - converted = ConvertUtils.convert(String.valueOf(value), propClass); - } - else if (propClass.isArray()) - { - if (value instanceof Collection) - value = ((Collection) value).toArray(new String[0]); - else if (!value.getClass().isArray()) - value = new String[] {String.valueOf(value)}; - converted = ConvertUtils.convert((String[])value, propClass); - } - PropertyUtils.setProperty(command, propertyName, converted); - } - catch (ConversionException x) - { - errors.addError(new FieldError(commandName, propertyName, value, true, new String[] {"ConversionError", "typeMismatch"}, null, "Could not convert to value: " + value)); - } - catch (Exception x) - { - errors.addError(new ObjectError(commandName, new String[]{"Error"}, new Object[] {value}, x.getMessage())); - logger.error("unexpected error", x); - } - } - return errors; - } - - @Override - public boolean supports(Class clazz) - { - return getCommandClass().isAssignableFrom(clazz); - } - - public Map getTransactionAuditDetails() - { - return getTransactionAuditDetails(getViewContext()); - } - - public static Map getTransactionAuditDetails(ViewContext viewContext) - { - Map map = new HashMap<>(); - map.put(TransactionAuditProvider.TransactionDetail.Action, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); - String clientLibrary = HttpUtil.getClientLibrary(viewContext.getRequest()); - if (null != clientLibrary) - map.put(TransactionAuditProvider.TransactionDetail.ClientLibrary, clientLibrary); - else - { - String productName = HttpUtil.getProductNameFromReferer(viewContext.getRequest()); // app - if (null != productName) - map.put(TransactionAuditProvider.TransactionDetail.Product, productName); - else // LKS - { - String refererRelativeURL = HttpUtil.getRefererRelativeURL(viewContext.getRequest()); - map.put(TransactionAuditProvider.TransactionDetail.Product, refererRelativeURL); - } - } - return map; - } - - /* for TableViewForm, uses BeanUtils to work with DynaBeans */ - static public class BeanUtilsPropertyBindingResult extends BeanPropertyBindingResult - { - public BeanUtilsPropertyBindingResult(Object target, String objectName) - { - super(target, objectName); - } - - @Override - protected BeanWrapper createBeanWrapper() - { - return new BeanUtilsWrapperImpl((DynaBean)getTarget()); - } - } - - static public class BeanUtilsWrapperImpl extends AbstractPropertyAccessor implements BeanWrapper - { - private Object object; - private boolean autoGrowNestedPaths = false; - private int autoGrowCollectionLimit = 0; - - public BeanUtilsWrapperImpl() - { - // registerDefaultEditors(); - } - - public BeanUtilsWrapperImpl(DynaBean target) - { - this(); - object = target; - } - - @Override - public Object getPropertyValue(String propertyName) throws BeansException - { - try - { - return PropertyUtils.getProperty(object, propertyName); - } - catch (Exception e) - { - throw new NotReadablePropertyException(object.getClass(), propertyName); - } - } - - @Override - public void setPropertyValue(String propertyName, Object value) throws BeansException - { - try - { - PropertyUtils.setProperty(object, propertyName, value); - } - catch (Exception e) - { - throw new NotWritablePropertyException(object.getClass(), propertyName); - } - } - - @Override - public boolean isReadableProperty(String propertyName) - { - return true; - } - - @Override - public boolean isWritableProperty(String propertyName) - { - return true; - } - - @Override - public TypeDescriptor getPropertyTypeDescriptor(String s) throws BeansException - { - return null; - } - - public void setWrappedInstance(Object obj) - { - object = obj; - } - - @Override - public Object getWrappedInstance() - { - return object; - } - - @Override - public Class getWrappedClass() - { - return object.getClass(); - } - - @Override - public PropertyDescriptor[] getPropertyDescriptors() - { - throw new UnsupportedOperationException(); - } - - @Override - public PropertyDescriptor getPropertyDescriptor(String propertyName) throws BeansException - { - throw new UnsupportedOperationException(); - } - - @Override - public void setAutoGrowNestedPaths(boolean b) - { - this.autoGrowNestedPaths = b; - } - - @Override - public boolean isAutoGrowNestedPaths() - { - return this.autoGrowNestedPaths; - } - - @Override - public void setAutoGrowCollectionLimit(int i) - { - this.autoGrowCollectionLimit = i; - } - - @Override - public int getAutoGrowCollectionLimit() - { - return this.autoGrowCollectionLimit; - } - - @Override - public T convertIfNecessary(Object value, Class requiredType) throws TypeMismatchException - { - if (value == null) - return null; - return (T)ConvertUtils.convert(String.valueOf(value), requiredType); - } - - @Override - public T convertIfNecessary(Object value, Class requiredType, MethodParameter methodParam) throws TypeMismatchException - { - return convertIfNecessary(value, requiredType); - } - } - - /** - * @return a map from form element name to uploaded files - */ - protected Map getFileMap() - { - if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) - return ((MultipartHttpServletRequest)getViewContext().getRequest()).getFileMap(); - return Collections.emptyMap(); - } - - protected List getAttachmentFileList() - { - return SpringAttachmentFile.createList(getFileMap()); - } - - public boolean isRobot() - { - return _robot; - } - - public boolean isPrint() - { - return _print; - } - - public boolean isDebug() - { - return _debug; - } - - public @NotNull Class getCommandClass() - { - if (null == _commandClass) - throw new IllegalStateException("NULL _commandClass in " + getClass().getName()); - return _commandClass; - } - - public void setCommandClass(@NotNull Class commandClass) - { - _commandClass = commandClass; - } - - protected final @NotNull Object createCommand() - { - return BeanUtils.instantiateClass(getCommandClass()); - } - - public void setCommandName(String commandName) - { - _commandName = commandName; - } - - public String getCommandName() - { - return _commandName; - } - - /** - * Cacheable resources can calculate a last modified timestamp to send to the browser. - */ - protected long getLastModified(FORM form) - { - return Long.MIN_VALUE; - } - - /** - * Cacheable resources can calculate an ETag header to send to the browser. - */ - protected String getETag(FORM form) - { - return null; - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.action; + +import jakarta.servlet.ServletRequest; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.beanutils.DynaBean; +import org.apache.commons.beanutils.PropertyUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.data.Container; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.security.User; +import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HttpUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.writer.ContainerUser; +import org.springframework.beans.AbstractPropertyAccessor; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.BeanWrapper; +import org.springframework.beans.BeansException; +import org.springframework.beans.InvalidPropertyException; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.NotReadablePropertyException; +import org.springframework.beans.NotWritablePropertyException; +import org.springframework.beans.PropertyAccessException; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.beans.TypeMismatchException; +import org.springframework.core.MethodParameter; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.lang.Nullable; +import org.springframework.validation.BeanPropertyBindingResult; +import org.springframework.validation.BindException; +import org.springframework.validation.BindingErrorProcessor; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.ObjectError; +import org.springframework.validation.Validator; +import org.springframework.web.bind.ServletRequestDataBinder; +import org.springframework.web.bind.ServletRequestParameterPropertyValues; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.servlet.ModelAndView; + +import java.beans.PropertyDescriptor; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; + +public abstract class BaseViewAction extends PermissionCheckableAction implements Validator, HasPageConfig, ContainerUser +{ + protected static final Logger logger = LogHelper.getLogger(BaseViewAction.class, "BaseViewAction"); + + private PageConfig _pageConfig = null; + private PropertyValues _pvs; + private boolean _robot = false; // Is this request from GoogleBot or some other crawler? + private boolean _debug = false; + + protected boolean _print = false; + protected Class _commandClass; + protected String _commandName = "form"; + + protected BaseViewAction() + { + String methodName = getCommandClassMethodName(); + + if (null == methodName) + return; + + // inspect the action's *public* methods to determine form class + Class typeBest = null; + for (Method m : this.getClass().getMethods()) + { + if (methodName.equals(m.getName())) + { + Class[] types = m.getParameterTypes(); + if (types.length < 1) + continue; + Class typeCurrent = types[0]; + assert null == _commandClass || typeCurrent.equals(_commandClass); + + // Using templated classes to extend a base action can lead to multiple + // versions of a method with acceptable types, so take the most extended + // type we can find. + if (typeBest == null || typeBest.isAssignableFrom(typeCurrent)) + typeBest = typeCurrent; + } + } + if (typeBest != null) + setCommandClass(typeBest); + } + + + protected abstract String getCommandClassMethodName(); + + + protected BaseViewAction(@NotNull Class commandClass) + { + setCommandClass(commandClass); + } + + + public void setProperties(PropertyValues pvs) + { + _pvs = pvs; + } + + + public void setProperties(Map m) + { + _pvs = new MutablePropertyValues(m); + } + + + /* Doesn't guarantee non-null, non-empty */ + public Object getProperty(String key, String d) + { + PropertyValue pv = _pvs.getPropertyValue(key); + return pv == null ? d : pv.getValue(); + } + + + public Object getProperty(Enum key) + { + PropertyValue pv = _pvs.getPropertyValue(key.name()); + return pv == null ? null : pv.getValue(); + } + + + public Object getProperty(String key) + { + PropertyValue pv = _pvs.getPropertyValue(key); + return pv == null ? null : pv.getValue(); + } + + public PropertyValues getPropertyValues() + { + return _pvs; + } + + + public static PropertyValues getPropertyValuesForFormBinding(PropertyValues pvs, @NotNull Predicate allowBind) + { + if (null == pvs) + return null; + MutablePropertyValues ret = new MutablePropertyValues(); + for (PropertyValue pv : pvs.getPropertyValues()) + { + if (allowBind.test(pv.getName())) + ret.addPropertyValue(pv); + } + return ret; + } + + static final String FORM_DATE_ENCODED_PARAM = "formDataEncoded"; + + /** + * When a double quote is encountered in a multipart/form-data context, it is encoded as %22 using URL-encoding by browsers. + * This process replaces the double quote with its hexadecimal equivalent in a URL-safe format, preventing it from being misinterpreted as the end of a value or a boundary. + * The consequence of such encoding is we can't distinguish '"' from the actual '%22' in parameter name. + * As a workaround, a client-side util `encodeFormDataQuote` is used to convert %22 to %2522 and " to %22 explicitly, while passing in an additional param formDataEncoded=true. + * This class converts those encoded param names back to its decoded form during PropertyValues binding. + * See Issue 52827, 52925 and 52119 for more information. + */ + static public class ViewActionParameterPropertyValues extends ServletRequestParameterPropertyValues + { + + public ViewActionParameterPropertyValues(ServletRequest request) { + this(request, null, null); + } + + public ViewActionParameterPropertyValues(ServletRequest request, @Nullable String prefix, @Nullable String prefixSeparator) + { + super(request, prefix, prefixSeparator); + if (isFormDataEncoded()) + { + for (int i = 0; i < getPropertyValues().length; i++) + { + PropertyValue formDataPropValue = getPropertyValues()[i]; + String propValueName = formDataPropValue.getName(); + String decoded = PageFlowUtil.decodeQuoteEncodedFormDataKey(propValueName); + if (!propValueName.equals(decoded)) + setPropertyValueAt(new PropertyValue(decoded, formDataPropValue.getValue()), i); + } + } + } + + private boolean isFormDataEncoded() + { + PropertyValue formDataPropValue = getPropertyValue(FORM_DATE_ENCODED_PARAM); + if (formDataPropValue != null) + { + Object v = formDataPropValue.getValue(); + String formDataPropValueStr = v == null ? null : String.valueOf(v); + if (StringUtils.isNotBlank(formDataPropValueStr)) + return (Boolean) ConvertUtils.convert(formDataPropValueStr, Boolean.class); + } + + return false; + } + } + + @Override + public ModelAndView handleRequest(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) throws Exception + { + if (null == getPropertyValues()) + setProperties(new ViewActionParameterPropertyValues(request)); + getViewContext().setBindPropertyValues(getPropertyValues()); + handleSpecialProperties(); + + return handleRequest(); + } + + + private void handleSpecialProperties() + { + _robot = PageFlowUtil.isRobotUserAgent(getViewContext().getRequest().getHeader("User-Agent")); + + // Special flag puts actions in "debug" mode, during which they should log extra information that would be + // helpful for testing or debugging problems + if (!_robot && hasStringValue("_debug")) + { + _debug = true; + } + + // SpringActionController.defaultPageConfig() has logic for isPrint, don't need to duplicate here + _print = PageConfig.Template.Print == HttpView.currentPageConfig().getTemplate(); + } + + private boolean hasStringValue(String propertyName) + { + Object o = getProperty(propertyName); + if (o == null) + { + return false; + } + if (o instanceof String s) + { + return null != StringUtils.trimToNull(s); + } + if (o instanceof String[] strings) + { + for (String s : strings) + { + if (null != StringUtils.trimToNull(s)) + { + return true; + } + } + } + return false; + } + + public abstract ModelAndView handleRequest() throws Exception; + + + @Override + public void setPageConfig(PageConfig page) + { + _pageConfig = page; + } + + + @Override + public Container getContainer() + { + return getViewContext().getContainer(); + } + + + @Override + public User getUser() + { + return getViewContext().getUser(); + } + + + @Override + public PageConfig getPageConfig() + { + return _pageConfig; + } + + + public void setTitle(String title) + { + assert null != getPageConfig() : "action not initialized property"; + getPageConfig().setTitle(title); + } + + + public void setHelpTopic(String topicName) + { + setHelpTopic(new HelpTopic(topicName)); + } + + + public void setHelpTopic(HelpTopic topic) + { + assert null != getPageConfig() : "action not initialized properly"; + getPageConfig().setHelpTopic(topic); + } + + + protected Object newInstance(Class c) + { + try + { + return c == null ? null : c.getConstructor().newInstance(); + } + catch (Exception x) + { + if (x instanceof RuntimeException) + throw ((RuntimeException)x); + else + throw new RuntimeException(x); + } + } + + + protected @NotNull FORM getCommand(HttpServletRequest request) throws Exception + { + FORM command = (FORM) createCommand(); + + if (command instanceof HasViewContext) + ((HasViewContext)command).setViewContext(getViewContext()); + + return command; + } + + + protected @NotNull FORM getCommand() throws Exception + { + return getCommand(getViewContext().getRequest()); + } + + + // + // PARAMETER BINDING + // + // don't assume parameters always come from a request, use PropertyValues interface + // + + public @NotNull BindException defaultBindParameters(FORM form, PropertyValues params) + { + return defaultBindParameters(form, getCommandName(), params); + } + + + public static @NotNull BindException defaultBindParameters(Object form, String commandName, PropertyValues params) + { + /* check for do-it-myself forms */ + if (form instanceof HasBindParameters) + { + return ((HasBindParameters)form).bindParameters(params); + } + + if (form instanceof DynaBean) + { + return simpleBindParameters(form, commandName, params); + } + else + { + return springBindParameters(form, commandName, params); + } + } + + public static @NotNull BindException springBindParameters(Object command, String commandName, PropertyValues params) + { + Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); + ServletRequestDataBinder binder = new ServletRequestDataBinder(command, commandName); + + String[] fields = binder.getDisallowedFields(); + List fieldList = new ArrayList<>(fields != null ? Arrays.asList(fields) : Collections.emptyList()); + fieldList.addAll(Arrays.asList("class.*", "Class.*", "*.class.*", "*.Class.*")); + binder.setDisallowedFields(fieldList.toArray(new String[] {})); + + ConvertHelper.getPropertyEditorRegistrar().registerCustomEditors(binder); + BindingErrorProcessor defaultBEP = binder.getBindingErrorProcessor(); + binder.setBindingErrorProcessor(getBindingErrorProcessor(defaultBEP)); + binder.setFieldMarkerPrefix(SpringActionController.FIELD_MARKER); + try + { + // most paths probably called getPropertyValuesForFormBinding() already, but this is a public static method, so call it again + binder.bind(getPropertyValuesForFormBinding(params, allow)); + BindException errors = new NullSafeBindException(binder.getBindingResult()); + return errors; + } + catch (InvalidPropertyException x) + { + // Maybe we should propagate exception and return SC_BAD_REQUEST (in ExceptionUtil.handleException()) + // most POST handlers check errors.hasErrors(), but not all GET handlers do + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding property: " + x.getPropertyName()); + return errors; + } + catch (NumberFormatException x) + { + // Malformed array parameter throws this exception, unfortunately. Just reject the request. #21931 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; invalid array index (" + x.getMessage() + ")"); + return errors; + } + catch (NegativeArraySizeException x) + { + // Another malformed array parameter throws this exception. #23929 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding array property; negative array size (" + x.getMessage() + ")"); + return errors; + } + catch (IllegalArgumentException x) + { + // General bean binding problem. #23929 + BindException errors = new BindException(command, commandName); + errors.reject(SpringActionController.ERROR_MSG, "Error binding property; (" + x.getMessage() + ")"); + return errors; + } + } + + + static BindingErrorProcessor getBindingErrorProcessor(final BindingErrorProcessor defaultBEP) + { + return new BindingErrorProcessor() + { + @Override + public void processMissingFieldError(String missingField, BindingResult bindingResult) + { + defaultBEP.processMissingFieldError(missingField, bindingResult); + } + + @Override + public void processPropertyAccessException(PropertyAccessException ex, BindingResult bindingResult) + { + Object newValue = ex.getPropertyChangeEvent().getNewValue(); + if (newValue instanceof String) + newValue = StringUtils.trimToNull((String)newValue); + + // convert NULL conversion errors to required errors + if (null == newValue) + defaultBEP.processMissingFieldError(ex.getPropertyChangeEvent().getPropertyName(), bindingResult); + else + defaultBEP.processPropertyAccessException(ex, bindingResult); + } + }; + } + + + /* + * This binder doesn't have much to offer over the standard spring data binding except that it will + * handle DynaBeans. + */ + public static @NotNull BindException simpleBindParameters(Object command, String commandName, PropertyValues params) + { + Predicate allow = command instanceof HasAllowBindParameter allowBP ? allowBP.allowBindParameter() : HasAllowBindParameter.getDefaultPredicate(); + + BindException errors = new NullSafeBindException(command, "Form"); + + // unfortunately ObjectFactory and BeanObjectFactory are not good about reporting errors + // do this by hand + for (PropertyValue pv : params.getPropertyValues()) + { + String propertyName = pv.getName(); + Object value = pv.getValue(); + if (!allow.test(propertyName)) + continue; + + try + { + Object converted = value; + Class propClass = PropertyUtils.getPropertyType(command, propertyName); + if (null == propClass) + continue; + if (value == null) + { + /* */ + } + else if (propClass.isPrimitive()) + { + converted = ConvertUtils.convert(String.valueOf(value), propClass); + } + else if (propClass.isArray()) + { + if (value instanceof Collection) + value = ((Collection) value).toArray(new String[0]); + else if (!value.getClass().isArray()) + value = new String[] {String.valueOf(value)}; + converted = ConvertUtils.convert((String[])value, propClass); + } + PropertyUtils.setProperty(command, propertyName, converted); + } + catch (ConversionException x) + { + errors.addError(new FieldError(commandName, propertyName, value, true, new String[] {"ConversionError", "typeMismatch"}, null, "Could not convert to value: " + value)); + } + catch (Exception x) + { + errors.addError(new ObjectError(commandName, new String[]{"Error"}, new Object[] {value}, x.getMessage())); + logger.error("unexpected error", x); + } + } + return errors; + } + + @Override + public boolean supports(Class clazz) + { + return getCommandClass().isAssignableFrom(clazz); + } + + public Map getTransactionAuditDetails() + { + return getTransactionAuditDetails(getViewContext()); + } + + public static Map getTransactionAuditDetails(ViewContext viewContext) + { + Map map = new HashMap<>(); + map.put(TransactionAuditProvider.TransactionDetail.Action, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); + String clientLibrary = HttpUtil.getClientLibrary(viewContext.getRequest()); + if (null != clientLibrary) + map.put(TransactionAuditProvider.TransactionDetail.ClientLibrary, clientLibrary); + else + { + String productName = HttpUtil.getProductNameFromReferer(viewContext.getRequest()); // app + if (null != productName) + map.put(TransactionAuditProvider.TransactionDetail.Product, productName); + else // LKS + { + String refererRelativeURL = HttpUtil.getRefererRelativeURL(viewContext.getRequest()); + map.put(TransactionAuditProvider.TransactionDetail.Product, refererRelativeURL); + } + } + return map; + } + + /* for TableViewForm, uses BeanUtils to work with DynaBeans */ + static public class BeanUtilsPropertyBindingResult extends BeanPropertyBindingResult + { + public BeanUtilsPropertyBindingResult(Object target, String objectName) + { + super(target, objectName); + } + + @Override + protected BeanWrapper createBeanWrapper() + { + return new BeanUtilsWrapperImpl((DynaBean)getTarget()); + } + } + + static public class BeanUtilsWrapperImpl extends AbstractPropertyAccessor implements BeanWrapper + { + private Object object; + private boolean autoGrowNestedPaths = false; + private int autoGrowCollectionLimit = 0; + + public BeanUtilsWrapperImpl() + { + // registerDefaultEditors(); + } + + public BeanUtilsWrapperImpl(DynaBean target) + { + this(); + object = target; + } + + @Override + public Object getPropertyValue(String propertyName) throws BeansException + { + try + { + return PropertyUtils.getProperty(object, propertyName); + } + catch (Exception e) + { + throw new NotReadablePropertyException(object.getClass(), propertyName); + } + } + + @Override + public void setPropertyValue(String propertyName, Object value) throws BeansException + { + try + { + PropertyUtils.setProperty(object, propertyName, value); + } + catch (Exception e) + { + throw new NotWritablePropertyException(object.getClass(), propertyName); + } + } + + @Override + public boolean isReadableProperty(String propertyName) + { + return true; + } + + @Override + public boolean isWritableProperty(String propertyName) + { + return true; + } + + @Override + public TypeDescriptor getPropertyTypeDescriptor(String s) throws BeansException + { + return null; + } + + public void setWrappedInstance(Object obj) + { + object = obj; + } + + @Override + public Object getWrappedInstance() + { + return object; + } + + @Override + public Class getWrappedClass() + { + return object.getClass(); + } + + @Override + public PropertyDescriptor[] getPropertyDescriptors() + { + throw new UnsupportedOperationException(); + } + + @Override + public PropertyDescriptor getPropertyDescriptor(String propertyName) throws BeansException + { + throw new UnsupportedOperationException(); + } + + @Override + public void setAutoGrowNestedPaths(boolean b) + { + this.autoGrowNestedPaths = b; + } + + @Override + public boolean isAutoGrowNestedPaths() + { + return this.autoGrowNestedPaths; + } + + @Override + public void setAutoGrowCollectionLimit(int i) + { + this.autoGrowCollectionLimit = i; + } + + @Override + public int getAutoGrowCollectionLimit() + { + return this.autoGrowCollectionLimit; + } + + @Override + public T convertIfNecessary(Object value, Class requiredType) throws TypeMismatchException + { + if (value == null) + return null; + return (T)ConvertUtils.convert(String.valueOf(value), requiredType); + } + + @Override + public T convertIfNecessary(Object value, Class requiredType, MethodParameter methodParam) throws TypeMismatchException + { + return convertIfNecessary(value, requiredType); + } + } + + /** + * @return a map from form element name to uploaded files + */ + protected Map getFileMap() + { + if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) + return ((MultipartHttpServletRequest)getViewContext().getRequest()).getFileMap(); + return Collections.emptyMap(); + } + + protected List getAttachmentFileList() + { + return SpringAttachmentFile.createList(getFileMap()); + } + + public boolean isRobot() + { + return _robot; + } + + public boolean isPrint() + { + return _print; + } + + public boolean isDebug() + { + return _debug; + } + + public @NotNull Class getCommandClass() + { + if (null == _commandClass) + throw new IllegalStateException("NULL _commandClass in " + getClass().getName()); + return _commandClass; + } + + public void setCommandClass(@NotNull Class commandClass) + { + _commandClass = commandClass; + } + + protected final @NotNull Object createCommand() + { + return BeanUtils.instantiateClass(getCommandClass()); + } + + public void setCommandName(String commandName) + { + _commandName = commandName; + } + + public String getCommandName() + { + return _commandName; + } + + /** + * Cacheable resources can calculate a last modified timestamp to send to the browser. + */ + protected long getLastModified(FORM form) + { + return Long.MIN_VALUE; + } + + /** + * Cacheable resources can calculate an ETag header to send to the browser. + */ + protected String getETag(FORM form) + { + return null; + } +} diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index 8f273f1aba9..c5046892d29 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -1,1271 +1,1271 @@ -/* - * Copyright (c) 2011-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.assay; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.assay.actions.AssayRunUploadForm; -import org.labkey.api.assay.pipeline.AssayRunAsyncContext; -import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; -import org.labkey.api.assay.sample.AssaySampleLookupContext; -import org.labkey.api.assay.transform.DataTransformService; -import org.labkey.api.assay.transform.TransformDataHandler; -import org.labkey.api.assay.transform.TransformResult; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.RemapCache; -import org.labkey.api.data.validator.ColumnValidator; -import org.labkey.api.data.validator.ColumnValidators; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.exp.ExperimentDataHandler; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.XarContext; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataRunInput; -import org.labkey.api.exp.api.ExpExperiment; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpObject; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpRunItem; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.ProvenanceService; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineValidationException; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.study.assay.ParticipantVisitResolver; -import org.labkey.api.study.assay.ParticipantVisitResolverType; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.writer.ContainerUser; -import org.labkey.vfs.FileLike; -import org.labkey.vfs.FileSystemLike; - -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - -import static java.util.Collections.unmodifiableCollection; -import static org.labkey.api.assay.AssayFileWriter.TEMP_DIR_NAME; - -public class DefaultAssayRunCreator implements AssayRunCreator -{ - private static final Logger LOG = LogManager.getLogger(DefaultAssayRunCreator.class); - public static final String CROSS_RUN_DATA_INPUT_ROLE = "cross run input"; - - private final ProviderType _provider; - - public DefaultAssayRunCreator(ProviderType provider) - { - _provider = provider; - } - - public TransformResult transform(AssayRunUploadContext context, ExpRun run) throws ValidationException - { - return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); - } - /** - * Create and save an experiment run synchronously or asynchronously in a background job depending upon the assay design. - * - * @param context The context used to create and save the batch and run. - * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. - * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. - */ - @Override - public Pair saveExperimentRun( - AssayRunUploadContext context, - @Nullable Long batchId, - boolean forceAsync, - Map transactionDetails - ) throws ExperimentException, ValidationException - { - ExpExperiment exp = null; - if (batchId != null) - { - exp = ExperimentService.get().getExpExperiment(batchId); - } - - AssayProvider provider = context.getProvider(); - ExpProtocol protocol = context.getProtocol(); - ExpRun run = null; - - try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) - { - if (transaction.getAuditId() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); - } - context.init(); - // Check if assay protocol is configured to import in the background. - // Issue 26811: If we don't have a view, assume that we are on a background job thread already. - boolean importInBackground = forceAsync || (provider.isBackgroundUpload(protocol) && HttpView.hasCurrentView()); - if (!importInBackground) - { - if ((Object) context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE) instanceof File errFile) - { - throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); - } - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); - run.setComments(context.getComments()); - run.setWorkflowTaskId(context.getWorkflowTask()); - - exp = saveExperimentRun(context, exp, run, false, transactionDetails); - - // re-fetch the run after it has been fully constructed - run = ExperimentService.get().getExpRun(run.getRowId()); - - context.uploadComplete(run); - } - else - { - context.uploadComplete(null); - context.setTransactionAuditId(transaction.getAuditId()); - exp = saveExperimentRunAsync(context, exp); - } - transaction.commit(); - } - - return Pair.of(exp, run); - } - - private ExpExperiment saveExperimentRunAsync(AssayRunUploadContext context, @Nullable ExpExperiment batch) throws ExperimentException - { - try - { - // Whether we need to save batch properties - boolean forceSaveBatchProps = false; - if (batch == null) - { - // No batch yet, so make one - batch = AssayService.get().createStandardBatch(context.getContainer(), null, context.getProtocol()); - batch.save(context.getUser()); - // It's brand new, so we need to eventually set its properties - forceSaveBatchProps = true; - } - - // Queue up a pipeline job to do the actual import in the background - ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); - - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - // Check if the primary file from the previous import is no longer present for a re-run - if (primaryFile == null && !context.getUploadedData().isEmpty()) - { - // Choose another file as the primary - primaryFile = context.getUploadedData().entrySet().iterator().next().getValue(); - } - primaryFile = Objects.requireNonNull(primaryFile); - AssayRunAsyncContext asyncContext = context.getProvider().createRunAsyncContext(context); - final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob( - asyncContext, - info, - batch, - forceSaveBatchProps, - PipelineService.get().getPipelineRootSetting(context.getContainer()), - primaryFile.toNioPathForRead().toFile() - ); - - context.setPipelineJobGUID(pipelineJob.getJobGUID()); - - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), null, pipelineJob.getJobGUID()); - resultsFileWriter.savePostedFiles(context); - - // Don't queue the job until the transaction is committed, since otherwise the thread - // that's running the job might start before it can access the job's row in the database. - ExperimentService.get().getSchema().getScope().addCommitTask(() -> { - try - { - PipelineService.get().queueJob(pipelineJob, asyncContext.getJobNotificationProvider()); - } - catch (PipelineValidationException e) - { - throw UnexpectedException.wrap(e); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - - return batch; - } - - /** - * @param batch if not null, the run group that's already created for this batch. If null, a new one needs to be created - * @param run The run to save - * @return the run and batch that were inserted - */ - @Override - public ExpExperiment saveExperimentRun( - final AssayRunUploadContext context, - @Nullable ExpExperiment batch, - @NotNull ExpRun run, - boolean forceSaveBatchProps, - @Nullable Map transactionDetails - ) throws ExperimentException, ValidationException - { - context.setAutoFillDefaultResultColumns(run.getRowId() > 0); // need to setAutoFillDefaultResultColumns before run is saved - - final Container container = context.getContainer(); - - Map inputMaterials = new HashMap<>(); - Map inputDatas = new HashMap<>(); - Map outputMaterials = new HashMap<>(); - Map outputDatas = new HashMap<>(); - Map transformedDatas = new HashMap<>(); - - Map runProperties = context.getRunProperties(); - Map unresolvedRunProperties = context.getUnresolvedRunProperties(); - Map batchProperties = context.getBatchProperties(); - - Map allProperties = new HashMap<>(); - allProperties.putAll(runProperties); - allProperties.putAll(batchProperties); - - ParticipantVisitResolverType resolverType = null; - for (Map.Entry entry : allProperties.entrySet()) - { - if (entry.getKey().getName().equals(AbstractAssayProvider.PARTICIPANT_VISIT_RESOLVER_PROPERTY_NAME)) - { - resolverType = AbstractAssayProvider.findType(entry.getValue(), getProvider().getParticipantVisitResolverTypes()); - if (resolverType != null) - { - resolverType.configureRun(context, run, inputDatas); - } - break; - } - } - - // TODO: Share these RemapCache and materialCache instances with AbstractAssayTsvDataHandler.checkData and ExpressionMatrixDataHandler.importFile - // Cache of resolved alternate lookup keys -> rowId - final RemapCache cache = new RemapCache(true); - // Cache of rowId -> ExpMaterial - final Map materialCache = new LongHashMap<>(); - - addInputMaterials(context, inputMaterials, cache, materialCache); - addInputDatas(context, inputDatas); - addOutputMaterials(context, outputMaterials, cache, materialCache); - addOutputDatas(context, inputDatas, outputDatas); - - boolean success = false; - DbScope scope = ExperimentService.get().getSchema().getScope(); - try (DbScope.Transaction transaction = scope.ensureTransaction(ExperimentService.get().getProtocolImportLock())) - { - if (transaction.getAuditId() == null) - { - var auditAction = context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT; - if (context.getTransactionAuditId() != null) - { - var auditEvent = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, context.getTransactionAuditId()); - transaction.setAuditEvent(auditEvent); - } - else - { - var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction, transactionDetails); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); - } - } - boolean saveBatchProps = forceSaveBatchProps; - - // Add any material/data inputs related to the specimen IDs, etc in the incoming data. - // Some subclasses may actually create ExpMaterials or do other database changes, so do this inside the - // overall transaction - resolveParticipantVisits(context, inputMaterials, inputDatas, outputMaterials, outputDatas, allProperties, resolverType); - - // Check for circular inputs/outputs - checkForCycles(inputMaterials, outputMaterials); - checkForCycles(inputDatas, outputDatas); - - // Create the batch, if needed - if (batch == null) - { - // Make sure that we have a batch to associate with this run - batch = AssayService.get().createStandardBatch(run.getContainer(), null, context.getProtocol()); - batch.save(context.getUser()); - saveBatchProps = true; - } - run.save(context.getUser()); - // Add the run to the batch so that we can find it when we're loading the data files - batch.addRuns(context.getUser(), run); - assert batch.equals(run.getBatch()) : "Run's batch should be the current batch"; - - ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); - XarContext xarContext = new AssayUploadXarContext("Simple Run Creation", context); - - run = ExperimentService.get().saveSimpleExperimentRun( - run, - inputMaterials, - inputDatas, - outputMaterials, - outputDatas, - transformedDatas, - info, - context.getLogger() != null ? context.getLogger() : LOG, - false - ); - - // handle data transformation - TransformResult transformResult = transform(context, run); - - if (transformResult.getWarnings() != null && context instanceof AssayRunUploadForm uploadForm) - { - context.setTransformResult(transformResult); - uploadForm.setName(run.getName()); - uploadForm.setComments(run.getComments()); - throw new ValidationException(" "); - } - - if (saveBatchProps) - saveProperties(context, batch, transformResult.getBatchProperties(), batchProperties); - if (null != transformResult.getAssayId()) - run.setName(transformResult.getAssayId()); - if (null != transformResult.getComments()) - run.setComments(transformResult.getComments()); - saveProperties(context, run, transformResult.getRunProperties(), runProperties); - - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); - resultsFileWriter.savePostedFiles(context); - - Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run); - if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir)) - { - FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir); - if (assayResultFileRoot != null) - QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot); - } - - importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult); - - var reRunId = context.getReRunId(); - if (reRunId != null && getProvider().getReRunSupport() == AssayProvider.ReRunSupport.ReRunAndReplace) - { - final ExpRun replacedRun = ExperimentService.get().getExpRun(reRunId); - if (replacedRun == null) - throw new ExperimentException(String.format("Unable to find run to be replaced (RowId %d)", reRunId)); - - if (replacedRun.getContainer().hasPermission(context.getUser(), UpdatePermission.class)) - { - replacedRun.setReplacedByRun(run); - replacedRun.save(context.getUser()); - } - - String auditMessage = String.format("Run id %d was replaced by run id %d", replacedRun.getRowId(), run.getRowId()); - ExperimentService.get().auditRunEvent(context.getUser(), context.getProtocol(), replacedRun, null, auditMessage, context.getAuditUserComment()); - - transaction.addCommitTask(() -> replacedRun.archiveDataFiles(context.getUser()), DbScope.CommitTaskOption.POSTCOMMIT); - // Issue 51710: Remove replaced assay runs from the search index - transaction.addCommitTask(() -> AssayService.get().deindexAssayRuns(List.of(replacedRun)), DbScope.CommitTaskOption.POSTCOMMIT); - } - - AssayService.get().ensureUniqueBatchName(batch, context.getProtocol(), context.getUser()); - - ExperimentService.get().onRunDataCreated(context.getProtocol(), run, container, context.getUser()); - - transaction.commit(); - success = true; - - // Inspect the run properties for a “prov:objectInputs” property that is a list of LSID strings. - // Attach run's starting protocol application with starting input LSIDs. - Object provInputsProperty = unresolvedRunProperties.get(ProvenanceService.PROVENANCE_INPUT_PROPERTY); - if (provInputsProperty != null) - { - ProvenanceService pvs = ProvenanceService.get(); - Set runInputLSIDs = null; - if (provInputsProperty instanceof String provInputs) - { - // parse as a JSONArray of values or a comma-separated list of values - if (provInputs.startsWith("[") && provInputs.endsWith("]")) - provInputsProperty = new JSONArray(provInputs); - else - runInputLSIDs = Set.of(provInputs.split(",")); - } - - if (provInputsProperty instanceof JSONArray jsonArray) - { - runInputLSIDs = jsonArray.toList().stream() - .map(String::valueOf) - .collect(Collectors.toSet()); - } - - if (runInputLSIDs != null && !runInputLSIDs.isEmpty()) - { - ExpProtocolApplication inputProtocolApp = run.getInputProtocolApplication(); - pvs.addProvenanceInputs(container, inputProtocolApp, runInputLSIDs); - } - } - - ExperimentService.get().queueSyncRunEdges(run); - - return batch; - } - catch (IOException | ConvertHelper.FileConversionException | BatchValidationException e) - { - // HACK: Rethrowing these as ApiUsageException avoids any upstream consequences of wrapping them in ExperimentException. - // Namely, that they are logged to the server/mothership. There has to be a better way. - if (e instanceof ConvertHelper.FileConversionException fce) - throw new ApiUsageException(fce.getMessage(), fce); - else if (e instanceof BatchValidationException bve) - throw new ApiUsageException(bve.getMessage(), bve); - - throw new ExperimentException(e); - } - finally - { - if (!success) - { - // clean up the run results file dir here if it was created, for non-async imports - AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); - resultsFileWriter.cleanupPostedFiles(context.getContainer(), false); - - cleanPrimaryFile(context); - } - } - } - - private void cleanPrimaryFile(AssayRunUploadContext context) throws ExperimentException - { - // Do not clear the primary file for run re-imports - if (context.getReRunId() != null) - return; - - try - { - // Issue 51300: don't keep the primary file if the new run failed to save - FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - - // If the uploaded file is in the temp directory, then do not delete it as it may be reused in the next import attempt. - if (primaryFile != null && primaryFile.exists() && !primaryFile.getPath().contains(TEMP_DIR_NAME)) - primaryFile.delete(); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - } - - private void resolveParticipantVisits( - AssayRunUploadContext context, - Map inputMaterials, - Map inputDatas, - Map outputMaterials, - Map outputDatas, - Map allProperties, - @Nullable ParticipantVisitResolverType resolverType - ) throws ExperimentException - { - try - { - ParticipantVisitResolver resolver = null; - if (resolverType != null) - { - String targetStudyId = null; - for (Map.Entry property : allProperties.entrySet()) - { - if (AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME.equals(property.getKey().getName())) - { - targetStudyId = property.getValue(); - break; - } - } - Container targetStudy = null; - if (targetStudyId != null && !targetStudyId.isEmpty()) - targetStudy = ContainerManager.getForId(targetStudyId); - - resolver = resolverType.createResolver( - unmodifiableCollection(inputMaterials.keySet()), - unmodifiableCollection(inputDatas.keySet()), - unmodifiableCollection(outputMaterials.keySet()), - unmodifiableCollection(outputDatas.keySet()), - context.getContainer(), - targetStudy, context.getUser()); - } - - resolveExtraRunData(resolver, context, inputMaterials, inputDatas, outputMaterials, outputDatas); - } - catch (IOException e) - { - throw new ExperimentException(e); - } - } - - protected void importStandardResultData( - AssayRunUploadContext context, - ExpRun run, - Map inputDatas, - Map outputDatas, - ViewBackgroundInfo info, - XarContext xarContext - ) throws ExperimentException, BatchValidationException - { - DataIteratorBuilder rawData = context.getRawData(); - List insertedDatas = new ArrayList<>(); - - if (rawData != null) - { - insertedDatas.addAll(outputDatas.keySet()); - - ExpData primaryData = null; - // Decide which file to treat as the primary, to which the data rows will be attached - for (Map.Entry entry : outputDatas.entrySet()) - { - if (ExpDataRunInput.DEFAULT_ROLE.equalsIgnoreCase(entry.getValue())) - { - primaryData = entry.getKey(); - } - } - if (primaryData == null && !insertedDatas.isEmpty()) - primaryData = insertedDatas.get(0); - - if (primaryData != null) - { - TsvDataHandler dataHandler = new TsvDataHandler(); - dataHandler.setAllowEmptyData(true); - dataHandler.importRows(primaryData, context.getUser(), run, context.getProtocol(), getProvider(), rawData, null, context.shouldAutoFillDefaultResultColumns(), context); - } - } - else - { - for (Map.Entry entry : inputDatas.entrySet()) - { - // skip any of the cross run inputData that are already in the outputData - if (CROSS_RUN_DATA_INPUT_ROLE.equals(entry.getValue())) - continue; - - insertedDatas.add(entry.getKey()); - } - - insertedDatas.addAll(outputDatas.keySet()); - - Logger logger = context.getLogger() != null ? context.getLogger() : LOG; - for (ExpData insertedData : insertedDatas) - { - ExperimentDataHandler dataHandler = insertedData.findDataHandler(); - - FileLike fileLike = FileSystemLike.wrapFile(insertedData.getFile()); - if (dataHandler instanceof AbstractAssayTsvDataHandler tsvHandler) - { - tsvHandler.importFile(insertedData, fileLike, info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); - } - else - { - dataHandler.importFile(insertedData, fileLike, info, logger, xarContext); - } - } - } - } - - private void importResultData( - AssayRunUploadContext context, - ExpRun run, - Map inputDatas, - Map outputDatas, - ViewBackgroundInfo info, - XarContext xarContext, - TransformResult transformResult - ) throws ExperimentException, BatchValidationException - { - if (transformResult.getTransformedData().isEmpty()) - { - importStandardResultData(context, run, inputDatas, outputDatas, info, xarContext); - return; - } - - DataType dataType = context.getProvider().getDataType(); - if (dataType == null) - { - // we know that we are importing transformed data at this point - dataType = TsvDataHandler.RELATED_TRANSFORM_FILE_DATA_TYPE; - } - - ExpData data = ExperimentService.get().createData(context.getContainer(), dataType); - ExperimentDataHandler handler = data.findDataHandler(); - - // this should assert to always be true - if (handler instanceof TransformDataHandler transformDataHandler) - { - for (Map.Entry entry : transformResult.getTransformedData().entrySet()) - { - ExpData expData = entry.getKey(); - // The object may have already been claimed by - if (expData.getSourceApplication() == null) - { - expData.setSourceApplication(run.getOutputProtocolApplication()); - } - expData.save(context.getUser()); - - run.getOutputProtocolApplication().addDataInput(context.getUser(), expData, ExpDataRunInput.IMPORTED_DATA_ROLE); - // Add to the cached list of outputs - run.getDataOutputs().add(expData); - - transformDataHandler.importTransformDataMap(expData, context, run, entry.getValue()); - } - } - } - - protected void addInputMaterials( - AssayRunUploadContext context, - Map inputMaterials, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - addMaterials(context, inputMaterials, context.getInputMaterials(), null, cache, materialCache); - - // Find lookups to a SampleType and add the resolved material as an input sample - for (Map.Entry entry : context.getRunProperties().entrySet()) - { - String value = StringUtils.trimToNull(entry.getValue()); - if (value == null) - continue; - - // Lookup must point at "Samples.*", "exp.materials.*", or "exp.Materials" - DomainProperty dp = entry.getKey(); - var sampleLookup = AssaySampleLookupContext.checkSampleLookup(context.getContainer(), context.getUser(), dp); - if (!sampleLookup.isLookup()) - continue; - - String role = AssayService.get().getPropertyInputLineageRole(dp); - addMaterials(context, inputMaterials, Map.of(value, role), sampleLookup.expSampleType(), cache, materialCache); - } - } - - protected void addInputDatas( - AssayRunUploadContext context, - @NotNull Map inputDatas - ) throws ExperimentException, ValidationException - { - Logger log = context.getLogger() != null ? context.getLogger() : LOG; - - Map inputs = context.getInputDatas(); - addDatas(context.getContainer(), inputDatas, inputs, log); - - // Inspect the uploaded files which will be added as outputs of the run - if (context.isAllowCrossRunFileInputs()) - { - Map files = context.getUploadedData(); - for (Map.Entry entry : files.entrySet()) - { - String key = entry.getKey(); - if (AssayDataCollector.PRIMARY_FILE.equals(key)) - { - FileLike file = entry.getValue(); - - // Check if the file is created by a run - // Don't use getExpDataByURL(String). That method expects string in a very particular format. - ExpData existingData = ExperimentService.get().getExpDataByURL(file.toNioPathForRead(), context.getContainer()); - if (existingData != null && existingData.getRunId() != null && !inputDatas.containsKey(existingData)) - { - // Add this file as an input to the run. When we add the outputs to the run, we will detect - // that this file was already added as an input and create a new exp.data for the same file - // path and attach it as an output. - log.debug("found existing cross run file input: name={}, rowId={}, dataFileUrl={}", existingData.getName(), existingData.getRowId(), existingData.getDataFileUrl()); - inputDatas.put(existingData, CROSS_RUN_DATA_INPUT_ROLE); - } - } - } - } - } - - // CONSIDER: Move this to ExperimentService - // Resolve submitted values into ExpData objects - protected void addDatas(Container c, @NotNull Map resolved, @NotNull Map unresolved, @Nullable Logger log) throws ValidationException - { - for (Map.Entry entry : unresolved.entrySet()) - { - Object o = entry.getKey(); - String role = entry.getValue(); - - if (o instanceof ExpData expData) - { - resolved.put(expData, role); - } - else - { - File file = ExpDataFileConverter.convert(o); - if (file != null) - { - ExpData data = ExperimentService.get().getExpDataByURL(file, c); - if (data == null) - { - DataType dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - data = createData(c, file, file.getName(), dataType, false, true, log); - } - - resolved.put(data, role); - } - } - } - } - - public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData) throws ValidationException - { - return generateResultData(user, container, provider, dataArray, outputData, null); - } - - public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData, @Nullable Logger log) throws ValidationException - { - if (log == null) - log = LOG; - - ExpData newData = null; - - // Don't create an empty result data file if there are other outputs from this run, or if the user didn't - // include any data rows - if (!dataArray.isEmpty() && outputData.isEmpty()) - { - DataType dataType = provider.getDataType(); - if (dataType == null) - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - - newData = createData(container, "Analysis Results", dataType, log); - newData.save(user); - outputData.put(newData, ExpDataRunInput.DEFAULT_ROLE); - } - - return newData; - } - - // Find an existing ExpData for the File or null. - public static @Nullable ExpData findExistingData(Container c, @Nullable File file, @Nullable Logger log) - { - if (file == null) - return null; - - if (log == null) - log = LOG; - - List existing = ExperimentService.get().getAllExpDataByURL(file, c); - if (!existing.isEmpty()) - { - for (ExpData d : existing) - { - log.debug("found existing exp.data for file, rowId={}, runId={}, dataFileUrl={}", d.getRowId(), d.getRunId(), d.getDataFileUrl()); - } - - // pick the most recently created one - return existing.get(0); - } - - return null; - } - - public static @NotNull ExpData createData(Container c, String name, @NotNull DataType dataType, @Nullable Logger log) throws ValidationException - { - // NOTE: reuseExistingData and errorOnDataOwned flags are irrelevant when we aren't providing a File - return createData(c, null, name, dataType, false, false, log); - } - - public static @NotNull ExpData createData( - Container c, - File file, - String name, - @Nullable DataType dataType, - boolean reuseExistingData, - boolean errorIfDataOwned, - @Nullable Logger log - ) throws ValidationException - { - if (log == null) - log = LOG; - - ExpData data = findExistingData(c, file, log); - - ExpRun previousRun; - if (data != null && null != (previousRun = data.getRun())) - { - // There's an existing data, but it's already marked as being created by another run - String msg = "File '" + data.getName() + "' has been previously imported in run '" + previousRun.getName() + "' (" + previousRun.getRowId() + ")"; - if (reuseExistingData && errorIfDataOwned) - throw new ValidationException(msg); - - log.debug(msg); - - // Create a new one for the same path so the new run can claim it as its own - if (!reuseExistingData) - { - log.debug("ignoring existing exp.data, will create a new one"); - data = null; - } - } - - if (data == null) - { - if (dataType == null) - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - - log.debug("creating assay exp.data for file. dataType={}, file={}", dataType.getNamespacePrefix(), file); - data = ExperimentService.get().createData(c, dataType, name); - data.setLSID(ExperimentService.get().generateGuidLSID(c, dataType)); - if (file != null) - { - data.setDataFileURI(FileUtil.getAbsoluteCaseSensitiveFile(file).toURI()); - } - } - else - { - if (dataType != null && !dataType.matches(new Lsid(data.getLSID()))) - { - // Reset its LSID so that it's the correct type // CONSIDER: creating a new ExpData with the correct type instead - String newLsid = ExperimentService.get().generateGuidLSID(c, dataType); - log.debug("LSID doesn't match desired type. Changed the LSID from '{}' to '{}'", data.getLSID(), newLsid); - data.setLSID(newLsid); - } - } - return data; - } - - protected void addOutputMaterials( - AssayRunUploadContext context, - Map outputMaterials, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - addMaterials(context, outputMaterials, context.getOutputMaterials(), null, cache, materialCache); - } - - protected void addMaterials( - AssayRunUploadContext context, - @NotNull Map resolved, - @NotNull Map unresolved, - @Nullable ExpSampleType sampleType, - @NotNull RemapCache cache, - @NotNull Map materialCache - ) throws ExperimentException, ValidationException - { - for (Map.Entry entry : unresolved.entrySet()) - { - Object sampleIdentifier = entry.getKey(); - ExpMaterial material = ExperimentService.get().findExpMaterial(context.getContainer(), context.getUser(), sampleIdentifier, sampleType, cache, materialCache); - if (material == null) - throw new ExperimentException("Unable to resolve sample: " + sampleIdentifier); - - if (!resolved.containsKey(material)) - { - if (!material.isOperationPermitted(SampleTypeService.SampleOperations.AddAssayData)) - throw new ExperimentException(SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddAssayData)); - if (sampleType == null || sampleType.getLSID().equals(material.getCpasType())) - resolved.put(material, entry.getValue()); - } - } - } - - protected void addOutputDatas( - AssayRunUploadContext context, - Map inputDatas, - Map outputDatas - ) throws ExperimentException, ValidationException - { - Logger log = context.getLogger() != null ? context.getLogger() : LOG; - - // Create set of existing input files - Set inputFiles = new HashSet<>(); - for (ExpData inputData : inputDatas.keySet()) - { - FileLike f = inputData.getFileLike(); - if (f != null) - inputFiles.add(f); - } - - Map files = context.getUploadedData(); - - AssayDataType dataType; - for (Map.Entry entry : files.entrySet()) - { - FileLike file = entry.getValue(); - dataType = context.getProvider().getDataType(); - - // Reuse existing exp.data as the assay output file unless: - // - we are re-importing the run - // - or the output file is already one of the input files and if we are allowing cross-run file inputs - boolean reuseExistingData = true; - if (context.getReRunId() != null) - reuseExistingData = false; - if (context.isAllowCrossRunFileInputs() && inputFiles.contains(file)) - reuseExistingData = false; - - // For Luminex re-import, we want to reuse the existing exp.data but not - // throw an error when we discover that the exp.data is already owned. The - // original run will be duplicated for re-import and then will be deleted. - boolean errorIfDataOwned = getProvider().getReRunSupport() != AssayProvider.ReRunSupport.ReRunAndDelete; - - log.debug("adding output data: file={}", file.toNioPathForRead()); - log.debug(" context.getReRunId()={}", context.getReRunId()); - log.debug(" provider.getReRunSupport()={}", getProvider().getReRunSupport()); - log.debug(" context.allowCrossRunFileInputs={}", context.isAllowCrossRunFileInputs()); - log.debug(" inputFiles.contains(file)={}", inputFiles.contains(file)); - log.debug("==> reuseExistingData = {}", reuseExistingData); - log.debug("==> errorIfDataOwned = {}", errorIfDataOwned); - - ExpData data = DefaultAssayRunCreator.createData(context.getContainer(), file.toNioPathForRead().toFile(), file.getName(), dataType, reuseExistingData, errorIfDataOwned, log); - String role = ExpDataRunInput.DEFAULT_ROLE; - if (dataType != null && dataType.getFileType().isType(file)) - { - if (dataType.getRole() != null) - { - role = dataType.getRole(); - } - } - outputDatas.put(data, role); - } - - FileLike primaryFile = files.get(AssayDataCollector.PRIMARY_FILE); - if (primaryFile != null) - { - addRelatedOutputDatas(context, inputFiles, outputDatas, primaryFile); - } - - Map outputs = context.getOutputDatas(); - addDatas(context.getContainer(), outputDatas, outputs, log); - } - - /** - * Add files that follow the general naming convention (same basename) as the primary file - */ - public void addRelatedOutputDatas( - AssayRunUploadContext context, - Set inputFiles, - Map outputDatas, - final FileLike primaryFile - ) throws ValidationException - { - AssayDataType dataType = getProvider().getDataType(); - final String baseName = dataType == null ? null : dataType.getFileType().getBaseName(primaryFile.toNioPathForRead()); - if (baseName != null) - { - // Grab all the files that are related based on naming convention - File primary = primaryFile.toNioPathForRead().toFile(); - File parent = primary.getParentFile(); - // converting to File land to reuse the FileFilter - File[] relatedFiles = parent.listFiles(getRelatedOutputDataFileFilter(primary, baseName)); - if (relatedFiles != null) - { - for (File f : relatedFiles) - { - FileLike relatedFile = primaryFile.getParent().resolveChild(f.getName()); - // Ignore files already considered inputs to the run - if (inputFiles.contains(relatedFile)) - continue; - - Pair dataOutput = createdRelatedOutputData(context, baseName, f); - if (dataOutput != null) - { - outputDatas.put(dataOutput.getKey(), dataOutput.getValue()); - } - } - } - } - } - - protected void resolveExtraRunData( - ParticipantVisitResolver resolver, - AssayRunUploadContext context, - Map inputMaterials, - Map inputDatas, - Map outputMaterials, - Map outputDatas - ) throws ExperimentException - { - } - - /** - * Create an ExpData object for the file, and figure out what its role name should be - * @return null if the file is already linked to another run - */ - @Nullable - public static Pair createdRelatedOutputData(AssayRunUploadContext context, String baseName, File relatedFile) throws ValidationException - { - String roleName = null; - DataType dataType = null; - for (AssayDataType inputType : context.getProvider().getRelatedDataTypes()) - { - // Check if we recognize it as a specially handled file type - if (inputType.getFileType().isMatch(relatedFile.getName(), baseName)) - { - roleName = inputType.getRole(); - dataType = inputType; - break; - } - } - // If not, make up a new type and role for it - if (roleName == null) - { - roleName = relatedFile.getName().substring(baseName.length()); - while (!roleName.isEmpty() && (roleName.startsWith(".") || roleName.startsWith("-") || roleName.startsWith("_") || roleName.startsWith(" "))) - { - roleName = roleName.substring(1); - } - if (roleName.isEmpty()) - { - roleName = null; - } - } - if (dataType == null) - { - dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; - } - - // Find an existing data that isn't owned by another run or create a new own - ExpData data = findExistingData(context.getContainer(), relatedFile, context.getLogger()); - if (data != null) - { - if (data.getSourceApplication() == null) - return new Pair<>(data, roleName); - - // The file is already linked to another run, so this one must have not created it - return null; - } - - data = createData(context.getContainer(), relatedFile, relatedFile.getName(), dataType, true, true, context.getLogger()); - assert data.getSourceApplication() == null; - return Pair.of(data, roleName); - } - - // Disallow creating a run with inputs which are also outputs - protected void checkForCycles( - Map inputs, - Map outputs - ) throws ExperimentException - { - for (ExpRunItem input : inputs.keySet()) - { - if (outputs.containsKey(input)) - { - String role = outputs.get(input); - throw new ExperimentException("Circular input/output '" + input.getName() + "' with role '" + role + "'"); - } - } - } - - private void saveProperties( - final AssayRunUploadContext context, - ExpObject expObject, - Map transformResultProperties, - Map properties - ) throws ValidationException - { - Map propsToSave = transformResultProperties.isEmpty() ? properties : transformResultProperties; - List errors = validateProperties(context, propsToSave); - if (!errors.isEmpty()) - throw new ValidationException(errors); - - savePropertyObject(expObject, propsToSave, context.getUser()); - } - - protected void savePropertyObject(ExpObject object, Map properties, User user) throws ValidationException - { - for (Map.Entry entry : properties.entrySet()) - { - DomainProperty pd = entry.getKey(); - String value = entry.getValue(); - - // resolve any file links for batch or run properties - if (PropertyType.FILE_LINK.getTypeUri().equals(pd.getType().getTypeURI())) - { - File resolvedFile = ExpDataFileConverter.convert(value); - if (resolvedFile != null) - value = resolvedFile.getAbsolutePath(); - } - - // Treat the empty string as a null in the database, which is our normal behavior when receiving data - // from HTML forms. - if (StringUtils.trimToNull(value) == null) - { - value = null; - } - if (value != null) - { - object.setProperty(user, pd.getPropertyDescriptor(), value); - } - else - { - // We still need to validate blanks - List errors = new ArrayList<>(); - OntologyManager.validateProperty(pd.getValidators(), pd.getPropertyDescriptor(), new ObjectProperty(object.getLSID(), object.getContainer(), pd.getPropertyDescriptor(), value), errors, new ValidatorContext(pd.getContainer(), user)); - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - } - } - - public static List validateColumnProperties(ContainerUser context, Map properties) - { - List errors = new ArrayList<>(); - RemapCache cache = new RemapCache(); - for (Map.Entry entry : properties.entrySet()) - { - validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); - } - return errors; - } - - public static List validateProperties(ContainerUser context, Map properties) - { - List errors = new ArrayList<>(); - RemapCache cache = new RemapCache(); - for (Map.Entry entry : properties.entrySet()) - { - validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); - } - return errors; - } - - private static void validateProperty(ContainerUser context, ColumnInfo columnInfo, String value, RemapCache cache, List errors) - { - Lookup lookup = null; - if (columnInfo.isLookup()) - { - ForeignKey fk = columnInfo.getFk(); - lookup = new Lookup(fk.getLookupContainer(), fk.getLookupSchemaName(), fk.getLookupTableName()); - } - validateProperty(context, ColumnValidators.create(columnInfo, null), value, columnInfo.getName(), - false, lookup, columnInfo.getJavaClass(), cache, errors); - } - - private static void validateProperty(ContainerUser context, DomainProperty dp, String value, RemapCache cache, List errors) - { - String label = dp.getPropertyDescriptor().getNonBlankCaption(); - PropertyType type = dp.getPropertyDescriptor().getPropertyType(); - validateProperty(context, ColumnValidators.create(null, dp), value, label, dp.isRequired(), - dp.getLookup(), type.getJavaType(), cache, errors); - } - - private static void validateProperty( - ContainerUser context, - List validators, - String value, - String label, - Boolean required, - Lookup lookup, - Class type, - RemapCache cache, - List errors - ) - { - boolean missing = (value == null || value.isEmpty()); - int rowNum = 0; - - if (required && missing) - { - errors.add(new SimpleValidationError(label + " is required and must be of type " + ColumnInfo.getFriendlyTypeName(type) + ".")); - } - else if (!missing) - { - try - { - Object o; - if (type == File.class) - o = ExpDataFileConverter.convert(value); - else - o = ConvertUtils.convert(value, type); - ValidatorContext validatorContext = new ValidatorContext(context.getContainer(), context.getUser()); - for (ColumnValidator validator : validators) - { - String msg = validator.validate(rowNum, o, validatorContext); - if (msg != null) - errors.add(new PropertyValidationError(msg, label)); - } - } - catch (ConversionException e) - { - String message; - if (e instanceof ConvertHelper.FileConversionException fce) - message = fce.getMessage(); - else - { - message = ConvertHelper.getStandardConversionErrorMessage(value, label, type); - if (e.getCause() instanceof ArithmeticException) - message += ": " + e.getCause().getLocalizedMessage(); - else - message += "."; - } - - // Attempt to resolve lookups by display value - boolean skipError = false; - if (lookup != null) - { - Object remappedValue = OntologyManager.getRemappedValueForLookup(context.getUser(), context.getContainer(), cache, lookup, value); - if (remappedValue != null) - skipError = true; - } - - if (!skipError) - errors.add(new SimpleValidationError(message)); - } - } - } - - protected FileFilter getRelatedOutputDataFileFilter(final File primaryFile, final String baseName) - { - // baseName doesn't include the trailing '.', so add it here. We want to associate myRun.jpg - // with myRun.xls, but we don't want to associate myRun2.xls with myRun.xls (which will happen without - // the trailing dot in the check). - return f -> f.getName().startsWith(baseName + ".") && !primaryFile.equals(f); - } - - protected ProviderType getProvider() - { - return _provider; - } -} +/* + * Copyright (c) 2011-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.assay; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.assay.actions.AssayRunUploadForm; +import org.labkey.api.assay.pipeline.AssayRunAsyncContext; +import org.labkey.api.assay.pipeline.AssayUploadPipelineJob; +import org.labkey.api.assay.sample.AssaySampleLookupContext; +import org.labkey.api.assay.transform.DataTransformService; +import org.labkey.api.assay.transform.TransformDataHandler; +import org.labkey.api.assay.transform.TransformResult; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.RemapCache; +import org.labkey.api.data.validator.ColumnValidator; +import org.labkey.api.data.validator.ColumnValidators; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.exp.ExperimentDataHandler; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.XarContext; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataRunInput; +import org.labkey.api.exp.api.ExpExperiment; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpObject; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpRunItem; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineValidationException; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.study.assay.ParticipantVisitResolver; +import org.labkey.api.study.assay.ParticipantVisitResolverType; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.writer.ContainerUser; +import org.labkey.vfs.FileLike; +import org.labkey.vfs.FileSystemLike; + +import java.io.File; +import java.io.FileFilter; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.unmodifiableCollection; +import static org.labkey.api.assay.AssayFileWriter.TEMP_DIR_NAME; + +public class DefaultAssayRunCreator implements AssayRunCreator +{ + private static final Logger LOG = LogManager.getLogger(DefaultAssayRunCreator.class); + public static final String CROSS_RUN_DATA_INPUT_ROLE = "cross run input"; + + private final ProviderType _provider; + + public DefaultAssayRunCreator(ProviderType provider) + { + _provider = provider; + } + + public TransformResult transform(AssayRunUploadContext context, ExpRun run) throws ValidationException + { + return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); + } + /** + * Create and save an experiment run synchronously or asynchronously in a background job depending upon the assay design. + * + * @param context The context used to create and save the batch and run. + * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. + * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. + */ + @Override + public Pair saveExperimentRun( + AssayRunUploadContext context, + @Nullable Long batchId, + boolean forceAsync, + Map transactionDetails + ) throws ExperimentException, ValidationException + { + ExpExperiment exp = null; + if (batchId != null) + { + exp = ExperimentService.get().getExpExperiment(batchId); + } + + AssayProvider provider = context.getProvider(); + ExpProtocol protocol = context.getProtocol(); + ExpRun run = null; + + try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) + { + if (transaction.getAuditId() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); + } + context.init(); + // Check if assay protocol is configured to import in the background. + // Issue 26811: If we don't have a view, assume that we are on a background job thread already. + boolean importInBackground = forceAsync || (provider.isBackgroundUpload(protocol) && HttpView.hasCurrentView()); + if (!importInBackground) + { + if ((Object) context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE) instanceof File errFile) + { + throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); + } + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); + run.setComments(context.getComments()); + run.setWorkflowTaskId(context.getWorkflowTask()); + + exp = saveExperimentRun(context, exp, run, false, transactionDetails); + + // re-fetch the run after it has been fully constructed + run = ExperimentService.get().getExpRun(run.getRowId()); + + context.uploadComplete(run); + } + else + { + context.uploadComplete(null); + context.setTransactionAuditId(transaction.getAuditId()); + exp = saveExperimentRunAsync(context, exp); + } + transaction.commit(); + } + + return Pair.of(exp, run); + } + + private ExpExperiment saveExperimentRunAsync(AssayRunUploadContext context, @Nullable ExpExperiment batch) throws ExperimentException + { + try + { + // Whether we need to save batch properties + boolean forceSaveBatchProps = false; + if (batch == null) + { + // No batch yet, so make one + batch = AssayService.get().createStandardBatch(context.getContainer(), null, context.getProtocol()); + batch.save(context.getUser()); + // It's brand new, so we need to eventually set its properties + forceSaveBatchProps = true; + } + + // Queue up a pipeline job to do the actual import in the background + ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); + + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + // Check if the primary file from the previous import is no longer present for a re-run + if (primaryFile == null && !context.getUploadedData().isEmpty()) + { + // Choose another file as the primary + primaryFile = context.getUploadedData().entrySet().iterator().next().getValue(); + } + primaryFile = Objects.requireNonNull(primaryFile); + AssayRunAsyncContext asyncContext = context.getProvider().createRunAsyncContext(context); + final AssayUploadPipelineJob pipelineJob = new AssayUploadPipelineJob( + asyncContext, + info, + batch, + forceSaveBatchProps, + PipelineService.get().getPipelineRootSetting(context.getContainer()), + primaryFile.toNioPathForRead().toFile() + ); + + context.setPipelineJobGUID(pipelineJob.getJobGUID()); + + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), null, pipelineJob.getJobGUID()); + resultsFileWriter.savePostedFiles(context); + + // Don't queue the job until the transaction is committed, since otherwise the thread + // that's running the job might start before it can access the job's row in the database. + ExperimentService.get().getSchema().getScope().addCommitTask(() -> { + try + { + PipelineService.get().queueJob(pipelineJob, asyncContext.getJobNotificationProvider()); + } + catch (PipelineValidationException e) + { + throw UnexpectedException.wrap(e); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + + return batch; + } + + /** + * @param batch if not null, the run group that's already created for this batch. If null, a new one needs to be created + * @param run The run to save + * @return the run and batch that were inserted + */ + @Override + public ExpExperiment saveExperimentRun( + final AssayRunUploadContext context, + @Nullable ExpExperiment batch, + @NotNull ExpRun run, + boolean forceSaveBatchProps, + @Nullable Map transactionDetails + ) throws ExperimentException, ValidationException + { + context.setAutoFillDefaultResultColumns(run.getRowId() > 0); // need to setAutoFillDefaultResultColumns before run is saved + + final Container container = context.getContainer(); + + Map inputMaterials = new HashMap<>(); + Map inputDatas = new HashMap<>(); + Map outputMaterials = new HashMap<>(); + Map outputDatas = new HashMap<>(); + Map transformedDatas = new HashMap<>(); + + Map runProperties = context.getRunProperties(); + Map unresolvedRunProperties = context.getUnresolvedRunProperties(); + Map batchProperties = context.getBatchProperties(); + + Map allProperties = new HashMap<>(); + allProperties.putAll(runProperties); + allProperties.putAll(batchProperties); + + ParticipantVisitResolverType resolverType = null; + for (Map.Entry entry : allProperties.entrySet()) + { + if (entry.getKey().getName().equals(AbstractAssayProvider.PARTICIPANT_VISIT_RESOLVER_PROPERTY_NAME)) + { + resolverType = AbstractAssayProvider.findType(entry.getValue(), getProvider().getParticipantVisitResolverTypes()); + if (resolverType != null) + { + resolverType.configureRun(context, run, inputDatas); + } + break; + } + } + + // TODO: Share these RemapCache and materialCache instances with AbstractAssayTsvDataHandler.checkData and ExpressionMatrixDataHandler.importFile + // Cache of resolved alternate lookup keys -> rowId + final RemapCache cache = new RemapCache(true); + // Cache of rowId -> ExpMaterial + final Map materialCache = new LongHashMap<>(); + + addInputMaterials(context, inputMaterials, cache, materialCache); + addInputDatas(context, inputDatas); + addOutputMaterials(context, outputMaterials, cache, materialCache); + addOutputDatas(context, inputDatas, outputDatas); + + boolean success = false; + DbScope scope = ExperimentService.get().getSchema().getScope(); + try (DbScope.Transaction transaction = scope.ensureTransaction(ExperimentService.get().getProtocolImportLock())) + { + if (transaction.getAuditId() == null) + { + var auditAction = context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT; + if (context.getTransactionAuditId() != null) + { + var auditEvent = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, context.getTransactionAuditId()); + transaction.setAuditEvent(auditEvent); + } + else + { + var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction, transactionDetails); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); + } + } + boolean saveBatchProps = forceSaveBatchProps; + + // Add any material/data inputs related to the specimen IDs, etc in the incoming data. + // Some subclasses may actually create ExpMaterials or do other database changes, so do this inside the + // overall transaction + resolveParticipantVisits(context, inputMaterials, inputDatas, outputMaterials, outputDatas, allProperties, resolverType); + + // Check for circular inputs/outputs + checkForCycles(inputMaterials, outputMaterials); + checkForCycles(inputDatas, outputDatas); + + // Create the batch, if needed + if (batch == null) + { + // Make sure that we have a batch to associate with this run + batch = AssayService.get().createStandardBatch(run.getContainer(), null, context.getProtocol()); + batch.save(context.getUser()); + saveBatchProps = true; + } + run.save(context.getUser()); + // Add the run to the batch so that we can find it when we're loading the data files + batch.addRuns(context.getUser(), run); + assert batch.equals(run.getBatch()) : "Run's batch should be the current batch"; + + ViewBackgroundInfo info = new ViewBackgroundInfo(context.getContainer(), context.getUser(), context.getActionURL()); + XarContext xarContext = new AssayUploadXarContext("Simple Run Creation", context); + + run = ExperimentService.get().saveSimpleExperimentRun( + run, + inputMaterials, + inputDatas, + outputMaterials, + outputDatas, + transformedDatas, + info, + context.getLogger() != null ? context.getLogger() : LOG, + false + ); + + // handle data transformation + TransformResult transformResult = transform(context, run); + + if (transformResult.getWarnings() != null && context instanceof AssayRunUploadForm uploadForm) + { + context.setTransformResult(transformResult); + uploadForm.setName(run.getName()); + uploadForm.setComments(run.getComments()); + throw new ValidationException(" "); + } + + if (saveBatchProps) + saveProperties(context, batch, transformResult.getBatchProperties(), batchProperties); + if (null != transformResult.getAssayId()) + run.setName(transformResult.getAssayId()); + if (null != transformResult.getComments()) + run.setComments(transformResult.getComments()); + saveProperties(context, run, transformResult.getRunProperties(), runProperties); + + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); + resultsFileWriter.savePostedFiles(context); + + Path assayResultsRunDir = AssayResultsFileWriter.getAssayFilesDirectoryPath(run); + if (null != assayResultsRunDir && !FileUtil.hasCloudScheme(assayResultsRunDir)) + { + FileLike assayResultFileRoot = FileSystemLike.wrapFile(assayResultsRunDir); + if (assayResultFileRoot != null) + QueryService.get().setEnvironment(QueryService.Environment.ASSAYFILESPATH, assayResultFileRoot); + } + + importResultData(context, run, inputDatas, outputDatas, info, xarContext, transformResult); + + var reRunId = context.getReRunId(); + if (reRunId != null && getProvider().getReRunSupport() == AssayProvider.ReRunSupport.ReRunAndReplace) + { + final ExpRun replacedRun = ExperimentService.get().getExpRun(reRunId); + if (replacedRun == null) + throw new ExperimentException(String.format("Unable to find run to be replaced (RowId %d)", reRunId)); + + if (replacedRun.getContainer().hasPermission(context.getUser(), UpdatePermission.class)) + { + replacedRun.setReplacedByRun(run); + replacedRun.save(context.getUser()); + } + + String auditMessage = String.format("Run id %d was replaced by run id %d", replacedRun.getRowId(), run.getRowId()); + ExperimentService.get().auditRunEvent(context.getUser(), context.getProtocol(), replacedRun, null, auditMessage, context.getAuditUserComment()); + + transaction.addCommitTask(() -> replacedRun.archiveDataFiles(context.getUser()), DbScope.CommitTaskOption.POSTCOMMIT); + // Issue 51710: Remove replaced assay runs from the search index + transaction.addCommitTask(() -> AssayService.get().deindexAssayRuns(List.of(replacedRun)), DbScope.CommitTaskOption.POSTCOMMIT); + } + + AssayService.get().ensureUniqueBatchName(batch, context.getProtocol(), context.getUser()); + + ExperimentService.get().onRunDataCreated(context.getProtocol(), run, container, context.getUser()); + + transaction.commit(); + success = true; + + // Inspect the run properties for a “prov:objectInputs” property that is a list of LSID strings. + // Attach run's starting protocol application with starting input LSIDs. + Object provInputsProperty = unresolvedRunProperties.get(ProvenanceService.PROVENANCE_INPUT_PROPERTY); + if (provInputsProperty != null) + { + ProvenanceService pvs = ProvenanceService.get(); + Set runInputLSIDs = null; + if (provInputsProperty instanceof String provInputs) + { + // parse as a JSONArray of values or a comma-separated list of values + if (provInputs.startsWith("[") && provInputs.endsWith("]")) + provInputsProperty = new JSONArray(provInputs); + else + runInputLSIDs = Set.of(provInputs.split(",")); + } + + if (provInputsProperty instanceof JSONArray jsonArray) + { + runInputLSIDs = jsonArray.toList().stream() + .map(String::valueOf) + .collect(Collectors.toSet()); + } + + if (runInputLSIDs != null && !runInputLSIDs.isEmpty()) + { + ExpProtocolApplication inputProtocolApp = run.getInputProtocolApplication(); + pvs.addProvenanceInputs(container, inputProtocolApp, runInputLSIDs); + } + } + + ExperimentService.get().queueSyncRunEdges(run); + + return batch; + } + catch (IOException | ConvertHelper.FileConversionException | BatchValidationException e) + { + // HACK: Rethrowing these as ApiUsageException avoids any upstream consequences of wrapping them in ExperimentException. + // Namely, that they are logged to the server/mothership. There has to be a better way. + if (e instanceof ConvertHelper.FileConversionException fce) + throw new ApiUsageException(fce.getMessage(), fce); + else if (e instanceof BatchValidationException bve) + throw new ApiUsageException(bve.getMessage(), bve); + + throw new ExperimentException(e); + } + finally + { + if (!success) + { + // clean up the run results file dir here if it was created, for non-async imports + AssayResultsFileWriter> resultsFileWriter = new AssayResultsFileWriter<>(context.getProtocol(), run, null); + resultsFileWriter.cleanupPostedFiles(context.getContainer(), false); + + cleanPrimaryFile(context); + } + } + } + + private void cleanPrimaryFile(AssayRunUploadContext context) throws ExperimentException + { + // Do not clear the primary file for run re-imports + if (context.getReRunId() != null) + return; + + try + { + // Issue 51300: don't keep the primary file if the new run failed to save + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + + // If the uploaded file is in the temp directory, then do not delete it as it may be reused in the next import attempt. + if (primaryFile != null && primaryFile.exists() && !primaryFile.getPath().contains(TEMP_DIR_NAME)) + primaryFile.delete(); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + } + + private void resolveParticipantVisits( + AssayRunUploadContext context, + Map inputMaterials, + Map inputDatas, + Map outputMaterials, + Map outputDatas, + Map allProperties, + @Nullable ParticipantVisitResolverType resolverType + ) throws ExperimentException + { + try + { + ParticipantVisitResolver resolver = null; + if (resolverType != null) + { + String targetStudyId = null; + for (Map.Entry property : allProperties.entrySet()) + { + if (AbstractAssayProvider.TARGET_STUDY_PROPERTY_NAME.equals(property.getKey().getName())) + { + targetStudyId = property.getValue(); + break; + } + } + Container targetStudy = null; + if (targetStudyId != null && !targetStudyId.isEmpty()) + targetStudy = ContainerManager.getForId(targetStudyId); + + resolver = resolverType.createResolver( + unmodifiableCollection(inputMaterials.keySet()), + unmodifiableCollection(inputDatas.keySet()), + unmodifiableCollection(outputMaterials.keySet()), + unmodifiableCollection(outputDatas.keySet()), + context.getContainer(), + targetStudy, context.getUser()); + } + + resolveExtraRunData(resolver, context, inputMaterials, inputDatas, outputMaterials, outputDatas); + } + catch (IOException e) + { + throw new ExperimentException(e); + } + } + + protected void importStandardResultData( + AssayRunUploadContext context, + ExpRun run, + Map inputDatas, + Map outputDatas, + ViewBackgroundInfo info, + XarContext xarContext + ) throws ExperimentException, BatchValidationException + { + DataIteratorBuilder rawData = context.getRawData(); + List insertedDatas = new ArrayList<>(); + + if (rawData != null) + { + insertedDatas.addAll(outputDatas.keySet()); + + ExpData primaryData = null; + // Decide which file to treat as the primary, to which the data rows will be attached + for (Map.Entry entry : outputDatas.entrySet()) + { + if (ExpDataRunInput.DEFAULT_ROLE.equalsIgnoreCase(entry.getValue())) + { + primaryData = entry.getKey(); + } + } + if (primaryData == null && !insertedDatas.isEmpty()) + primaryData = insertedDatas.get(0); + + if (primaryData != null) + { + TsvDataHandler dataHandler = new TsvDataHandler(); + dataHandler.setAllowEmptyData(true); + dataHandler.importRows(primaryData, context.getUser(), run, context.getProtocol(), getProvider(), rawData, null, context.shouldAutoFillDefaultResultColumns(), context); + } + } + else + { + for (Map.Entry entry : inputDatas.entrySet()) + { + // skip any of the cross run inputData that are already in the outputData + if (CROSS_RUN_DATA_INPUT_ROLE.equals(entry.getValue())) + continue; + + insertedDatas.add(entry.getKey()); + } + + insertedDatas.addAll(outputDatas.keySet()); + + Logger logger = context.getLogger() != null ? context.getLogger() : LOG; + for (ExpData insertedData : insertedDatas) + { + ExperimentDataHandler dataHandler = insertedData.findDataHandler(); + + FileLike fileLike = FileSystemLike.wrapFile(insertedData.getFile()); + if (dataHandler instanceof AbstractAssayTsvDataHandler tsvHandler) + { + tsvHandler.importFile(insertedData, fileLike, info, logger, xarContext, context.isAllowLookupByAlternateKey(), context.shouldAutoFillDefaultResultColumns()); + } + else + { + dataHandler.importFile(insertedData, fileLike, info, logger, xarContext); + } + } + } + } + + private void importResultData( + AssayRunUploadContext context, + ExpRun run, + Map inputDatas, + Map outputDatas, + ViewBackgroundInfo info, + XarContext xarContext, + TransformResult transformResult + ) throws ExperimentException, BatchValidationException + { + if (transformResult.getTransformedData().isEmpty()) + { + importStandardResultData(context, run, inputDatas, outputDatas, info, xarContext); + return; + } + + DataType dataType = context.getProvider().getDataType(); + if (dataType == null) + { + // we know that we are importing transformed data at this point + dataType = TsvDataHandler.RELATED_TRANSFORM_FILE_DATA_TYPE; + } + + ExpData data = ExperimentService.get().createData(context.getContainer(), dataType); + ExperimentDataHandler handler = data.findDataHandler(); + + // this should assert to always be true + if (handler instanceof TransformDataHandler transformDataHandler) + { + for (Map.Entry entry : transformResult.getTransformedData().entrySet()) + { + ExpData expData = entry.getKey(); + // The object may have already been claimed by + if (expData.getSourceApplication() == null) + { + expData.setSourceApplication(run.getOutputProtocolApplication()); + } + expData.save(context.getUser()); + + run.getOutputProtocolApplication().addDataInput(context.getUser(), expData, ExpDataRunInput.IMPORTED_DATA_ROLE); + // Add to the cached list of outputs + run.getDataOutputs().add(expData); + + transformDataHandler.importTransformDataMap(expData, context, run, entry.getValue()); + } + } + } + + protected void addInputMaterials( + AssayRunUploadContext context, + Map inputMaterials, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + addMaterials(context, inputMaterials, context.getInputMaterials(), null, cache, materialCache); + + // Find lookups to a SampleType and add the resolved material as an input sample + for (Map.Entry entry : context.getRunProperties().entrySet()) + { + String value = StringUtils.trimToNull(entry.getValue()); + if (value == null) + continue; + + // Lookup must point at "Samples.*", "exp.materials.*", or "exp.Materials" + DomainProperty dp = entry.getKey(); + var sampleLookup = AssaySampleLookupContext.checkSampleLookup(context.getContainer(), context.getUser(), dp); + if (!sampleLookup.isLookup()) + continue; + + String role = AssayService.get().getPropertyInputLineageRole(dp); + addMaterials(context, inputMaterials, Map.of(value, role), sampleLookup.expSampleType(), cache, materialCache); + } + } + + protected void addInputDatas( + AssayRunUploadContext context, + @NotNull Map inputDatas + ) throws ExperimentException, ValidationException + { + Logger log = context.getLogger() != null ? context.getLogger() : LOG; + + Map inputs = context.getInputDatas(); + addDatas(context.getContainer(), inputDatas, inputs, log); + + // Inspect the uploaded files which will be added as outputs of the run + if (context.isAllowCrossRunFileInputs()) + { + Map files = context.getUploadedData(); + for (Map.Entry entry : files.entrySet()) + { + String key = entry.getKey(); + if (AssayDataCollector.PRIMARY_FILE.equals(key)) + { + FileLike file = entry.getValue(); + + // Check if the file is created by a run + // Don't use getExpDataByURL(String). That method expects string in a very particular format. + ExpData existingData = ExperimentService.get().getExpDataByURL(file.toNioPathForRead(), context.getContainer()); + if (existingData != null && existingData.getRunId() != null && !inputDatas.containsKey(existingData)) + { + // Add this file as an input to the run. When we add the outputs to the run, we will detect + // that this file was already added as an input and create a new exp.data for the same file + // path and attach it as an output. + log.debug("found existing cross run file input: name={}, rowId={}, dataFileUrl={}", existingData.getName(), existingData.getRowId(), existingData.getDataFileUrl()); + inputDatas.put(existingData, CROSS_RUN_DATA_INPUT_ROLE); + } + } + } + } + } + + // CONSIDER: Move this to ExperimentService + // Resolve submitted values into ExpData objects + protected void addDatas(Container c, @NotNull Map resolved, @NotNull Map unresolved, @Nullable Logger log) throws ValidationException + { + for (Map.Entry entry : unresolved.entrySet()) + { + Object o = entry.getKey(); + String role = entry.getValue(); + + if (o instanceof ExpData expData) + { + resolved.put(expData, role); + } + else + { + File file = ExpDataFileConverter.convert(o); + if (file != null) + { + ExpData data = ExperimentService.get().getExpDataByURL(file, c); + if (data == null) + { + DataType dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + data = createData(c, file, file.getName(), dataType, false, true, log); + } + + resolved.put(data, role); + } + } + } + } + + public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData) throws ValidationException + { + return generateResultData(user, container, provider, dataArray, outputData, null); + } + + public static ExpData generateResultData(User user, Container container, AssayProvider provider, List> dataArray, Map outputData, @Nullable Logger log) throws ValidationException + { + if (log == null) + log = LOG; + + ExpData newData = null; + + // Don't create an empty result data file if there are other outputs from this run, or if the user didn't + // include any data rows + if (!dataArray.isEmpty() && outputData.isEmpty()) + { + DataType dataType = provider.getDataType(); + if (dataType == null) + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + + newData = createData(container, "Analysis Results", dataType, log); + newData.save(user); + outputData.put(newData, ExpDataRunInput.DEFAULT_ROLE); + } + + return newData; + } + + // Find an existing ExpData for the File or null. + public static @Nullable ExpData findExistingData(Container c, @Nullable File file, @Nullable Logger log) + { + if (file == null) + return null; + + if (log == null) + log = LOG; + + List existing = ExperimentService.get().getAllExpDataByURL(file, c); + if (!existing.isEmpty()) + { + for (ExpData d : existing) + { + log.debug("found existing exp.data for file, rowId={}, runId={}, dataFileUrl={}", d.getRowId(), d.getRunId(), d.getDataFileUrl()); + } + + // pick the most recently created one + return existing.get(0); + } + + return null; + } + + public static @NotNull ExpData createData(Container c, String name, @NotNull DataType dataType, @Nullable Logger log) throws ValidationException + { + // NOTE: reuseExistingData and errorOnDataOwned flags are irrelevant when we aren't providing a File + return createData(c, null, name, dataType, false, false, log); + } + + public static @NotNull ExpData createData( + Container c, + File file, + String name, + @Nullable DataType dataType, + boolean reuseExistingData, + boolean errorIfDataOwned, + @Nullable Logger log + ) throws ValidationException + { + if (log == null) + log = LOG; + + ExpData data = findExistingData(c, file, log); + + ExpRun previousRun; + if (data != null && null != (previousRun = data.getRun())) + { + // There's an existing data, but it's already marked as being created by another run + String msg = "File '" + data.getName() + "' has been previously imported in run '" + previousRun.getName() + "' (" + previousRun.getRowId() + ")"; + if (reuseExistingData && errorIfDataOwned) + throw new ValidationException(msg); + + log.debug(msg); + + // Create a new one for the same path so the new run can claim it as its own + if (!reuseExistingData) + { + log.debug("ignoring existing exp.data, will create a new one"); + data = null; + } + } + + if (data == null) + { + if (dataType == null) + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + + log.debug("creating assay exp.data for file. dataType={}, file={}", dataType.getNamespacePrefix(), file); + data = ExperimentService.get().createData(c, dataType, name); + data.setLSID(ExperimentService.get().generateGuidLSID(c, dataType)); + if (file != null) + { + data.setDataFileURI(FileUtil.getAbsoluteCaseSensitiveFile(file).toURI()); + } + } + else + { + if (dataType != null && !dataType.matches(new Lsid(data.getLSID()))) + { + // Reset its LSID so that it's the correct type // CONSIDER: creating a new ExpData with the correct type instead + String newLsid = ExperimentService.get().generateGuidLSID(c, dataType); + log.debug("LSID doesn't match desired type. Changed the LSID from '{}' to '{}'", data.getLSID(), newLsid); + data.setLSID(newLsid); + } + } + return data; + } + + protected void addOutputMaterials( + AssayRunUploadContext context, + Map outputMaterials, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + addMaterials(context, outputMaterials, context.getOutputMaterials(), null, cache, materialCache); + } + + protected void addMaterials( + AssayRunUploadContext context, + @NotNull Map resolved, + @NotNull Map unresolved, + @Nullable ExpSampleType sampleType, + @NotNull RemapCache cache, + @NotNull Map materialCache + ) throws ExperimentException, ValidationException + { + for (Map.Entry entry : unresolved.entrySet()) + { + Object sampleIdentifier = entry.getKey(); + ExpMaterial material = ExperimentService.get().findExpMaterial(context.getContainer(), context.getUser(), sampleIdentifier, sampleType, cache, materialCache); + if (material == null) + throw new ExperimentException("Unable to resolve sample: " + sampleIdentifier); + + if (!resolved.containsKey(material)) + { + if (!material.isOperationPermitted(SampleTypeService.SampleOperations.AddAssayData)) + throw new ExperimentException(SampleTypeService.get().getOperationNotPermittedMessage(Collections.singleton(material), SampleTypeService.SampleOperations.AddAssayData)); + if (sampleType == null || sampleType.getLSID().equals(material.getCpasType())) + resolved.put(material, entry.getValue()); + } + } + } + + protected void addOutputDatas( + AssayRunUploadContext context, + Map inputDatas, + Map outputDatas + ) throws ExperimentException, ValidationException + { + Logger log = context.getLogger() != null ? context.getLogger() : LOG; + + // Create set of existing input files + Set inputFiles = new HashSet<>(); + for (ExpData inputData : inputDatas.keySet()) + { + FileLike f = inputData.getFileLike(); + if (f != null) + inputFiles.add(f); + } + + Map files = context.getUploadedData(); + + AssayDataType dataType; + for (Map.Entry entry : files.entrySet()) + { + FileLike file = entry.getValue(); + dataType = context.getProvider().getDataType(); + + // Reuse existing exp.data as the assay output file unless: + // - we are re-importing the run + // - or the output file is already one of the input files and if we are allowing cross-run file inputs + boolean reuseExistingData = true; + if (context.getReRunId() != null) + reuseExistingData = false; + if (context.isAllowCrossRunFileInputs() && inputFiles.contains(file)) + reuseExistingData = false; + + // For Luminex re-import, we want to reuse the existing exp.data but not + // throw an error when we discover that the exp.data is already owned. The + // original run will be duplicated for re-import and then will be deleted. + boolean errorIfDataOwned = getProvider().getReRunSupport() != AssayProvider.ReRunSupport.ReRunAndDelete; + + log.debug("adding output data: file={}", file.toNioPathForRead()); + log.debug(" context.getReRunId()={}", context.getReRunId()); + log.debug(" provider.getReRunSupport()={}", getProvider().getReRunSupport()); + log.debug(" context.allowCrossRunFileInputs={}", context.isAllowCrossRunFileInputs()); + log.debug(" inputFiles.contains(file)={}", inputFiles.contains(file)); + log.debug("==> reuseExistingData = {}", reuseExistingData); + log.debug("==> errorIfDataOwned = {}", errorIfDataOwned); + + ExpData data = DefaultAssayRunCreator.createData(context.getContainer(), file.toNioPathForRead().toFile(), file.getName(), dataType, reuseExistingData, errorIfDataOwned, log); + String role = ExpDataRunInput.DEFAULT_ROLE; + if (dataType != null && dataType.getFileType().isType(file)) + { + if (dataType.getRole() != null) + { + role = dataType.getRole(); + } + } + outputDatas.put(data, role); + } + + FileLike primaryFile = files.get(AssayDataCollector.PRIMARY_FILE); + if (primaryFile != null) + { + addRelatedOutputDatas(context, inputFiles, outputDatas, primaryFile); + } + + Map outputs = context.getOutputDatas(); + addDatas(context.getContainer(), outputDatas, outputs, log); + } + + /** + * Add files that follow the general naming convention (same basename) as the primary file + */ + public void addRelatedOutputDatas( + AssayRunUploadContext context, + Set inputFiles, + Map outputDatas, + final FileLike primaryFile + ) throws ValidationException + { + AssayDataType dataType = getProvider().getDataType(); + final String baseName = dataType == null ? null : dataType.getFileType().getBaseName(primaryFile.toNioPathForRead()); + if (baseName != null) + { + // Grab all the files that are related based on naming convention + File primary = primaryFile.toNioPathForRead().toFile(); + File parent = primary.getParentFile(); + // converting to File land to reuse the FileFilter + File[] relatedFiles = parent.listFiles(getRelatedOutputDataFileFilter(primary, baseName)); + if (relatedFiles != null) + { + for (File f : relatedFiles) + { + FileLike relatedFile = primaryFile.getParent().resolveChild(f.getName()); + // Ignore files already considered inputs to the run + if (inputFiles.contains(relatedFile)) + continue; + + Pair dataOutput = createdRelatedOutputData(context, baseName, f); + if (dataOutput != null) + { + outputDatas.put(dataOutput.getKey(), dataOutput.getValue()); + } + } + } + } + } + + protected void resolveExtraRunData( + ParticipantVisitResolver resolver, + AssayRunUploadContext context, + Map inputMaterials, + Map inputDatas, + Map outputMaterials, + Map outputDatas + ) throws ExperimentException + { + } + + /** + * Create an ExpData object for the file, and figure out what its role name should be + * @return null if the file is already linked to another run + */ + @Nullable + public static Pair createdRelatedOutputData(AssayRunUploadContext context, String baseName, File relatedFile) throws ValidationException + { + String roleName = null; + DataType dataType = null; + for (AssayDataType inputType : context.getProvider().getRelatedDataTypes()) + { + // Check if we recognize it as a specially handled file type + if (inputType.getFileType().isMatch(relatedFile.getName(), baseName)) + { + roleName = inputType.getRole(); + dataType = inputType; + break; + } + } + // If not, make up a new type and role for it + if (roleName == null) + { + roleName = relatedFile.getName().substring(baseName.length()); + while (!roleName.isEmpty() && (roleName.startsWith(".") || roleName.startsWith("-") || roleName.startsWith("_") || roleName.startsWith(" "))) + { + roleName = roleName.substring(1); + } + if (roleName.isEmpty()) + { + roleName = null; + } + } + if (dataType == null) + { + dataType = AbstractAssayProvider.RELATED_FILE_DATA_TYPE; + } + + // Find an existing data that isn't owned by another run or create a new own + ExpData data = findExistingData(context.getContainer(), relatedFile, context.getLogger()); + if (data != null) + { + if (data.getSourceApplication() == null) + return new Pair<>(data, roleName); + + // The file is already linked to another run, so this one must have not created it + return null; + } + + data = createData(context.getContainer(), relatedFile, relatedFile.getName(), dataType, true, true, context.getLogger()); + assert data.getSourceApplication() == null; + return Pair.of(data, roleName); + } + + // Disallow creating a run with inputs which are also outputs + protected void checkForCycles( + Map inputs, + Map outputs + ) throws ExperimentException + { + for (ExpRunItem input : inputs.keySet()) + { + if (outputs.containsKey(input)) + { + String role = outputs.get(input); + throw new ExperimentException("Circular input/output '" + input.getName() + "' with role '" + role + "'"); + } + } + } + + private void saveProperties( + final AssayRunUploadContext context, + ExpObject expObject, + Map transformResultProperties, + Map properties + ) throws ValidationException + { + Map propsToSave = transformResultProperties.isEmpty() ? properties : transformResultProperties; + List errors = validateProperties(context, propsToSave); + if (!errors.isEmpty()) + throw new ValidationException(errors); + + savePropertyObject(expObject, propsToSave, context.getUser()); + } + + protected void savePropertyObject(ExpObject object, Map properties, User user) throws ValidationException + { + for (Map.Entry entry : properties.entrySet()) + { + DomainProperty pd = entry.getKey(); + String value = entry.getValue(); + + // resolve any file links for batch or run properties + if (PropertyType.FILE_LINK.getTypeUri().equals(pd.getType().getTypeURI())) + { + File resolvedFile = ExpDataFileConverter.convert(value); + if (resolvedFile != null) + value = resolvedFile.getAbsolutePath(); + } + + // Treat the empty string as a null in the database, which is our normal behavior when receiving data + // from HTML forms. + if (StringUtils.trimToNull(value) == null) + { + value = null; + } + if (value != null) + { + object.setProperty(user, pd.getPropertyDescriptor(), value); + } + else + { + // We still need to validate blanks + List errors = new ArrayList<>(); + OntologyManager.validateProperty(pd.getValidators(), pd.getPropertyDescriptor(), new ObjectProperty(object.getLSID(), object.getContainer(), pd.getPropertyDescriptor(), value), errors, new ValidatorContext(pd.getContainer(), user)); + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + } + } + + public static List validateColumnProperties(ContainerUser context, Map properties) + { + List errors = new ArrayList<>(); + RemapCache cache = new RemapCache(); + for (Map.Entry entry : properties.entrySet()) + { + validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); + } + return errors; + } + + public static List validateProperties(ContainerUser context, Map properties) + { + List errors = new ArrayList<>(); + RemapCache cache = new RemapCache(); + for (Map.Entry entry : properties.entrySet()) + { + validateProperty(context, entry.getKey(), entry.getValue(), cache, errors); + } + return errors; + } + + private static void validateProperty(ContainerUser context, ColumnInfo columnInfo, String value, RemapCache cache, List errors) + { + Lookup lookup = null; + if (columnInfo.isLookup()) + { + ForeignKey fk = columnInfo.getFk(); + lookup = new Lookup(fk.getLookupContainer(), fk.getLookupSchemaName(), fk.getLookupTableName()); + } + validateProperty(context, ColumnValidators.create(columnInfo, null), value, columnInfo.getName(), + false, lookup, columnInfo.getJavaClass(), cache, errors); + } + + private static void validateProperty(ContainerUser context, DomainProperty dp, String value, RemapCache cache, List errors) + { + String label = dp.getPropertyDescriptor().getNonBlankCaption(); + PropertyType type = dp.getPropertyDescriptor().getPropertyType(); + validateProperty(context, ColumnValidators.create(null, dp), value, label, dp.isRequired(), + dp.getLookup(), type.getJavaType(), cache, errors); + } + + private static void validateProperty( + ContainerUser context, + List validators, + String value, + String label, + Boolean required, + Lookup lookup, + Class type, + RemapCache cache, + List errors + ) + { + boolean missing = (value == null || value.isEmpty()); + int rowNum = 0; + + if (required && missing) + { + errors.add(new SimpleValidationError(label + " is required and must be of type " + ColumnInfo.getFriendlyTypeName(type) + ".")); + } + else if (!missing) + { + try + { + Object o; + if (type == File.class) + o = ExpDataFileConverter.convert(value); + else + o = ConvertUtils.convert(value, type); + ValidatorContext validatorContext = new ValidatorContext(context.getContainer(), context.getUser()); + for (ColumnValidator validator : validators) + { + String msg = validator.validate(rowNum, o, validatorContext); + if (msg != null) + errors.add(new PropertyValidationError(msg, label)); + } + } + catch (ConversionException e) + { + String message; + if (e instanceof ConvertHelper.FileConversionException fce) + message = fce.getMessage(); + else + { + message = ConvertHelper.getStandardConversionErrorMessage(value, label, type); + if (e.getCause() instanceof ArithmeticException) + message += ": " + e.getCause().getLocalizedMessage(); + else + message += "."; + } + + // Attempt to resolve lookups by display value + boolean skipError = false; + if (lookup != null) + { + Object remappedValue = OntologyManager.getRemappedValueForLookup(context.getUser(), context.getContainer(), cache, lookup, value); + if (remappedValue != null) + skipError = true; + } + + if (!skipError) + errors.add(new SimpleValidationError(message)); + } + } + } + + protected FileFilter getRelatedOutputDataFileFilter(final File primaryFile, final String baseName) + { + // baseName doesn't include the trailing '.', so add it here. We want to associate myRun.jpg + // with myRun.xls, but we don't want to associate myRun2.xls with myRun.xls (which will happen without + // the trailing dot in the check). + return f -> f.getName().startsWith(baseName + ".") && !primaryFile.equals(f); + } + + protected ProviderType getProvider() + { + return _provider; + } +} diff --git a/api/src/org/labkey/api/audit/AuditTypeEvent.java b/api/src/org/labkey/api/audit/AuditTypeEvent.java index 48ad70b0e57..15778411cab 100644 --- a/api/src/org/labkey/api/audit/AuditTypeEvent.java +++ b/api/src/org/labkey/api/audit/AuditTypeEvent.java @@ -1,267 +1,267 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.security.Group; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ExceptionUtil; - -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Bean object to capture audit log entries. Will be used to populate the database tables via get/set methods that - * align with column names in the corresponding provisioned table. - */ -public class AuditTypeEvent -{ - protected static final String CREATED_BY_KEY = "auditEventCreatedBy"; - protected static final String IMPERSONATED_BY_KEY = "impersonatedBy"; - protected static final String CONTAINER_KEY = "container"; - protected static final String PROJECT_KEY = "project"; - protected static final String COMMENT_KEY = "comment"; - private static final String USER_COMMENT_KEY = "userComment"; - - // long type used here to allow for DbSequences to supply the rowId - private long _rowId; - private Integer _impersonatedBy; - private String _comment; - private Container _projectId; - private Container _container; - private String _eventType; - private Date _created; - private User _createdBy; - private Date _modified; - private User _modifiedBy; - private String _userComment; - private Long _transactionId; - - public AuditTypeEvent(@NotNull String eventType, @NotNull Container container, @Nullable String comment) - { - _eventType = eventType; - if (container == null) - { - ExceptionUtil.logExceptionToMothership(null, new IllegalStateException("Audit event container is null")); - } - _container = container; - _comment = comment; - _projectId = container.getProject(); - } - - /** Important for reflection-based instantiation */ - public AuditTypeEvent() - { - } - - public long getRowId() - { - return _rowId; - } - - public void setRowId(long rowId) - { - _rowId = rowId; - } - - public Integer getImpersonatedBy() - { - return _impersonatedBy; - } - - public void setImpersonatedBy(Integer impersonatedBy) - { - _impersonatedBy = impersonatedBy; - } - - public String getComment() - { - return _comment; - } - - public void setComment(String comment) - { - _comment = comment; - } - - public Container getProjectId() - { - return _projectId; - } - - public void setProjectId(Container projectId) - { - _projectId = projectId; - } - - public Container getContainer() - { - return _container; - } - - public void setContainer(Container container) - { - _container = container; - } - - public String getEventType() - { - return _eventType; - } - - public void setEventType(String eventType) - { - _eventType = eventType; - } - - public Date getCreated() - { - return _created; - } - - public void setCreated(Date created) - { - _created = created; - } - - public User getCreatedBy() - { - return _createdBy; - } - - public void setCreatedBy(User createdBy) - { - _createdBy = createdBy; - } - - public Date getModified() - { - return _modified; - } - - public void setModified(Date modified) - { - _modified = modified; - } - - public User getModifiedBy() - { - return _modifiedBy; - } - - public void setModifiedBy(User modifiedBy) - { - _modifiedBy = modifiedBy; - } - - public void setUserComment(String userComment) - { - _userComment = userComment; - } - - public String getUserComment() - { - return _userComment; - } - - public Long getTransactionId() - { - return _transactionId; - } - - public void setTransactionId(Long transactionId) - { - _transactionId = transactionId; - } - - public void setTransactionEvent(@Nullable TransactionAuditProvider.TransactionAuditEvent transactionEvent, String auditEventType) - { - if (transactionEvent == null) - return; - - _transactionId = transactionEvent.getRowId(); - transactionEvent.addDetail(TransactionAuditProvider.TransactionDetail.AuditEvents, auditEventType); - } - - protected String getContainerMessageElement(@NotNull Container container) - { - String value = " (" + container.getId() + ")"; - value = container.getPath() + value; - return value; - } - - protected String getUserMessageElement(@NotNull User user) - { - return user.getEmail() + " (" + user.getUserId() + ")"; - } - - protected String getUserMessageElement(int userId) - { - String value = " (" + userId + ")"; - User user = UserManager.getUser(userId); - if (user != null) - value = user.getEmail() + value; - return value; - } - - protected String getGroupMessageElement(@NotNull Group group) - { - return group.getName() + " (" + group.getUserId() + ")"; - } - - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - User createdBy = getCreatedBy(); - if (createdBy != null) - { - String message = createdBy.getEmail() != null ? createdBy.getEmail() : ""; - message += " (" + createdBy.getUserId() + ")"; - elements.put(CREATED_BY_KEY, message); - } - Integer impersonatorId = getImpersonatedBy(); - if (impersonatorId != null) - elements.put(IMPERSONATED_BY_KEY, getUserMessageElement(impersonatorId)); - Container container = getContainer(); - elements.put(CONTAINER_KEY, getContainerMessageElement(container)); - Container projectId = getProjectId(); - if (projectId != null) - elements.put(PROJECT_KEY, getContainerMessageElement(projectId)); - if (getComment() != null) - elements.put(COMMENT_KEY, getComment()); - if (getUserComment() != null) - elements.put(USER_COMMENT_KEY, getUserComment()); - - return elements; - } - - public String getAuditLogMessage() - { - StringBuilder builder = new StringBuilder(); - builder.append(getEventType()).append(" - "); - - Map messageElements = getAuditLogMessageElements(); - for (String key : messageElements.keySet()) - { - builder.append(key).append(": ").append(messageElements.get(key)).append(" | "); - } - return builder.toString(); - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.security.Group; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ExceptionUtil; + +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Bean object to capture audit log entries. Will be used to populate the database tables via get/set methods that + * align with column names in the corresponding provisioned table. + */ +public class AuditTypeEvent +{ + protected static final String CREATED_BY_KEY = "auditEventCreatedBy"; + protected static final String IMPERSONATED_BY_KEY = "impersonatedBy"; + protected static final String CONTAINER_KEY = "container"; + protected static final String PROJECT_KEY = "project"; + protected static final String COMMENT_KEY = "comment"; + private static final String USER_COMMENT_KEY = "userComment"; + + // long type used here to allow for DbSequences to supply the rowId + private long _rowId; + private Integer _impersonatedBy; + private String _comment; + private Container _projectId; + private Container _container; + private String _eventType; + private Date _created; + private User _createdBy; + private Date _modified; + private User _modifiedBy; + private String _userComment; + private Long _transactionId; + + public AuditTypeEvent(@NotNull String eventType, @NotNull Container container, @Nullable String comment) + { + _eventType = eventType; + if (container == null) + { + ExceptionUtil.logExceptionToMothership(null, new IllegalStateException("Audit event container is null")); + } + _container = container; + _comment = comment; + _projectId = container.getProject(); + } + + /** Important for reflection-based instantiation */ + public AuditTypeEvent() + { + } + + public long getRowId() + { + return _rowId; + } + + public void setRowId(long rowId) + { + _rowId = rowId; + } + + public Integer getImpersonatedBy() + { + return _impersonatedBy; + } + + public void setImpersonatedBy(Integer impersonatedBy) + { + _impersonatedBy = impersonatedBy; + } + + public String getComment() + { + return _comment; + } + + public void setComment(String comment) + { + _comment = comment; + } + + public Container getProjectId() + { + return _projectId; + } + + public void setProjectId(Container projectId) + { + _projectId = projectId; + } + + public Container getContainer() + { + return _container; + } + + public void setContainer(Container container) + { + _container = container; + } + + public String getEventType() + { + return _eventType; + } + + public void setEventType(String eventType) + { + _eventType = eventType; + } + + public Date getCreated() + { + return _created; + } + + public void setCreated(Date created) + { + _created = created; + } + + public User getCreatedBy() + { + return _createdBy; + } + + public void setCreatedBy(User createdBy) + { + _createdBy = createdBy; + } + + public Date getModified() + { + return _modified; + } + + public void setModified(Date modified) + { + _modified = modified; + } + + public User getModifiedBy() + { + return _modifiedBy; + } + + public void setModifiedBy(User modifiedBy) + { + _modifiedBy = modifiedBy; + } + + public void setUserComment(String userComment) + { + _userComment = userComment; + } + + public String getUserComment() + { + return _userComment; + } + + public Long getTransactionId() + { + return _transactionId; + } + + public void setTransactionId(Long transactionId) + { + _transactionId = transactionId; + } + + public void setTransactionEvent(@Nullable TransactionAuditProvider.TransactionAuditEvent transactionEvent, String auditEventType) + { + if (transactionEvent == null) + return; + + _transactionId = transactionEvent.getRowId(); + transactionEvent.addDetail(TransactionAuditProvider.TransactionDetail.AuditEvents, auditEventType); + } + + protected String getContainerMessageElement(@NotNull Container container) + { + String value = " (" + container.getId() + ")"; + value = container.getPath() + value; + return value; + } + + protected String getUserMessageElement(@NotNull User user) + { + return user.getEmail() + " (" + user.getUserId() + ")"; + } + + protected String getUserMessageElement(int userId) + { + String value = " (" + userId + ")"; + User user = UserManager.getUser(userId); + if (user != null) + value = user.getEmail() + value; + return value; + } + + protected String getGroupMessageElement(@NotNull Group group) + { + return group.getName() + " (" + group.getUserId() + ")"; + } + + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + User createdBy = getCreatedBy(); + if (createdBy != null) + { + String message = createdBy.getEmail() != null ? createdBy.getEmail() : ""; + message += " (" + createdBy.getUserId() + ")"; + elements.put(CREATED_BY_KEY, message); + } + Integer impersonatorId = getImpersonatedBy(); + if (impersonatorId != null) + elements.put(IMPERSONATED_BY_KEY, getUserMessageElement(impersonatorId)); + Container container = getContainer(); + elements.put(CONTAINER_KEY, getContainerMessageElement(container)); + Container projectId = getProjectId(); + if (projectId != null) + elements.put(PROJECT_KEY, getContainerMessageElement(projectId)); + if (getComment() != null) + elements.put(COMMENT_KEY, getComment()); + if (getUserComment() != null) + elements.put(USER_COMMENT_KEY, getUserComment()); + + return elements; + } + + public String getAuditLogMessage() + { + StringBuilder builder = new StringBuilder(); + builder.append(getEventType()).append(" - "); + + Map messageElements = getAuditLogMessageElements(); + for (String key : messageElements.keySet()) + { + builder.append(key).append(": ").append(messageElements.get(key)).append(" | "); + } + return builder.toString(); + } +} diff --git a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java index 83195e9e0a4..441f5862e21 100644 --- a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java +++ b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java @@ -1,234 +1,234 @@ -/* - * Copyright (c) 2015-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit.provider; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.data.Container; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/19/13 - */ -public class FileSystemAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String EVENT_TYPE = "FileSystem"; - - public static final String COLUMN_NAME_DIRECTORY = "Directory"; - public static final String COLUMN_NAME_FILE = "File"; - public static final String COLUMN_NAME_PROVIDED_FILE = "ProvidedFileName"; - public static final String COLUMN_NAME_FIELD_NAME = "FieldName"; - public static final String COLUMN_NAME_RESOURCE_PATH = "ResourcePath"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_DIRECTORY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FILE)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROVIDED_FILE)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FIELD_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - } - - public FileSystemAuditProvider() - { - super(new FileSystemAuditDomainKind()); - } - - @Override - public String getEventName() - { - return EVENT_TYPE; - } - - @Override - public String getLabel() - { - return "File events"; - } - - @Override - public String getDescription() - { - return "Displays information about file uploads and modifications."; - } - - @Override - public Map legacyNameMap() - { - Map legacyNames = super.legacyNameMap(); - legacyNames.put(FieldKey.fromParts("key1"), COLUMN_NAME_DIRECTORY); - legacyNames.put(FieldKey.fromParts("key2"), COLUMN_NAME_FILE); - legacyNames.put(FieldKey.fromParts("key3"), COLUMN_NAME_RESOURCE_PATH); - return legacyNames; - } - - @Override - public Class getEventClass() - { - return (Class)FileSystemAuditEvent.class; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - public static class FileSystemAuditEvent extends AuditTypeEvent - { - private String _directory; // the directory name - private String _file; // the file name - private String _resourcePath; // the webdav resource path - private String _providedFileName; // the name of the file as provided by the user, before renaming to make it unique and/or legal - private String _fieldName; // name of the field associated with the file, if any - - /** Important for reflection-based instantiation */ - public FileSystemAuditEvent() - { - super(); - setEventType(EVENT_TYPE); - } - - public FileSystemAuditEvent(Container container, String comment) - { - super(EVENT_TYPE, container, comment); - setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); - } - - public String getDirectory() - { - return _directory; - } - - public void setDirectory(String directory) - { - _directory = directory; - } - - public String getFile() - { - return _file; - } - - public void setFile(String file) - { - _file = file; - } - - public String getResourcePath() - { - return _resourcePath; - } - - public void setResourcePath(String resourcePath) - { - _resourcePath = resourcePath; - } - - public String getProvidedFileName() - { - return _providedFileName; - } - - public void setProvidedFileName(String providedFileName) - { - _providedFileName = providedFileName; - } - - public String getFieldName() - { - return _fieldName; - } - - public void setFieldName(String fieldName) - { - _fieldName = fieldName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("directory", getDirectory()); - elements.put("file", getFile()); - elements.put("resourcePath", getResourcePath()); - elements.put("providedFileName", getProvidedFileName()); - elements.put("fieldName", getFieldName()); - elements.put("transactionId", getTransactionId()); - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class FileSystemAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "FileSystemAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public FileSystemAuditDomainKind() - { - super(EVENT_TYPE); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_DIRECTORY, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_FILE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_PROVIDED_FILE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_FIELD_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_RESOURCE_PATH, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2015-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit.provider; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.data.Container; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: 7/19/13 + */ +public class FileSystemAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String EVENT_TYPE = "FileSystem"; + + public static final String COLUMN_NAME_DIRECTORY = "Directory"; + public static final String COLUMN_NAME_FILE = "File"; + public static final String COLUMN_NAME_PROVIDED_FILE = "ProvidedFileName"; + public static final String COLUMN_NAME_FIELD_NAME = "FieldName"; + public static final String COLUMN_NAME_RESOURCE_PATH = "ResourcePath"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_DIRECTORY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FILE)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROVIDED_FILE)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_FIELD_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + } + + public FileSystemAuditProvider() + { + super(new FileSystemAuditDomainKind()); + } + + @Override + public String getEventName() + { + return EVENT_TYPE; + } + + @Override + public String getLabel() + { + return "File events"; + } + + @Override + public String getDescription() + { + return "Displays information about file uploads and modifications."; + } + + @Override + public Map legacyNameMap() + { + Map legacyNames = super.legacyNameMap(); + legacyNames.put(FieldKey.fromParts("key1"), COLUMN_NAME_DIRECTORY); + legacyNames.put(FieldKey.fromParts("key2"), COLUMN_NAME_FILE); + legacyNames.put(FieldKey.fromParts("key3"), COLUMN_NAME_RESOURCE_PATH); + return legacyNames; + } + + @Override + public Class getEventClass() + { + return (Class)FileSystemAuditEvent.class; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + public static class FileSystemAuditEvent extends AuditTypeEvent + { + private String _directory; // the directory name + private String _file; // the file name + private String _resourcePath; // the webdav resource path + private String _providedFileName; // the name of the file as provided by the user, before renaming to make it unique and/or legal + private String _fieldName; // name of the field associated with the file, if any + + /** Important for reflection-based instantiation */ + public FileSystemAuditEvent() + { + super(); + setEventType(EVENT_TYPE); + } + + public FileSystemAuditEvent(Container container, String comment) + { + super(EVENT_TYPE, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); + } + + public String getDirectory() + { + return _directory; + } + + public void setDirectory(String directory) + { + _directory = directory; + } + + public String getFile() + { + return _file; + } + + public void setFile(String file) + { + _file = file; + } + + public String getResourcePath() + { + return _resourcePath; + } + + public void setResourcePath(String resourcePath) + { + _resourcePath = resourcePath; + } + + public String getProvidedFileName() + { + return _providedFileName; + } + + public void setProvidedFileName(String providedFileName) + { + _providedFileName = providedFileName; + } + + public String getFieldName() + { + return _fieldName; + } + + public void setFieldName(String fieldName) + { + _fieldName = fieldName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("directory", getDirectory()); + elements.put("file", getFile()); + elements.put("resourcePath", getResourcePath()); + elements.put("providedFileName", getProvidedFileName()); + elements.put("fieldName", getFieldName()); + elements.put("transactionId", getTransactionId()); + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class FileSystemAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "FileSystemAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public FileSystemAuditDomainKind() + { + super(EVENT_TYPE); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_DIRECTORY, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_FILE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_PROVIDED_FILE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_FIELD_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_RESOURCE_PATH, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index 0afea156a79..edc190429e2 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -1,1520 +1,1520 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.assay.AssayFileWriter; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.collections.ArrayListMap; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConvertHelper; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.ExpDataFileConverter; -import org.labkey.api.data.ImportAliasable; -import org.labkey.api.data.MultiValuedForeignKey; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.Sort; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.AttachmentDataIterator; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.ExistingRecordDataIterator; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.dataiterator.Pump; -import org.labkey.api.dataiterator.StandardDataIteratorBuilder; -import org.labkey.api.dataiterator.TriggerDataBuilderHelper; -import org.labkey.api.dataiterator.WrapperDataIterator; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.MvColumn; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.reader.TabLoader; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URIUtil; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.vfs.FileLike; -import org.springframework.web.multipart.MultipartFile; - -import java.io.File; -import java.io.IOException; -import java.io.StringReader; -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.function.Function; - -import static java.util.Objects.requireNonNull; -import static org.labkey.api.audit.TransactionAuditProvider.DB_SEQUENCE_NAME; -import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior; -import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment; -import static org.labkey.api.files.FileContentService.UPLOADED_FILE; -import static org.labkey.api.util.FileUtil.toFileForRead; -import static org.labkey.api.util.FileUtil.toFileForWrite; - -public abstract class AbstractQueryUpdateService implements QueryUpdateService -{ - protected final TableInfo _queryTable; - - private boolean _bulkLoad = false; - private CaseInsensitiveHashMap _columnImportMap = null; - private VirtualFile _att = null; - - /* AbstractQueryUpdateService is generally responsible for some shared functionality - * - triggers - * - coercion/validation - * - detailed logging - * - attachments - * - * If a subclass wants to disable some of these features (w/o subclassing), put flags here... - */ - protected boolean _enableExistingRecordsDataIterator = true; - protected Set _previouslyUpdatedRows = new HashSet<>(); - - protected AbstractQueryUpdateService(TableInfo queryTable) - { - if (queryTable == null) - throw new IllegalArgumentException(); - _queryTable = queryTable; - } - - protected TableInfo getQueryTable() - { - return _queryTable; - } - - public @NotNull Set getPreviouslyUpdatedRows() - { - return _previouslyUpdatedRows == null ? new HashSet<>() : _previouslyUpdatedRows; - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) - { - return getQueryTable().hasPermission(user, acl); - } - - protected Map getRow(User user, Container container, Map keys, boolean allowCrossContainer) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - return getRow(user, container, keys); - } - - protected abstract Map getRow(User user, Container container, Map keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException; - - @Override - public List> getRows(User user, Container container, List> keys) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - - List> result = new ArrayList<>(); - for (Map rowKeys : keys) - { - Map row = getRow(user, container, rowKeys); - if (row != null) - result.add(row); - } - return result; - } - - @Override - public Map> getExistingRows(User user, Container container, Map> keys, boolean verifyNoCrossFolderData, boolean verifyExisting, @Nullable Set columns) - throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - - Map> result = new LinkedHashMap<>(); - for (Map.Entry> key : keys.entrySet()) - { - Map row = getRow(user, container, key.getValue(), verifyNoCrossFolderData); - if (row != null && !row.isEmpty()) - { - result.put(key.getKey(), row); - if (verifyNoCrossFolderData) - { - String dataContainer = (String) row.get("container"); - if (StringUtils.isEmpty(dataContainer)) - dataContainer = (String) row.get("folder"); - if (!container.getId().equals(dataContainer)) - throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values()); - } - } - else if (verifyExisting) - throw new InvalidKeyException("Data not found for " + key.getValue().values()); - } - return result; - } - - @Override - public boolean hasExistingRowsInOtherContainers(Container container, Map> keys) - { - return false; - } - - public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction) - { - return createTransactionAuditEvent(container, auditAction, null); - } - - public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction, @Nullable Map details) - { - long auditId = DbSequenceManager.get(ContainerManager.getRoot(), DB_SEQUENCE_NAME).next(); - TransactionAuditProvider.TransactionAuditEvent event = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); - if (details != null) - event.addDetails(details); - return event; - } - - public static void addTransactionAuditEvent(DbScope.Transaction transaction, User user, TransactionAuditProvider.TransactionAuditEvent auditEvent) - { - UserSchema schema = AuditLogService.getAuditLogSchema(user, ContainerManager.getRoot()); - - if (schema != null) - { - // This is a little hack to ensure that the audit table has actually been created and gets put into the table cache by the time the - // pre-commit task is executed. Otherwise, since the creation of the table happens while within the commit for the - // outermost transaction, it looks like there is a close that hasn't happened when trying to commit the transaction for creating the - // table. - schema.getTable(auditEvent.getEventType(), false); - - transaction.addCommitTask(() -> AuditLogService.get().addEvent(user, auditEvent), DbScope.CommitTaskOption.PRECOMMIT); - - transaction.setAuditEvent(auditEvent); - } - } - - protected final DataIteratorContext getDataIteratorContext(BatchValidationException errors, InsertOption forImport, Map configParameters) - { - if (null == errors) - errors = new BatchValidationException(); - DataIteratorContext context = new DataIteratorContext(errors); - context.setInsertOption(forImport); - context.setConfigParameters(configParameters); - configureDataIteratorContext(context); - return context; - } - - /** - * If QUS wants to use something other than PKs to select existing rows for merge, it can override this method. - * Used only for generating ExistingRecordDataIterator at the moment. - */ - protected Set getSelectKeys(DataIteratorContext context) - { - if (!context.getAlternateKeys().isEmpty()) - return context.getAlternateKeys(); - return null; - } - - /* - * construct the core DataIterator transformation pipeline for this table, may be just StandardDataIteratorBuilder. - * does NOT handle triggers or the insert/update iterator. - */ - public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) - { - DataIteratorBuilder dib = StandardDataIteratorBuilder.forInsert(getQueryTable(), data, container, user); - - if (_enableExistingRecordsDataIterator || context.getInsertOption().updateOnly) - { - // some tables need to generate PKs, so they need to add ExistingRecordDataIterator in persistRows() (after generating PK, before inserting) - dib = ExistingRecordDataIterator.createBuilder(dib, getQueryTable(), getSelectKeys(context)); - } - - dib = ((UpdateableTableInfo) getQueryTable()).persistRows(dib, context); - dib = AttachmentDataIterator.getAttachmentDataIteratorBuilder(getQueryTable(), dib, user, context.getInsertOption().batch ? getAttachmentDirectory() : null, container, getAttachmentParentFactory()); - dib = DetailedAuditLogDataIterator.getDataIteratorBuilder(getQueryTable(), dib, context.getInsertOption(), user, container, null); - return dib; - } - - - /** - * Implementation to use insertRows() while we migrate to using DIB for all code paths - *

- * DataIterator should/must use the same error collection as passed in - */ - @Deprecated - protected int _importRowsUsingInsertRows(User user, Container container, DataIterator rows, BatchValidationException errors, Map extraScriptContext) - { - MapDataIterator mapIterator = DataIteratorUtil.wrapMap(rows, true); - List> list = new ArrayList<>(); - List> ret; - Exception rowException; - - try - { - while (mapIterator.next()) - list.add(mapIterator.getMap()); - ret = insertRows(user, container, list, errors, null, extraScriptContext); - if (errors.hasErrors()) - return 0; - return ret.size(); - } - catch (BatchValidationException x) - { - assert x == errors; - assert x.hasErrors(); - return 0; - } - catch (QueryUpdateServiceException | DuplicateKeyException | SQLException x) - { - rowException = x; - } - finally - { - DataIteratorUtil.closeQuietly(mapIterator); - } - errors.addRowError(new ValidationException(rowException.getMessage())); - return 0; - } - - protected boolean hasImportRowsPermission(User user, Container container, DataIteratorContext context) - { - return hasPermission(user, context.getInsertOption().updateOnly ? UpdatePermission.class : InsertPermission.class); - } - - // override this - protected void preImportDIBValidation(@Nullable DataIteratorBuilder in, @Nullable Collection inputColumns) - { - } - - protected int _importRowsUsingDIB(User user, Container container, DataIteratorBuilder in, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasImportRowsPermission(user, container, context)) - throw new UnauthorizedException("You do not have permission to " + (context.getInsertOption().updateOnly ? "update data in this table." : "insert data into this table.")); - - if (!context.getConfigParameterBoolean(ConfigParameters.SkipInsertOptionValidation)) - assert(getQueryTable().supportsInsertOption(context.getInsertOption())); - - context.getErrors().setExtraContext(extraScriptContext); - if (extraScriptContext != null) - { - context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE)); - } - - preImportDIBValidation(in, null); - - boolean skipTriggers = context.getConfigParameterBoolean(ConfigParameters.SkipTriggers) || context.isCrossTypeImport() || context.isCrossFolderImport(); - boolean hasTableScript = hasTableScript(container); - TriggerDataBuilderHelper helper = new TriggerDataBuilderHelper(getQueryTable(), container, user, extraScriptContext, context.getInsertOption().useImportAliases); - if (!skipTriggers) - { - in = preTriggerDataIterator(in, context); - if (hasTableScript) - in = helper.before(in); - } - DataIteratorBuilder importDIB = createImportDIB(user, container, in, context); - DataIteratorBuilder out = importDIB; - - if (!skipTriggers) - { - if (hasTableScript) - out = helper.after(importDIB); - - out = postTriggerDataIterator(out, context); - } - - if (hasTableScript) - { - context.setFailFast(false); - context.setMaxRowErrors(Math.max(context.getMaxRowErrors(),1000)); - } - int count = _pump(out, outputRows, context); - - if (context.getErrors().hasErrors()) - return 0; - else - { - if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level - { - AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); - String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); - boolean skipAuditLevelCheck = false; - if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) - { - if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad - skipAuditLevelCheck = true; - } - getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); - } - return count; - } - } - - protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) - { - return in; - } - - protected DataIteratorBuilder postTriggerDataIterator(DataIteratorBuilder out, DataIteratorContext context) - { - return out; - } - - - /** this is extracted so subclasses can add wrap */ - protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList> rows, DataIteratorContext context) - { - DataIterator it = etl.getDataIterator(context); - - try - { - if (null != rows) - { - MapDataIterator maps = DataIteratorUtil.wrapMap(it, false); - it = new WrapperDataIterator(maps) - { - @Override - public boolean next() throws BatchValidationException - { - boolean ret = super.next(); - if (ret) - rows.add(((MapDataIterator)_delegate).getMap()); - return ret; - } - }; - } - - Pump pump = new Pump(it, context); - pump.run(); - - return pump.getRowCount(); - } - finally - { - DataIteratorUtil.closeQuietly(it); - } - } - - /* can be used for simple bookkeeping tasks, per row processing belongs in a data iterator */ - protected void afterInsertUpdate(int count, BatchValidationException errors, boolean isUpdate) - { - afterInsertUpdate(count, errors); - } - - protected void afterInsertUpdate(int count, BatchValidationException errors) - {} - - @Override - public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - return loadRows(user, container, rows, null, context, extraScriptContext); - } - - public int loadRows(User user, Container container, DataIteratorBuilder rows, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - configureDataIteratorContext(context); - int count = _importRowsUsingDIB(user, container, rows, outputRows, context, extraScriptContext); - afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); - return count; - } - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, @Nullable Map extraScriptContext) - { - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingInsertRows(user, container, rows.getDataIterator(context), errors, extraScriptContext); - afterInsertUpdate(count, errors, context.getInsertOption().updateOnly); - return count; - } - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - { - throw new UnsupportedOperationException("merge is not supported for all tables"); - } - - private boolean hasTableScript(Container container) - { - return getQueryTable().hasTriggers(container); - } - - - protected Map insertRow(User user, Container container, Map row) - throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException("Not implemented by this QueryUpdateService"); - } - - - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, InsertPermission.class)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); - } - - protected @Nullable List> _insertUpdateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - DataIteratorBuilder dib = _toDataIteratorBuilder(getClass().getSimpleName() + (context.getInsertOption().updateOnly ? ".updateRows" : ".insertRows()"), rows); - ArrayList> outputRows = new ArrayList<>(); - int count = _importRowsUsingDIB(user, container, dib, outputRows, context, extraScriptContext); - afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); - - if (context.getErrors().hasErrors()) - return null; - - return outputRows; - } - - // not yet supported - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); - } - - - protected DataIteratorBuilder _toDataIteratorBuilder(String debugName, List> rows) - { - // TODO probably can't assume all rows have all columns - // TODO can we assume that all rows refer to columns consistently? (not PTID and MouseId for the same column) - // TODO optimize ArrayListMap? - Set colNames; - - if (!rows.isEmpty() && rows.get(0) instanceof ArrayListMap) - { - colNames = ((ArrayListMap)rows.get(0)).getFindMap().keySet(); - } - else - { - // Preserve casing by using wrapped CaseInsensitiveHashMap instead of CaseInsensitiveHashSet - colNames = Sets.newCaseInsensitiveHashSet(); - for (Map row : rows) - colNames.addAll(row.keySet()); - } - - preImportDIBValidation(null, colNames); - return MapDataIterator.of(colNames, rows, debugName); - } - - - /** @deprecated switch to using DIB based method */ - @Deprecated - protected List> _insertRowsUsingInsertRow(User user, Container container, List> rows, BatchValidationException errors, Map extraScriptContext) - throws DuplicateKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, InsertPermission.class)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - assert(getQueryTable().supportsInsertOption(InsertOption.INSERT)); - - boolean hasTableScript = hasTableScript(container); - - errors.setExtraContext(extraScriptContext); - if (hasTableScript) - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, true, errors, extraScriptContext); - - List> result = new ArrayList<>(rows.size()); - List> providedValues = new ArrayList<>(rows.size()); - for (int i = 0; i < rows.size(); i++) - { - Map row = rows.get(i); - row = normalizeColumnNames(row); - try - { - providedValues.add(new CaseInsensitiveHashMap<>()); - row = coerceTypes(row, providedValues.get(i), false); - if (hasTableScript) - { - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, true, i, row, null, extraScriptContext); - } - row = insertRow(user, container, row); - if (row == null) - continue; - - if (hasTableScript) - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, false, i, row, null, extraScriptContext); - result.add(row); - } - catch (SQLException sqlx) - { - if (StringUtils.startsWith(sqlx.getSQLState(), "22") || RuntimeSQLException.isConstraintException(sqlx)) - { - ValidationException vex = new ValidationException(sqlx.getMessage()); - vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i+1); - errors.addRowError(vex); - } - else if (SqlDialect.isTransactionException(sqlx) && errors.hasErrors()) - { - // if we already have some errors, just break - break; - } - else - { - throw sqlx; - } - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - } - - if (hasTableScript) - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, false, errors, extraScriptContext); - - addAuditEvent(user, container, QueryService.AuditAction.INSERT, null, result, null, providedValues); - - return result; - } - - protected void addAuditEvent(User user, Container container, QueryService.AuditAction auditAction, @Nullable Map configParameters, @Nullable List> rows, @Nullable List> existingRows, @Nullable List> providedValues) - { - if (!isBulkLoad()) - { - AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(AuditBehavior) : null; - String userComment = configParameters == null ? null : (String) configParameters.get(AuditUserComment); - getQueryTable().getAuditHandler(auditBehavior) - .addAuditEvent(user, container, getQueryTable(), auditBehavior, userComment, auditAction, rows, existingRows, providedValues); - } - } - - private Map normalizeColumnNames(Map row) - { - if(_columnImportMap == null) - { - _columnImportMap = (CaseInsensitiveHashMap)ImportAliasable.Helper.createImportMap(getQueryTable().getColumns(), false); - } - - Map newRow = new CaseInsensitiveHashMap<>(); - CaseInsensitiveHashSet columns = new CaseInsensitiveHashSet(); - columns.addAll(row.keySet()); - - String newName; - for(String key : row.keySet()) - { - if(_columnImportMap.containsKey(key)) - { - //it is possible for a normalized name to conflict with an existing property. if so, defer to the original - newName = _columnImportMap.get(key).getName(); - if(!columns.contains(newName)){ - newRow.put(newName, row.get(key)); - continue; - } - } - newRow.put(key, row.get(key)); - } - - return newRow; - } - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws DuplicateKeyException, QueryUpdateServiceException, SQLException - { - try - { - List> ret = _insertRowsUsingInsertRow(user, container, rows, errors, extraScriptContext); - afterInsertUpdate(null==ret?0:ret.size(), errors); - if (errors.hasErrors()) - return null; - return ret; - } - catch (BatchValidationException x) - { - assert x == errors; - assert x.hasErrors(); - } - return null; - } - - - /** Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */ - @Deprecated - protected Map coerceTypes(Map row, Map providedValues, boolean isUpdate) - { - Map result = new CaseInsensitiveHashMap<>(row.size()); - Map columnMap = ImportAliasable.Helper.createImportMap(_queryTable.getColumns(), true); - for (Map.Entry entry : row.entrySet()) - { - ColumnInfo col = columnMap.get(entry.getKey()); - - Object value = entry.getValue(); - if (col != null && value != null && - !col.getJavaObjectClass().isInstance(value) && - !(value instanceof AttachmentFile) && - !(value instanceof MultipartFile) && - !(value instanceof String[]) && - !(col.getFk() instanceof MultiValuedForeignKey)) - { - try - { - if (PropertyType.FILE_LINK.equals(col.getPropertyType())) - value = ExpDataFileConverter.convert(value); - else if (col.getKindOfQuantity() != null) - { - providedValues.put(entry.getKey(), value); - value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit()); - } - else - value = col.getConvertFn().apply(value); - } - catch (ConvertHelper.FileConversionException e) - { - throw e; - } - catch (ConversionException e) - { - // That's OK, the transformation script may be able to fix up the value before it gets inserted - } - } - result.put(entry.getKey(), value); - } - return result; - } - - - protected abstract Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; - - - protected boolean firstUpdateRow = true; - Function,Map> updateTransform = Function.identity(); - - /* Do standard AQUS stuff here, then call the subclass specific implementation of updateRow() */ - final protected Map updateOneRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - if (firstUpdateRow) - { - firstUpdateRow = false; - if (null != OntologyService.get()) - { - var t = OntologyService.get().getConceptUpdateHandler(_queryTable); - if (null != t) - updateTransform = t; - } - } - row = updateTransform.apply(row); - return updateRow(user, container, row, oldRow, configParameters); - } - - // used by updateRows to check if all rows have the same set of keys - // prepared statement can only be used to updateRows if all rows have the same set of keys - protected boolean hasUniformKeys(List> rowsToUpdate) - { - if (rowsToUpdate == null || rowsToUpdate.isEmpty()) - return false; - - if (rowsToUpdate.size() == 1) - return true; - - Set keys = rowsToUpdate.get(0).keySet(); - int keySize = keys.size(); - - for (int i = 1 ; i < rowsToUpdate.size(); i ++) - { - Set otherKeys = rowsToUpdate.get(i).keySet(); - if (otherKeys.size() != keySize) - return false; - if (!otherKeys.containsAll(keys)) - return false; - } - - return true; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - if (oldKeys != null && rows.size() != oldKeys.size()) - throw new IllegalArgumentException("rows and oldKeys are required to be the same length, but were " + rows.size() + " and " + oldKeys + " in length, respectively"); - - assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); - - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, true, errors, extraScriptContext); - - List> result = new ArrayList<>(rows.size()); - List> oldRows = new ArrayList<>(rows.size()); - List> providedValues = new ArrayList<>(rows.size()); - // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container - boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; - - for (int i = 0; i < rows.size(); i++) - { - Map row = rows.get(i); - providedValues.add(new CaseInsensitiveHashMap<>()); - row = coerceTypes(row, providedValues.get(i), true); - try - { - Map oldKey = oldKeys == null ? row : oldKeys.get(i); - Map oldRow = null; - if (!streaming) - { - oldRow = getRow(user, container, oldKey); - if (oldRow == null) - throw new NotFoundException("The existing row was not found."); - } - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, true, i, row, oldRow, extraScriptContext); - Map updatedRow = updateOneRow(user, container, row, oldRow, configParameters); - if (!streaming && updatedRow == null) - continue; - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, false, i, updatedRow, oldRow, extraScriptContext); - if (!streaming) - { - result.add(updatedRow); - oldRows.add(oldRow); - } - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); - } - catch (OptimisticConflictException e) - { - errors.addRowError(new ValidationException("Unable to update. Row may have been deleted.")); - } - } - - // Fire triggers, if any, and also throw if there are any errors - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, false, errors, extraScriptContext); - afterInsertUpdate(null==result?0:result.size(), errors, true); - - if (errors.hasErrors()) - throw errors; - - addAuditEvent(user, container, QueryService.AuditAction.UPDATE, configParameters, result, oldRows, providedValues); - - return result; - } - - protected void checkDuplicateUpdate(Object pkVals) throws ValidationException - { - if (pkVals == null) - return; - - Set updatedRows = getPreviouslyUpdatedRows(); - - Object[] keysObj; - if (pkVals.getClass().isArray()) - keysObj = (Object[]) pkVals; - else if (pkVals instanceof Map map) - { - List orderedKeyVals = new ArrayList<>(); - SortedSet sortedKeys = new TreeSet<>(map.keySet()); - for (String key : sortedKeys) - orderedKeyVals.add(map.get(key)); - keysObj = orderedKeyVals.toArray(); - } - else - keysObj = new Object[]{pkVals}; - - if (keysObj.length == 1) - { - if (updatedRows.contains(keysObj[0])) - throw new ValidationException("Duplicate key provided: " + keysObj[0]); - updatedRows.add(keysObj[0]); - return; - } - - List keys = new ArrayList<>(); - for (Object key : keysObj) - keys.add(String.valueOf(key)); - if (updatedRows.contains(keys)) - throw new ValidationException("Duplicate key provided: " + StringUtils.join(keys, ", ")); - updatedRows.add(keys); - } - - @Override - public Map moveRows(User user, Container container, Container targetContainer, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException("Move is not supported for this table type."); - } - - protected abstract Map deleteRow(User user, Container container, Map oldRow) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; - - protected Map deleteRow(User user, Container container, Map oldRow, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - return deleteRow(user, container, oldRow); - } - - @Override - public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!hasPermission(user, DeletePermission.class)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - BatchValidationException errors = new BatchValidationException(); - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, true, errors, extraScriptContext); - - // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container - boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; - - List> result = new ArrayList<>(keys.size()); - for (int i = 0; i < keys.size(); i++) - { - Map key = keys.get(i); - try - { - Map oldRow = null; - if (!streaming) - { - oldRow = getRow(user, container, key); - // if row doesn't exist, bail early - if (oldRow == null) - continue; - } - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, true, i, null, oldRow, extraScriptContext); - Map updatedRow = deleteRow(user, container, oldRow, configParameters, extraScriptContext); - if (!streaming && updatedRow == null) - continue; - - getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, false, i, null, updatedRow, extraScriptContext); - result.add(updatedRow); - } - catch (InvalidKeyException ex) - { - ValidationException vex = new ValidationException(ex.getMessage()); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - catch (ValidationException vex) - { - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - catch (RuntimeValidationException rvex) - { - ValidationException vex = rvex.getValidationException(); - errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); - } - } - - // Fire triggers, if any, and also throw if there are any errors - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, false, errors, extraScriptContext); - - addAuditEvent(user, container, QueryService.AuditAction.DELETE, configParameters, result, null, null); - - return result; - } - - protected int truncateRows(User user, Container container) - throws QueryUpdateServiceException, SQLException - { - throw new UnsupportedOperationException(); - } - - @Override - public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!container.hasPermission(user, AdminPermission.class) && !hasPermission(user, DeletePermission.class)) - throw new UnauthorizedException("You do not have permission to truncate this table."); - - BatchValidationException errors = new BatchValidationException(); - errors.setExtraContext(extraScriptContext); - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, true, errors, extraScriptContext); - - int result = truncateRows(user, container); - - getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, false, errors, extraScriptContext); - addAuditEvent(user, container, QueryService.AuditAction.TRUNCATE, configParameters, null, null, null); - - return result; - } - - @Override - public void setBulkLoad(boolean bulkLoad) - { - _bulkLoad = bulkLoad; - } - - @Override - public boolean isBulkLoad() - { - return _bulkLoad; - } - - public static Object saveFile(User user, Container container, String name, Object value, @Nullable String dirName) throws ValidationException, QueryUpdateServiceException - { - FileLike dirPath = AssayFileWriter.getUploadDirectoryPath(container, dirName); - return saveFile(user, container, name, value, dirPath); - } - - /** - * Save uploaded file to dirName directory under file or pipeline root. - */ - public static Object saveFile(User user, Container container, String name, Object value, @Nullable FileLike dirPath) throws ValidationException, QueryUpdateServiceException - { - if (!(value instanceof MultipartFile) && !(value instanceof SpringAttachmentFile)) - throw new ValidationException("Invalid file value"); - - String auditMessageFormat = "Saved file '%s' for field '%s' in folder %s."; - FileLike file = null; - try - { - FileLike dir = AssayFileWriter.ensureUploadDirectory(dirPath); - - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(container, null); - if (value instanceof MultipartFile multipartFile) - { - // Once we've found one, write it to disk and replace the row's value with just the File reference to it - if (multipartFile.isEmpty()) - { - throw new ValidationException("File " + multipartFile.getOriginalFilename() + " for field " + name + " has no content"); - } - file = FileUtil.findUniqueFileName(multipartFile.getOriginalFilename(), dir); - checkFileUnderRoot(container, file); - multipartFile.transferTo(toFileForWrite(file)); - event.setComment(String.format(auditMessageFormat, multipartFile.getOriginalFilename(), name, container.getPath())); - event.setProvidedFileName(multipartFile.getOriginalFilename()); - } - else - { - SpringAttachmentFile saf = (SpringAttachmentFile) value; - file = FileUtil.findUniqueFileName(saf.getFilename(), dir); - checkFileUnderRoot(container, file); - saf.saveTo(file); - event.setComment(String.format(auditMessageFormat, saf.getFilename(), name, container.getPath())); - event.setProvidedFileName(saf.getFilename()); - } - event.setFile(file.getName()); - event.setFieldName(name); - event.setDirectory(file.getParent().toURI().getPath()); - AuditLogService.get().addEvent(user, event); - } - catch (IOException | ExperimentException e) - { - throw new QueryUpdateServiceException(e); - } - - ensureExpData(user, container, file.toNioPathForRead().toFile()); - return file; - } - - public static ExpData ensureExpData(User user, Container container, File file) - { - ExpData existingData = ExperimentService.get().getExpDataByURL(file, container); - // create exp.data record - if (existingData == null) - { - File canonicalFile = FileUtil.getAbsoluteCaseSensitiveFile(file); - ExpData data = ExperimentService.get().createData(container, UPLOADED_FILE); - data.setName(file.getName()); - data.setDataFileURI(canonicalFile.toPath().toUri()); - if (data.getDataFileUrl() != null && data.getDataFileUrl().length() <= ExperimentService.get().getTinfoData().getColumn("DataFileURL").getScale()) - { - // If the path is too long to store, bail out without creating an exp.data row - data.save(user); - } - - return data; - } - - return existingData; - } - - // For security reasons, make sure the user hasn't tried to reference a file that's not under - // the pipeline root or @assayfiles root. Otherwise, they could get access to any file on the server - static FileLike checkFileUnderRoot(Container container, FileLike file) throws ExperimentException - { - Path assayFilesRoot = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.assayfiles); - if (assayFilesRoot != null && URIUtil.isDescendant(assayFilesRoot.toUri(), file.toURI())) - return file; - - PipeRoot root = PipelineService.get().findPipelineRoot(container); - if (root == null) - throw new ExperimentException("Pipeline root not available in container " + container.getPath()); - - if (!root.isUnderRoot(toFileForRead(file))) - { - throw new ExperimentException("Cannot reference file '" + file + "' from " + container.getPath()); - } - - return file; - } - - /** - * Is used by the AttachmentDataIterator to point to the location of the serialized - * attachment files. - */ - public void setAttachmentDirectory(VirtualFile att) - { - _att = att; - } - - @Nullable - protected VirtualFile getAttachmentDirectory() - { - return _att; - } - - /** - * QUS instances that allow import of attachments through the AttachmentDataIterator should furnish a factory - * implementation in order to resolve the attachment parent on incoming attachment files. - */ - @Nullable - protected AttachmentParentFactory getAttachmentParentFactory() - { - return null; - } - - /** Translate between the column name that query is exposing to the column name that actually lives in the database */ - protected static void aliasColumns(Map columnMapping, Map row) - { - for (Map.Entry entry : columnMapping.entrySet()) - { - if (row.containsKey(entry.getValue()) && !row.containsKey(entry.getKey())) - { - row.put(entry.getKey(), row.get(entry.getValue())); - } - } - } - - /** - * The database table has underscores for MV column names, but we expose a column without the underscore. - * Therefore, we need to translate between the two sets of column names. - * @return database column name -> exposed TableInfo column name - */ - protected static Map createMVMapping(Domain domain) - { - Map result = new CaseInsensitiveHashMap<>(); - if (domain != null) - { - for (DomainProperty domainProperty : domain.getProperties()) - { - if (domainProperty.isMvEnabled()) - { - result.put(PropertyStorageSpec.getMvIndicatorStorageColumnName(domainProperty.getPropertyDescriptor()), domainProperty.getName() + MvColumn.MV_INDICATOR_SUFFIX); - } - } - } - return result; - } - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert - { - private boolean _useAlias = false; - - static TabLoader getTestData() throws IOException - { - TabLoader testData = new TabLoader(new StringReader("pk,i,s\n0,0,zero\n1,1,one\n2,2,two"),true); - testData.parseAsCSV(); - testData.getColumns()[0].clazz = Integer.class; - testData.getColumns()[1].clazz = Integer.class; - testData.getColumns()[2].clazz = String.class; - return testData; - } - - @BeforeClass - public static void createList() throws Exception - { - if (null == ListService.get()) - return; - deleteList(); - - TabLoader testData = getTestData(); - String hash = GUID.makeHash(); - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - ListService s = ListService.get(); - UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); - assertNotNull(lists); - - ListDefinition R = s.createList(c, "R", ListDefinition.KeyType.Integer); - R.setKeyName("pk"); - Domain d = requireNonNull(R.getDomain()); - for (int i=0 ; i> getRows() - { - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); - TableInfo rTableInfo = requireNonNull(lists.getTable("R", null)); - return Arrays.asList(new TableSelector(rTableInfo, TableSelector.ALL_COLUMNS, null, new Sort("PK")).getMapArray()); - } - - @Before - public void resetList() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - qus.truncateRows(user, c, null, null); - } - - @AfterClass - public static void deleteList() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - ListService s = ListService.get(); - Map m = s.getLists(c); - if (m.containsKey("R")) - m.get("R").delete(user); - } - - void validateDefaultData(List> rows) - { - assertEquals(3, rows.size()); - - assertEquals(0, rows.get(0).get("pk")); - assertEquals(1, rows.get(1).get("pk")); - assertEquals(2, rows.get(2).get("pk")); - - assertEquals(0, rows.get(0).get("i")); - assertEquals(1, rows.get(1).get("i")); - assertEquals(2, rows.get(2).get("i")); - - assertEquals("zero", rows.get(0).get("s")); - assertEquals("one", rows.get(1).get("s")); - assertEquals("two", rows.get(2).get("s")); - } - - @Test - public void INSERT() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - assert(getRows().isEmpty()); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - BatchValidationException errors = new BatchValidationException(); - var rows = qus.insertRows(user, c, getTestData().load(), errors, null, null); - assertFalse(errors.hasErrors()); - validateDefaultData(rows); - validateDefaultData(getRows()); - - qus.insertRows(user, c, getTestData().load(), errors, null, null); - assertTrue(errors.hasErrors()); - } - - @Test - public void UPSERT() throws Exception - { - if (null == ListService.get()) - return; - /* not sure how you use/test ImportOptions.UPSERT - * the only row returning QUS method is insertRows(), which doesn't let you specify the InsertOption? - */ - } - - @Test - public void IMPORT() throws Exception - { - if (null == ListService.get()) - return; - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - assert(getRows().isEmpty()); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - BatchValidationException errors = new BatchValidationException(); - var count = qus.importRows(user, c, getTestData(), errors, null, null); - assertFalse(errors.hasErrors()); - assert(count == 3); - validateDefaultData(getRows()); - - qus.importRows(user, c, getTestData(), errors, null, null); - assertTrue(errors.hasErrors()); - } - - @Test - public void MERGE() throws Exception - { - if (null == ListService.get()) - return; - INSERT(); - assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var mergeRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); - BatchValidationException errors = new BatchValidationException() - { - @Override - public void addRowError(ValidationException vex) - { - LogManager.getLogger(AbstractQueryUpdateService.class).error("test error", vex); - fail(vex.getMessage()); - } - }; - int count=0; - try (var tx = rTableInfo.getSchema().getScope().ensureTransaction()) - { - var ret = qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); - if (!errors.hasErrors()) - { - tx.commit(); - count = ret; - } - } - assertFalse("mergeRows error(s): " + errors.getMessage(), errors.hasErrors()); - assertEquals(2, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO", rows.get(2).get("s")); - // test existing row value is not updated - assertEquals(2, rows.get(2).get("i")); - // test new row - assertEquals("THREE", rows.get(3).get("s")); - assertNull(rows.get(3).get("i")); - - // merge should fail if duplicate keys are provided - errors = new BatchValidationException(); - mergeRows = new ArrayList<>(); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); - qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); - assertTrue(errors.hasErrors()); - assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); - } - - @Test - public void UPDATE() throws Exception - { - if (null == ListService.get()) - return; - INSERT(); - assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var updateRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - - // update using data iterator - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP")); - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(InsertOption.UPDATE); - var count = qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertFalse(context.getErrors().hasErrors()); - assertEquals(1, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO-UP", rows.get(2).get("s")); - // test existing row value is not updated/erased - assertEquals(2, rows.get(2).get("i")); - - // update should fail if a new record is provided - updateRows = new ArrayList<>(); - updateRows.add(CaseInsensitiveHashMap.of(pkName,123,colName,"NEW")); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertTrue(context.getErrors().hasErrors()); - - // Issue 52728: update should fail if duplicate key is provide - updateRows = new ArrayList<>(); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); - updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); - - // use DIB - context = new DataIteratorContext(); - context.setInsertOption(InsertOption.UPDATE); - qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); - assertTrue(context.getErrors().hasErrors()); - assertTrue("Duplicate key error: " + context.getErrors().getMessage(), context.getErrors().getMessage().contains("Duplicate key provided: 2")); - - // use updateRows - if (!_useAlias) // _update using alias is not supported - { - BatchValidationException errors = new BatchValidationException(); - try - { - qus.updateRows(user, c, updateRows, null, errors, null, null); - } - catch (Exception e) - { - - } - assertTrue(errors.hasErrors()); - assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); - - } - } - - @Test - public void REPLACE() throws Exception - { - if (null == ListService.get()) - return; - assert(getRows().isEmpty()); - INSERT(); - - User user = TestContext.get().getUser(); - Container c = JunitUtil.getTestContainer(); - TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); - QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); - var mergeRows = new ArrayList>(); - String colName = _useAlias ? "s_alias" : "s"; - String pkName = _useAlias ? "pk_alias" : "pk"; - mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); - mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); - DataIteratorContext context = new DataIteratorContext(); - context.setInsertOption(InsertOption.REPLACE); - var count = qus.loadRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), context, null); - assertFalse(context.getErrors().hasErrors()); - assertEquals(2, count); - var rows = getRows(); - // test existing row value is updated - assertEquals("TWO", rows.get(2).get("s")); - // test existing row value is updated - assertNull(rows.get(2).get("i")); - // test new row - assertEquals("THREE", rows.get(3).get("s")); - assertNull(rows.get(3).get("i")); - } - - @Test - public void IMPORT_IDENTITY() - { - if (null == ListService.get()) - return; - // TODO - } - - @Test - public void ALIAS_MERGE() throws Exception - { - _useAlias = true; - MERGE(); - } - - @Test - public void ALIAS_REPLACE() throws Exception - { - _useAlias = true; - REPLACE(); - } - - @Test - public void ALIAS_UPDATE() throws Exception - { - _useAlias = true; - UPDATE(); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.collections.ArrayListMap; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConvertHelper; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.ExpDataFileConverter; +import org.labkey.api.data.ImportAliasable; +import org.labkey.api.data.MultiValuedForeignKey; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Sort; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.AttachmentDataIterator; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.ExistingRecordDataIterator; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.dataiterator.Pump; +import org.labkey.api.dataiterator.StandardDataIteratorBuilder; +import org.labkey.api.dataiterator.TriggerDataBuilderHelper; +import org.labkey.api.dataiterator.WrapperDataIterator; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.MvColumn; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.reader.TabLoader; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URIUtil; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.vfs.FileLike; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.Function; + +import static java.util.Objects.requireNonNull; +import static org.labkey.api.audit.TransactionAuditProvider.DB_SEQUENCE_NAME; +import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior; +import static org.labkey.api.dataiterator.DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment; +import static org.labkey.api.files.FileContentService.UPLOADED_FILE; +import static org.labkey.api.util.FileUtil.toFileForRead; +import static org.labkey.api.util.FileUtil.toFileForWrite; + +public abstract class AbstractQueryUpdateService implements QueryUpdateService +{ + protected final TableInfo _queryTable; + + private boolean _bulkLoad = false; + private CaseInsensitiveHashMap _columnImportMap = null; + private VirtualFile _att = null; + + /* AbstractQueryUpdateService is generally responsible for some shared functionality + * - triggers + * - coercion/validation + * - detailed logging + * - attachments + * + * If a subclass wants to disable some of these features (w/o subclassing), put flags here... + */ + protected boolean _enableExistingRecordsDataIterator = true; + protected Set _previouslyUpdatedRows = new HashSet<>(); + + protected AbstractQueryUpdateService(TableInfo queryTable) + { + if (queryTable == null) + throw new IllegalArgumentException(); + _queryTable = queryTable; + } + + protected TableInfo getQueryTable() + { + return _queryTable; + } + + public @NotNull Set getPreviouslyUpdatedRows() + { + return _previouslyUpdatedRows == null ? new HashSet<>() : _previouslyUpdatedRows; + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) + { + return getQueryTable().hasPermission(user, acl); + } + + protected Map getRow(User user, Container container, Map keys, boolean allowCrossContainer) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + return getRow(user, container, keys); + } + + protected abstract Map getRow(User user, Container container, Map keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException; + + @Override + public List> getRows(User user, Container container, List> keys) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + + List> result = new ArrayList<>(); + for (Map rowKeys : keys) + { + Map row = getRow(user, container, rowKeys); + if (row != null) + result.add(row); + } + return result; + } + + @Override + public Map> getExistingRows(User user, Container container, Map> keys, boolean verifyNoCrossFolderData, boolean verifyExisting, @Nullable Set columns) + throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + + Map> result = new LinkedHashMap<>(); + for (Map.Entry> key : keys.entrySet()) + { + Map row = getRow(user, container, key.getValue(), verifyNoCrossFolderData); + if (row != null && !row.isEmpty()) + { + result.put(key.getKey(), row); + if (verifyNoCrossFolderData) + { + String dataContainer = (String) row.get("container"); + if (StringUtils.isEmpty(dataContainer)) + dataContainer = (String) row.get("folder"); + if (!container.getId().equals(dataContainer)) + throw new InvalidKeyException("Data doesn't belong to folder '" + container.getName() + "': " + key.getValue().values()); + } + } + else if (verifyExisting) + throw new InvalidKeyException("Data not found for " + key.getValue().values()); + } + return result; + } + + @Override + public boolean hasExistingRowsInOtherContainers(Container container, Map> keys) + { + return false; + } + + public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction) + { + return createTransactionAuditEvent(container, auditAction, null); + } + + public static TransactionAuditProvider.TransactionAuditEvent createTransactionAuditEvent(Container container, QueryService.AuditAction auditAction, @Nullable Map details) + { + long auditId = DbSequenceManager.get(ContainerManager.getRoot(), DB_SEQUENCE_NAME).next(); + TransactionAuditProvider.TransactionAuditEvent event = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); + if (details != null) + event.addDetails(details); + return event; + } + + public static void addTransactionAuditEvent(DbScope.Transaction transaction, User user, TransactionAuditProvider.TransactionAuditEvent auditEvent) + { + UserSchema schema = AuditLogService.getAuditLogSchema(user, ContainerManager.getRoot()); + + if (schema != null) + { + // This is a little hack to ensure that the audit table has actually been created and gets put into the table cache by the time the + // pre-commit task is executed. Otherwise, since the creation of the table happens while within the commit for the + // outermost transaction, it looks like there is a close that hasn't happened when trying to commit the transaction for creating the + // table. + schema.getTable(auditEvent.getEventType(), false); + + transaction.addCommitTask(() -> AuditLogService.get().addEvent(user, auditEvent), DbScope.CommitTaskOption.PRECOMMIT); + + transaction.setAuditEvent(auditEvent); + } + } + + protected final DataIteratorContext getDataIteratorContext(BatchValidationException errors, InsertOption forImport, Map configParameters) + { + if (null == errors) + errors = new BatchValidationException(); + DataIteratorContext context = new DataIteratorContext(errors); + context.setInsertOption(forImport); + context.setConfigParameters(configParameters); + configureDataIteratorContext(context); + return context; + } + + /** + * If QUS wants to use something other than PKs to select existing rows for merge, it can override this method. + * Used only for generating ExistingRecordDataIterator at the moment. + */ + protected Set getSelectKeys(DataIteratorContext context) + { + if (!context.getAlternateKeys().isEmpty()) + return context.getAlternateKeys(); + return null; + } + + /* + * construct the core DataIterator transformation pipeline for this table, may be just StandardDataIteratorBuilder. + * does NOT handle triggers or the insert/update iterator. + */ + public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) + { + DataIteratorBuilder dib = StandardDataIteratorBuilder.forInsert(getQueryTable(), data, container, user); + + if (_enableExistingRecordsDataIterator || context.getInsertOption().updateOnly) + { + // some tables need to generate PKs, so they need to add ExistingRecordDataIterator in persistRows() (after generating PK, before inserting) + dib = ExistingRecordDataIterator.createBuilder(dib, getQueryTable(), getSelectKeys(context)); + } + + dib = ((UpdateableTableInfo) getQueryTable()).persistRows(dib, context); + dib = AttachmentDataIterator.getAttachmentDataIteratorBuilder(getQueryTable(), dib, user, context.getInsertOption().batch ? getAttachmentDirectory() : null, container, getAttachmentParentFactory()); + dib = DetailedAuditLogDataIterator.getDataIteratorBuilder(getQueryTable(), dib, context.getInsertOption(), user, container, null); + return dib; + } + + + /** + * Implementation to use insertRows() while we migrate to using DIB for all code paths + *

+ * DataIterator should/must use the same error collection as passed in + */ + @Deprecated + protected int _importRowsUsingInsertRows(User user, Container container, DataIterator rows, BatchValidationException errors, Map extraScriptContext) + { + MapDataIterator mapIterator = DataIteratorUtil.wrapMap(rows, true); + List> list = new ArrayList<>(); + List> ret; + Exception rowException; + + try + { + while (mapIterator.next()) + list.add(mapIterator.getMap()); + ret = insertRows(user, container, list, errors, null, extraScriptContext); + if (errors.hasErrors()) + return 0; + return ret.size(); + } + catch (BatchValidationException x) + { + assert x == errors; + assert x.hasErrors(); + return 0; + } + catch (QueryUpdateServiceException | DuplicateKeyException | SQLException x) + { + rowException = x; + } + finally + { + DataIteratorUtil.closeQuietly(mapIterator); + } + errors.addRowError(new ValidationException(rowException.getMessage())); + return 0; + } + + protected boolean hasImportRowsPermission(User user, Container container, DataIteratorContext context) + { + return hasPermission(user, context.getInsertOption().updateOnly ? UpdatePermission.class : InsertPermission.class); + } + + // override this + protected void preImportDIBValidation(@Nullable DataIteratorBuilder in, @Nullable Collection inputColumns) + { + } + + protected int _importRowsUsingDIB(User user, Container container, DataIteratorBuilder in, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasImportRowsPermission(user, container, context)) + throw new UnauthorizedException("You do not have permission to " + (context.getInsertOption().updateOnly ? "update data in this table." : "insert data into this table.")); + + if (!context.getConfigParameterBoolean(ConfigParameters.SkipInsertOptionValidation)) + assert(getQueryTable().supportsInsertOption(context.getInsertOption())); + + context.getErrors().setExtraContext(extraScriptContext); + if (extraScriptContext != null) + { + context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE)); + } + + preImportDIBValidation(in, null); + + boolean skipTriggers = context.getConfigParameterBoolean(ConfigParameters.SkipTriggers) || context.isCrossTypeImport() || context.isCrossFolderImport(); + boolean hasTableScript = hasTableScript(container); + TriggerDataBuilderHelper helper = new TriggerDataBuilderHelper(getQueryTable(), container, user, extraScriptContext, context.getInsertOption().useImportAliases); + if (!skipTriggers) + { + in = preTriggerDataIterator(in, context); + if (hasTableScript) + in = helper.before(in); + } + DataIteratorBuilder importDIB = createImportDIB(user, container, in, context); + DataIteratorBuilder out = importDIB; + + if (!skipTriggers) + { + if (hasTableScript) + out = helper.after(importDIB); + + out = postTriggerDataIterator(out, context); + } + + if (hasTableScript) + { + context.setFailFast(false); + context.setMaxRowErrors(Math.max(context.getMaxRowErrors(),1000)); + } + int count = _pump(out, outputRows, context); + + if (context.getErrors().hasErrors()) + return 0; + else + { + if (!context.isCrossTypeImport() && !context.isCrossFolderImport()) // audit handled at table level + { + AuditBehaviorType auditType = (AuditBehaviorType) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior); + String auditUserComment = (String) context.getConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment); + boolean skipAuditLevelCheck = false; + if (context.getConfigParameterBoolean(QueryUpdateService.ConfigParameters.BulkLoad)) + { + if (getQueryTable().getEffectiveAuditBehavior(auditType) == AuditBehaviorType.DETAILED) // allow ETL to demote audit level for bulkLoad + skipAuditLevelCheck = true; + } + getQueryTable().getAuditHandler(auditType).addSummaryAuditEvent(user, container, getQueryTable(), context.getInsertOption().auditAction, count, auditType, auditUserComment, skipAuditLevelCheck); + } + return count; + } + } + + protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) + { + return in; + } + + protected DataIteratorBuilder postTriggerDataIterator(DataIteratorBuilder out, DataIteratorContext context) + { + return out; + } + + + /** this is extracted so subclasses can add wrap */ + protected int _pump(DataIteratorBuilder etl, final @Nullable ArrayList> rows, DataIteratorContext context) + { + DataIterator it = etl.getDataIterator(context); + + try + { + if (null != rows) + { + MapDataIterator maps = DataIteratorUtil.wrapMap(it, false); + it = new WrapperDataIterator(maps) + { + @Override + public boolean next() throws BatchValidationException + { + boolean ret = super.next(); + if (ret) + rows.add(((MapDataIterator)_delegate).getMap()); + return ret; + } + }; + } + + Pump pump = new Pump(it, context); + pump.run(); + + return pump.getRowCount(); + } + finally + { + DataIteratorUtil.closeQuietly(it); + } + } + + /* can be used for simple bookkeeping tasks, per row processing belongs in a data iterator */ + protected void afterInsertUpdate(int count, BatchValidationException errors, boolean isUpdate) + { + afterInsertUpdate(count, errors); + } + + protected void afterInsertUpdate(int count, BatchValidationException errors) + {} + + @Override + public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + return loadRows(user, container, rows, null, context, extraScriptContext); + } + + public int loadRows(User user, Container container, DataIteratorBuilder rows, @Nullable final ArrayList> outputRows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + configureDataIteratorContext(context); + int count = _importRowsUsingDIB(user, container, rows, outputRows, context, extraScriptContext); + afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); + return count; + } + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, @Nullable Map extraScriptContext) + { + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingInsertRows(user, container, rows.getDataIterator(context), errors, extraScriptContext); + afterInsertUpdate(count, errors, context.getInsertOption().updateOnly); + return count; + } + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + { + throw new UnsupportedOperationException("merge is not supported for all tables"); + } + + private boolean hasTableScript(Container container) + { + return getQueryTable().hasTriggers(container); + } + + + protected Map insertRow(User user, Container container, Map row) + throws DuplicateKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException("Not implemented by this QueryUpdateService"); + } + + + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, InsertPermission.class)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); + } + + protected @Nullable List> _insertUpdateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + DataIteratorBuilder dib = _toDataIteratorBuilder(getClass().getSimpleName() + (context.getInsertOption().updateOnly ? ".updateRows" : ".insertRows()"), rows); + ArrayList> outputRows = new ArrayList<>(); + int count = _importRowsUsingDIB(user, container, dib, outputRows, context, extraScriptContext); + afterInsertUpdate(count, context.getErrors(), context.getInsertOption().updateOnly); + + if (context.getErrors().hasErrors()) + return null; + + return outputRows; + } + + // not yet supported + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return _insertUpdateRowsUsingDIB(user, container, rows, context, extraScriptContext); + } + + + protected DataIteratorBuilder _toDataIteratorBuilder(String debugName, List> rows) + { + // TODO probably can't assume all rows have all columns + // TODO can we assume that all rows refer to columns consistently? (not PTID and MouseId for the same column) + // TODO optimize ArrayListMap? + Set colNames; + + if (!rows.isEmpty() && rows.get(0) instanceof ArrayListMap) + { + colNames = ((ArrayListMap)rows.get(0)).getFindMap().keySet(); + } + else + { + // Preserve casing by using wrapped CaseInsensitiveHashMap instead of CaseInsensitiveHashSet + colNames = Sets.newCaseInsensitiveHashSet(); + for (Map row : rows) + colNames.addAll(row.keySet()); + } + + preImportDIBValidation(null, colNames); + return MapDataIterator.of(colNames, rows, debugName); + } + + + /** @deprecated switch to using DIB based method */ + @Deprecated + protected List> _insertRowsUsingInsertRow(User user, Container container, List> rows, BatchValidationException errors, Map extraScriptContext) + throws DuplicateKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, InsertPermission.class)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + assert(getQueryTable().supportsInsertOption(InsertOption.INSERT)); + + boolean hasTableScript = hasTableScript(container); + + errors.setExtraContext(extraScriptContext); + if (hasTableScript) + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, true, errors, extraScriptContext); + + List> result = new ArrayList<>(rows.size()); + List> providedValues = new ArrayList<>(rows.size()); + for (int i = 0; i < rows.size(); i++) + { + Map row = rows.get(i); + row = normalizeColumnNames(row); + try + { + providedValues.add(new CaseInsensitiveHashMap<>()); + row = coerceTypes(row, providedValues.get(i), false); + if (hasTableScript) + { + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, true, i, row, null, extraScriptContext); + } + row = insertRow(user, container, row); + if (row == null) + continue; + + if (hasTableScript) + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.INSERT, false, i, row, null, extraScriptContext); + result.add(row); + } + catch (SQLException sqlx) + { + if (StringUtils.startsWith(sqlx.getSQLState(), "22") || RuntimeSQLException.isConstraintException(sqlx)) + { + ValidationException vex = new ValidationException(sqlx.getMessage()); + vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i+1); + errors.addRowError(vex); + } + else if (SqlDialect.isTransactionException(sqlx) && errors.hasErrors()) + { + // if we already have some errors, just break + break; + } + else + { + throw sqlx; + } + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + } + + if (hasTableScript) + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.INSERT, false, errors, extraScriptContext); + + addAuditEvent(user, container, QueryService.AuditAction.INSERT, null, result, null, providedValues); + + return result; + } + + protected void addAuditEvent(User user, Container container, QueryService.AuditAction auditAction, @Nullable Map configParameters, @Nullable List> rows, @Nullable List> existingRows, @Nullable List> providedValues) + { + if (!isBulkLoad()) + { + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(AuditBehavior) : null; + String userComment = configParameters == null ? null : (String) configParameters.get(AuditUserComment); + getQueryTable().getAuditHandler(auditBehavior) + .addAuditEvent(user, container, getQueryTable(), auditBehavior, userComment, auditAction, rows, existingRows, providedValues); + } + } + + private Map normalizeColumnNames(Map row) + { + if(_columnImportMap == null) + { + _columnImportMap = (CaseInsensitiveHashMap)ImportAliasable.Helper.createImportMap(getQueryTable().getColumns(), false); + } + + Map newRow = new CaseInsensitiveHashMap<>(); + CaseInsensitiveHashSet columns = new CaseInsensitiveHashSet(); + columns.addAll(row.keySet()); + + String newName; + for(String key : row.keySet()) + { + if(_columnImportMap.containsKey(key)) + { + //it is possible for a normalized name to conflict with an existing property. if so, defer to the original + newName = _columnImportMap.get(key).getName(); + if(!columns.contains(newName)){ + newRow.put(newName, row.get(key)); + continue; + } + } + newRow.put(key, row.get(key)); + } + + return newRow; + } + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws DuplicateKeyException, QueryUpdateServiceException, SQLException + { + try + { + List> ret = _insertRowsUsingInsertRow(user, container, rows, errors, extraScriptContext); + afterInsertUpdate(null==ret?0:ret.size(), errors); + if (errors.hasErrors()) + return null; + return ret; + } + catch (BatchValidationException x) + { + assert x == errors; + assert x.hasErrors(); + } + return null; + } + + + /** Attempt to make the passed in types match the expected types so the script doesn't have to do the conversion */ + @Deprecated + protected Map coerceTypes(Map row, Map providedValues, boolean isUpdate) + { + Map result = new CaseInsensitiveHashMap<>(row.size()); + Map columnMap = ImportAliasable.Helper.createImportMap(_queryTable.getColumns(), true); + for (Map.Entry entry : row.entrySet()) + { + ColumnInfo col = columnMap.get(entry.getKey()); + + Object value = entry.getValue(); + if (col != null && value != null && + !col.getJavaObjectClass().isInstance(value) && + !(value instanceof AttachmentFile) && + !(value instanceof MultipartFile) && + !(value instanceof String[]) && + !(col.getFk() instanceof MultiValuedForeignKey)) + { + try + { + if (PropertyType.FILE_LINK.equals(col.getPropertyType())) + value = ExpDataFileConverter.convert(value); + else if (col.getKindOfQuantity() != null) + { + providedValues.put(entry.getKey(), value); + value = Quantity.convert(value, col.getKindOfQuantity().getStorageUnit()); + } + else + value = col.getConvertFn().apply(value); + } + catch (ConvertHelper.FileConversionException e) + { + throw e; + } + catch (ConversionException e) + { + // That's OK, the transformation script may be able to fix up the value before it gets inserted + } + } + result.put(entry.getKey(), value); + } + return result; + } + + + protected abstract Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; + + + protected boolean firstUpdateRow = true; + Function,Map> updateTransform = Function.identity(); + + /* Do standard AQUS stuff here, then call the subclass specific implementation of updateRow() */ + final protected Map updateOneRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + if (firstUpdateRow) + { + firstUpdateRow = false; + if (null != OntologyService.get()) + { + var t = OntologyService.get().getConceptUpdateHandler(_queryTable); + if (null != t) + updateTransform = t; + } + } + row = updateTransform.apply(row); + return updateRow(user, container, row, oldRow, configParameters); + } + + // used by updateRows to check if all rows have the same set of keys + // prepared statement can only be used to updateRows if all rows have the same set of keys + protected boolean hasUniformKeys(List> rowsToUpdate) + { + if (rowsToUpdate == null || rowsToUpdate.isEmpty()) + return false; + + if (rowsToUpdate.size() == 1) + return true; + + Set keys = rowsToUpdate.get(0).keySet(); + int keySize = keys.size(); + + for (int i = 1 ; i < rowsToUpdate.size(); i ++) + { + Set otherKeys = rowsToUpdate.get(i).keySet(); + if (otherKeys.size() != keySize) + return false; + if (!otherKeys.containsAll(keys)) + return false; + } + + return true; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + if (oldKeys != null && rows.size() != oldKeys.size()) + throw new IllegalArgumentException("rows and oldKeys are required to be the same length, but were " + rows.size() + " and " + oldKeys + " in length, respectively"); + + assert(getQueryTable().supportsInsertOption(InsertOption.UPDATE)); + + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, true, errors, extraScriptContext); + + List> result = new ArrayList<>(rows.size()); + List> oldRows = new ArrayList<>(rows.size()); + List> providedValues = new ArrayList<>(rows.size()); + // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container + boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; + + for (int i = 0; i < rows.size(); i++) + { + Map row = rows.get(i); + providedValues.add(new CaseInsensitiveHashMap<>()); + row = coerceTypes(row, providedValues.get(i), true); + try + { + Map oldKey = oldKeys == null ? row : oldKeys.get(i); + Map oldRow = null; + if (!streaming) + { + oldRow = getRow(user, container, oldKey); + if (oldRow == null) + throw new NotFoundException("The existing row was not found."); + } + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, true, i, row, oldRow, extraScriptContext); + Map updatedRow = updateOneRow(user, container, row, oldRow, configParameters); + if (!streaming && updatedRow == null) + continue; + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.UPDATE, false, i, updatedRow, oldRow, extraScriptContext); + if (!streaming) + { + result.add(updatedRow); + oldRows.add(oldRow); + } + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), row, i)); + } + catch (OptimisticConflictException e) + { + errors.addRowError(new ValidationException("Unable to update. Row may have been deleted.")); + } + } + + // Fire triggers, if any, and also throw if there are any errors + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.UPDATE, false, errors, extraScriptContext); + afterInsertUpdate(null==result?0:result.size(), errors, true); + + if (errors.hasErrors()) + throw errors; + + addAuditEvent(user, container, QueryService.AuditAction.UPDATE, configParameters, result, oldRows, providedValues); + + return result; + } + + protected void checkDuplicateUpdate(Object pkVals) throws ValidationException + { + if (pkVals == null) + return; + + Set updatedRows = getPreviouslyUpdatedRows(); + + Object[] keysObj; + if (pkVals.getClass().isArray()) + keysObj = (Object[]) pkVals; + else if (pkVals instanceof Map map) + { + List orderedKeyVals = new ArrayList<>(); + SortedSet sortedKeys = new TreeSet<>(map.keySet()); + for (String key : sortedKeys) + orderedKeyVals.add(map.get(key)); + keysObj = orderedKeyVals.toArray(); + } + else + keysObj = new Object[]{pkVals}; + + if (keysObj.length == 1) + { + if (updatedRows.contains(keysObj[0])) + throw new ValidationException("Duplicate key provided: " + keysObj[0]); + updatedRows.add(keysObj[0]); + return; + } + + List keys = new ArrayList<>(); + for (Object key : keysObj) + keys.add(String.valueOf(key)); + if (updatedRows.contains(keys)) + throw new ValidationException("Duplicate key provided: " + StringUtils.join(keys, ", ")); + updatedRows.add(keys); + } + + @Override + public Map moveRows(User user, Container container, Container targetContainer, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException("Move is not supported for this table type."); + } + + protected abstract Map deleteRow(User user, Container container, Map oldRow) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException; + + protected Map deleteRow(User user, Container container, Map oldRow, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + return deleteRow(user, container, oldRow); + } + + @Override + public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!hasPermission(user, DeletePermission.class)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + BatchValidationException errors = new BatchValidationException(); + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, true, errors, extraScriptContext); + + // TODO: Support update/delete without selecting the existing row -- unfortunately, we currently get the existing row to check its container matches the incoming container + boolean streaming = false; //_queryTable.canStreamTriggers(container) && _queryTable.getAuditBehavior() != AuditBehaviorType.NONE; + + List> result = new ArrayList<>(keys.size()); + for (int i = 0; i < keys.size(); i++) + { + Map key = keys.get(i); + try + { + Map oldRow = null; + if (!streaming) + { + oldRow = getRow(user, container, key); + // if row doesn't exist, bail early + if (oldRow == null) + continue; + } + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, true, i, null, oldRow, extraScriptContext); + Map updatedRow = deleteRow(user, container, oldRow, configParameters, extraScriptContext); + if (!streaming && updatedRow == null) + continue; + + getQueryTable().fireRowTrigger(container, user, TableInfo.TriggerType.DELETE, false, i, null, updatedRow, extraScriptContext); + result.add(updatedRow); + } + catch (InvalidKeyException ex) + { + ValidationException vex = new ValidationException(ex.getMessage()); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + catch (ValidationException vex) + { + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + catch (RuntimeValidationException rvex) + { + ValidationException vex = rvex.getValidationException(); + errors.addRowError(vex.fillIn(getQueryTable().getPublicSchemaName(), getQueryTable().getName(), key, i)); + } + } + + // Fire triggers, if any, and also throw if there are any errors + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.DELETE, false, errors, extraScriptContext); + + addAuditEvent(user, container, QueryService.AuditAction.DELETE, configParameters, result, null, null); + + return result; + } + + protected int truncateRows(User user, Container container) + throws QueryUpdateServiceException, SQLException + { + throw new UnsupportedOperationException(); + } + + @Override + public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!container.hasPermission(user, AdminPermission.class) && !hasPermission(user, DeletePermission.class)) + throw new UnauthorizedException("You do not have permission to truncate this table."); + + BatchValidationException errors = new BatchValidationException(); + errors.setExtraContext(extraScriptContext); + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, true, errors, extraScriptContext); + + int result = truncateRows(user, container); + + getQueryTable().fireBatchTrigger(container, user, TableInfo.TriggerType.TRUNCATE, false, errors, extraScriptContext); + addAuditEvent(user, container, QueryService.AuditAction.TRUNCATE, configParameters, null, null, null); + + return result; + } + + @Override + public void setBulkLoad(boolean bulkLoad) + { + _bulkLoad = bulkLoad; + } + + @Override + public boolean isBulkLoad() + { + return _bulkLoad; + } + + public static Object saveFile(User user, Container container, String name, Object value, @Nullable String dirName) throws ValidationException, QueryUpdateServiceException + { + FileLike dirPath = AssayFileWriter.getUploadDirectoryPath(container, dirName); + return saveFile(user, container, name, value, dirPath); + } + + /** + * Save uploaded file to dirName directory under file or pipeline root. + */ + public static Object saveFile(User user, Container container, String name, Object value, @Nullable FileLike dirPath) throws ValidationException, QueryUpdateServiceException + { + if (!(value instanceof MultipartFile) && !(value instanceof SpringAttachmentFile)) + throw new ValidationException("Invalid file value"); + + String auditMessageFormat = "Saved file '%s' for field '%s' in folder %s."; + FileLike file = null; + try + { + FileLike dir = AssayFileWriter.ensureUploadDirectory(dirPath); + + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(container, null); + if (value instanceof MultipartFile multipartFile) + { + // Once we've found one, write it to disk and replace the row's value with just the File reference to it + if (multipartFile.isEmpty()) + { + throw new ValidationException("File " + multipartFile.getOriginalFilename() + " for field " + name + " has no content"); + } + file = FileUtil.findUniqueFileName(multipartFile.getOriginalFilename(), dir); + checkFileUnderRoot(container, file); + multipartFile.transferTo(toFileForWrite(file)); + event.setComment(String.format(auditMessageFormat, multipartFile.getOriginalFilename(), name, container.getPath())); + event.setProvidedFileName(multipartFile.getOriginalFilename()); + } + else + { + SpringAttachmentFile saf = (SpringAttachmentFile) value; + file = FileUtil.findUniqueFileName(saf.getFilename(), dir); + checkFileUnderRoot(container, file); + saf.saveTo(file); + event.setComment(String.format(auditMessageFormat, saf.getFilename(), name, container.getPath())); + event.setProvidedFileName(saf.getFilename()); + } + event.setFile(file.getName()); + event.setFieldName(name); + event.setDirectory(file.getParent().toURI().getPath()); + AuditLogService.get().addEvent(user, event); + } + catch (IOException | ExperimentException e) + { + throw new QueryUpdateServiceException(e); + } + + ensureExpData(user, container, file.toNioPathForRead().toFile()); + return file; + } + + public static ExpData ensureExpData(User user, Container container, File file) + { + ExpData existingData = ExperimentService.get().getExpDataByURL(file, container); + // create exp.data record + if (existingData == null) + { + File canonicalFile = FileUtil.getAbsoluteCaseSensitiveFile(file); + ExpData data = ExperimentService.get().createData(container, UPLOADED_FILE); + data.setName(file.getName()); + data.setDataFileURI(canonicalFile.toPath().toUri()); + if (data.getDataFileUrl() != null && data.getDataFileUrl().length() <= ExperimentService.get().getTinfoData().getColumn("DataFileURL").getScale()) + { + // If the path is too long to store, bail out without creating an exp.data row + data.save(user); + } + + return data; + } + + return existingData; + } + + // For security reasons, make sure the user hasn't tried to reference a file that's not under + // the pipeline root or @assayfiles root. Otherwise, they could get access to any file on the server + static FileLike checkFileUnderRoot(Container container, FileLike file) throws ExperimentException + { + Path assayFilesRoot = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.assayfiles); + if (assayFilesRoot != null && URIUtil.isDescendant(assayFilesRoot.toUri(), file.toURI())) + return file; + + PipeRoot root = PipelineService.get().findPipelineRoot(container); + if (root == null) + throw new ExperimentException("Pipeline root not available in container " + container.getPath()); + + if (!root.isUnderRoot(toFileForRead(file))) + { + throw new ExperimentException("Cannot reference file '" + file + "' from " + container.getPath()); + } + + return file; + } + + /** + * Is used by the AttachmentDataIterator to point to the location of the serialized + * attachment files. + */ + public void setAttachmentDirectory(VirtualFile att) + { + _att = att; + } + + @Nullable + protected VirtualFile getAttachmentDirectory() + { + return _att; + } + + /** + * QUS instances that allow import of attachments through the AttachmentDataIterator should furnish a factory + * implementation in order to resolve the attachment parent on incoming attachment files. + */ + @Nullable + protected AttachmentParentFactory getAttachmentParentFactory() + { + return null; + } + + /** Translate between the column name that query is exposing to the column name that actually lives in the database */ + protected static void aliasColumns(Map columnMapping, Map row) + { + for (Map.Entry entry : columnMapping.entrySet()) + { + if (row.containsKey(entry.getValue()) && !row.containsKey(entry.getKey())) + { + row.put(entry.getKey(), row.get(entry.getValue())); + } + } + } + + /** + * The database table has underscores for MV column names, but we expose a column without the underscore. + * Therefore, we need to translate between the two sets of column names. + * @return database column name -> exposed TableInfo column name + */ + protected static Map createMVMapping(Domain domain) + { + Map result = new CaseInsensitiveHashMap<>(); + if (domain != null) + { + for (DomainProperty domainProperty : domain.getProperties()) + { + if (domainProperty.isMvEnabled()) + { + result.put(PropertyStorageSpec.getMvIndicatorStorageColumnName(domainProperty.getPropertyDescriptor()), domainProperty.getName() + MvColumn.MV_INDICATOR_SUFFIX); + } + } + } + return result; + } + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert + { + private boolean _useAlias = false; + + static TabLoader getTestData() throws IOException + { + TabLoader testData = new TabLoader(new StringReader("pk,i,s\n0,0,zero\n1,1,one\n2,2,two"),true); + testData.parseAsCSV(); + testData.getColumns()[0].clazz = Integer.class; + testData.getColumns()[1].clazz = Integer.class; + testData.getColumns()[2].clazz = String.class; + return testData; + } + + @BeforeClass + public static void createList() throws Exception + { + if (null == ListService.get()) + return; + deleteList(); + + TabLoader testData = getTestData(); + String hash = GUID.makeHash(); + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + ListService s = ListService.get(); + UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); + assertNotNull(lists); + + ListDefinition R = s.createList(c, "R", ListDefinition.KeyType.Integer); + R.setKeyName("pk"); + Domain d = requireNonNull(R.getDomain()); + for (int i=0 ; i> getRows() + { + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + UserSchema lists = (UserSchema)DefaultSchema.get(user, c).getSchema("lists"); + TableInfo rTableInfo = requireNonNull(lists.getTable("R", null)); + return Arrays.asList(new TableSelector(rTableInfo, TableSelector.ALL_COLUMNS, null, new Sort("PK")).getMapArray()); + } + + @Before + public void resetList() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + qus.truncateRows(user, c, null, null); + } + + @AfterClass + public static void deleteList() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + ListService s = ListService.get(); + Map m = s.getLists(c); + if (m.containsKey("R")) + m.get("R").delete(user); + } + + void validateDefaultData(List> rows) + { + assertEquals(3, rows.size()); + + assertEquals(0, rows.get(0).get("pk")); + assertEquals(1, rows.get(1).get("pk")); + assertEquals(2, rows.get(2).get("pk")); + + assertEquals(0, rows.get(0).get("i")); + assertEquals(1, rows.get(1).get("i")); + assertEquals(2, rows.get(2).get("i")); + + assertEquals("zero", rows.get(0).get("s")); + assertEquals("one", rows.get(1).get("s")); + assertEquals("two", rows.get(2).get("s")); + } + + @Test + public void INSERT() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + assert(getRows().isEmpty()); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + BatchValidationException errors = new BatchValidationException(); + var rows = qus.insertRows(user, c, getTestData().load(), errors, null, null); + assertFalse(errors.hasErrors()); + validateDefaultData(rows); + validateDefaultData(getRows()); + + qus.insertRows(user, c, getTestData().load(), errors, null, null); + assertTrue(errors.hasErrors()); + } + + @Test + public void UPSERT() throws Exception + { + if (null == ListService.get()) + return; + /* not sure how you use/test ImportOptions.UPSERT + * the only row returning QUS method is insertRows(), which doesn't let you specify the InsertOption? + */ + } + + @Test + public void IMPORT() throws Exception + { + if (null == ListService.get()) + return; + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + assert(getRows().isEmpty()); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + BatchValidationException errors = new BatchValidationException(); + var count = qus.importRows(user, c, getTestData(), errors, null, null); + assertFalse(errors.hasErrors()); + assert(count == 3); + validateDefaultData(getRows()); + + qus.importRows(user, c, getTestData(), errors, null, null); + assertTrue(errors.hasErrors()); + } + + @Test + public void MERGE() throws Exception + { + if (null == ListService.get()) + return; + INSERT(); + assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var mergeRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); + BatchValidationException errors = new BatchValidationException() + { + @Override + public void addRowError(ValidationException vex) + { + LogManager.getLogger(AbstractQueryUpdateService.class).error("test error", vex); + fail(vex.getMessage()); + } + }; + int count=0; + try (var tx = rTableInfo.getSchema().getScope().ensureTransaction()) + { + var ret = qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); + if (!errors.hasErrors()) + { + tx.commit(); + count = ret; + } + } + assertFalse("mergeRows error(s): " + errors.getMessage(), errors.hasErrors()); + assertEquals(2, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO", rows.get(2).get("s")); + // test existing row value is not updated + assertEquals(2, rows.get(2).get("i")); + // test new row + assertEquals("THREE", rows.get(3).get("s")); + assertNull(rows.get(3).get("i")); + + // merge should fail if duplicate keys are provided + errors = new BatchValidationException(); + mergeRows = new ArrayList<>(); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); + qus.mergeRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), errors, null, null); + assertTrue(errors.hasErrors()); + assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); + } + + @Test + public void UPDATE() throws Exception + { + if (null == ListService.get()) + return; + INSERT(); + assertEquals("Wrong number of rows after INSERT", 3, getRows().size()); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var updateRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + + // update using data iterator + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP")); + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(InsertOption.UPDATE); + var count = qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertFalse(context.getErrors().hasErrors()); + assertEquals(1, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO-UP", rows.get(2).get("s")); + // test existing row value is not updated/erased + assertEquals(2, rows.get(2).get("i")); + + // update should fail if a new record is provided + updateRows = new ArrayList<>(); + updateRows.add(CaseInsensitiveHashMap.of(pkName,123,colName,"NEW")); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertTrue(context.getErrors().hasErrors()); + + // Issue 52728: update should fail if duplicate key is provide + updateRows = new ArrayList<>(); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-2")); + updateRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO-UP-UP-2")); + + // use DIB + context = new DataIteratorContext(); + context.setInsertOption(InsertOption.UPDATE); + qus.loadRows(user, c, MapDataIterator.of(updateRows.get(0).keySet(), updateRows), context, null); + assertTrue(context.getErrors().hasErrors()); + assertTrue("Duplicate key error: " + context.getErrors().getMessage(), context.getErrors().getMessage().contains("Duplicate key provided: 2")); + + // use updateRows + if (!_useAlias) // _update using alias is not supported + { + BatchValidationException errors = new BatchValidationException(); + try + { + qus.updateRows(user, c, updateRows, null, errors, null, null); + } + catch (Exception e) + { + + } + assertTrue(errors.hasErrors()); + assertTrue("Duplicate key error: " + errors.getMessage(), errors.getMessage().contains("Duplicate key provided: 2")); + + } + } + + @Test + public void REPLACE() throws Exception + { + if (null == ListService.get()) + return; + assert(getRows().isEmpty()); + INSERT(); + + User user = TestContext.get().getUser(); + Container c = JunitUtil.getTestContainer(); + TableInfo rTableInfo = ((UserSchema)DefaultSchema.get(user, c).getSchema("lists")).getTable("R", null); + QueryUpdateService qus = requireNonNull(rTableInfo.getUpdateService()); + var mergeRows = new ArrayList>(); + String colName = _useAlias ? "s_alias" : "s"; + String pkName = _useAlias ? "pk_alias" : "pk"; + mergeRows.add(CaseInsensitiveHashMap.of(pkName,2,colName,"TWO")); + mergeRows.add(CaseInsensitiveHashMap.of(pkName,3,colName,"THREE")); + DataIteratorContext context = new DataIteratorContext(); + context.setInsertOption(InsertOption.REPLACE); + var count = qus.loadRows(user, c, MapDataIterator.of(mergeRows.get(0).keySet(), mergeRows), context, null); + assertFalse(context.getErrors().hasErrors()); + assertEquals(2, count); + var rows = getRows(); + // test existing row value is updated + assertEquals("TWO", rows.get(2).get("s")); + // test existing row value is updated + assertNull(rows.get(2).get("i")); + // test new row + assertEquals("THREE", rows.get(3).get("s")); + assertNull(rows.get(3).get("i")); + } + + @Test + public void IMPORT_IDENTITY() + { + if (null == ListService.get()) + return; + // TODO + } + + @Test + public void ALIAS_MERGE() throws Exception + { + _useAlias = true; + MERGE(); + } + + @Test + public void ALIAS_REPLACE() throws Exception + { + _useAlias = true; + REPLACE(); + } + + @Test + public void ALIAS_UPDATE() throws Exception + { + _useAlias = true; + UPDATE(); + } + } +} diff --git a/api/src/org/labkey/api/query/UserSchemaAction.java b/api/src/org/labkey/api/query/UserSchemaAction.java index 3479b23bc72..4f267a9c678 100644 --- a/api/src/org/labkey/api/query/UserSchemaAction.java +++ b/api/src/org/labkey/api/query/UserSchemaAction.java @@ -1,335 +1,335 @@ -/* - * Copyright (c) 2012-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.query; - -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.NullSafeBindException; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveMapWrapper; -import org.labkey.api.data.ActionButton; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.TableInfo; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.ViewContext; -import org.springframework.beans.PropertyValues; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.multipart.MultipartFile; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: 9/29/12 - */ -public abstract class UserSchemaAction extends FormViewAction -{ - protected QueryForm _form; - protected UserSchema _schema; - protected TableInfo _table; - - @Override - public BindException bindParameters(PropertyValues m) throws Exception - { - _form = createQueryForm(getViewContext()); - _schema = _form.getSchema(); - if (null == _schema) - { - throw new NotFoundException("Schema not found"); - } - _table = _schema.getTable(_form.getQueryName(), getBindParametersContainerFilter(), true, true); - if (null == _table) - { - throw new NotFoundException("Query not found"); - } - QueryUpdateForm command = new QueryUpdateForm(_table, getViewContext(), null); - if (command.isBulkUpdate()) - command.setValidateRequired(false); - BindException errors = new NullSafeBindException(new BeanUtilsPropertyBindingResult(command, "form")); - command.validateBind(errors); - return errors; - } - - // This ContainerFilter is applied to the underlying table that backs this UserSchemaAction. - // As a result all lookup fields, that respect container filters, in these views will populate - // with this container filter applied. - protected @Nullable ContainerFilter getBindParametersContainerFilter() - { - return QueryService.get().getContainerFilterForLookups(getContainer(), getUser()); - } - - protected QueryForm createQueryForm(ViewContext context) - { - QueryForm form = new QueryForm(); - form.setViewContext(getViewContext()); - form.bindParameters(getViewContext().getBindPropertyValues()); - - return form; - } - - @Override - public void validateCommand(QueryUpdateForm target, Errors errors) - { - } - - protected ButtonBar createSubmitCancelButtonBar(QueryUpdateForm form) - { - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - - ActionButton btnSubmit = new ActionButton(getViewContext().getActionURL(), "Submit") - .setActionType(ActionButton.Action.POST) - .setDisableOnClick(true); - - return bb.add( - btnSubmit, - new ButtonBuilder("Cancel").href(getCancelURL(form)).build() - ); - } - - private ActionURL getActionURLParam(ActionURL.Param param) - { - String url = getViewContext().getActionURL().getParameter(param); - if (url != null) - { - try - { - return new ActionURL(url); - } - catch (IllegalArgumentException ignored) {} - } - return null; - } - - /* - * NOTE (MAB) UserSchemaAction.appendNavTrail() uses getSuccessURL(null) for the nav trail link. - * That's not really right, since the success url and the back/cancel url could be different. - * - * I changed getSuccessURL(null) to return cancelUrl if it is provided. - */ - @Override - public ActionURL getSuccessURL(QueryUpdateForm form) - { - return resolveReturnUrl(form == null ? getActionURLParam(ActionURL.Param.cancelUrl) : null, form); - } - - private ActionURL resolveReturnUrl(@Nullable ActionURL returnUrl, QueryUpdateForm form) - { - if (null == returnUrl) - returnUrl = getActionURLParam(ActionURL.Param.returnUrl); - if (null == returnUrl) - { - if (_schema != null && _table != null) - returnUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); - else - returnUrl = QueryService.get().urlDefault(form.getContainer(), QueryAction.executeQuery, null, null); - } - return returnUrl; - } - - public ActionURL getCancelURL(QueryUpdateForm form) - { - ActionURL cancelURL = getActionURLParam(ActionURL.Param.cancelUrl); - return resolveReturnUrl(cancelURL, form); - } - - @Override - public void addNavTrail(NavTree root) - { - if (_table != null) - root.addChild(_table.getName(), getSuccessURL(null)); - } - - protected List> doInsertUpdate(QueryUpdateForm form, BindException errors, boolean insert) - { - TableInfo table = form.getTable(); - if (!table.hasPermission(form.getUser(), insert ? InsertPermission.class : UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - Map values = form.getTypedColumns(); - - // Allow for attachment-based columns - Map fileMap = getFileMap(); - if (null != fileMap) - { - for (String key : fileMap.keySet()) - { - // Check if the column has already been processed - if (!values.containsKey(key)) - { - SpringAttachmentFile file = new SpringAttachmentFile(fileMap.get(key)); - form.setTypedValue(key, file.isEmpty() ? null : file); - } - } - } - - values = form.getTypedColumns(); - - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - throw new IllegalArgumentException("The query '" + _table.getName() + "' in the schema '" + _schema.getName() + "' is not updatable."); - - - List> rows; - List> ret = null; - - if (form.isBulkUpdate()) - { - rows = new ArrayList<>(); - - // Merge the bulk edits back into the selected rows + validate - String[] pkValues = form.getSelectedRows(); - - if (pkValues == null || pkValues.length == 0) - errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Please reselect the rows to update."); - else if (table.getPkColumnNames().size() > 1) - { - errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Does not support update for multi-keyed tables."); - } - else - { - Map row; - String pkName = table.getPkColumnNames().get(0); - for (String pkValue : pkValues) - { - row = new CaseInsensitiveHashMap<>(); - for (Map.Entry entry : values.entrySet()) - { - // If a value is left as null it is considered untouched for a given row - if (entry.getValue() != null) - row.put(entry.getKey(), entry.getValue()); - } - - row.put(pkName, pkValue); - rows.add(row); - } - } - } - else - { - rows = Collections.singletonList(values); - } - - DbSchema dbschema = table.getSchema(); - try - { - try (DbScope.Transaction transaction = dbschema.getScope().ensureTransaction()) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; - QueryService.AuditAction auditAction = insert ? QueryService.AuditAction.INSERT : QueryService.AuditAction.UPDATE; - - // transaction audit - BatchValidationException batchErrors = new BatchValidationException(); - - AuditBehaviorType auditBehaviorType = table.getEffectiveAuditBehavior(); - if (auditBehaviorType != AuditBehaviorType.NONE) - { - if (transaction.getAuditEvent() != null) - auditEvent = transaction.getAuditEvent(); - else - { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction, getTransactionAuditDetails()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); - } - } - - if (insert) - { - ret = qus.insertRows(form.getUser(), form.getContainer(), rows, batchErrors, null, null); - } - else - { - // Currently, bulkUpdate doesn't support oldValues due to the need to re-query... - if (form.isBulkUpdate()) - { - ret = qus.updateRows(form.getUser(), form.getContainer(), rows, null, batchErrors, null, null); - } - else - { - Map oldValues = null; - if (form.getOldValues() instanceof Map) - { - oldValues = (Map) form.getOldValues(); - if (!(oldValues instanceof CaseInsensitiveMapWrapper)) - oldValues = new CaseInsensitiveMapWrapper<>(oldValues); - } - - // 18292 - updateRows expects a null list in the case of an "empty" or null map. - List> oldKeys = (oldValues == null || oldValues.isEmpty()) ? null : Collections.singletonList(oldValues); - ret = qus.updateRows(form.getUser(), form.getContainer(), rows, oldKeys, batchErrors, null, null); - } - } - if (batchErrors.hasErrors()) - batchErrors.addToErrors(errors); - - if (!errors.hasErrors()) - { - if (auditEvent != null) - auditEvent.addComment(auditAction, ret.size()); - transaction.commit(); // Only commit if there were no errors - } - - } - catch (SQLException x) - { - if (!RuntimeSQLException.isConstraintException(x)) - throw x; - errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); - } - catch (InvalidKeyException | DuplicateKeyException | DataIntegrityViolationException | RuntimeSQLException x) - { - errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); - } - catch (BatchValidationException x) - { - x.addToErrors(errors); - } - return ret; - } - catch (Exception x) - { - // Do this in a separate, outer try/catch so that we will have already committed or rolled back - // the transaction we started. Otherwise, our database connection is likely in a bad state and can't be - // reused when submitting the exception report. - errors.reject(SpringActionController.ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); - return null; - } - } -} +/* + * Copyright (c) 2012-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.query; + +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.NullSafeBindException; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveMapWrapper; +import org.labkey.api.data.ActionButton; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.TableInfo; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.ViewContext; +import org.springframework.beans.PropertyValues; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.multipart.MultipartFile; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: 9/29/12 + */ +public abstract class UserSchemaAction extends FormViewAction +{ + protected QueryForm _form; + protected UserSchema _schema; + protected TableInfo _table; + + @Override + public BindException bindParameters(PropertyValues m) throws Exception + { + _form = createQueryForm(getViewContext()); + _schema = _form.getSchema(); + if (null == _schema) + { + throw new NotFoundException("Schema not found"); + } + _table = _schema.getTable(_form.getQueryName(), getBindParametersContainerFilter(), true, true); + if (null == _table) + { + throw new NotFoundException("Query not found"); + } + QueryUpdateForm command = new QueryUpdateForm(_table, getViewContext(), null); + if (command.isBulkUpdate()) + command.setValidateRequired(false); + BindException errors = new NullSafeBindException(new BeanUtilsPropertyBindingResult(command, "form")); + command.validateBind(errors); + return errors; + } + + // This ContainerFilter is applied to the underlying table that backs this UserSchemaAction. + // As a result all lookup fields, that respect container filters, in these views will populate + // with this container filter applied. + protected @Nullable ContainerFilter getBindParametersContainerFilter() + { + return QueryService.get().getContainerFilterForLookups(getContainer(), getUser()); + } + + protected QueryForm createQueryForm(ViewContext context) + { + QueryForm form = new QueryForm(); + form.setViewContext(getViewContext()); + form.bindParameters(getViewContext().getBindPropertyValues()); + + return form; + } + + @Override + public void validateCommand(QueryUpdateForm target, Errors errors) + { + } + + protected ButtonBar createSubmitCancelButtonBar(QueryUpdateForm form) + { + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + + ActionButton btnSubmit = new ActionButton(getViewContext().getActionURL(), "Submit") + .setActionType(ActionButton.Action.POST) + .setDisableOnClick(true); + + return bb.add( + btnSubmit, + new ButtonBuilder("Cancel").href(getCancelURL(form)).build() + ); + } + + private ActionURL getActionURLParam(ActionURL.Param param) + { + String url = getViewContext().getActionURL().getParameter(param); + if (url != null) + { + try + { + return new ActionURL(url); + } + catch (IllegalArgumentException ignored) {} + } + return null; + } + + /* + * NOTE (MAB) UserSchemaAction.appendNavTrail() uses getSuccessURL(null) for the nav trail link. + * That's not really right, since the success url and the back/cancel url could be different. + * + * I changed getSuccessURL(null) to return cancelUrl if it is provided. + */ + @Override + public ActionURL getSuccessURL(QueryUpdateForm form) + { + return resolveReturnUrl(form == null ? getActionURLParam(ActionURL.Param.cancelUrl) : null, form); + } + + private ActionURL resolveReturnUrl(@Nullable ActionURL returnUrl, QueryUpdateForm form) + { + if (null == returnUrl) + returnUrl = getActionURLParam(ActionURL.Param.returnUrl); + if (null == returnUrl) + { + if (_schema != null && _table != null) + returnUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); + else + returnUrl = QueryService.get().urlDefault(form.getContainer(), QueryAction.executeQuery, null, null); + } + return returnUrl; + } + + public ActionURL getCancelURL(QueryUpdateForm form) + { + ActionURL cancelURL = getActionURLParam(ActionURL.Param.cancelUrl); + return resolveReturnUrl(cancelURL, form); + } + + @Override + public void addNavTrail(NavTree root) + { + if (_table != null) + root.addChild(_table.getName(), getSuccessURL(null)); + } + + protected List> doInsertUpdate(QueryUpdateForm form, BindException errors, boolean insert) + { + TableInfo table = form.getTable(); + if (!table.hasPermission(form.getUser(), insert ? InsertPermission.class : UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + Map values = form.getTypedColumns(); + + // Allow for attachment-based columns + Map fileMap = getFileMap(); + if (null != fileMap) + { + for (String key : fileMap.keySet()) + { + // Check if the column has already been processed + if (!values.containsKey(key)) + { + SpringAttachmentFile file = new SpringAttachmentFile(fileMap.get(key)); + form.setTypedValue(key, file.isEmpty() ? null : file); + } + } + } + + values = form.getTypedColumns(); + + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + throw new IllegalArgumentException("The query '" + _table.getName() + "' in the schema '" + _schema.getName() + "' is not updatable."); + + + List> rows; + List> ret = null; + + if (form.isBulkUpdate()) + { + rows = new ArrayList<>(); + + // Merge the bulk edits back into the selected rows + validate + String[] pkValues = form.getSelectedRows(); + + if (pkValues == null || pkValues.length == 0) + errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Please reselect the rows to update."); + else if (table.getPkColumnNames().size() > 1) + { + errors.reject(SpringActionController.ERROR_MSG, "Unable to update multiple rows. Does not support update for multi-keyed tables."); + } + else + { + Map row; + String pkName = table.getPkColumnNames().get(0); + for (String pkValue : pkValues) + { + row = new CaseInsensitiveHashMap<>(); + for (Map.Entry entry : values.entrySet()) + { + // If a value is left as null it is considered untouched for a given row + if (entry.getValue() != null) + row.put(entry.getKey(), entry.getValue()); + } + + row.put(pkName, pkValue); + rows.add(row); + } + } + } + else + { + rows = Collections.singletonList(values); + } + + DbSchema dbschema = table.getSchema(); + try + { + try (DbScope.Transaction transaction = dbschema.getScope().ensureTransaction()) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = null; + QueryService.AuditAction auditAction = insert ? QueryService.AuditAction.INSERT : QueryService.AuditAction.UPDATE; + + // transaction audit + BatchValidationException batchErrors = new BatchValidationException(); + + AuditBehaviorType auditBehaviorType = table.getEffectiveAuditBehavior(); + if (auditBehaviorType != AuditBehaviorType.NONE) + { + if (transaction.getAuditEvent() != null) + auditEvent = transaction.getAuditEvent(); + else + { + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction, getTransactionAuditDetails()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); + } + } + + if (insert) + { + ret = qus.insertRows(form.getUser(), form.getContainer(), rows, batchErrors, null, null); + } + else + { + // Currently, bulkUpdate doesn't support oldValues due to the need to re-query... + if (form.isBulkUpdate()) + { + ret = qus.updateRows(form.getUser(), form.getContainer(), rows, null, batchErrors, null, null); + } + else + { + Map oldValues = null; + if (form.getOldValues() instanceof Map) + { + oldValues = (Map) form.getOldValues(); + if (!(oldValues instanceof CaseInsensitiveMapWrapper)) + oldValues = new CaseInsensitiveMapWrapper<>(oldValues); + } + + // 18292 - updateRows expects a null list in the case of an "empty" or null map. + List> oldKeys = (oldValues == null || oldValues.isEmpty()) ? null : Collections.singletonList(oldValues); + ret = qus.updateRows(form.getUser(), form.getContainer(), rows, oldKeys, batchErrors, null, null); + } + } + if (batchErrors.hasErrors()) + batchErrors.addToErrors(errors); + + if (!errors.hasErrors()) + { + if (auditEvent != null) + auditEvent.addComment(auditAction, ret.size()); + transaction.commit(); // Only commit if there were no errors + } + + } + catch (SQLException x) + { + if (!RuntimeSQLException.isConstraintException(x)) + throw x; + errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); + } + catch (InvalidKeyException | DuplicateKeyException | DataIntegrityViolationException | RuntimeSQLException x) + { + errors.reject(SpringActionController.ERROR_MSG, x.getMessage()); + } + catch (BatchValidationException x) + { + x.addToErrors(errors); + } + return ret; + } + catch (Exception x) + { + // Do this in a separate, outer try/catch so that we will have already committed or rolled back + // the transaction we started. Otherwise, our database connection is likely in a bad state and can't be + // reused when submitting the exception report. + errors.reject(SpringActionController.ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); + return null; + } + } +} diff --git a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java index 41730f6a38e..3a5f1ea4fa7 100644 --- a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java +++ b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java @@ -1,220 +1,220 @@ -/* - * Copyright (c) 2013-2017 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * User: klum - * Date: 7/21/13 - */ -public class SampleTypeAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String EVENT_TYPE = "SampleSetAuditEvent"; - - public static final String COLUMN_NAME_SOURCE_LSID = "SourceLsid"; - public static final String COLUMN_NAME_SAMPLE_TYPE_NAME = "SampleSetName"; - public static final String COLUMN_NAME_INSERT_UPDATE_CHOICE = "InsertUpdateChoice"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SAMPLE_TYPE_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public SampleTypeAuditProvider() - { - super(new SampleTypeAuditDomainKind()); - } - - @Override - public String getEventName() - { - return EVENT_TYPE; - } - - @Override - public String getLabel() - { - return "Sample Type events"; - } - - @Override - public String getDescription() - { - return "Summarizes events from sample type inserts or updates"; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SAMPLE_TYPE_NAME.equalsIgnoreCase(col.getName())) - col.setLabel("Sample Type"); - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - col.setLabel("Reason"); - } - }; - } - - @Override - public Class getEventClass() - { - return (Class) SampleTypeAuditEvent.class; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - public static class SampleTypeAuditEvent extends AuditTypeEvent - { - private String _sourceLsid; - private String _sampleSetName; - private String _insertUpdateChoice; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public SampleTypeAuditEvent() - { - super(); - } - - public SampleTypeAuditEvent(Container container, String comment) - { - super(EVENT_TYPE, container, comment); - setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); - } - - public String getSourceLsid() - { - return _sourceLsid; - } - - public void setSourceLsid(String sourceLsid) - { - _sourceLsid = sourceLsid; - } - - public String getSampleSetName() - { - return _sampleSetName; - } - - public void setSampleSetName(String sampleSetName) - { - _sampleSetName = sampleSetName; - } - - public String getInsertUpdateChoice() - { - return _insertUpdateChoice; - } - - public void setInsertUpdateChoice(String insertUpdateChoice) - { - _insertUpdateChoice = insertUpdateChoice; - } - - - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("sourceLsid", getSourceLsid()); - elements.put("sampleSetName", getSampleSetName()); - elements.put("insertUpdateChoice", getInsertUpdateChoice()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class SampleTypeAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "SampleSetAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - private final Set _fields; - - public SampleTypeAuditDomainKind() - { - super(EVENT_TYPE); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_SOURCE_LSID, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SAMPLE_TYPE_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_INSERT_UPDATE_CHOICE, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2017 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * User: klum + * Date: 7/21/13 + */ +public class SampleTypeAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String EVENT_TYPE = "SampleSetAuditEvent"; + + public static final String COLUMN_NAME_SOURCE_LSID = "SourceLsid"; + public static final String COLUMN_NAME_SAMPLE_TYPE_NAME = "SampleSetName"; + public static final String COLUMN_NAME_INSERT_UPDATE_CHOICE = "InsertUpdateChoice"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SAMPLE_TYPE_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public SampleTypeAuditProvider() + { + super(new SampleTypeAuditDomainKind()); + } + + @Override + public String getEventName() + { + return EVENT_TYPE; + } + + @Override + public String getLabel() + { + return "Sample Type events"; + } + + @Override + public String getDescription() + { + return "Summarizes events from sample type inserts or updates"; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SAMPLE_TYPE_NAME.equalsIgnoreCase(col.getName())) + col.setLabel("Sample Type"); + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + col.setLabel("Reason"); + } + }; + } + + @Override + public Class getEventClass() + { + return (Class) SampleTypeAuditEvent.class; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + public static class SampleTypeAuditEvent extends AuditTypeEvent + { + private String _sourceLsid; + private String _sampleSetName; + private String _insertUpdateChoice; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public SampleTypeAuditEvent() + { + super(); + } + + public SampleTypeAuditEvent(Container container, String comment) + { + super(EVENT_TYPE, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); + } + + public String getSourceLsid() + { + return _sourceLsid; + } + + public void setSourceLsid(String sourceLsid) + { + _sourceLsid = sourceLsid; + } + + public String getSampleSetName() + { + return _sampleSetName; + } + + public void setSampleSetName(String sampleSetName) + { + _sampleSetName = sampleSetName; + } + + public String getInsertUpdateChoice() + { + return _insertUpdateChoice; + } + + public void setInsertUpdateChoice(String insertUpdateChoice) + { + _insertUpdateChoice = insertUpdateChoice; + } + + + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("sourceLsid", getSourceLsid()); + elements.put("sampleSetName", getSampleSetName()); + elements.put("insertUpdateChoice", getInsertUpdateChoice()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class SampleTypeAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "SampleSetAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + private final Set _fields; + + public SampleTypeAuditDomainKind() + { + super(EVENT_TYPE); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_SOURCE_LSID, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SAMPLE_TYPE_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_INSERT_UPDATE_CHOICE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 80c2cdf9d4d..013bf5d01ac 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1,2370 +1,2370 @@ -/* - * Copyright (c) 2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.experiment.api; - -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.math3.util.Precision; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.audit.AbstractAuditHandler; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.FileSystemAuditProvider; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.LongArrayList; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.collections.LongHashSet; -import org.labkey.api.data.AuditConfigurable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ConversionExceptionWithMessage; -import org.labkey.api.data.DatabaseCache; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DbSequence; -import org.labkey.api.data.DbSequenceManager; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.Parameter; -import org.labkey.api.data.ParameterMapStatement; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.OntologyObject; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpMaterialRunInput; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolApplication; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.NameExpressionOptionService; -import org.labkey.api.exp.api.SampleTypeDomainKindProperties; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.query.ExpMaterialTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.inventory.InventoryService; -import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.miniprofiler.Timing; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.qc.DataState; -import org.labkey.api.qc.SampleStatusService; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.publish.StudyPublishService; -import org.labkey.api.util.CPUTimer; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.experiment.SampleTypeAuditProvider; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.singleton; -import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; -import static org.labkey.api.data.CompareType.STARTS_WITH; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; -import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; -import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; -import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; -import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; -import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; -import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; -import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; -import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; -import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; - - -public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService -{ - public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; - public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; - - public static final List SUPPORTED_UNITS = new ArrayList<>(); - public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; - - static - { - SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); - SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); - } - - // columns that may appear in a row when only the sample status is updating. - public static final Set statusUpdateColumns = Set.of( - ExpMaterialTable.Column.Modified.name().toLowerCase(), - ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), - ExpMaterialTable.Column.SampleState.name().toLowerCase(), - ExpMaterialTable.Column.Folder.name().toLowerCase() - ); - - public static SampleTypeServiceImpl get() - { - return (SampleTypeServiceImpl) SampleTypeService.get(); - } - - private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); - - /** SampleType LSID -> Container cache */ - private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); - - /** ContainerId -> MaterialSources */ - private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> - { - Container c = ContainerManager.getForId(container); - if (c == null) - return Collections.emptySortedSet(); - - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); - }); - - Cache> getMaterialSourceCache() - { - return materialSourceCache; - } - - @Override @NotNull - public List getSupportedUnits() - { - return SUPPORTED_UNITS; - } - - @Nullable @Override - public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) - { - if (rawUnits == null) - return null; - if (rawUnits instanceof Unit u) - { - if (defaultUnits == null) - return u; - else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - else - return u; - } - if (!(rawUnits instanceof String rawUnitsString)) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - if (!StringUtils.isBlank(rawUnitsString)) - { - rawUnitsString = rawUnitsString.trim(); - - Unit mUnit = Unit.fromName(rawUnitsString); - List commonUnits = getSupportedUnits(); - if (mUnit == null || !commonUnits.contains(mUnit)) - { - throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); - } - if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) - throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); - return mUnit; - } - return null; - } - - public void clearMaterialSourceCache(@Nullable Container c) - { - LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); - if (c == null) - materialSourceCache.clear(); - else - materialSourceCache.remove(c.getId()); - } - - - private TableInfo getTinfoMaterialSource() - { - return ExperimentServiceImpl.get().getTinfoSampleType(); - } - - private TableInfo getTinfoMaterial() - { - return ExperimentServiceImpl.get().getTinfoMaterial(); - } - - private TableInfo getTinfoProtocolApplication() - { - return ExperimentServiceImpl.get().getTinfoProtocolApplication(); - } - - private TableInfo getTinfoProtocol() - { - return ExperimentServiceImpl.get().getTinfoProtocol(); - } - - private TableInfo getTinfoMaterialInput() - { - return ExperimentServiceImpl.get().getTinfoMaterialInput(); - } - - private TableInfo getTinfoExperimentRun() - { - return ExperimentServiceImpl.get().getTinfoExperimentRun(); - } - - private TableInfo getTinfoDataClass() - { - return ExperimentServiceImpl.get().getTinfoDataClass(); - } - - private TableInfo getTinfoProtocolInput() - { - return ExperimentServiceImpl.get().getTinfoProtocolInput(); - } - - private TableInfo getTinfoMaterialAliasMap() - { - return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); - } - - private DbSchema getExpSchema() - { - return ExperimentServiceImpl.getExpSchema(); - } - - @Override - public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - if (sampleType == null) - return; - - queue.addRunnable((q) -> { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT * FROM ") - .append(getTinfoMaterialSource(), "ms") - .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND ms.LSID = ?").add(sampleType.getLSID()) - .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") - .add(sampleType.getModified()); - - MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); - if (materialSource != null) - { - ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); - impl.index(q, null); - } - - indexSampleTypeMaterials(sampleType, q); - }); - } - - private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) - { - // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed - SQLFragment sql = new SQLFragment("SELECT m.* FROM ") - .append(getTinfoMaterial(), "m") - .append(" LEFT OUTER JOIN ") - .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") - .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) - .append(" AND m.cpasType = ?").add(sampleType.getLSID()) - .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") - .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock - .add(sampleType.getModified()); - - new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { - for (Material m : batch) - { - ExpMaterialImpl impl = new ExpMaterialImpl(m); - impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); - } - }); - } - - - @Override - public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) - { - SQLFragment sql = new SQLFragment(); - sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); - sql.append(getTinfoMaterial(), "m"); - sql.append(", "); - sql.append(getTinfoMaterialInput(), "mi"); - sql.append(", "); - sql.append(getTinfoProtocolApplication(), "pa"); - sql.append(", "); - sql.append(getTinfoExperimentRun(), "r"); - - if (type != null) - { - sql.append(", "); - sql.append(getTinfoProtocol(), "p"); - sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); - sql.add(type.toString()); - } - else - { - sql.append(" WHERE "); - } - - sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + - "pa.RunId = r.RowId AND "); - sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); - sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); - - Map result = new LinkedHashMap<>(); - for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) - { - ExpSampleType sampleType = null; - String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); - String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); - - // Check if we have a sample type that was being referenced - if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) - { - // If the min and the max are the same, it means all rows share the same value so we know that there's - // a single sample type being targeted - sampleType = getSampleType(container, maxSampleTypeLSID); - } - result.put((String) queryResult.get("Role"), sampleType); - } - return result; - } - - @Override - public void removeAutoLinkedStudy(@NotNull Container studyContainer) - { - SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) - .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") - .add(studyContainer.getId()); - new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); - } - - public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) - { - OntologyObject obj = OntologyManager.getOntologyObject(objectId); - if (obj == null) - return null; - - return getSampleType(obj.getObjectURI()); - } - - @Override - public @Nullable ExpSampleType getEffectiveSampleType( - @NotNull Container definitionContainer, - @NotNull String sampleTypeName, - @NotNull Date effectiveDate, - @Nullable ContainerFilter cf - ) - { - Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); - if (legacyObjectId != null) - return getSampleTypeByObjectId(legacyObjectId); - - boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; - ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); - if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) - return sampleType; - - return null; - } - - @Override - public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); - - // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres - TreeSet result = new TreeSet<>(); - for (String containerId : containerIds) - { - for (MaterialSource source : getMaterialSourceCache().get(containerId)) - { - result.add(new ExpSampleTypeImpl(source)); - } - } - - return List.copyOf(result); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) - { - return getSampleType(c, false, sampleTypeName); - } - - // NOTE: This method used to not take a user or check permissions - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) - { - return getSampleType(c, true, sampleTypeName); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) - { - return getSampleType(c, rowId, false); - } - - @Override - public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) - { - return getSampleType(c, rowId, true); - } - - @Override - public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) - { - Container c = hint; - String id = sampleTypeCache.get(lsid); - if (null != id && (null == hint || !id.equals(hint.getId()))) - c = ContainerManager.getForId(id); - ExpSampleTypeImpl st = null; - if (null != c) - st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); - if (null == st) - st = _getSampleType(lsid); - if (null != st && null==id) - sampleTypeCache.put(lsid,st.getContainer().getId()); - return st; - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) - { - return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); - } - - private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) - { - List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); - for (String containerId : containerIds) - { - Collection sampleTypes = getMaterialSourceCache().get(containerId); - for (MaterialSource materialSource : sampleTypes) - { - if (predicate.test(materialSource)) - return new ExpSampleTypeImpl(materialSource); - } - } - - return null; - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(long rowId) - { - // TODO: Cache - MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); - if (materialSource == null) - return null; - - return new ExpSampleTypeImpl(materialSource); - } - - @Nullable - @Override - public ExpSampleTypeImpl getSampleType(String lsid) - { - return getSampleTypeByType(lsid, null); - } - - @Nullable - @Override - public DataState getSampleState(Container container, Long stateRowId) - { - return SampleStatusService.get().getStateForRowId(container, stateRowId); - } - - private ExpSampleTypeImpl _getSampleType(String lsid) - { - MaterialSource ms = getMaterialSource(lsid); - if (ms == null) - return null; - - return new ExpSampleTypeImpl(ms); - } - - public MaterialSource getMaterialSource(String lsid) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); - return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); - } - - public DbScope.Transaction ensureTransaction() - { - return getExpSchema().getScope().ensureTransaction(); - } - - @Override - public Lsid getSampleTypeLsid(String sourceName, Container container) - { - return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); - } - - @Override - public Pair getSampleTypeSamplePrefixLsids(Container container) - { - Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); - String sampleTypeLsidStr = lsidDbSeq.first; - Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); - - String dbSeqStr = lsidDbSeq.second; - String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); - - return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); - } - - /** - * Delete all exp.Material from the SampleType. If container is not provided, - * all rows from the SampleType will be deleted regardless of container. - */ - public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) - { - assert getExpSchema().getScope().isTransactionActive(); - - Set containers = new HashSet<>(); - if (c == null) - { - SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); - containerSql.append(getTinfoMaterial(), "m"); - containerSql.append(" WHERE CpasType = ?"); - containerSql.add(source.getLSID()); - new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); - } - else - { - containers.add(c); - } - - int count = 0; - for (Container toDelete : containers) - { - SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); - sqlFilter.add(source.getLSID()); - sqlFilter.add(toDelete); - count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); - } - return count; - } - - @Override - public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException - { - CPUTimer timer = new CPUTimer("delete sample type"); - timer.start(); - - ExpSampleTypeImpl source = getSampleType(c, user, rowId); - if (null == source) - throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); - if (!source.getContainer().equals(c)) - throw new ExperimentException("Trying to delete a SampleType from a different container"); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway - // TODO do we need both truncateSampleType() and deleteDomainObjects()? - truncateSampleType(source, user, null); - - StudyService studyService = StudyService.get(); - if (studyService != null) - { - for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) - { - dataset.delete(user, auditUserComment); - } - } - else - { - LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); - } - - Domain d = source.getDomain(); - d.delete(user, auditUserComment); - - ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); - - SqlExecutor executor = new SqlExecutor(getExpSchema()); - executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); - executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); - - addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); - - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); - ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); - - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.commit(); - } - - // Delete sequences (genId and the unique counters) - DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); - - SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); - QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); - - SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); - QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); - - // Remove SampleType from search index - try (Timing ignored = MiniProfiler.step("search docs")) - { - SearchService.get().deleteResource(source.getDocumentId()); - } - - timer.stop(); - LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); - } - - private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) - { - addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); - } - - private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) - { - SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); - event.setUserComment(auditUserComment); - - if (sampleType != null) - { - event.setSourceLsid(sampleType.getLSID()); - event.setSampleSetName(sampleType.getName()); - } - event.setInsertUpdateChoice(insertUpdateChoice); - AuditLogService.get().addEvent(user, event); - } - - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType() - { - return new ExpSampleTypeImpl(new MaterialSource()); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) - throws ExperimentException - { - return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, @Nullable TemplateInfo templateInfo) - throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, - parentCol, nameExpression, null, templateInfo, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException - { - return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); - } - - @NotNull - @Override - public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, - String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, - @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, - @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) - throws ExperimentException - { - validateSampleTypeName(c, u, name, false); - - if (properties == null || properties.isEmpty()) - throw new ApiUsageException("At least one property is required"); - - if (idCol2 != -1 && idCol1 == idCol2) - throw new ApiUsageException("You cannot use the same id column twice."); - - if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) - throw new ApiUsageException("You cannot use the same id column twice."); - - if ((idCol1 > -1 && idCol1 >= properties.size()) || - (idCol2 > -1 && idCol2 >= properties.size()) || - (idCol3 > -1 && idCol3 >= properties.size()) || - (parentCol > -1 && parentCol >= properties.size())) - throw new ApiUsageException("column index out of range"); - - // Name expression is only allowed when no idCol is set - if (nameExpression != null && idCol1 > -1) - throw new ApiUsageException("Name expression cannot be used with id columns"); - - NameExpressionOptionService svc = NameExpressionOptionService.get(); - if (!svc.allowUserSpecifiedNames(c)) - { - if (nameExpression == null) - throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - if (svc.getExpressionPrefix(c) != null) - { - // automatically apply the configured prefix to the name expression - nameExpression = svc.createPrefixedExpression(c, nameExpression, false); - aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); - } - - // Validate the name expression length - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); - if (nameExpression != null && nameExpression.length() > nameExpMax) - throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); - - // Validate the aliquot name expression length - int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); - if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) - throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); - - // Validate the label color length - int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); - if (labelColor != null && labelColor.length() > labelColorMax) - throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); - - // Validate the metricUnit length - int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); - if (metricUnit != null && metricUnit.length() > metricUnitMax) - throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); - - // Validate the category length - int categoryMax = materialSourceTable.getColumn("Category").getScale(); - if (category != null && category.length() > categoryMax) - throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); - - Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); - String lsid = dbSeqLsids.first; - String materialPrefixLsid = dbSeqLsids.second; - Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); - DomainKind kind = domain.getDomainKind(); - if (kind != null) - domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); - Set reservedNames = kind.getReservedPropertyNames(domain, u); - Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - boolean hasNameProperty = false; - String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - List calculatedFields = new ArrayList<>(); - for (int i = 0; i < properties.size(); i++) - { - GWTPropertyDescriptor pd = properties.get(i); - String propertyName = pd.getName().toLowerCase(); - - // calculatedFields will be handled separately - if (pd.getValueExpression() != null) - { - calculatedFields.add(pd); - continue; - } - - if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) - { - hasNameProperty = true; - } - else - { - if (!reservedPrefixes.isEmpty()) - { - Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); - reservedPrefix.ifPresent(s -> { - throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); - }); - } - - if (lowerReservedNames.contains(propertyName)) - { - throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); - } - - DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); - - if (dp != null) - { - if (idCol1 == i) idUri1 = dp.getPropertyURI(); - if (idCol2 == i) idUri2 = dp.getPropertyURI(); - if (idCol3 == i) idUri3 = dp.getPropertyURI(); - if (parentCol == i) parentUri = dp.getPropertyURI(); - } - } - } - - domain.setPropertyIndices(indices, lowerReservedNames); - - if (!hasNameProperty && idUri1 == null) - throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); - - if (hasNameProperty && idUri1 != null) - throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); - - String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); - - MaterialSource source = new MaterialSource(); - source.setLSID(lsid); - source.setName(name); - source.setDescription(description); - source.setMaterialLSIDPrefix(materialPrefixLsid); - if (nameExpression != null) - source.setNameExpression(nameExpression); - if (aliquotNameExpression != null) - source.setAliquotNameExpression(aliquotNameExpression); - source.setLabelColor(labelColor); - source.setMetricUnit(metricUnit); - source.setAutoLinkTargetContainer(autoLinkTargetContainer); - source.setAutoLinkCategory(autoLinkCategory); - source.setCategory(category); - source.setContainer(c); - source.setMaterialParentImportAliasMap(importAliasJson); - - if (hasNameProperty) - { - source.setIdCol1(ExpMaterialTable.Column.Name.name()); - } - else - { - source.setIdCol1(idUri1); - if (idUri2 != null) - source.setIdCol2(idUri2); - if (idUri3 != null) - source.setIdCol3(idUri3); - } - if (parentUri != null) - source.setParentCol(parentUri); - - final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); - - try - { - getExpSchema().getScope().executeWithRetry(transaction -> - { - try - { - domain.save(u, changeDetails, calculatedFields); - st.save(u); - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); - DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); - if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); - if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) - ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); - else - ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); - transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); - - return st; - } - catch (ExperimentException | MetadataUnavailableException eex) - { - throw new DbScope.RetryPassthroughException(eex); - } - }); - } - catch (DbScope.RetryPassthroughException x) - { - x.rethrow(ExperimentException.class); - throw x; - } - - return st; - } - - public enum SampleSequenceType - { - DAILY("yyyy-MM-dd"), - WEEKLY("YYYY-'W'ww"), - MONTHLY("yyyy-MM"), - YEARLY("yyyy"); - - final DateTimeFormatter _formatter; - - SampleSequenceType(String pattern) - { - _formatter = DateTimeFormatter.ofPattern(pattern); - } - - public Pair getSequenceName(@Nullable Date date) - { - LocalDateTime ldt; - if (date == null) - ldt = LocalDateTime.now(); - else - ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); - String suffix = _formatter.format(ldt); - // NOTE: it would make sense to use the dbsequence "id" feature here. - // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 - // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 - // however, that would require a fix up on upgrade. - return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); - } - - public long next(Date date) - { - return getDbSequence(date).next(); - } - - public DbSequence getDbSequence(Date date) - { - Pair seqName = getSequenceName(date); - return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); - } - } - - - @Override - public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) - { - final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); - final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); - final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); - final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); - - return (counts) -> - { - if (null==counts) - counts = new HashMap<>(); - counts.put("dailySampleCount", dailySampleCount.next()); - counts.put("weeklySampleCount", weeklySampleCount.next()); - counts.put("monthlySampleCount", monthlySampleCount.next()); - counts.put("yearlySampleCount", yearlySampleCount.next()); - return counts; - }; - } - - @Override - public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) - { - if (name == null || StringUtils.isBlank(name)) - throw new ApiUsageException("Sample Type name is required."); - - TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); - int nameMax = materialSourceTable.getColumn("Name").getScale(); - if (name.length() > nameMax) - throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); - - if (!skipExistingCheck) - { - if (getSampleType(container, user, name) != null) - throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); - } - - String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); - if (reservedError != null) - throw new ApiUsageException(reservedError); - } - - @Override - public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException errors; - - ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); - - StringBuilder changeDetails = new StringBuilder(); - - Map oldProps = new LinkedHashMap<>(); - Map newProps = new LinkedHashMap<>(); - - String newName = StringUtils.trimToNull(update.getName()); - String oldSampleTypeName = st.getName(); - oldProps.put("Name", oldSampleTypeName); - newProps.put("Name", newName); - - boolean hasNameChange = false; - if (!oldSampleTypeName.equals(newName)) - { - validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); - hasNameChange = true; - st.setName(newName); - changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); - } - - String newDescription = StringUtils.trimToNull(update.getDescription()); - String description = st.getDescription(); - if (StringUtils.isNotBlank(description)) - oldProps.put("Description", description); - if (StringUtils.isNotBlank(newDescription)) - newProps.put("Description", newDescription); - if (description == null || !description.equals(newDescription)) - st.setDescription(newDescription); - - Map oldProps_ = st.getAuditRecordMap(); - Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; - newProps.putAll(newProps_); - oldProps.putAll(oldProps_); - - if (options != null) - { - String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); - String oldPattern = st.getNameExpression(); - if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) - { - st.setNameExpression(sampleIdPattern); - if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) - throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); - } - - String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); - String oldAliquotPattern = st.getAliquotNameExpression(); - if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) - st.setAliquotNameExpression(aliquotIdPattern); - - st.setLabelColor(options.getLabelColor()); - st.setMetricUnit(options.getMetricUnit()); - - if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) - { - try - { - Map> newAliases = options.getImportAliases(); - Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); - String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); - if (invalidParentType != null) - throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); - - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } - - st.setImportAliasMap(options.getImportAliases()); - String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); - st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); - st.setAutoLinkCategory(options.getAutoLinkCategory()); - if (options.getCategory() != null) // update sample type category is currently not supported - st.setCategory(options.getCategory()); - } - - try (DbScope.Transaction transaction = ensureTransaction()) - { - st.save(user); - if (hasNameChange) - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); - - if (options != null && options.getExcludedContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); - oldProps.put("ContainerExclusions", exclusionChanges.first); - newProps.put("ContainerExclusions", exclusionChanges.second); - } - if (options != null && options.getExcludedDashboardContainerIds() != null) - { - Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); - oldProps.put("DashboardContainerExclusions", exclusionChanges.first); - newProps.put("DashboardContainerExclusions", exclusionChanges.second); - } - - errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); - - if (!errors.hasErrors()) - { - QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (hasNameChange) - ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); - - transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); - transaction.commit(); - refreshSampleTypeMaterializedView(st, SampleChangeType.schema); - } - } - catch (MetadataUnavailableException e) - { - errors = new ValidationException(); - errors.addError(new SimpleValidationError(e.getMessage())); - } - - return errors; - } - - public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) - { - String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); - return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; - } - - @Override - public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) - { - return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); - } - - @Override - protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) - { - return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) - { - return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); - } - - private boolean isInputFieldKey(String fieldKey) - { - int slash = fieldKey.indexOf('/'); - return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || - slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); - } - - private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); - event.setUserComment(userComment); - - var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; - if (row != null) - { - Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); - event.setLineageUpdate(parentFields.isPresent()); - - if (staticsRow.containsKey(LSID)) - event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); - if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) - event.setSampleId((Integer) staticsRow.get(ROW_ID)); - if (staticsRow.containsKey(NAME)) - event.setSampleName(String.valueOf(staticsRow.get(NAME))); - - String sampleTypeLsid = null; - if (staticsRow.containsKey(CPAS_TYPE)) - sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); - // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" - if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) - sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); - - ExpSampleType sampleType = null; - if (sampleTypeLsid != null) - sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); - else if (event.getSampleId() > 0) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); - if (sample != null) sampleType = sample.getSampleType(); - } - else if (event.getSampleLsid() != null) - { - ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); - if (sample != null) sampleType = sample.getSampleType(); - } - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - - // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) - row.put(ROW_ID,staticsRow.get(ROW_ID)); - } - else if (tInfo != null) - { - UserSchema schema = tInfo.getUserSchema(); - if (schema != null) - { - ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); - if (sampleType != null) - { - event.setSampleType(sampleType.getName()); - event.setSampleTypeId(sampleType.getRowId()); - } - } - } - - // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns - if (existingRow != null && !existingRow.isEmpty()) - { - if (existingRow.containsKey(RawAmount.name())) - existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); - if (existingRow.containsKey(RawUnits.name())) - existingRow.put(Units.name(), existingRow.get(RawUnits.name())); - } - - // Add providedValues to eventMetadata - Map eventMetadata = new HashMap<>(); - if (providedValues != null) - { - eventMetadata.putAll(providedValues); - } - if (action != null) - { - SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); - if (timelineEventType != null) - eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); - } - if (!eventMetadata.isEmpty()) - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); - - return event; - } - - private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) - { - SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); - event.setSampleName(sample.getName()); - event.setSampleLsid(sample.getLSID()); - event.setSampleId(sample.getRowId()); - ExpSampleType type = sample.getSampleType(); - if (type != null) - { - event.setSampleType(type.getName()); - event.setSampleTypeId(type.getRowId()); - } - event.setUserComment(userComment); - event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); - return event; - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) - { - AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); - } - - @Override - public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) - { - SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); - event.setInventoryUpdateType(updateType); - event.setUserComment(userComment); - AuditLogService.get().addEvent(user, event); - } - - @Override - public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) - { - long max = 0; - String aliquotNamePrefix = sampleName + "-"; - - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); - filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); - - TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); - final List aliquotIds = new ArrayList<>(); - selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); - - for (String aliquotId : aliquotIds) - { - try - { - long id = Long.parseLong(aliquotId); - if (id > max) - max = id; - } - catch (NumberFormatException ignored) { - } - } - - return max; - } - - @Override - public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) - { - return samples.stream() - .filter(sample -> !sample.isOperationPermitted(operation)) - .collect(Collectors.toList()); - } - - @Override - public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) - { - String message; - if (samples.size() == 1) - { - ExpMaterial sample = samples.iterator().next(); - message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; - } - else - { - message = samples.size() + " samples ("; - message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); - if (samples.size() > 10) - message += " ..."; - message += ") have statuses that prevent"; - } - return message + " " + operation.getDescription() + "."; - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException - { - Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); - Collection allParents = parentsGroup.first; - Collection withAmountsParents = parentsGroup.second; - return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - @Override - public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); - } - - public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} - - public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException - { - return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container, false); - } - - /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ - public int recomputeSamplesRollup( - Collection parents, - @Nullable Collection availableParents, - Collection withAmountsParents, - String sampleTypeUnit, - Container container, - boolean useRootMaterialLSID - ) throws IllegalStateException, SQLException - { - Map sampleUnits = new LongHashMap<>(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); - DbScope scope = materialTable.getSchema().getScope(); - - List availableSampleStates = new LongArrayList(); - - if (SampleStatusService.get().supportsSampleStatus()) - { - for (DataState state: SampleStatusService.get().getAllProjectStates(container)) - { - if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) - availableSampleStates.add(state.getRowId()); - } - } - - if (!parents.isEmpty()) - { - Map> sampleAliquotCounts = getSampleAliquotCounts(parents, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotCount: sublist) - { - Long sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) - { - Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates, useRootMaterialLSID); - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); - - List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); - - ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> - { - for (var sampleAliquotCount: sublist) - { - var sampleId = sampleAliquotCount.getKey(); - Integer aliquotCount = sampleAliquotCount.getValue().first; - String sampleUnit = sampleAliquotCount.getValue().second; - sampleUnits.put(sampleId, sampleUnit); - - rowid.setValue(sampleId); - count.setValue(aliquotCount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - if (!withAmountsParents.isEmpty()) - { - Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates, useRootMaterialLSID); - - try (Connection c = scope.getConnection()) - { - Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); - Parameter amount = new Parameter("amount", JdbcType.DOUBLE); - Parameter unit = new Parameter("unit", JdbcType.VARCHAR); - Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); - - ParameterMapStatement pm = new ParameterMapStatement(scope, c, - new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); - - List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); - - ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> - { - for (Map.Entry> sampleAliquotAmounts: sublist) - { - Long sampleId = sampleAliquotAmounts.getKey(); - List aliquotAmounts = sampleAliquotAmounts.getValue(); - - if (aliquotAmounts == null || aliquotAmounts.isEmpty()) - continue; - AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); - rowid.setValue(sampleId); - amount.setValue(amountUnit.amount); - unit.setValue(amountUnit.unit); - availableAmount.setValue(amountUnit.availableAmount); - - pm.addBatch(); - } - pm.executeBatch(); - }); - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - } - - return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); - } - - @Override - public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException - { - Set rootSamplesToRecalc = new LongHashSet(); - if (rootRowIds != null) - rootSamplesToRecalc.addAll(rootRowIds); - if (parentNames != null) - rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); - - return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); - } - - private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) - { - if (parentNames == null || parentNames.isEmpty()) - return Collections.emptySet(); - - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - - SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") - .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) - .append(" AND rowid IN (") - .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") - .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) - .append(")"); - - return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); - } - - private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) - { - if (volumeUnits == null || volumeUnits.isEmpty()) - return null; - - Unit totalUnit = null; - String totalUnitsStr; - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnitsStr = sampleTypeUnitsStr; - else if (!StringUtils.isEmpty(sampleItemUnitsStr)) - totalUnitsStr = sampleItemUnitsStr; - else // use the unit of the first aliquot if there are no other indications - totalUnitsStr = volumeUnits.get(0).unit; - if (!StringUtils.isEmpty(totalUnitsStr)) - { - try - { - if (!StringUtils.isEmpty(sampleTypeUnitsStr)) - totalUnit = Unit.valueOf(totalUnitsStr).getBase(); - else - totalUnit = Unit.valueOf(totalUnitsStr); - } - catch (IllegalArgumentException e) - { - // do nothing; leave unit as null - } - } - - double totalVolume = 0.0; - double totalAvailableVolume = 0.0; - - for (AliquotAmountUnitResult volumeUnit : volumeUnits) - { - Unit unit = null; - try - { - double storedAmount = volumeUnit.amount; - String aliquotUnit = volumeUnit.unit; - boolean isAvailable = volumeUnit.isAvailable; - - try - { - unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); - } - catch (IllegalArgumentException ignore) - { - } - - double convertedAmount = 0; - // include in total volume only if aliquot unit is compatible - if (totalUnit != null && totalUnit.isCompatible(unit)) - convertedAmount = Unit.convert(storedAmount, unit, totalUnit); - else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit - { - if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty - convertedAmount = storedAmount; - else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') - convertedAmount = storedAmount; - } - - totalVolume += convertedAmount; - if (isAvailable) - totalAvailableVolume += convertedAmount; - } - catch (IllegalArgumentException ignore) // invalid volume - { - - } - } - int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); - totalVolume = Precision.round(totalVolume, scale); - totalAvailableVolume = Precision.round(totalAvailableVolume, scale); - - return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); - } - - public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException - { - Collection parents = getAliquotParents(sampleTypeLsid, container); - Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); - return new Pair<>(parents, withAmountsParents); - } - - private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, false, container); - } - - private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException - { - return getAliquotParents(sampleTypeLsid, true, container); - } - - private SQLFragment getParentsOfAliquotsWithAmountsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE aliquot.storedAmount IS NOT NULL AND\s - """); - } - - private SQLFragment getParentsOfAliquotsSql() - { - return new SQLFragment( - """ - SELECT DISTINCT parent.rowId, parent.cpastype - FROM exp.material AS aliquot - JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId - WHERE - """); - } - - private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - - SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); - - sql.append("parent.cpastype = ?"); - sql.add(sampleTypeLsid); - sql.append(" AND parent.container = ?"); - sql.add(container.getId()); - - Set parentIds = new LongHashSet(); - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - parentIds.add(rs.getLong(1)); - } - - return parentIds; - } - - private Map> getSampleAliquotCounts(Collection sampleIds, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") - .append(useRootMaterialLSID ? "a.rootMaterialLsid = m.lsid" : "a.rootMaterialRowId = m.rowId") - .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema dbSchema = getExpSchema(); - SqlDialect dialect = dbSchema.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql; - if (useRootMaterialLSID) - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialLSID as rootLsid, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialLSID <> LSID AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialLSID - ) AS c ON m.lsid = c.rootLsid - WHERE m.rootmateriallsid = m.LSID AND m.rowid\s"""); - } - else - { - sql = new SQLFragment( - """ - SELECT m.RowId as SampleId, m.Units, - (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount - FROM exp.material AS m - LEFT JOIN ( - SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount - FROM exp.material - WHERE RootMaterialRowId <> RowId AND SampleState\s""") - .appendInClause(availableSampleStates, dialect) - .append(""" - GROUP BY RootMaterialRowId - ) AS c ON m.rowId = c.rootRowId - WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); - } - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer - try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - String sampleUnit = rs.getString(2); - int aliquotCount = rs.getInt(3); - - sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); - } - } - - return sampleAliquotCounts; - } - - private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates, boolean useRootMaterialLSID) throws SQLException - { - DbSchema exp = getExpSchema(); - SqlDialect dialect = exp.getSqlDialect(); - - // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an - // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not - // have run yet. - SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") - .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") - .append(useRootMaterialLSID ? "parent.lsid = aliquot.rootmateriallsid" : "parent.rowid = aliquot.rootmaterialrowid") - .append(" WHERE ") - .append(useRootMaterialLSID ? "aliquot.rootmateriallsid <> aliquot.lsid" : "aliquot.rootmaterialrowid <> aliquot.rowid") - .append(" AND parent.rowid "); - dialect.appendInClauseSql(sql, sampleIds); - - Map> sampleAliquotAmounts = new LongHashMap<>(); - - try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) - { - while (rs.next()) - { - long parentId = rs.getLong(1); - Double volume = rs.getDouble(2); - String unit = rs.getString(3); - long sampleState = rs.getLong(4); - - if (!sampleAliquotAmounts.containsKey(parentId)) - sampleAliquotAmounts.put(parentId, new ArrayList<>()); - - sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); - } - } - // for any parents with no remaining aliquots, set the amounts to 0 - for (var parentId : sampleIds) - { - if (!sampleAliquotAmounts.containsKey(parentId)) - { - List aliquotAmounts = new ArrayList<>(); - aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); - sampleAliquotAmounts.put(parentId, aliquotAmounts); - } - } - - return sampleAliquotAmounts; - } - - record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } - - @Override - public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException - { - if (samples == null || samples.isEmpty()) - throw new IllegalArgumentException("No samples provided to move operation."); - - Map> sampleTypesMap = new HashMap<>(); - samples.forEach(sample -> - sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); - Map updateCounts = new HashMap<>(); - updateCounts.put("samples", 0); - updateCounts.put("sampleAliases", 0); - updateCounts.put("sampleAuditEvents", 0); - Map> fileMovesBySampleId = new LongHashMap<>(); - ExperimentService expService = ExperimentService.get(); - - try (DbScope.Transaction transaction = ensureTransaction()) - { - if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(samples.size()); - AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); - } - - for (Map.Entry> entry: sampleTypesMap.entrySet()) - { - ExpSampleType sampleType = entry.getKey(); - SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); - TableInfo samplesTable = schema.getTable(sampleType, null); - - List typeSamples = entry.getValue(); - List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); - - // update for exp.material.container - updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); - - // update for exp.object.container - expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); - - // update the paths to files associated with individual samples - fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); - - // update for exp.materialaliasmap.container - updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); - - // update inventory.item.container - InventoryService inventoryService = InventoryService.get(); - if (inventoryService != null) - { - Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); - inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); - } - - // create summary audit entries for the source and target containers - String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); - addSampleTypeAuditEvent(user, sourceContainer, sampleType, - "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); - addSampleTypeAuditEvent(user, targetContainer, sampleType, - "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); - - // move the events associated with the samples that have moved - SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); - int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); - updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); - - AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); - // create new events for each sample that was moved. - if (stAuditBehavior == AuditBehaviorType.DETAILED) - { - for (ExpMaterial sample : typeSamples) - { - SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); - Map oldRecordMap = new HashMap<>(); - // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't - // use "Folder" here because this sample-type field is filtered out of timeline events by default - oldRecordMap.put("ContainerName", sourceContainer.getName()); - Map newRecordMap = new HashMap<>(); - newRecordMap.put("ContainerName", targetContainer.getName()); - if (fileMovesBySampleId.containsKey(sample.getRowId())) - { - fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { - oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); - newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); - }); - } - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); - AuditLogService.get().addEvent(user, event); - } - } - } - - updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); - - transaction.addCommitTask(() -> { - for (ExpSampleType sampleType : sampleTypesMap.keySet()) - { - // force refresh of materialized view - SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); - // update search index for moved samples via indexSampleType() helper, it filters for samples to index - // based on the modified date - SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); - } - }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); - - // add up the size of the value arrays in the fileMovesBySampleId map - int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); - updateCounts.put("sampleFiles", fileMoveCount); - transaction.addCommitTask(() -> { - for (List sampleFileRenameData : fileMovesBySampleId.values()) - { - for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); - } - }, POSTCOMMIT); - - transaction.commit(); - } - - return updateCounts; - } - - private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - // collect unique runIds mapped to the samples that are moving that have that runId - Map> runIdSamples = new LongHashMap<>(); - samples.forEach(sample -> { - if (sample.getRunId() != null) - runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); - }); - ExperimentService expService = ExperimentService.get(); - // find the set of runs associated with samples that are moving - List runs = expService.getExpRuns(runIdSamples.keySet()); - List toUpdate = new ArrayList<>(); - List toSplit = new ArrayList<>(); - for (ExpRun run : runs) - { - Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); - if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) - toUpdate.add(run); - else - toSplit.add(run); - } - - int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); - int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); - return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); - } - - private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException - { - final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); - ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); - int runCount = 0; - for (ExpRun run : runs) - { - ExpProtocolApplication sourceApplication = null; - ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); - boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); - - Set movingSet = movingSamples.get(run.getRowId()); - int numStaying = 0; - Map movingOutputsMap = new HashMap<>(); - ExpMaterial aliquotParent = null; - // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) - for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) - { - ExpMaterial material = materialInput.getMaterial(); - if (movingSet.contains(material)) - { - // clear out the run and source application so a new derivation run can be created. - material.setRun(null); - material.setSourceApplication(null); - movingOutputsMap.put(material, materialInput.getRole()); - } - else - { - if (sourceApplication == null) - sourceApplication = material.getSourceApplication(); - numStaying++; - } - if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) - { - aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); - } - } - - try - { - if (isAliquot && aliquotParent != null) - { - ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); - expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); - } - else - { - // create a new derivation run for the samples that are moving - expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); - // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs - run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); - } - } - catch (ValidationException e) - { - BatchValidationException errors = new BatchValidationException(); - errors.addRowError(e); - throw errors; - } - run.save(user); - List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); - - outputApp.removeMaterialInputs(user, movingSampleIds); - if (sourceApplication != null) - sourceApplication.removeMaterialInputs(user, movingSampleIds); - - runCount++; - } - return runCount; - } - - record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} - - // return the map of file renames - private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException - { - Map> sampleFileRenames = new LongHashMap<>(); - - FileContentService fileService = FileContentService.get(); - if (fileService == null) - { - LOG.warn("No file service available. Sample files cannot be moved."); - return sampleFileRenames; - } - - if (fileService.getFileRoot(targetContainer) == null) - { - LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); - return sampleFileRenames; - } - - List fileDomainProps = sampleType.getDomain() - .getProperties().stream() - .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); - if (fileDomainProps.isEmpty()) - return sampleFileRenames; - - Map hasFileRoot = new HashMap<>(); - Map fileMoveCounts = new HashMap<>(); - Map fileMoveReferences = new HashMap<>(); - for (ExpMaterial sample : samples) - { - boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); - if (!hasSourceRoot) - LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); - else - for (DomainProperty fileProp : fileDomainProps ) - { - String sourceFileName = (String) sample.getProperty(fileProp); - if (StringUtils.isBlank(sourceFileName)) - continue; - File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); - if (updatedFile != null) - { - - if (!fileMoveReferences.containsKey(sourceFileName)) - fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); - if (!fileMoveCounts.containsKey(sourceFileName)) - fileMoveCounts.put(sourceFileName, 0); - fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); - - File sourceFile = new File(sourceFileName); - FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); - sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); - List fieldRenameData = sampleFileRenames.get(sample.getRowId()); - fieldRenameData.add(renameData); - } - } - } - - for (String filePath : fileMoveReferences.keySet()) - { - SampleFileMoveReference ref = fileMoveReferences.get(filePath); - File sourceFile = new File(filePath); - if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) - throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); - - // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls - fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); - FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); - event.setProvidedFileName(sourceFile.getName()); - event.setFile(ref.targetFile.getName()); - event.setDirectory(ref.targetFile.getParent()); - event.setFieldName(ref.fieldName); - AuditLogService.get().addEvent(user, event); - } - - return sampleFileRenames; - } - - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) - { - if (!renameData.targetFile.getParentFile().exists()) - { - String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", - renameData.targetFile.getParent(), - renameData.sourceFile.getAbsolutePath(), - renameData.sampleType.getName(), - renameData.sampleName, - renameData.fieldName); - try - { - if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) - { - LOG.warn(errorMsg); - return false; - } - } - catch (IOException e) - { - LOG.warn(errorMsg + e.getMessage()); - } - } - - String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); - } - - @Override - @Nullable - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) - { - return getSampleCountSequence(container, isRootSampleOnly, true); - } - - public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) - { - Container seqContainer = container.getProject(); - if (seqContainer == null) - return null; - - String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - - if (!create) - { - // check if sequence already exist so we don't create one just for querying - Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); - if (null == seqRowId) - return null; - } - - if (ExperimentService.get().useStrictCounter()) - return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); - - return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); - } - - @Override - public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - - DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); - if (seq == null) - return; - - long current = seq.current(); - if (newSeqValue < current) - { - if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) - throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); - - if (newSeqValue <= 0) - { - deleteSampleCounterSequence(container, isRootOnly); - return; - } - } - - seq.ensureMinimum(newSeqValue); - seq.sync(); - } - - public void deleteSampleCounterSequences(Container container) - { - deleteSampleCounterSequence(container, false); - deleteSampleCounterSequence(container, true); - } - - private void deleteSampleCounterSequence(Container container, boolean isRootOnly) - { - String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; - Container seqContainer = container.getProject(); - DbSequenceManager.delete(seqContainer, seqName); - DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); - } - - @Override - public long getProjectSampleCount(Container container) - { - return getProjectSampleCount(container, false); - } - - @Override - public long getProjectRootSampleCount(Container container) - { - return getProjectSampleCount(container, true); - } - - private long getProjectSampleCount(Container container, boolean isRootOnly) - { - User searchUser = User.getSearchUser(); - ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); - Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); - TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); - SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); - sql.append(tableInfo); - sql.append(" WHERE "); - if (isRootOnly) - sql.append(" AliquotedFromLsid IS NULL AND "); - sql.append("Container "); - sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); - return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); - } - - @Override - public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) - { - boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; - DbSequence seq = getSampleCountSequence(container, isRootOnly, false); - if (seq != null) - { - long current = seq.current(); - if (current > 0) - return current; - } - - return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); - } - - public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } - - public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) - { - ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); - } - - - public static class TestCase extends Assert - { - @Test - public void testGetValidatedUnit() - { - SampleTypeService service = SampleTypeService.get(); - try - { - service.getValidatedUnit("g", Unit.mg, "Sample Type"); - service.getValidatedUnit("g ", Unit.mg, "Sample Type"); - service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); - } - catch (ConversionExceptionWithMessage e) - { - fail("Compatible unit should not throw exception."); - } - try - { - assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("null units should be null"); - } - try - { - assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); - } - catch (ConversionExceptionWithMessage e) - { - fail("empty units should be null"); - } - try - { - service.getValidatedUnit("g", Unit.unit, "Sample Type"); - fail("Units that are not comparable should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - try - { - service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); - fail("Invalid units should throw exception."); - } - catch (ConversionExceptionWithMessage ignore) - { - - } - - } - } -} +/* + * Copyright (c) 2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.experiment.api; + +import org.apache.commons.collections4.ListUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.math3.util.Precision; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.audit.AbstractAuditHandler; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.FileSystemAuditProvider; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.LongArrayList; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.collections.LongHashSet; +import org.labkey.api.data.AuditConfigurable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ConversionExceptionWithMessage; +import org.labkey.api.data.DatabaseCache; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DbSequence; +import org.labkey.api.data.DbSequenceManager; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.Parameter; +import org.labkey.api.data.ParameterMapStatement; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.OntologyObject; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpMaterialRunInput; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolApplication; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.NameExpressionOptionService; +import org.labkey.api.exp.api.SampleTypeDomainKindProperties; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.query.ExpMaterialTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.inventory.InventoryService; +import org.labkey.api.miniprofiler.MiniProfiler; +import org.labkey.api.miniprofiler.Timing; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.qc.DataState; +import org.labkey.api.qc.SampleStatusService; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.util.CPUTimer; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.experiment.SampleTypeAuditProvider; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; + +import java.io.File; +import java.io.IOException; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static java.util.Collections.singleton; +import static org.labkey.api.audit.SampleTimelineAuditEvent.SAMPLE_TIMELINE_EVENT_TYPE; +import static org.labkey.api.data.CompareType.STARTS_WITH; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTCOMMIT; +import static org.labkey.api.data.DbScope.CommitTaskOption.POSTROLLBACK; +import static org.labkey.api.exp.api.ExperimentJSONConverter.CPAS_TYPE; +import static org.labkey.api.exp.api.ExperimentJSONConverter.LSID; +import static org.labkey.api.exp.api.ExperimentJSONConverter.NAME; +import static org.labkey.api.exp.api.ExperimentJSONConverter.ROW_ID; +import static org.labkey.api.exp.api.ExperimentService.SAMPLE_ALIQUOT_PROTOCOL_LSID; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG; +import static org.labkey.api.exp.api.NameExpressionOptionService.NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.RawUnits; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.StoredAmount; +import static org.labkey.api.exp.query.ExpMaterialTable.Column.Units; +import static org.labkey.api.exp.query.ExpSchema.NestedSchemas.materials; + + +public class SampleTypeServiceImpl extends AbstractAuditHandler implements SampleTypeService +{ + public static final String SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:sampleCount"; + public static final String ROOT_SAMPLE_COUNT_SEQ_NAME = "org.labkey.api.exp.api.ExpMaterial:rootSampleCount"; + + public static final List SUPPORTED_UNITS = new ArrayList<>(); + public static final String CONVERSION_EXCEPTION_MESSAGE ="Units value (%s) is not compatible with the %s display units (%s)."; + + static + { + SUPPORTED_UNITS.addAll(KindOfQuantity.Volume.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Mass.getCommonUnits()); + SUPPORTED_UNITS.addAll(KindOfQuantity.Count.getCommonUnits()); + } + + // columns that may appear in a row when only the sample status is updating. + public static final Set statusUpdateColumns = Set.of( + ExpMaterialTable.Column.Modified.name().toLowerCase(), + ExpMaterialTable.Column.ModifiedBy.name().toLowerCase(), + ExpMaterialTable.Column.SampleState.name().toLowerCase(), + ExpMaterialTable.Column.Folder.name().toLowerCase() + ); + + public static SampleTypeServiceImpl get() + { + return (SampleTypeServiceImpl) SampleTypeService.get(); + } + + private static final Logger LOG = LogHelper.getLogger(SampleTypeServiceImpl.class, "Info about sample type operations"); + + /** SampleType LSID -> Container cache */ + private final Cache sampleTypeCache = CacheManager.getStringKeyCache(CacheManager.UNLIMITED, CacheManager.DAY, "SampleType to container"); + + /** ContainerId -> MaterialSources */ + private final Cache> materialSourceCache = DatabaseCache.get(ExperimentServiceImpl.get().getSchema().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "Material sources", (container, argument) -> + { + Container c = ContainerManager.getForId(container); + if (c == null) + return Collections.emptySortedSet(); + + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + return Collections.unmodifiableSortedSet(new TreeSet<>(new TableSelector(getTinfoMaterialSource(), filter, null).getCollection(MaterialSource.class))); + }); + + Cache> getMaterialSourceCache() + { + return materialSourceCache; + } + + @Override @NotNull + public List getSupportedUnits() + { + return SUPPORTED_UNITS; + } + + @Nullable @Override + public Unit getValidatedUnit(@Nullable Object rawUnits, @Nullable Unit defaultUnits, String sampleTypeName) + { + if (rawUnits == null) + return null; + if (rawUnits instanceof Unit u) + { + if (defaultUnits == null) + return u; + else if (u.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + else + return u; + } + if (!(rawUnits instanceof String rawUnitsString)) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + if (!StringUtils.isBlank(rawUnitsString)) + { + rawUnitsString = rawUnitsString.trim(); + + Unit mUnit = Unit.fromName(rawUnitsString); + List commonUnits = getSupportedUnits(); + if (mUnit == null || !commonUnits.contains(mUnit)) + { + throw new ConversionExceptionWithMessage("Unsupported Units value (" + rawUnitsString + "). Supported values are: " + StringUtils.join(commonUnits, ", ") + "."); + } + if (defaultUnits != null && mUnit.getKindOfQuantity() != defaultUnits.getKindOfQuantity()) + throw new ConversionExceptionWithMessage(String.format(CONVERSION_EXCEPTION_MESSAGE, rawUnits, sampleTypeName == null ? "" : sampleTypeName, defaultUnits)); + return mUnit; + } + return null; + } + + public void clearMaterialSourceCache(@Nullable Container c) + { + LOG.debug("clearMaterialSourceCache: " + (c == null ? "all" : c.getPath())); + if (c == null) + materialSourceCache.clear(); + else + materialSourceCache.remove(c.getId()); + } + + + private TableInfo getTinfoMaterialSource() + { + return ExperimentServiceImpl.get().getTinfoSampleType(); + } + + private TableInfo getTinfoMaterial() + { + return ExperimentServiceImpl.get().getTinfoMaterial(); + } + + private TableInfo getTinfoProtocolApplication() + { + return ExperimentServiceImpl.get().getTinfoProtocolApplication(); + } + + private TableInfo getTinfoProtocol() + { + return ExperimentServiceImpl.get().getTinfoProtocol(); + } + + private TableInfo getTinfoMaterialInput() + { + return ExperimentServiceImpl.get().getTinfoMaterialInput(); + } + + private TableInfo getTinfoExperimentRun() + { + return ExperimentServiceImpl.get().getTinfoExperimentRun(); + } + + private TableInfo getTinfoDataClass() + { + return ExperimentServiceImpl.get().getTinfoDataClass(); + } + + private TableInfo getTinfoProtocolInput() + { + return ExperimentServiceImpl.get().getTinfoProtocolInput(); + } + + private TableInfo getTinfoMaterialAliasMap() + { + return ExperimentServiceImpl.get().getTinfoMaterialAliasMap(); + } + + private DbSchema getExpSchema() + { + return ExperimentServiceImpl.getExpSchema(); + } + + @Override + public void indexSampleType(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + if (sampleType == null) + return; + + queue.addRunnable((q) -> { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT * FROM ") + .append(getTinfoMaterialSource(), "ms") + .append(" WHERE ms.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND ms.LSID = ?").add(sampleType.getLSID()) + .append(" AND (ms.lastIndexed IS NULL OR ms.lastIndexed < ? OR (ms.modified IS NOT NULL AND ms.lastIndexed < ms.modified))") + .add(sampleType.getModified()); + + MaterialSource materialSource = new SqlSelector(getExpSchema().getScope(), sql).getObject(MaterialSource.class); + if (materialSource != null) + { + ExpSampleTypeImpl impl = new ExpSampleTypeImpl(materialSource); + impl.index(q, null); + } + + indexSampleTypeMaterials(sampleType, q); + }); + } + + private void indexSampleTypeMaterials(ExpSampleType sampleType, SearchService.TaskIndexingQueue queue) + { + // Index all ExpMaterial that have never been indexed OR where either the ExpSampleType definition or ExpMaterial itself has changed since last indexed + SQLFragment sql = new SQLFragment("SELECT m.* FROM ") + .append(getTinfoMaterial(), "m") + .append(" LEFT OUTER JOIN ") + .append(ExperimentServiceImpl.get().getTinfoMaterialIndexed(), "mi") + .append(" ON m.RowId = mi.MaterialId WHERE m.LSID NOT LIKE ").appendValue("%:" + StudyService.SPECIMEN_NAMESPACE_PREFIX + "%", getExpSchema().getSqlDialect()) + .append(" AND m.cpasType = ?").add(sampleType.getLSID()) + .append(" AND (mi.lastIndexed IS NULL OR mi.lastIndexed < ? OR (m.modified IS NOT NULL AND mi.lastIndexed < m.modified))") + .append(" ORDER BY m.RowId") // Issue 51263: order by RowId to reduce deadlock + .add(sampleType.getModified()); + + new SqlSelector(getExpSchema().getScope(), sql).forEachBatch(Material.class, 1000, batch -> { + for (Material m : batch) + { + ExpMaterialImpl impl = new ExpMaterialImpl(m); + impl.index(queue, null /* null tableInfo since samples may belong to multiple containers*/); + } + }); + } + + + @Override + public Map getSampleTypesForRoles(Container container, ContainerFilter filter, ExpProtocol.ApplicationType type) + { + SQLFragment sql = new SQLFragment(); + sql.append("SELECT mi.Role, MAX(m.CpasType) AS MaxSampleSetLSID, MIN (m.CpasType) AS MinSampleSetLSID FROM "); + sql.append(getTinfoMaterial(), "m"); + sql.append(", "); + sql.append(getTinfoMaterialInput(), "mi"); + sql.append(", "); + sql.append(getTinfoProtocolApplication(), "pa"); + sql.append(", "); + sql.append(getTinfoExperimentRun(), "r"); + + if (type != null) + { + sql.append(", "); + sql.append(getTinfoProtocol(), "p"); + sql.append(" WHERE p.lsid = pa.protocollsid AND p.applicationtype = ? AND "); + sql.add(type.toString()); + } + else + { + sql.append(" WHERE "); + } + + sql.append(" m.RowId = mi.MaterialId AND mi.TargetApplicationId = pa.RowId AND " + + "pa.RunId = r.RowId AND "); + sql.append(filter.getSQLFragment(getExpSchema(), new SQLFragment("r.Container"))); + sql.append(" GROUP BY mi.Role ORDER BY mi.Role"); + + Map result = new LinkedHashMap<>(); + for (Map queryResult : new SqlSelector(getExpSchema(), sql).getMapCollection()) + { + ExpSampleType sampleType = null; + String maxSampleTypeLSID = (String) queryResult.get("MaxSampleSetLSID"); + String minSampleTypeLSID = (String) queryResult.get("MinSampleSetLSID"); + + // Check if we have a sample type that was being referenced + if (maxSampleTypeLSID != null && maxSampleTypeLSID.equalsIgnoreCase(minSampleTypeLSID)) + { + // If the min and the max are the same, it means all rows share the same value so we know that there's + // a single sample type being targeted + sampleType = getSampleType(container, maxSampleTypeLSID); + } + result.put((String) queryResult.get("Role"), sampleType); + } + return result; + } + + @Override + public void removeAutoLinkedStudy(@NotNull Container studyContainer) + { + SQLFragment sql = new SQLFragment("UPDATE ").append(getTinfoMaterialSource()) + .append(" SET autolinkTargetContainer = NULL WHERE autolinkTargetContainer = ?") + .add(studyContainer.getId()); + new SqlExecutor(ExperimentService.get().getSchema()).execute(sql); + } + + public ExpSampleTypeImpl getSampleTypeByObjectId(Long objectId) + { + OntologyObject obj = OntologyManager.getOntologyObject(objectId); + if (obj == null) + return null; + + return getSampleType(obj.getObjectURI()); + } + + @Override + public @Nullable ExpSampleType getEffectiveSampleType( + @NotNull Container definitionContainer, + @NotNull String sampleTypeName, + @NotNull Date effectiveDate, + @Nullable ContainerFilter cf + ) + { + Long legacyObjectId = ExperimentService.get().getObjectIdWithLegacyName(sampleTypeName, ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), effectiveDate, definitionContainer, cf); + if (legacyObjectId != null) + return getSampleTypeByObjectId(legacyObjectId); + + boolean includeOtherContainers = cf != null && cf.getType() != ContainerFilter.Type.Current; + ExpSampleTypeImpl sampleType = getSampleType(definitionContainer, includeOtherContainers, sampleTypeName); + if (sampleType != null && sampleType.getCreated().compareTo(effectiveDate) <= 0) + return sampleType; + + return null; + } + + @Override + public List getSampleTypes(@NotNull Container container, boolean includeOtherContainers) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(container, includeOtherContainers); + + // Do the sort on the Java side to make sure it's always case-insensitive, even on Postgres + TreeSet result = new TreeSet<>(); + for (String containerId : containerIds) + { + for (MaterialSource source : getMaterialSourceCache().get(containerId)) + { + result.add(new ExpSampleTypeImpl(source)); + } + } + + return List.copyOf(result); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull String sampleTypeName) + { + return getSampleType(c, false, sampleTypeName); + } + + // NOTE: This method used to not take a user or check permissions + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, @NotNull String sampleTypeName) + { + return getSampleType(c, true, sampleTypeName); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, String sampleTypeName) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getName().equalsIgnoreCase(sampleTypeName))); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId) + { + return getSampleType(c, rowId, false); + } + + @Override + public ExpSampleTypeImpl getSampleType(@NotNull Container c, @NotNull User user, long rowId) + { + return getSampleType(c, rowId, true); + } + + @Override + public ExpSampleTypeImpl getSampleTypeByType(@NotNull String lsid, Container hint) + { + Container c = hint; + String id = sampleTypeCache.get(lsid); + if (null != id && (null == hint || !id.equals(hint.getId()))) + c = ContainerManager.getForId(id); + ExpSampleTypeImpl st = null; + if (null != c) + st = getSampleType(c, false, ms -> lsid.equals(ms.getLSID()) ); + if (null == st) + st = _getSampleType(lsid); + if (null != st && null==id) + sampleTypeCache.put(lsid,st.getContainer().getId()); + return st; + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, long rowId, boolean includeOtherContainers) + { + return getSampleType(c, includeOtherContainers, (materialSource -> materialSource.getRowId() == rowId)); + } + + private ExpSampleTypeImpl getSampleType(@NotNull Container c, boolean includeOtherContainers, Predicate predicate) + { + List containerIds = ExperimentServiceImpl.get().createContainerList(c, includeOtherContainers); + for (String containerId : containerIds) + { + Collection sampleTypes = getMaterialSourceCache().get(containerId); + for (MaterialSource materialSource : sampleTypes) + { + if (predicate.test(materialSource)) + return new ExpSampleTypeImpl(materialSource); + } + } + + return null; + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(long rowId) + { + // TODO: Cache + MaterialSource materialSource = new TableSelector(getTinfoMaterialSource()).getObject(rowId, MaterialSource.class); + if (materialSource == null) + return null; + + return new ExpSampleTypeImpl(materialSource); + } + + @Nullable + @Override + public ExpSampleTypeImpl getSampleType(String lsid) + { + return getSampleTypeByType(lsid, null); + } + + @Nullable + @Override + public DataState getSampleState(Container container, Long stateRowId) + { + return SampleStatusService.get().getStateForRowId(container, stateRowId); + } + + private ExpSampleTypeImpl _getSampleType(String lsid) + { + MaterialSource ms = getMaterialSource(lsid); + if (ms == null) + return null; + + return new ExpSampleTypeImpl(ms); + } + + public MaterialSource getMaterialSource(String lsid) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("LSID"), lsid); + return new TableSelector(getTinfoMaterialSource(), filter, null).getObject(MaterialSource.class); + } + + public DbScope.Transaction ensureTransaction() + { + return getExpSchema().getScope().ensureTransaction(); + } + + @Override + public Lsid getSampleTypeLsid(String sourceName, Container container) + { + return Lsid.parse(ExperimentService.get().generateLSID(container, ExpSampleType.class, sourceName)); + } + + @Override + public Pair getSampleTypeSamplePrefixLsids(Container container) + { + Pair lsidDbSeq = ExperimentService.get().generateLSIDWithDBSeq(container, ExpSampleType.class); + String sampleTypeLsidStr = lsidDbSeq.first; + Lsid sampleTypeLsid = Lsid.parse(sampleTypeLsidStr); + + String dbSeqStr = lsidDbSeq.second; + String samplePrefixLsid = new Lsid.LsidBuilder("Sample", "Folder-" + container.getRowId() + "." + dbSeqStr, "").toString(); + + return new Pair<>(sampleTypeLsid.toString(), samplePrefixLsid); + } + + /** + * Delete all exp.Material from the SampleType. If container is not provided, + * all rows from the SampleType will be deleted regardless of container. + */ + public int truncateSampleType(ExpSampleTypeImpl source, User user, @Nullable Container c) + { + assert getExpSchema().getScope().isTransactionActive(); + + Set containers = new HashSet<>(); + if (c == null) + { + SQLFragment containerSql = new SQLFragment("SELECT DISTINCT Container FROM "); + containerSql.append(getTinfoMaterial(), "m"); + containerSql.append(" WHERE CpasType = ?"); + containerSql.add(source.getLSID()); + new SqlSelector(getExpSchema(), containerSql).forEach(String.class, cId -> containers.add(ContainerManager.getForId(cId))); + } + else + { + containers.add(c); + } + + int count = 0; + for (Container toDelete : containers) + { + SQLFragment sqlFilter = new SQLFragment("CpasType = ? AND Container = ?"); + sqlFilter.add(source.getLSID()); + sqlFilter.add(toDelete); + count += ExperimentServiceImpl.get().deleteMaterialBySqlFilter(user, toDelete, sqlFilter, true, false, source, true, true); + } + return count; + } + + @Override + public void deleteSampleType(long rowId, Container c, User user, @Nullable String auditUserComment) throws ExperimentException + { + CPUTimer timer = new CPUTimer("delete sample type"); + timer.start(); + + ExpSampleTypeImpl source = getSampleType(c, user, rowId); + if (null == source) + throw new IllegalArgumentException("Can't find SampleType with rowId " + rowId); + if (!source.getContainer().equals(c)) + throw new ExperimentException("Trying to delete a SampleType from a different container"); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + // TODO: option to skip deleting rows from the materialized table since we're about to delete it anyway + // TODO do we need both truncateSampleType() and deleteDomainObjects()? + truncateSampleType(source, user, null); + + StudyService studyService = StudyService.get(); + if (studyService != null) + { + for (Dataset dataset : StudyPublishService.get().getDatasetsForPublishSource(rowId, Dataset.PublishSource.SampleType)) + { + dataset.delete(user, auditUserComment); + } + } + else + { + LOG.warn("Could not delete datasets associated with this protocol: Study service not available."); + } + + Domain d = source.getDomain(); + d.delete(user, auditUserComment); + + ExperimentServiceImpl.get().deleteDomainObjects(source.getContainer(), source.getLSID()); + + SqlExecutor executor = new SqlExecutor(getExpSchema()); + executor.execute("UPDATE " + getTinfoDataClass() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("UPDATE " + getTinfoProtocolInput() + " SET materialSourceId = NULL WHERE materialSourceId = ?", source.getRowId()); + executor.execute("DELETE FROM " + getTinfoMaterialSource() + " WHERE RowId = ?", rowId); + + addSampleTypeDeletedAuditEvent(user, c, source, auditUserComment); + + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.SampleType); + ExperimentService.get().removeDataTypeExclusion(Collections.singleton(rowId), ExperimentService.DataTypeForExclusion.DashboardSampleType); + + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.commit(); + } + + // Delete sequences (genId and the unique counters) + DbSequenceManager.deleteLike(c, ExpSampleType.SEQUENCE_PREFIX, (int)source.getRowId(), getExpSchema().getSqlDialect()); + + SchemaKey samplesSchema = SchemaKey.fromParts(SamplesSchema.SCHEMA_NAME); + QueryService.get().fireQueryDeleted(user, c, null, samplesSchema, singleton(source.getName())); + + SchemaKey expMaterialsSchema = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME, materials.toString()); + QueryService.get().fireQueryDeleted(user, c, null, expMaterialsSchema, singleton(source.getName())); + + // Remove SampleType from search index + try (Timing ignored = MiniProfiler.step("search docs")) + { + SearchService.get().deleteResource(source.getDocumentId()); + } + + timer.stop(); + LOG.info("Deleted SampleType '" + source.getName() + "' from '" + c.getPath() + "' in " + timer.getDuration()); + } + + private void addSampleTypeDeletedAuditEvent(User user, Container c, ExpSampleType sampleType, @Nullable String auditUserComment) + { + addSampleTypeAuditEvent(user, c, sampleType, String.format("Sample Type deleted: %1$s", sampleType.getName()),auditUserComment, "delete type"); + } + + private void addSampleTypeAuditEvent(User user, Container c, ExpSampleType sampleType, String comment, @Nullable String auditUserComment, String insertUpdateChoice) + { + SampleTypeAuditProvider.SampleTypeAuditEvent event = new SampleTypeAuditProvider.SampleTypeAuditEvent(c, comment); + event.setUserComment(auditUserComment); + + if (sampleType != null) + { + event.setSourceLsid(sampleType.getLSID()); + event.setSampleSetName(sampleType.getName()); + } + event.setInsertUpdateChoice(insertUpdateChoice); + AuditLogService.get().addEvent(user, event); + } + + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType() + { + return new ExpSampleTypeImpl(new MaterialSource()); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, String nameExpression) + throws ExperimentException + { + return createSampleType(c,u,name,description,properties,indices,idCol1,idCol2,idCol3,parentCol,nameExpression, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, @Nullable TemplateInfo templateInfo) + throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, + parentCol, nameExpression, null, templateInfo, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit) throws ExperimentException + { + return createSampleType(c, u, name, description, properties, indices, idCol1, idCol2, idCol3, parentCol, nameExpression, aliquotNameExpression, templateInfo, importAliases, labelColor, metricUnit, null, null, null, null, null, null, null); + } + + @NotNull + @Override + public ExpSampleTypeImpl createSampleType(Container c, User u, String name, String description, List properties, List indices, int idCol1, int idCol2, int idCol3, int parentCol, + String nameExpression, String aliquotNameExpression, @Nullable TemplateInfo templateInfo, @Nullable Map> importAliases, @Nullable String labelColor, @Nullable String metricUnit, + @Nullable Container autoLinkTargetContainer, @Nullable String autoLinkCategory, @Nullable String category, @Nullable List disabledSystemField, + @Nullable List excludedContainerIds, @Nullable List excludedDashboardContainerIds, @Nullable Map changeDetails) + throws ExperimentException + { + validateSampleTypeName(c, u, name, false); + + if (properties == null || properties.isEmpty()) + throw new ApiUsageException("At least one property is required"); + + if (idCol2 != -1 && idCol1 == idCol2) + throw new ApiUsageException("You cannot use the same id column twice."); + + if (idCol3 != -1 && (idCol1 == idCol3 || idCol2 == idCol3)) + throw new ApiUsageException("You cannot use the same id column twice."); + + if ((idCol1 > -1 && idCol1 >= properties.size()) || + (idCol2 > -1 && idCol2 >= properties.size()) || + (idCol3 > -1 && idCol3 >= properties.size()) || + (parentCol > -1 && parentCol >= properties.size())) + throw new ApiUsageException("column index out of range"); + + // Name expression is only allowed when no idCol is set + if (nameExpression != null && idCol1 > -1) + throw new ApiUsageException("Name expression cannot be used with id columns"); + + NameExpressionOptionService svc = NameExpressionOptionService.get(); + if (!svc.allowUserSpecifiedNames(c)) + { + if (nameExpression == null) + throw new ApiUsageException(c.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + if (svc.getExpressionPrefix(c) != null) + { + // automatically apply the configured prefix to the name expression + nameExpression = svc.createPrefixedExpression(c, nameExpression, false); + aliquotNameExpression = svc.createPrefixedExpression(c, aliquotNameExpression, true); + } + + // Validate the name expression length + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameExpMax = materialSourceTable.getColumn("NameExpression").getScale(); + if (nameExpression != null && nameExpression.length() > nameExpMax) + throw new ApiUsageException("Name expression may not exceed " + nameExpMax + " characters."); + + // Validate the aliquot name expression length + int aliquotNameExpMax = materialSourceTable.getColumn("AliquotNameExpression").getScale(); + if (aliquotNameExpression != null && aliquotNameExpression.length() > aliquotNameExpMax) + throw new ApiUsageException("Aliquot naming patten may not exceed " + aliquotNameExpMax + " characters."); + + // Validate the label color length + int labelColorMax = materialSourceTable.getColumn("LabelColor").getScale(); + if (labelColor != null && labelColor.length() > labelColorMax) + throw new ApiUsageException("Label color may not exceed " + labelColorMax + " characters."); + + // Validate the metricUnit length + int metricUnitMax = materialSourceTable.getColumn("MetricUnit").getScale(); + if (metricUnit != null && metricUnit.length() > metricUnitMax) + throw new ApiUsageException("Metric unit may not exceed " + metricUnitMax + " characters."); + + // Validate the category length + int categoryMax = materialSourceTable.getColumn("Category").getScale(); + if (category != null && category.length() > categoryMax) + throw new ApiUsageException("Category may not exceed " + categoryMax + " characters."); + + Pair dbSeqLsids = getSampleTypeSamplePrefixLsids(c); + String lsid = dbSeqLsids.first; + String materialPrefixLsid = dbSeqLsids.second; + Domain domain = PropertyService.get().createDomain(c, lsid, name, templateInfo); + DomainKind kind = domain.getDomainKind(); + if (kind != null) + domain.setDisabledSystemFields(kind.getDisabledSystemFields(disabledSystemField)); + Set reservedNames = kind.getReservedPropertyNames(domain, u); + Set reservedPrefixes = kind.getReservedPropertyNamePrefixes(); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + boolean hasNameProperty = false; + String idUri1 = null, idUri2 = null, idUri3 = null, parentUri = null; + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + List calculatedFields = new ArrayList<>(); + for (int i = 0; i < properties.size(); i++) + { + GWTPropertyDescriptor pd = properties.get(i); + String propertyName = pd.getName().toLowerCase(); + + // calculatedFields will be handled separately + if (pd.getValueExpression() != null) + { + calculatedFields.add(pd); + continue; + } + + if (ExpMaterialTable.Column.Name.name().equalsIgnoreCase(propertyName)) + { + hasNameProperty = true; + } + else + { + if (!reservedPrefixes.isEmpty()) + { + Optional reservedPrefix = reservedPrefixes.stream().filter(prefix -> propertyName.startsWith(prefix.toLowerCase())).findAny(); + reservedPrefix.ifPresent(s -> { + throw new IllegalArgumentException("The prefix '" + s + "' is reserved for system use."); + }); + } + + if (lowerReservedNames.contains(propertyName)) + { + throw new IllegalArgumentException("Property name '" + propertyName + "' is a reserved name."); + } + + DomainProperty dp = DomainUtil.addProperty(domain, pd, defaultValues, propertyUris, null); + + if (dp != null) + { + if (idCol1 == i) idUri1 = dp.getPropertyURI(); + if (idCol2 == i) idUri2 = dp.getPropertyURI(); + if (idCol3 == i) idUri3 = dp.getPropertyURI(); + if (parentCol == i) parentUri = dp.getPropertyURI(); + } + } + } + + domain.setPropertyIndices(indices, lowerReservedNames); + + if (!hasNameProperty && idUri1 == null) + throw new ApiUsageException("Either a 'Name' property or an index for idCol1 is required"); + + if (hasNameProperty && idUri1 != null) + throw new ApiUsageException("Either a 'Name' property or idCols can be used, but not both"); + + String importAliasJson = ExperimentJSONConverter.getAliasJson(importAliases, name); + + MaterialSource source = new MaterialSource(); + source.setLSID(lsid); + source.setName(name); + source.setDescription(description); + source.setMaterialLSIDPrefix(materialPrefixLsid); + if (nameExpression != null) + source.setNameExpression(nameExpression); + if (aliquotNameExpression != null) + source.setAliquotNameExpression(aliquotNameExpression); + source.setLabelColor(labelColor); + source.setMetricUnit(metricUnit); + source.setAutoLinkTargetContainer(autoLinkTargetContainer); + source.setAutoLinkCategory(autoLinkCategory); + source.setCategory(category); + source.setContainer(c); + source.setMaterialParentImportAliasMap(importAliasJson); + + if (hasNameProperty) + { + source.setIdCol1(ExpMaterialTable.Column.Name.name()); + } + else + { + source.setIdCol1(idUri1); + if (idUri2 != null) + source.setIdCol2(idUri2); + if (idUri3 != null) + source.setIdCol3(idUri3); + } + if (parentUri != null) + source.setParentCol(parentUri); + + final ExpSampleTypeImpl st = new ExpSampleTypeImpl(source); + + try + { + getExpSchema().getScope().executeWithRetry(transaction -> + { + try + { + domain.save(u, changeDetails, calculatedFields); + st.save(u); + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, name, null, calculatedFields, false, u, c); + DefaultValueService.get().setDefaultValues(domain.getContainer(), defaultValues); + if (excludedContainerIds != null && !excludedContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, excludedContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.SampleType, st.getRowId(), c, u); + if (excludedDashboardContainerIds != null && !excludedDashboardContainerIds.isEmpty()) + ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, excludedDashboardContainerIds, st.getRowId(), u); + else + ExperimentService.get().ensureDataTypeContainerExclusionsNonAdmin(ExperimentService.DataTypeForExclusion.DashboardSampleType, st.getRowId(), c, u); + transaction.addCommitTask(() -> clearMaterialSourceCache(c), DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + transaction.addCommitTask(() -> indexSampleType(SampleTypeService.get().getSampleType(domain.getTypeURI()), SearchService.get().defaultTask().getQueue(c, SearchService.PRIORITY.modified)), POSTCOMMIT); + + return st; + } + catch (ExperimentException | MetadataUnavailableException eex) + { + throw new DbScope.RetryPassthroughException(eex); + } + }); + } + catch (DbScope.RetryPassthroughException x) + { + x.rethrow(ExperimentException.class); + throw x; + } + + return st; + } + + public enum SampleSequenceType + { + DAILY("yyyy-MM-dd"), + WEEKLY("YYYY-'W'ww"), + MONTHLY("yyyy-MM"), + YEARLY("yyyy"); + + final DateTimeFormatter _formatter; + + SampleSequenceType(String pattern) + { + _formatter = DateTimeFormatter.ofPattern(pattern); + } + + public Pair getSequenceName(@Nullable Date date) + { + LocalDateTime ldt; + if (date == null) + ldt = LocalDateTime.now(); + else + ldt = LocalDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()); + String suffix = _formatter.format(ldt); + // NOTE: it would make sense to use the dbsequence "id" feature here. + // e.g. instead of name=org.labkey.api.exp.api.ExpMaterial:DAILY:2021-05-25 id=0 + // we could use name=org.labkey.api.exp.api.ExpMaterial:DAILY id=20210525 + // however, that would require a fix up on upgrade. + return new Pair<>("org.labkey.api.exp.api.ExpMaterial:" + name() + ":" + suffix, 0); + } + + public long next(Date date) + { + return getDbSequence(date).next(); + } + + public DbSequence getDbSequence(Date date) + { + Pair seqName = getSequenceName(date); + return DbSequenceManager.getPreallocatingSequence(ContainerManager.getRoot(), seqName.first, seqName.second, 100); + } + } + + + @Override + public Function,Map> getSampleCountsFunction(@Nullable Date counterDate) + { + final var dailySampleCount = SampleSequenceType.DAILY.getDbSequence(counterDate); + final var weeklySampleCount = SampleSequenceType.WEEKLY.getDbSequence(counterDate); + final var monthlySampleCount = SampleSequenceType.MONTHLY.getDbSequence(counterDate); + final var yearlySampleCount = SampleSequenceType.YEARLY.getDbSequence(counterDate); + + return (counts) -> + { + if (null==counts) + counts = new HashMap<>(); + counts.put("dailySampleCount", dailySampleCount.next()); + counts.put("weeklySampleCount", weeklySampleCount.next()); + counts.put("monthlySampleCount", monthlySampleCount.next()); + counts.put("yearlySampleCount", yearlySampleCount.next()); + return counts; + }; + } + + @Override + public void validateSampleTypeName(Container container, User user, String name, boolean skipExistingCheck) + { + if (name == null || StringUtils.isBlank(name)) + throw new ApiUsageException("Sample Type name is required."); + + TableInfo materialSourceTable = ExperimentService.get().getTinfoSampleType(); + int nameMax = materialSourceTable.getColumn("Name").getScale(); + if (name.length() > nameMax) + throw new ApiUsageException("Sample Type name may not exceed " + nameMax + " characters."); + + if (!skipExistingCheck) + { + if (getSampleType(container, user, name) != null) + throw new ApiUsageException("A Sample Type with name '" + name + "' already exists."); + } + + String reservedError = DomainUtil.validateReservedName(name, "Sample Type"); + if (reservedError != null) + throw new ApiUsageException(reservedError); + } + + @Override + public ValidationException updateSampleType(GWTDomain original, GWTDomain update, SampleTypeDomainKindProperties options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException errors; + + ExpSampleTypeImpl st = new ExpSampleTypeImpl(getMaterialSource(update.getDomainURI())); + + StringBuilder changeDetails = new StringBuilder(); + + Map oldProps = new LinkedHashMap<>(); + Map newProps = new LinkedHashMap<>(); + + String newName = StringUtils.trimToNull(update.getName()); + String oldSampleTypeName = st.getName(); + oldProps.put("Name", oldSampleTypeName); + newProps.put("Name", newName); + + boolean hasNameChange = false; + if (!oldSampleTypeName.equals(newName)) + { + validateSampleTypeName(container, user, newName, oldSampleTypeName.equalsIgnoreCase(newName)); + hasNameChange = true; + st.setName(newName); + changeDetails.append("The name of the sample type '").append(oldSampleTypeName).append("' was changed to '").append(newName).append("'."); + } + + String newDescription = StringUtils.trimToNull(update.getDescription()); + String description = st.getDescription(); + if (StringUtils.isNotBlank(description)) + oldProps.put("Description", description); + if (StringUtils.isNotBlank(newDescription)) + newProps.put("Description", newDescription); + if (description == null || !description.equals(newDescription)) + st.setDescription(newDescription); + + Map oldProps_ = st.getAuditRecordMap(); + Map newProps_ = options != null ? options.getAuditRecordMap() : st.getAuditRecordMap() /* no update */; + newProps.putAll(newProps_); + oldProps.putAll(oldProps_); + + if (options != null) + { + String sampleIdPattern = StringUtils.trimToNull(StringUtilsLabKey.replaceBadCharacters(options.getNameExpression())); + String oldPattern = st.getNameExpression(); + if (oldPattern == null || !oldPattern.equals(sampleIdPattern)) + { + st.setNameExpression(sampleIdPattern); + if (!NameExpressionOptionService.get().allowUserSpecifiedNames(container) && sampleIdPattern == null) + throw new ApiUsageException(container.hasProductFolders() ? NAME_EXPRESSION_REQUIRED_MSG_WITH_SUBFOLDERS : NAME_EXPRESSION_REQUIRED_MSG); + } + + String aliquotIdPattern = StringUtils.trimToNull(options.getAliquotNameExpression()); + String oldAliquotPattern = st.getAliquotNameExpression(); + if (oldAliquotPattern == null || !oldAliquotPattern.equals(aliquotIdPattern)) + st.setAliquotNameExpression(aliquotIdPattern); + + st.setLabelColor(options.getLabelColor()); + st.setMetricUnit(options.getMetricUnit()); + + if (options.getImportAliases() != null && !options.getImportAliases().isEmpty()) + { + try + { + Map> newAliases = options.getImportAliases(); + Set existingRequiredInputs = new HashSet<>(st.getRequiredImportAliases().values()); + String invalidParentType = ExperimentServiceImpl.get().getInvalidRequiredImportAliasUpdate(st.getLSID(), true, newAliases, existingRequiredInputs, container, user); + if (invalidParentType != null) + throw new ApiUsageException("'" + invalidParentType + "' cannot be required as a parent type when there are existing samples without a parent of this type."); + + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + st.setImportAliasMap(options.getImportAliases()); + String targetContainerId = StringUtils.trimToNull(options.getAutoLinkTargetContainerId()); + st.setAutoLinkTargetContainer(targetContainerId != null ? ContainerManager.getForId(targetContainerId) : null); + st.setAutoLinkCategory(options.getAutoLinkCategory()); + if (options.getCategory() != null) // update sample type category is currently not supported + st.setCategory(options.getCategory()); + } + + try (DbScope.Transaction transaction = ensureTransaction()) + { + st.save(user); + if (hasNameChange) + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldSampleTypeName, newName, new SchemaKey(null, SamplesSchema.SCHEMA_NAME), user, container); + + if (options != null && options.getExcludedContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.SampleType, options.getExcludedContainerIds(), st.getRowId(), user); + oldProps.put("ContainerExclusions", exclusionChanges.first); + newProps.put("ContainerExclusions", exclusionChanges.second); + } + if (options != null && options.getExcludedDashboardContainerIds() != null) + { + Pair, Collection> exclusionChanges = ExperimentService.get().ensureDataTypeContainerExclusions(ExperimentService.DataTypeForExclusion.DashboardSampleType, options.getExcludedDashboardContainerIds(), st.getRowId(), user); + oldProps.put("DashboardContainerExclusions", exclusionChanges.first); + newProps.put("DashboardContainerExclusions", exclusionChanges.second); + } + + errors = DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps); + + if (!errors.hasErrors()) + { + QueryService.get().saveCalculatedFieldsMetadata(SamplesSchema.SCHEMA_NAME, update.getQueryName(), hasNameChange ? newName : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (hasNameChange) + ExperimentService.get().addObjectLegacyName(st.getObjectId(), ExperimentServiceImpl.getNamespacePrefix(ExpSampleType.class), oldSampleTypeName, user); + + transaction.addCommitTask(() -> SampleTypeServiceImpl.get().indexSampleType(st, SearchService.get().defaultTask().getQueue(container, SearchService.PRIORITY.modified)), POSTCOMMIT); + transaction.commit(); + refreshSampleTypeMaterializedView(st, SampleChangeType.schema); + } + } + catch (MetadataUnavailableException e) + { + errors = new ValidationException(); + errors.addError(new SimpleValidationError(e.getMessage())); + } + + return errors; + } + + public String getCommentDetailed(QueryService.AuditAction action, boolean isUpdate) + { + String comment = SampleTimelineAuditEvent.SampleTimelineEventType.getActionCommentDetailed(action, isUpdate); + return StringUtils.isEmpty(comment) ? action.getCommentDetailed() : comment; + } + + @Override + public DetailedAuditTypeEvent createDetailedAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, @Nullable Map row, Map existingRow, Map providedValues) + { + return createAuditRecord(c, tInfo, getCommentDetailed(action, !existingRow.isEmpty()), userComment, action, row, existingRow, providedValues); + } + + @Override + protected AuditTypeEvent createSummaryAuditRecord(User user, Container c, AuditConfigurable tInfo, QueryService.AuditAction action, @Nullable String userComment, int rowCount, @Nullable Map row) + { + return createAuditRecord(c, tInfo, String.format(action.getCommentSummary(), rowCount), userComment, row); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable Map row) + { + return createAuditRecord(c, tInfo, comment, userComment, null, row, null, null); + } + + private boolean isInputFieldKey(String fieldKey) + { + int slash = fieldKey.indexOf('/'); + return slash==ExpData.DATA_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpData.DATA_INPUT_PARENT) || + slash==ExpMaterial.MATERIAL_INPUT_PARENT.length() && Strings.CI.startsWith(fieldKey,ExpMaterial.MATERIAL_INPUT_PARENT); + } + + private SampleTimelineAuditEvent createAuditRecord(Container c, AuditConfigurable tInfo, String comment, String userComment, @Nullable QueryService.AuditAction action, @Nullable Map row, @Nullable Map existingRow, @Nullable Map providedValues) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(c, comment); + event.setUserComment(userComment); + + var staticsRow = existingRow != null && !existingRow.isEmpty() ? existingRow : row; + if (row != null) + { + Optional parentFields = row.keySet().stream().filter(this::isInputFieldKey).findAny(); + event.setLineageUpdate(parentFields.isPresent()); + + if (staticsRow.containsKey(LSID)) + event.setSampleLsid(String.valueOf(staticsRow.get(LSID))); + if (staticsRow.containsKey(ROW_ID) && staticsRow.get(ROW_ID) != null) + event.setSampleId((Integer) staticsRow.get(ROW_ID)); + if (staticsRow.containsKey(NAME)) + event.setSampleName(String.valueOf(staticsRow.get(NAME))); + + String sampleTypeLsid = null; + if (staticsRow.containsKey(CPAS_TYPE)) + sampleTypeLsid = String.valueOf(staticsRow.get(CPAS_TYPE)); + // When a sample is deleted, the LSID is provided via the "sampleset" field instead of "LSID" + if (sampleTypeLsid == null && staticsRow.containsKey("sampleset")) + sampleTypeLsid = String.valueOf(staticsRow.get("sampleset")); + + ExpSampleType sampleType = null; + if (sampleTypeLsid != null) + sampleType = SampleTypeService.get().getSampleTypeByType(sampleTypeLsid, c); + else if (event.getSampleId() > 0) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleId()); + if (sample != null) sampleType = sample.getSampleType(); + } + else if (event.getSampleLsid() != null) + { + ExpMaterial sample = ExperimentService.get().getExpMaterial(event.getSampleLsid()); + if (sample != null) sampleType = sample.getSampleType(); + } + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + + // NOTE: to avoid a diff in the audit log make sure row("rowid") is correct! (not the unused generated value) + row.put(ROW_ID,staticsRow.get(ROW_ID)); + } + else if (tInfo != null) + { + UserSchema schema = tInfo.getUserSchema(); + if (schema != null) + { + ExpSampleType sampleType = getSampleType(c, schema.getUser(), tInfo.getName()); + if (sampleType != null) + { + event.setSampleType(sampleType.getName()); + event.setSampleTypeId(sampleType.getRowId()); + } + } + } + + // Put the raw amount and units into the stored amount and unit fields to override the conversion to display values that has happened via the expression columns + if (existingRow != null && !existingRow.isEmpty()) + { + if (existingRow.containsKey(RawAmount.name())) + existingRow.put(StoredAmount.name(), existingRow.get(RawAmount.name())); + if (existingRow.containsKey(RawUnits.name())) + existingRow.put(Units.name(), existingRow.get(RawUnits.name())); + } + + // Add providedValues to eventMetadata + Map eventMetadata = new HashMap<>(); + if (providedValues != null) + { + eventMetadata.putAll(providedValues); + } + if (action != null) + { + SampleTimelineAuditEvent.SampleTimelineEventType timelineEventType = SampleTimelineAuditEvent.SampleTimelineEventType.getTypeFromAction(action); + if (timelineEventType != null) + eventMetadata.put(SAMPLE_TIMELINE_EVENT_TYPE, action); + } + if (!eventMetadata.isEmpty()) + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(eventMetadata)); + + return event; + } + + private SampleTimelineAuditEvent createAuditRecord(Container container, String comment, String userComment, ExpMaterial sample, @Nullable Map metadata) + { + SampleTimelineAuditEvent event = new SampleTimelineAuditEvent(container, comment); + event.setSampleName(sample.getName()); + event.setSampleLsid(sample.getLSID()); + event.setSampleId(sample.getRowId()); + ExpSampleType type = sample.getSampleType(); + if (type != null) + { + event.setSampleType(type.getName()); + event.setSampleTypeId(type.getRowId()); + } + event.setUserComment(userComment); + event.setMetadata(AbstractAuditTypeProvider.encodeForDataMap(metadata)); + return event; + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata) + { + AuditLogService.get().addEvent(user, createAuditRecord(container, comment, userComment, sample, metadata)); + } + + @Override + public void addAuditEvent(User user, Container container, String comment, String userComment, ExpMaterial sample, Map metadata, String updateType) + { + SampleTimelineAuditEvent event = createAuditRecord(container, comment, userComment, sample, metadata); + event.setInventoryUpdateType(updateType); + event.setUserComment(userComment); + AuditLogService.get().addEvent(user, event); + } + + @Override + public long getMaxAliquotId(@NotNull String sampleName, @NotNull String sampleTypeLsid, Container container) + { + long max = 0; + String aliquotNamePrefix = sampleName + "-"; + + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("cpastype"), sampleTypeLsid); + filter.addCondition(FieldKey.fromParts("Name"), aliquotNamePrefix, STARTS_WITH); + + TableSelector selector = new TableSelector(getTinfoMaterial(), Collections.singleton("Name"), filter, null); + final List aliquotIds = new ArrayList<>(); + selector.forEach(String.class, fullname -> aliquotIds.add(fullname.replace(aliquotNamePrefix, ""))); + + for (String aliquotId : aliquotIds) + { + try + { + long id = Long.parseLong(aliquotId); + if (id > max) + max = id; + } + catch (NumberFormatException ignored) { + } + } + + return max; + } + + @Override + public Collection getSamplesNotPermitted(Collection samples, SampleOperations operation) + { + return samples.stream() + .filter(sample -> !sample.isOperationPermitted(operation)) + .collect(Collectors.toList()); + } + + @Override + public String getOperationNotPermittedMessage(Collection samples, SampleOperations operation) + { + String message; + if (samples.size() == 1) + { + ExpMaterial sample = samples.iterator().next(); + message = "Sample " + sample.getName() + " has status " + sample.getStateLabel() + ", which prevents"; + } + else + { + message = samples.size() + " samples ("; + message += samples.stream().limit(10).map(ExpMaterial::getNameAndStatus).collect(Collectors.joining(", ")); + if (samples.size() > 10) + message += " ..."; + message += ") have statuses that prevent"; + } + return message + " " + operation.getDescription() + "."; + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSampleTypeRollup(ExpSampleType sampleType, Container container) throws IllegalStateException, SQLException + { + Pair, Collection> parentsGroup = getAliquotParentsForRecalc(sampleType.getLSID(), container); + Collection allParents = parentsGroup.first; + Collection withAmountsParents = parentsGroup.second; + return recomputeSamplesRollup(allParents, withAmountsParents, sampleType.getMetricUnit(), container); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + @Override + public int recomputeSamplesRollup(Collection sampleIds, String sampleTypeMetricUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(sampleIds, sampleIds, sampleTypeMetricUnit, container); + } + + public record AliquotAmountUnitResult(Double amount, String unit, boolean isAvailable) {} + + public record AliquotAvailableAmountUnit(Double amount, String unit, Double availableAmount) {} + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + private int recomputeSamplesRollup(Collection parents, Collection withAmountsParents, String sampleTypeUnit, Container container) throws IllegalStateException, SQLException + { + return recomputeSamplesRollup(parents, null, withAmountsParents, sampleTypeUnit, container, false); + } + + /** This method updates exp.material, caller should call {@link SampleTypeServiceImpl#refreshSampleTypeMaterializedView} as appropriate. */ + public int recomputeSamplesRollup( + Collection parents, + @Nullable Collection availableParents, + Collection withAmountsParents, + String sampleTypeUnit, + Container container, + boolean useRootMaterialLSID + ) throws IllegalStateException, SQLException + { + Map sampleUnits = new LongHashMap<>(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + DbScope scope = materialTable.getSchema().getScope(); + + List availableSampleStates = new LongArrayList(); + + if (SampleStatusService.get().supportsSampleStatus()) + { + for (DataState state: SampleStatusService.get().getAllProjectStates(container)) + { + if (ExpSchema.SampleStateType.Available.name().equals(state.getStateType())) + availableSampleStates.add(state.getRowId()); + } + } + + if (!parents.isEmpty()) + { + Map> sampleAliquotCounts = getSampleAliquotCounts(parents, useRootMaterialLSID); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("rollupCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotCount: sublist) + { + Long sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!parents.isEmpty() || (availableParents != null && !availableParents.isEmpty())) + { + Map> sampleAliquotCounts = getSampleAvailableAliquotCounts(availableParents == null ? parents : availableParents, availableSampleStates, useRootMaterialLSID); + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter count = new Parameter("AvailableAliquotCount", JdbcType.INTEGER); + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AvailableAliquotCount = ? WHERE RowId = ?").addAll(count, rowid), null); + + List>> sampleAliquotCountList = new ArrayList<>(sampleAliquotCounts.entrySet()); + + ListUtils.partition(sampleAliquotCountList, 1000).forEach(sublist -> + { + for (var sampleAliquotCount: sublist) + { + var sampleId = sampleAliquotCount.getKey(); + Integer aliquotCount = sampleAliquotCount.getValue().first; + String sampleUnit = sampleAliquotCount.getValue().second; + sampleUnits.put(sampleId, sampleUnit); + + rowid.setValue(sampleId); + count.setValue(aliquotCount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + if (!withAmountsParents.isEmpty()) + { + Map> samplesAliquotAmounts = getSampleAliquotAmounts(withAmountsParents, availableSampleStates, useRootMaterialLSID); + + try (Connection c = scope.getConnection()) + { + Parameter rowid = new Parameter("rowid", JdbcType.INTEGER); + Parameter amount = new Parameter("amount", JdbcType.DOUBLE); + Parameter unit = new Parameter("unit", JdbcType.VARCHAR); + Parameter availableAmount = new Parameter("availableAmount", JdbcType.DOUBLE); + + ParameterMapStatement pm = new ParameterMapStatement(scope, c, + new SQLFragment("UPDATE ").append(materialTable).append(" SET AliquotVolume = ?, AliquotUnit = ? , AvailableAliquotVolume = ? WHERE RowId = ? ").addAll(amount, unit, availableAmount, rowid), null); + + List>> sampleAliquotAmountsList = new ArrayList<>(samplesAliquotAmounts.entrySet()); + + ListUtils.partition(sampleAliquotAmountsList, 1000).forEach(sublist -> + { + for (Map.Entry> sampleAliquotAmounts: sublist) + { + Long sampleId = sampleAliquotAmounts.getKey(); + List aliquotAmounts = sampleAliquotAmounts.getValue(); + + if (aliquotAmounts == null || aliquotAmounts.isEmpty()) + continue; + AliquotAvailableAmountUnit amountUnit = computeAliquotTotalAmounts(aliquotAmounts, sampleTypeUnit, sampleUnits.get(sampleId)); + rowid.setValue(sampleId); + amount.setValue(amountUnit.amount); + unit.setValue(amountUnit.unit); + availableAmount.setValue(amountUnit.availableAmount); + + pm.addBatch(); + } + pm.executeBatch(); + }); + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + } + + return !parents.isEmpty() ? parents.size() : (availableParents != null ? availableParents.size() : withAmountsParents.size()); + } + + @Override + public int recomputeSampleTypeRollup(@NotNull ExpSampleType sampleType, Set rootRowIds, Set parentNames, Container container) throws SQLException + { + Set rootSamplesToRecalc = new LongHashSet(); + if (rootRowIds != null) + rootSamplesToRecalc.addAll(rootRowIds); + if (parentNames != null) + rootSamplesToRecalc.addAll(getRootSampleIdsFromParentNames(sampleType.getLSID(), parentNames)); + + return recomputeSamplesRollup(rootSamplesToRecalc, rootSamplesToRecalc, sampleType.getMetricUnit(), container); + } + + private Set getRootSampleIdsFromParentNames(String sampleTypeLsid, Set parentNames) + { + if (parentNames == null || parentNames.isEmpty()) + return Collections.emptySet(); + + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sql = new SQLFragment("SELECT rowid FROM ").append(tableInfo, "") + .append(" WHERE cpastype = ").appendValue(sampleTypeLsid) + .append(" AND rowid IN (") + .append(" SELECT DISTINCT rootmaterialrowid FROM ").append(tableInfo, "") + .append(" WHERE Name").appendInClause(parentNames, tableInfo.getSqlDialect()) + .append(")"); + + return new SqlSelector(tableInfo.getSchema(), sql).fillSet(Long.class, new HashSet<>()); + } + + private AliquotAvailableAmountUnit computeAliquotTotalAmounts(List volumeUnits, String sampleTypeUnitsStr, String sampleItemUnitsStr) + { + if (volumeUnits == null || volumeUnits.isEmpty()) + return null; + + Unit totalUnit = null; + String totalUnitsStr; + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnitsStr = sampleTypeUnitsStr; + else if (!StringUtils.isEmpty(sampleItemUnitsStr)) + totalUnitsStr = sampleItemUnitsStr; + else // use the unit of the first aliquot if there are no other indications + totalUnitsStr = volumeUnits.get(0).unit; + if (!StringUtils.isEmpty(totalUnitsStr)) + { + try + { + if (!StringUtils.isEmpty(sampleTypeUnitsStr)) + totalUnit = Unit.valueOf(totalUnitsStr).getBase(); + else + totalUnit = Unit.valueOf(totalUnitsStr); + } + catch (IllegalArgumentException e) + { + // do nothing; leave unit as null + } + } + + double totalVolume = 0.0; + double totalAvailableVolume = 0.0; + + for (AliquotAmountUnitResult volumeUnit : volumeUnits) + { + Unit unit = null; + try + { + double storedAmount = volumeUnit.amount; + String aliquotUnit = volumeUnit.unit; + boolean isAvailable = volumeUnit.isAvailable; + + try + { + unit = StringUtils.isEmpty(aliquotUnit) ? totalUnit : Unit.fromName(aliquotUnit); + } + catch (IllegalArgumentException ignore) + { + } + + double convertedAmount = 0; + // include in total volume only if aliquot unit is compatible + if (totalUnit != null && totalUnit.isCompatible(unit)) + convertedAmount = Unit.convert(storedAmount, unit, totalUnit); + else if (totalUnit == null) // sample (or 1st aliquot) unit is not a supported unit + { + if (StringUtils.isEmpty(sampleTypeUnitsStr) && StringUtils.isEmpty(aliquotUnit)) //aliquot units are empty + convertedAmount = storedAmount; + else if (sampleTypeUnitsStr != null && sampleTypeUnitsStr.equalsIgnoreCase(aliquotUnit)) //aliquot units use the same unsupported unit ('cc') + convertedAmount = storedAmount; + } + + totalVolume += convertedAmount; + if (isAvailable) + totalAvailableVolume += convertedAmount; + } + catch (IllegalArgumentException ignore) // invalid volume + { + + } + } + int scale = totalUnit == null ? Quantity.DEFAULT_PRECISION_SCALE : totalUnit.getPrecisionScale(); + totalVolume = Precision.round(totalVolume, scale); + totalAvailableVolume = Precision.round(totalAvailableVolume, scale); + + return new AliquotAvailableAmountUnit(totalVolume, totalUnit == null ? null : totalUnit.name(), totalAvailableVolume); + } + + public Pair, Collection> getAliquotParentsForRecalc(String sampleTypeLsid, Container container) throws SQLException + { + Collection parents = getAliquotParents(sampleTypeLsid, container); + Collection withAmountsParents = parents.isEmpty() ? Collections.emptySet() : getAliquotsWithAmountsParents(sampleTypeLsid, container); + return new Pair<>(parents, withAmountsParents); + } + + private Collection getAliquotParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, false, container); + } + + private Collection getAliquotsWithAmountsParents(String sampleTypeLsid, Container container) throws IllegalStateException, SQLException + { + return getAliquotParents(sampleTypeLsid, true, container); + } + + private SQLFragment getParentsOfAliquotsWithAmountsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE aliquot.storedAmount IS NOT NULL AND\s + """); + } + + private SQLFragment getParentsOfAliquotsSql() + { + return new SQLFragment( + """ + SELECT DISTINCT parent.rowId, parent.cpastype + FROM exp.material AS aliquot + JOIN exp.material AS parent ON aliquot.rootMaterialRowId = parent.rowId AND aliquot.rootMaterialRowId <> aliquot.rowId + WHERE + """); + } + + private Collection getAliquotParents(String sampleTypeLsid, boolean withAmount, Container container) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + + SQLFragment sql = withAmount ? getParentsOfAliquotsWithAmountsSql() : getParentsOfAliquotsSql(); + + sql.append("parent.cpastype = ?"); + sql.add(sampleTypeLsid); + sql.append(" AND parent.container = ?"); + sql.add(container.getId()); + + Set parentIds = new LongHashSet(); + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + parentIds.add(rs.getLong(1)); + } + + return parentIds; + } + + private Map> getSampleAliquotCounts(Collection sampleIds, boolean useRootMaterialLSID) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql = new SQLFragment("SELECT m.RowId as SampleId, m.Units, (SELECT COUNT(*) FROM exp.material a WHERE ") + .append(useRootMaterialLSID ? "a.rootMaterialLsid = m.lsid" : "a.rootMaterialRowId = m.rowId") + .append(")-1 AS CreatedAliquotCount FROM exp.material AS m WHERE m.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order sample by rowId to reduce probability of deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAvailableAliquotCounts(Collection sampleIds, Collection availableSampleStates, boolean useRootMaterialLSID) throws SQLException + { + DbSchema dbSchema = getExpSchema(); + SqlDialect dialect = dbSchema.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql; + if (useRootMaterialLSID) + { + sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialLSID as rootLsid, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialLSID <> LSID AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialLSID + ) AS c ON m.lsid = c.rootLsid + WHERE m.rootmateriallsid = m.LSID AND m.rowid\s"""); + } + else + { + sql = new SQLFragment( + """ + SELECT m.RowId as SampleId, m.Units, + (CASE WHEN c.aliquotCount IS NULL THEN 0 ELSE c.aliquotCount END) as CreatedAliquotCount + FROM exp.material AS m + LEFT JOIN ( + SELECT RootMaterialRowId as rootRowId, COUNT(*) as aliquotCount + FROM exp.material + WHERE RootMaterialRowId <> RowId AND SampleState\s""") + .appendInClause(availableSampleStates, dialect) + .append(""" + GROUP BY RootMaterialRowId + ) AS c ON m.rowId = c.rootRowId + WHERE m.rootmaterialrowid = m.rowid AND m.rowid\s"""); + } + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotCounts = new TreeMap<>(); // Order by rowId to reduce deadlock with search indexer + try (ResultSet rs = new SqlSelector(dbSchema, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + String sampleUnit = rs.getString(2); + int aliquotCount = rs.getInt(3); + + sampleAliquotCounts.put(parentId, new Pair<>(aliquotCount, sampleUnit)); + } + } + + return sampleAliquotCounts; + } + + private Map> getSampleAliquotAmounts(Collection sampleIds, List availableSampleStates, boolean useRootMaterialLSID) throws SQLException + { + DbSchema exp = getExpSchema(); + SqlDialect dialect = exp.getSqlDialect(); + + // Issue 49150: In 23.12 we migrated from RootMaterialLSID to RootMaterialRowID, however, there is still an + // upgrade path that requires these queries be done with RootMaterialLSID since the 23.12 upgrade will not + // have run yet. + SQLFragment sql = new SQLFragment("SELECT parent.rowid AS parentSampleId, aliquot.StoredAmount, aliquot.Units, aliquot.samplestate\n") + .append("FROM exp.material AS aliquot JOIN exp.material AS parent ON ") + .append(useRootMaterialLSID ? "parent.lsid = aliquot.rootmateriallsid" : "parent.rowid = aliquot.rootmaterialrowid") + .append(" WHERE ") + .append(useRootMaterialLSID ? "aliquot.rootmateriallsid <> aliquot.lsid" : "aliquot.rootmaterialrowid <> aliquot.rowid") + .append(" AND parent.rowid "); + dialect.appendInClauseSql(sql, sampleIds); + + Map> sampleAliquotAmounts = new LongHashMap<>(); + + try (ResultSet rs = new SqlSelector(exp, sql).getResultSet()) + { + while (rs.next()) + { + long parentId = rs.getLong(1); + Double volume = rs.getDouble(2); + String unit = rs.getString(3); + long sampleState = rs.getLong(4); + + if (!sampleAliquotAmounts.containsKey(parentId)) + sampleAliquotAmounts.put(parentId, new ArrayList<>()); + + sampleAliquotAmounts.get(parentId).add(new AliquotAmountUnitResult(volume, unit, availableSampleStates.contains(sampleState))); + } + } + // for any parents with no remaining aliquots, set the amounts to 0 + for (var parentId : sampleIds) + { + if (!sampleAliquotAmounts.containsKey(parentId)) + { + List aliquotAmounts = new ArrayList<>(); + aliquotAmounts.add(new AliquotAmountUnitResult(0.0, null, false)); + sampleAliquotAmounts.put(parentId, aliquotAmounts); + } + } + + return sampleAliquotAmounts; + } + + record FileFieldRenameData(ExpSampleType sampleType, String sampleName, String fieldName, File sourceFile, File targetFile) { } + + @Override + public Map moveSamples(Collection samples, @NotNull Container sourceContainer, @NotNull Container targetContainer, @NotNull User user, @Nullable String userComment, @Nullable AuditBehaviorType auditBehavior) throws ExperimentException, BatchValidationException + { + if (samples == null || samples.isEmpty()) + throw new IllegalArgumentException("No samples provided to move operation."); + + Map> sampleTypesMap = new HashMap<>(); + samples.forEach(sample -> + sampleTypesMap.computeIfAbsent(sample.getSampleType(), t -> new ArrayList<>()).add(sample)); + Map updateCounts = new HashMap<>(); + updateCounts.put("samples", 0); + updateCounts.put("sampleAliases", 0); + updateCounts.put("sampleAuditEvents", 0); + Map> fileMovesBySampleId = new LongHashMap<>(); + ExperimentService expService = ExperimentService.get(); + + try (DbScope.Transaction transaction = ensureTransaction()) + { + if (AuditBehaviorType.NONE != auditBehavior && transaction.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(samples.size()); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, auditEvent); + } + + for (Map.Entry> entry: sampleTypesMap.entrySet()) + { + ExpSampleType sampleType = entry.getKey(); + SamplesSchema schema = new SamplesSchema(user, sampleType.getContainer()); + TableInfo samplesTable = schema.getTable(sampleType, null); + + List typeSamples = entry.getValue(); + List sampleIds = typeSamples.stream().map(ExpMaterial::getRowId).toList(); + + // update for exp.material.container + updateCounts.put("samples", updateCounts.get("samples") + Table.updateContainer(getTinfoMaterial(), "rowid", sampleIds, targetContainer, user, true)); + + // update for exp.object.container + expService.updateExpObjectContainers(getTinfoMaterial(), sampleIds, targetContainer); + + // update the paths to files associated with individual samples + fileMovesBySampleId.putAll(updateSampleFilePaths(sampleType, typeSamples, targetContainer, user)); + + // update for exp.materialaliasmap.container + updateCounts.put("sampleAliases", updateCounts.get("sampleAliases") + expService.aliasMapRowContainerUpdate(getTinfoMaterialAliasMap(), sampleIds, targetContainer)); + + // update inventory.item.container + InventoryService inventoryService = InventoryService.get(); + if (inventoryService != null) + { + Map inventoryCounts = inventoryService.moveSamples(sampleIds, targetContainer, user); + inventoryCounts.forEach((key, count) -> updateCounts.compute(key, (k, c) -> c == null ? count : c + count)); + } + + // create summary audit entries for the source and target containers + String samplesPhrase = StringUtilsLabKey.pluralize(sampleIds.size(), "sample"); + addSampleTypeAuditEvent(user, sourceContainer, sampleType, + "Moved " + samplesPhrase + " to " + targetContainer.getPath(), userComment, "moved from project"); + addSampleTypeAuditEvent(user, targetContainer, sampleType, + "Moved " + samplesPhrase + " from " + sourceContainer.getPath(), userComment, "moved to project"); + + // move the events associated with the samples that have moved + SampleTimelineAuditProvider auditProvider = new SampleTimelineAuditProvider(); + int auditEventCount = auditProvider.moveEvents(targetContainer, sampleIds); + updateCounts.compute("sampleAuditEvents", (k, c) -> c == null ? auditEventCount : c + auditEventCount); + + AuditBehaviorType stAuditBehavior = samplesTable.getEffectiveAuditBehavior(auditBehavior); + // create new events for each sample that was moved. + if (stAuditBehavior == AuditBehaviorType.DETAILED) + { + for (ExpMaterial sample : typeSamples) + { + SampleTimelineAuditEvent event = createAuditRecord(targetContainer, "Sample folder was updated.", userComment, sample, null); + Map oldRecordMap = new HashMap<>(); + // ContainerName is remapped to "Folder" within SampleTimelineEvent, but we don't + // use "Folder" here because this sample-type field is filtered out of timeline events by default + oldRecordMap.put("ContainerName", sourceContainer.getName()); + Map newRecordMap = new HashMap<>(); + newRecordMap.put("ContainerName", targetContainer.getName()); + if (fileMovesBySampleId.containsKey(sample.getRowId())) + { + fileMovesBySampleId.get(sample.getRowId()).forEach(fileUpdateData -> { + oldRecordMap.put(fileUpdateData.fieldName, fileUpdateData.sourceFile.getAbsolutePath()); + newRecordMap.put(fileUpdateData.fieldName, fileUpdateData.targetFile.getAbsolutePath()); + }); + } + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(oldRecordMap)); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(newRecordMap)); + AuditLogService.get().addEvent(user, event); + } + } + } + + updateCounts.putAll(moveDerivationRuns(samples, targetContainer, user)); + + transaction.addCommitTask(() -> { + for (ExpSampleType sampleType : sampleTypesMap.keySet()) + { + // force refresh of materialized view + SampleTypeServiceImpl.get().refreshSampleTypeMaterializedView(sampleType, SampleChangeType.update); + // update search index for moved samples via indexSampleType() helper, it filters for samples to index + // based on the modified date + SampleTypeServiceImpl.get().indexSampleType(sampleType, SearchService.get().defaultTask().getQueue(sampleType.getContainer(), SearchService.PRIORITY.modified)); + } + }, DbScope.CommitTaskOption.IMMEDIATE, POSTCOMMIT, POSTROLLBACK); + + // add up the size of the value arrays in the fileMovesBySampleId map + int fileMoveCount = fileMovesBySampleId.values().stream().mapToInt(List::size).sum(); + updateCounts.put("sampleFiles", fileMoveCount); + transaction.addCommitTask(() -> { + for (List sampleFileRenameData : fileMovesBySampleId.values()) + { + for (FileFieldRenameData renameData : sampleFileRenameData) + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); + } + }, POSTCOMMIT); + + transaction.commit(); + } + + return updateCounts; + } + + private Map moveDerivationRuns(Collection samples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + // collect unique runIds mapped to the samples that are moving that have that runId + Map> runIdSamples = new LongHashMap<>(); + samples.forEach(sample -> { + if (sample.getRunId() != null) + runIdSamples.computeIfAbsent(sample.getRunId(), t -> new HashSet<>()).add(sample); + }); + ExperimentService expService = ExperimentService.get(); + // find the set of runs associated with samples that are moving + List runs = expService.getExpRuns(runIdSamples.keySet()); + List toUpdate = new ArrayList<>(); + List toSplit = new ArrayList<>(); + for (ExpRun run : runs) + { + Set outputIds = run.getMaterialOutputs().stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + Set movingIds = runIdSamples.get(run.getRowId()).stream().map(ExpMaterial::getRowId).collect(Collectors.toSet()); + if (movingIds.size() == outputIds.size() && movingIds.containsAll(outputIds)) + toUpdate.add(run); + else + toSplit.add(run); + } + + int updateCount = expService.moveExperimentRuns(toUpdate, targetContainer, user); + int splitCount = splitExperimentRuns(toSplit, runIdSamples, targetContainer, user); + return Map.of("sampleDerivationRunsUpdated", updateCount, "sampleDerivationRunsSplit", splitCount); + } + + private int splitExperimentRuns(List runs, Map> movingSamples, Container targetContainer, User user) throws ExperimentException, BatchValidationException + { + final ViewBackgroundInfo targetInfo = new ViewBackgroundInfo(targetContainer, user, null); + ExperimentServiceImpl expService = (ExperimentServiceImpl) ExperimentService.get(); + int runCount = 0; + for (ExpRun run : runs) + { + ExpProtocolApplication sourceApplication = null; + ExpProtocolApplication outputApp = run.getOutputProtocolApplication(); + boolean isAliquot = SAMPLE_ALIQUOT_PROTOCOL_LSID.equals(run.getProtocol().getLSID()); + + Set movingSet = movingSamples.get(run.getRowId()); + int numStaying = 0; + Map movingOutputsMap = new HashMap<>(); + ExpMaterial aliquotParent = null; + // the derived samples (outputs of the run) are inputs to the output step of the run (obviously) + for (ExpMaterialRunInput materialInput : outputApp.getMaterialInputs()) + { + ExpMaterial material = materialInput.getMaterial(); + if (movingSet.contains(material)) + { + // clear out the run and source application so a new derivation run can be created. + material.setRun(null); + material.setSourceApplication(null); + movingOutputsMap.put(material, materialInput.getRole()); + } + else + { + if (sourceApplication == null) + sourceApplication = material.getSourceApplication(); + numStaying++; + } + if (isAliquot && aliquotParent == null && material.getAliquotedFromLSID() != null) + { + aliquotParent = expService.getExpMaterial(material.getAliquotedFromLSID()); + } + } + + try + { + if (isAliquot && aliquotParent != null) + { + ExpRunImpl expRun = expService.createAliquotRun(aliquotParent, movingOutputsMap.keySet(), targetInfo); + expService.saveSimpleExperimentRun(expRun, run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), Collections.emptyMap(), targetInfo, LOG, false); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getAliquotRunName(aliquotParent, numStaying)); + } + else + { + // create a new derivation run for the samples that are moving + expService.derive(run.getMaterialInputs(), run.getDataInputs(), movingOutputsMap, Collections.emptyMap(), targetInfo, LOG); + // Update the run for the samples that have stayed behind. Change the name and remove the moved samples as outputs + run.setName(ExperimentServiceImpl.getDerivationRunName(run.getMaterialInputs(), run.getDataInputs(), numStaying, run.getDataOutputs().size())); + } + } + catch (ValidationException e) + { + BatchValidationException errors = new BatchValidationException(); + errors.addRowError(e); + throw errors; + } + run.save(user); + List movingSampleIds = movingSet.stream().map(ExpMaterial::getRowId).toList(); + + outputApp.removeMaterialInputs(user, movingSampleIds); + if (sourceApplication != null) + sourceApplication.removeMaterialInputs(user, movingSampleIds); + + runCount++; + } + return runCount; + } + + record SampleFileMoveReference(String sourceFilePath, File targetFile, Container sourceContainer, String sampleName, String fieldName) {} + + // return the map of file renames + private Map> updateSampleFilePaths(ExpSampleType sampleType, List samples, Container targetContainer, User user) throws ExperimentException + { + Map> sampleFileRenames = new LongHashMap<>(); + + FileContentService fileService = FileContentService.get(); + if (fileService == null) + { + LOG.warn("No file service available. Sample files cannot be moved."); + return sampleFileRenames; + } + + if (fileService.getFileRoot(targetContainer) == null) + { + LOG.warn("No file root found for target container " + targetContainer + "'. Files cannot be moved."); + return sampleFileRenames; + } + + List fileDomainProps = sampleType.getDomain() + .getProperties().stream() + .filter(prop -> PropertyType.FILE_LINK.getTypeUri().equals(prop.getRangeURI())).toList(); + if (fileDomainProps.isEmpty()) + return sampleFileRenames; + + Map hasFileRoot = new HashMap<>(); + Map fileMoveCounts = new HashMap<>(); + Map fileMoveReferences = new HashMap<>(); + for (ExpMaterial sample : samples) + { + boolean hasSourceRoot = hasFileRoot.computeIfAbsent(sample.getContainer(), (container) -> fileService.getFileRoot(container) != null); + if (!hasSourceRoot) + LOG.warn("No file root found for source container " + sample.getContainer() + ". Files cannot be moved."); + else + for (DomainProperty fileProp : fileDomainProps ) + { + String sourceFileName = (String) sample.getProperty(fileProp); + if (StringUtils.isBlank(sourceFileName)) + continue; + File updatedFile = FileContentService.get().getMoveTargetFile(sourceFileName, sample.getContainer(), targetContainer); + if (updatedFile != null) + { + + if (!fileMoveReferences.containsKey(sourceFileName)) + fileMoveReferences.put(sourceFileName, new SampleFileMoveReference(sourceFileName, updatedFile, sample.getContainer(), sample.getName(), fileProp.getName())); + if (!fileMoveCounts.containsKey(sourceFileName)) + fileMoveCounts.put(sourceFileName, 0); + fileMoveCounts.put(sourceFileName, fileMoveCounts.get(sourceFileName) + 1); + + File sourceFile = new File(sourceFileName); + FileFieldRenameData renameData = new FileFieldRenameData(sampleType, sample.getName(), fileProp.getName(), sourceFile, updatedFile); + sampleFileRenames.putIfAbsent(sample.getRowId(), new ArrayList<>()); + List fieldRenameData = sampleFileRenames.get(sample.getRowId()); + fieldRenameData.add(renameData); + } + } + } + + for (String filePath : fileMoveReferences.keySet()) + { + SampleFileMoveReference ref = fileMoveReferences.get(filePath); + File sourceFile = new File(filePath); + if (!ExperimentServiceImpl.get().canMoveFileReference(user, ref.sourceContainer, sourceFile, fileMoveCounts.get(filePath))) + throw new ExperimentException("Sample " + ref.sampleName + " cannot be moved since it references a shared file: " + sourceFile.getName()); + + // TODO, support batch fireFileMoveEvent to avoid excessive FileLinkFileListener.hardTableFileLinkColumns calls + fileService.fireFileMoveEvent(sourceFile, ref.targetFile, user, targetContainer); + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(targetContainer, "File moved from " + ref.sourceContainer.getPath() + " to " + targetContainer.getPath() + "."); + event.setProvidedFileName(sourceFile.getName()); + event.setFile(ref.targetFile.getName()); + event.setDirectory(ref.targetFile.getParent()); + event.setFieldName(ref.fieldName); + AuditLogService.get().addEvent(user, event); + } + + return sampleFileRenames; + } + + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) + { + if (!renameData.targetFile.getParentFile().exists()) + { + String errorMsg = String.format("Creation of target directory '%s' to move file '%s' to, for '%s' sample '%s' (field: '%s') failed.", + renameData.targetFile.getParent(), + renameData.sourceFile.getAbsolutePath(), + renameData.sampleType.getName(), + renameData.sampleName, + renameData.fieldName); + try + { + if (!FileUtil.mkdirs(renameData.targetFile.getParentFile())) + { + LOG.warn(errorMsg); + return false; + } + } + catch (IOException e) + { + LOG.warn(errorMsg + e.getMessage()); + } + } + + String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); + } + + @Override + @Nullable + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly) + { + return getSampleCountSequence(container, isRootSampleOnly, true); + } + + public DbSequence getSampleCountSequence(Container container, boolean isRootSampleOnly, boolean create) + { + Container seqContainer = container.getProject(); + if (seqContainer == null) + return null; + + String seqName = isRootSampleOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + + if (!create) + { + // check if sequence already exist so we don't create one just for querying + Integer seqRowId = DbSequenceManager.getRowId(seqContainer, seqName, 0); + if (null == seqRowId) + return null; + } + + if (ExperimentService.get().useStrictCounter()) + return DbSequenceManager.getReclaimable(seqContainer, seqName, 0); + + return DbSequenceManager.getPreallocatingSequence(seqContainer, seqName, 0, 100); + } + + @Override + public void ensureMinSampleCount(long newSeqValue, NameGenerator.EntityCounter counterType, Container container) throws ExperimentException + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + + DbSequence seq = getSampleCountSequence(container, isRootOnly, newSeqValue >= 1); + if (seq == null) + return; + + long current = seq.current(); + if (newSeqValue < current) + { + if ((isRootOnly ? getProjectRootSampleCount(container) : getProjectSampleCount(container)) > 0) + throw new ExperimentException("Unable to set " + counterType.name() + " to " + newSeqValue + " due to conflict with existing samples."); + + if (newSeqValue <= 0) + { + deleteSampleCounterSequence(container, isRootOnly); + return; + } + } + + seq.ensureMinimum(newSeqValue); + seq.sync(); + } + + public void deleteSampleCounterSequences(Container container) + { + deleteSampleCounterSequence(container, false); + deleteSampleCounterSequence(container, true); + } + + private void deleteSampleCounterSequence(Container container, boolean isRootOnly) + { + String seqName = isRootOnly ? ROOT_SAMPLE_COUNT_SEQ_NAME : SAMPLE_COUNT_SEQ_NAME; + Container seqContainer = container.getProject(); + DbSequenceManager.delete(seqContainer, seqName); + DbSequenceManager.invalidatePreallocatingSequence(container, seqName, 0); + } + + @Override + public long getProjectSampleCount(Container container) + { + return getProjectSampleCount(container, false); + } + + @Override + public long getProjectRootSampleCount(Container container) + { + return getProjectSampleCount(container, true); + } + + private long getProjectSampleCount(Container container, boolean isRootOnly) + { + User searchUser = User.getSearchUser(); + ContainerFilter.ContainerFilterWithPermission cf = new ContainerFilter.AllInProject(container, searchUser); + Collection validContainerIds = cf.generateIds(container, ReadPermission.class, null); + TableInfo tableInfo = ExperimentService.get().getTinfoMaterial(); + SQLFragment sql = new SQLFragment("SELECT COUNT(*) FROM "); + sql.append(tableInfo); + sql.append(" WHERE "); + if (isRootOnly) + sql.append(" AliquotedFromLsid IS NULL AND "); + sql.append("Container "); + sql.appendInClause(validContainerIds, tableInfo.getSqlDialect()); + return new SqlSelector(ExperimentService.get().getSchema(), sql).getObject(Long.class).longValue(); + } + + @Override + public long getCurrentCount(NameGenerator.EntityCounter counterType, Container container) + { + boolean isRootOnly = counterType == NameGenerator.EntityCounter.rootSampleCount; + DbSequence seq = getSampleCountSequence(container, isRootOnly, false); + if (seq != null) + { + long current = seq.current(); + if (current > 0) + return current; + } + + return getProjectSampleCount(container, counterType == NameGenerator.EntityCounter.rootSampleCount); + } + + public enum SampleChangeType { insert, update, delete, rollup /* aliquot count */, schema } + + public void refreshSampleTypeMaterializedView(@NotNull ExpSampleType st, SampleChangeType reason) + { + ExpMaterialTableImpl.refreshMaterializedView(st.getLSID(), reason); + } + + + public static class TestCase extends Assert + { + @Test + public void testGetValidatedUnit() + { + SampleTypeService service = SampleTypeService.get(); + try + { + service.getValidatedUnit("g", Unit.mg, "Sample Type"); + service.getValidatedUnit("g ", Unit.mg, "Sample Type"); + service.getValidatedUnit(" g ", Unit.mg, "Sample Type"); + } + catch (ConversionExceptionWithMessage e) + { + fail("Compatible unit should not throw exception."); + } + try + { + assertNull(service.getValidatedUnit(null, Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("null units should be null"); + } + try + { + assertNull(service.getValidatedUnit("", Unit.unit, "Sample Type")); + } + catch (ConversionExceptionWithMessage e) + { + fail("empty units should be null"); + } + try + { + service.getValidatedUnit("g", Unit.unit, "Sample Type"); + fail("Units that are not comparable should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + try + { + service.getValidatedUnit("nonesuch", Unit.unit, "Sample Type"); + fail("Invalid units should throw exception."); + } + catch (ConversionExceptionWithMessage ignore) + { + + } + + } + } +} diff --git a/list/src/org/labkey/list/model/ListAuditProvider.java b/list/src/org/labkey/list/model/ListAuditProvider.java index f2294fa07bf..ac40e3dab9b 100644 --- a/list/src/org/labkey/list/model/ListAuditProvider.java +++ b/list/src/org/labkey/list/model/ListAuditProvider.java @@ -1,273 +1,273 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.util.StringExpressionFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class ListAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String COLUMN_NAME_LIST_ID = "ListId"; - public static final String COLUMN_NAME_LIST_DOMAIN_URI = "ListDomainUri"; - public static final String COLUMN_NAME_LIST_ITEM_ENTITY_ID = "ListItemEntityId"; - public static final String COLUMN_NAME_LIST_NAME = "ListName"; - - private static final List defaultVisibleColumns = new ArrayList<>(); - - static - { - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_LIST_DOMAIN_URI)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - } - - public ListAuditProvider() - { - super(new ListAuditDomainKind()); - } - - @Override - public String getEventName() - { - return ListManager.LIST_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "List events"; - } - - @Override - public String getDescription() - { - return "Data about list creation, deletion, insertion, etc."; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_LIST_DOMAIN_URI.equalsIgnoreCase(col.getName())) - { - col.setLabel("List"); - col.setDisplayColumnFactory(colInfo -> new DomainAuditProvider.DomainColumn(colInfo, COLUMN_NAME_CONTAINER, COLUMN_NAME_LIST_NAME)); - } - } - }; - appendValueMapColumns(table, null, true); - - // Render a details URL only for rows that have a listItemEntityId - DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&name=${listName}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); - table.setDetailsURL(url); - - return table; - } - - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("intKey1"), COLUMN_NAME_LIST_ID); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_LIST_DOMAIN_URI); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_LIST_ITEM_ENTITY_ID); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_LIST_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - // Unused Property/oldRecord and Property/newRecord columns should just be migrated to the oldRecordMap and newRecordMap columns - legacyMap.put(FieldKey.fromParts("Property", "OldRecord"), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", "NewRecord"), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)ListAuditEvent.class; - } - - public int moveEvents(Container targetContainer, List listRowEntityIds) - { - return moveEvents(targetContainer, COLUMN_NAME_LIST_ITEM_ENTITY_ID, listRowEntityIds); - } - - public static class ListAuditEvent extends DetailedAuditTypeEvent - { - private int _listId; - private String _listDomainUri; - private String _listItemEntityId; - private String _listName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public ListAuditEvent() - { - super(); - } - - public ListAuditEvent(Container container, String comment, ListDefinitionImpl list) - { - super(ListManager.LIST_AUDIT_EVENT, container, comment); - setListDomainUri(list.getDomain().getTypeURI()); - setListId(list.getListId()); - setListName(list.getName()); - setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), ListManager.LIST_AUDIT_EVENT); - } - - public int getListId() - { - return _listId; - } - - public void setListId(int listId) - { - _listId = listId; - } - - public String getListDomainUri() - { - return _listDomainUri; - } - - public void setListDomainUri(String listDomainUri) - { - _listDomainUri = listDomainUri; - } - - public String getListItemEntityId() - { - return _listItemEntityId; - } - - public void setListItemEntityId(String listItemEntityId) - { - _listItemEntityId = listItemEntityId; - } - - public String getListName() - { - return _listName; - } - - public void setListName(String listName) - { - _listName = listName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("list", getListName() + " (" + getListId() + ")"); - elements.put("listDomainUri", getListDomainUri()); - elements.put("listItemEntityId", getListItemEntityId()); - // N.B. oldRecordMap and newRecordMap can be very large and are not included here - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class ListAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "ListAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public ListAuditDomainKind() - { - super(ListManager.LIST_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ID, PropertyType.INTEGER)); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_DOMAIN_URI, PropertyType.STRING)); - // Choose a length that should be much larger than necessary to give extra buffer, but still small enough - // to be indexed - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ITEM_ENTITY_ID, PropertyType.STRING, 300)); // UNDONE: is needed ? .setEntityId(true)); - fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - - @Override - public Set getPropertyIndices(Domain domain) - { - Set indexes = super.getPropertyIndices(domain); - indexes.add(new PropertyStorageSpec.Index(false, COLUMN_NAME_LIST_ITEM_ENTITY_ID)); - return indexes; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.util.StringExpressionFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ListAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String COLUMN_NAME_LIST_ID = "ListId"; + public static final String COLUMN_NAME_LIST_DOMAIN_URI = "ListDomainUri"; + public static final String COLUMN_NAME_LIST_ITEM_ENTITY_ID = "ListItemEntityId"; + public static final String COLUMN_NAME_LIST_NAME = "ListName"; + + private static final List defaultVisibleColumns = new ArrayList<>(); + + static + { + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_LIST_DOMAIN_URI)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + } + + public ListAuditProvider() + { + super(new ListAuditDomainKind()); + } + + @Override + public String getEventName() + { + return ListManager.LIST_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "List events"; + } + + @Override + public String getDescription() + { + return "Data about list creation, deletion, insertion, etc."; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_LIST_DOMAIN_URI.equalsIgnoreCase(col.getName())) + { + col.setLabel("List"); + col.setDisplayColumnFactory(colInfo -> new DomainAuditProvider.DomainColumn(colInfo, COLUMN_NAME_CONTAINER, COLUMN_NAME_LIST_NAME)); + } + } + }; + appendValueMapColumns(table, null, true); + + // Render a details URL only for rows that have a listItemEntityId + DetailsURL url = DetailsURL.fromString("list/listItemDetails.view?listId=${listId}&name=${listName}&entityId=${listItemEntityId}&rowId=${rowId}", null, StringExpressionFactory.AbstractStringExpression.NullValueBehavior.NullResult); + table.setDetailsURL(url); + + return table; + } + + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("intKey1"), COLUMN_NAME_LIST_ID); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_LIST_DOMAIN_URI); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_LIST_ITEM_ENTITY_ID); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_LIST_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + // Unused Property/oldRecord and Property/newRecord columns should just be migrated to the oldRecordMap and newRecordMap columns + legacyMap.put(FieldKey.fromParts("Property", "OldRecord"), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", "NewRecord"), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)ListAuditEvent.class; + } + + public int moveEvents(Container targetContainer, List listRowEntityIds) + { + return moveEvents(targetContainer, COLUMN_NAME_LIST_ITEM_ENTITY_ID, listRowEntityIds); + } + + public static class ListAuditEvent extends DetailedAuditTypeEvent + { + private int _listId; + private String _listDomainUri; + private String _listItemEntityId; + private String _listName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public ListAuditEvent() + { + super(); + } + + public ListAuditEvent(Container container, String comment, ListDefinitionImpl list) + { + super(ListManager.LIST_AUDIT_EVENT, container, comment); + setListDomainUri(list.getDomain().getTypeURI()); + setListId(list.getListId()); + setListName(list.getName()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), ListManager.LIST_AUDIT_EVENT); + } + + public int getListId() + { + return _listId; + } + + public void setListId(int listId) + { + _listId = listId; + } + + public String getListDomainUri() + { + return _listDomainUri; + } + + public void setListDomainUri(String listDomainUri) + { + _listDomainUri = listDomainUri; + } + + public String getListItemEntityId() + { + return _listItemEntityId; + } + + public void setListItemEntityId(String listItemEntityId) + { + _listItemEntityId = listItemEntityId; + } + + public String getListName() + { + return _listName; + } + + public void setListName(String listName) + { + _listName = listName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("list", getListName() + " (" + getListId() + ")"); + elements.put("listDomainUri", getListDomainUri()); + elements.put("listItemEntityId", getListItemEntityId()); + // N.B. oldRecordMap and newRecordMap can be very large and are not included here + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class ListAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "ListAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public ListAuditDomainKind() + { + super(ListManager.LIST_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ID, PropertyType.INTEGER)); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_DOMAIN_URI, PropertyType.STRING)); + // Choose a length that should be much larger than necessary to give extra buffer, but still small enough + // to be indexed + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_ITEM_ENTITY_ID, PropertyType.STRING, 300)); // UNDONE: is needed ? .setEntityId(true)); + fields.add(createPropertyDescriptor(COLUMN_NAME_LIST_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + + @Override + public Set getPropertyIndices(Domain domain) + { + Set indexes = super.getPropertyIndices(domain); + indexes.add(new PropertyStorageSpec.Index(false, COLUMN_NAME_LIST_ITEM_ENTITY_ID)); + return indexes; + } + } +} diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index c5f84eea273..bb86fed1d37 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -1,890 +1,890 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.attachments.AttachmentParent; -import org.labkey.api.attachments.AttachmentParentFactory; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.LookupResolutionType; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.Selector.ForEachBatchBlock; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.exp.ObjectProperty; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListImportProgress; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.IPropertyValidator; -import org.labkey.api.exp.property.ValidatorContext; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.lists.permissions.ManagePicklistsPermission; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.PropertyValidationError; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.ValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reader.DataLoader; -import org.labkey.api.security.ElevatedUser; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.usageMetrics.SimpleMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.VirtualFile; -import org.labkey.list.view.ListItemAttachmentParent; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.labkey.api.util.IntegerUtils.isIntegral; - -/** - * Implementation of QueryUpdateService for Lists - */ -public class ListQueryUpdateService extends DefaultQueryUpdateService -{ - private final ListDefinitionImpl _list; - private static final String ID = "entityId"; - - public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) - { - super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); - _list = (ListDefinitionImpl) list; - } - - @Override - public void configureDataIteratorContext(DataIteratorContext context) - { - if (context.getInsertOption().batch) - { - context.setMaxRowErrors(100); - context.setFailFast(false); - } - - context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); - } - - @Override - protected @Nullable AttachmentParentFactory getAttachmentParentFactory() - { - return new ListItemAttachmentParentFactory(); - } - - @Override - protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException - { - Map ret = null; - - if (null != listRow) - { - SimpleFilter keyFilter = getKeyFilter(listRow); - - if (null != keyFilter) - { - TableInfo queryTable = getQueryTable(); - Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); - - if (null != raw && !raw.isEmpty()) - { - ret = new CaseInsensitiveHashMap<>(); - - // EntityId - ret.put("EntityId", raw.get("entityid")); - - for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) - { - String propName = prop.getName(); - ColumnInfo column = queryTable.getColumn(propName); - Object value = column.getValue(raw); - if (value != null) - ret.put(propName, value); - } - } - } - } - - return ret; - } - - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - for (Map row : rows) - { - aliasColumns(getColumnMapping(), row); - } - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - - if (null != result) - { - ListManager mgr = ListManager.get(); - - for (Map row : result) - { - if (null != row.get(ID)) - { - // Audit each row - String entityId = (String) row.get(ID); - String newRecord = mgr.formatAuditItem(_list, user, row); - - mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); - } - } - - if (!result.isEmpty() && !errors.hasErrors()) - mgr.indexList(_list); - } - - return result; - } - - private User getListUser(User user, Container container) - { - if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) - { - // if the list is a picklist and you have permission to manage picklists, that equates - // to having editor permission. - return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); - } - return user; - } - - @Override - protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - @Override - protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, - DataIteratorContext context, @Nullable Map extraScriptContext) - { - if (!hasPermission(user, UpdatePermission.class)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); - } - - public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, - @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - User updatedUser = getListUser(user, container); - DataIteratorContext context = new DataIteratorContext(errors); - context.setFailFast(false); - context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter - context.setSupportAutoIncrementKey(supportAutoIncrementKey); - context.setLookupResolutionType(lookupResolutionType); - setAttachmentDirectory(attachmentDir); - TableInfo ti = _list.getTable(updatedUser); - - if (null != ti) - { - try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) - { - int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); - - if (!errors.hasErrors()) - { - //Make entry to audit log if anything was inserted - if (imported > 0) - ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); - - transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - - return imported; - } - - return 0; - } - } - - return 0; - } - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - @Nullable Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data in this table."); - - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - } - - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, - Map configParameters, Map extraScriptContext) - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to insert data into this table."); - - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); - if (count > 0 && !errors.hasErrors()) - ListManager.get().indexList(_list); - return count; - } - - @Override - public List> updateRows(User user, Container container, List> rows, List> oldKeys, - BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to update data into this table."); - - List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (!result.isEmpty()) - ListManager.get().indexList(_list); - return result; - } - - @Override - protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) - throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException - { - // TODO: Check for equivalency so that attachments can be deleted etc. - - Map dps = new HashMap<>(); - for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) - { - dps.put(dp.getPropertyURI(), dp); - } - - ValidatorContext validatorCache = new ValidatorContext(container, user); - - ListItm itm = new ListItm(); - itm.setEntityId((String) oldRow.get(ID)); - itm.setListId(_list.getListId()); - itm.setKey(oldRow.get(_list.getKeyName())); - - ListItem item = new ListItemImpl(_list, itm); - - if (item.getProperties() != null) - { - List errors = new ArrayList<>(); - for (Map.Entry entry : dps.entrySet()) - { - Object value = row.get(entry.getValue().getName()); - validateProperty(entry.getValue(), value, row, errors, validatorCache); - } - - if (!errors.isEmpty()) - throw new ValidationException(errors); - } - - // MVIndicators - Map rowCopy = new CaseInsensitiveHashMap<>(); - ArrayList modifiedAttachmentColumns = new ArrayList<>(); - ArrayList attachmentFiles = new ArrayList<>(); - - TableInfo qt = getQueryTable(); - for (Map.Entry r : row.entrySet()) - { - ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); - rowCopy.put(r.getKey(), r.getValue()); - - // 22747: Attachment columns - if (null != column) - { - DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); - if (null != dp && isAttachmentProperty(dp)) - { - modifiedAttachmentColumns.add(column); - - // setup any new attachments - if (r.getValue() instanceof AttachmentFile file) - { - if (null != file.getFilename()) - attachmentFiles.add(file); - } - else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) - { - throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); - } - } - } - } - - // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) - Object newRowKey = getField(rowCopy, _list.getKeyName()); - Object oldRowKey = getField(oldRow, _list.getKeyName()); - - if (null == newRowKey && null != oldRowKey) - rowCopy.put(_list.getKeyName(), oldRowKey); - - Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); - - if (null != result) - { - result = getRow(user, container, result); - - if (null != result && null != result.get(ID)) - { - ListManager mgr = ListManager.get(); - String entityId = (String) result.get(ID); - - try - { - // Remove prior attachment -- only includes columns which are modified in this update - for (ColumnInfo col : modifiedAttachmentColumns) - { - Object value = oldRow.get(col.getName()); - if (null != value) - { - AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); - } - } - - // Update attachments - if (!attachmentFiles.isEmpty()) - AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); - } - catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) - { - // issues 21503, 28633: turn these into a validation exception to get a nicer error - throw new ValidationException(e.getMessage()); - } - catch (IOException e) - { - throw UnexpectedException.wrap(e); - } - finally - { - for (AttachmentFile attachmentFile : attachmentFiles) - { - try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} - } - } - - String oldRecord = mgr.formatAuditItem(_list, user, oldRow); - String newRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); - } - } - - return result; - } - - // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() - private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) - { - //check for isRequired - if (prop.isRequired()) - { - // for mv indicator columns either an indicator or a field value is sufficient - boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); - if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) - { - if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) - { - errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); - return false; - } - } - } - - if (null != value) - { - for (IPropertyValidator validator : prop.getValidators()) - { - if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) - return false; - } - } - - return true; - } - - private record ListRecord(Object key, String entityId) { } - - @Override - public Map moveRows( - User _user, - Container container, - Container targetContainer, - List> rows, - BatchValidationException errors, - @Nullable Map configParameters, - @Nullable Map extraScriptContext - ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException - { - // Ensure the list is in scope for the target container - if (null == ListService.get().getList(targetContainer, _list.getName(), true)) - { - errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); - throw errors; - } - - User user = getListUser(_user, container); - Map> containerRows = getListRowsForMoveRows(container, user, targetContainer, rows, errors); - if (errors.hasErrors()) - throw errors; - - int fileAttachmentsMovedCount = 0; - int listAuditEventsCreatedCount = 0; - int listAuditEventsMovedCount = 0; - int listRecordsCount = 0; - int queryAuditEventsMovedCount = 0; - - if (containerRows.isEmpty()) - return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); - - AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; - String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; - boolean hasAttachmentProperties = _list.getDomainOrThrow() - .getProperties() - .stream() - .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); - - ListAuditProvider listAuditProvider = new ListAuditProvider(); - final int BATCH_SIZE = 5_000; - boolean isAuditEnabled = auditBehavior != null && AuditBehaviorType.NONE != auditBehavior; - - try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) - { - if (isAuditEnabled && tx.getAuditEvent() == null) - { - TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); - auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); - AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); - } - - List listAuditEvents = new ArrayList<>(); - - for (GUID containerId : containerRows.keySet()) - { - Container sourceContainer = ContainerManager.getForId(containerId); - if (sourceContainer == null) - throw new InvalidKeyException("Container '" + containerId + "' does not exist."); - - if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) - throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); - - TableInfo listTable = _list.getTable(user, sourceContainer); - if (listTable == null) - throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); - - List records = containerRows.get(containerId); - int numRecords = records.size(); - - for (int start = 0; start < numRecords; start += BATCH_SIZE) - { - int end = Math.min(start + BATCH_SIZE, numRecords); - List batch = records.subList(start, end); - List rowPks = batch.stream().map(ListRecord::key).toList(); - - // Before trigger per batch - Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); - if (errors.hasErrors()) - throw errors; - - listRecordsCount += Table.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); - if (errors.hasErrors()) - throw errors; - - if (hasAttachmentProperties) - { - fileAttachmentsMovedCount += moveAttachments(user, sourceContainer, targetContainer, batch, errors); - if (errors.hasErrors()) - throw errors; - } - - queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, ListQuerySchema.NAME, _list.getName()); - listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, batch.stream().map(ListRecord::entityId).toList()); - - // Detailed audit events per row - if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) - listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, batch); - - // After trigger per batch - listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); - if (errors.hasErrors()) - throw errors; - } - - // Create a summary audit event for the source container - if (isAuditEnabled) - { - String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - - // Create a summary audit event for the target container - if (isAuditEnabled) - { - String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); - event.setUserComment(auditUserComment); - listAuditEvents.add(event); - } - } - - if (!listAuditEvents.isEmpty()) - { - AuditLogService.get().addEvents(user, listAuditEvents, true); - listAuditEventsCreatedCount += listAuditEvents.size(); - } - - tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); - - tx.commit(); - - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); - } - - return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); - } - - private Map moveRowsCounts(int fileAttachmentsMovedCount, int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) - { - return Map.of( - "fileAttachmentsMoved", fileAttachmentsMovedCount, - "listAuditEventsCreated", listAuditEventsCreated, - "listAuditEventsMoved", listAuditEventsMoved, - "listRecords", listRecords, - "queryAuditEventsMoved", queryAuditEventsMoved - ); - } - - private Map> getListRowsForMoveRows( - Container container, - User user, - Container targetContainer, - List> rows, - BatchValidationException errors - ) throws QueryUpdateServiceException - { - if (rows.isEmpty()) - return Collections.emptyMap(); - - String keyName = _list.getKeyName(); - List keys = new ArrayList<>(); - for (var row : rows) - { - Object key = getField(row, keyName); - if (key == null) - { - errors.addRowError(new ValidationException("Key field value required for moving list rows.")); - return Collections.emptyMap(); - } - - keys.add(getKeyFilterValue(key)); - } - - SimpleFilter filter = new SimpleFilter(); - FieldKey fieldKey = FieldKey.fromParts(keyName); - filter.addInClause(fieldKey, keys); - filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); - - // Request all rows without a container filter so that rows are more easily resolved across the list scope. - // Read permissions are subsequently checked upon loading a row. - TableInfo table = _list.getTable(user, container, ContainerFilter.getUnsafeEverythingFilter()); - if (table == null) - throw new QueryUpdateServiceException(String.format("Failed to resolve table for list %s in %s", _list.getName(), container.getPath())); - - Map> containerRows = new HashMap<>(); - try (var result = new TableSelector(table, PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) - { - while (result.next()) - { - GUID containerId = new GUID(result.getString("Container")); - if (!containerRows.containsKey(containerId)) - { - var c = ContainerManager.getForId(containerId); - if (c == null) - throw new QueryUpdateServiceException(String.format("Failed to resolve container for row in list %s in %s.", _list.getName(), container.getPath())); - else if (!c.hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read list rows in all source containers."); - } - - containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); - containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); - } - } - catch (SQLException e) - { - throw new RuntimeSQLException(e); - } - - return containerRows; - } - - private int moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) - { - List parents = new ArrayList<>(); - for (ListRecord record : records) - parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); - - int count = 0; - try - { - count = AttachmentService.get().moveAttachments(targetContainer, parents, user); - } - catch (IOException e) - { - errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); - } - - return count; - } - - private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) - { - List auditEvents = new ArrayList<>(records.size()); - String keyName = _list.getKeyName(); - String sourcePath = sourceContainer.getPath(); - String targetPath = targetContainer.getPath(); - - for (ListRecord record : records) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); - event.setListItemEntityId(record.entityId); - event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); - event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); - auditEvents.add(event); - } - - AuditLogService.get().addEvents(user, auditEvents, true); - - return auditEvents.size(); - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException - { - if (!_list.isVisible(user)) - throw new UnauthorizedException("You do not have permission to delete data from this table."); - - Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); - - if (null != result) - { - String entityId = (String) result.get(ID); - - if (null != entityId) - { - ListManager mgr = ListManager.get(); - String deletedRecord = mgr.formatAuditItem(_list, user, result); - - // Audit - mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); - - // Remove attachments - if (hasAttachmentProperties()) - AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); - - // Clean up Search indexer - if (!result.isEmpty()) - mgr.deleteItemIndex(_list, entityId); - } - } - - return result; - } - - - // Deletes attachments & discussions, and removes list documents from full-text search index. - public void deleteRelatedListData(final User user, final Container container) - { - // Unindex all item docs and the entire list doc - ListManager.get().deleteIndexedList(_list); - - // Delete attachments and discussions associated with a list in batches of 1,000 - new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() - { - @Override - public boolean accept(String entityId) - { - return null != entityId; - } - - @Override - public void exec(List entityIds) - { - // delete the related list data for this block - deleteRelatedListData(user, container, entityIds); - } - }); - } - - // delete the related list data for this block of entityIds - private void deleteRelatedListData(User user, Container container, List entityIds) - { - // Build up set of entityIds and AttachmentParents - List attachmentParents = new ArrayList<>(); - - // Delete Attachments - if (hasAttachmentProperties()) - { - for (String entityId : entityIds) - { - attachmentParents.add(new ListItemAttachmentParent(entityId, container)); - } - AttachmentService.get().deleteAttachments(attachmentParents); - } - } - - @Override - protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException - { - int result; - try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) - { - deleteRelatedListData(user, container); - result = super.truncateRows(getListUser(user, container), container); - transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); - transaction.commit(); - } - - return result; - } - - @Nullable - public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException - { - String keyName = _list.getKeyName(); - Object key = getField(map, keyName); - - if (null == key) - { - // Auto-increment lists might not provide a key so allow them to pass through - if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) - return null; - throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); - } - - return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); - } - - @NotNull - private Object getKeyFilterValue(@NotNull Object key) - { - ListDefinition.KeyType type = _list.getKeyType(); - - // Check the type of the list to ensure proper casting of the key type - if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) - return isIntegral(key) ? key : Integer.valueOf(key.toString()); - - return key.toString(); - } - - @Nullable - private Object getField(Map map, String key) - { - /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ - Object value = map.get(key); - - if (null == value) - value = map.get(key + "_"); - - if (null == value) - value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); - - return value; - } - - /** - * Delegate class to generate an AttachmentParent - */ - public static class ListItemAttachmentParentFactory implements AttachmentParentFactory - { - @Override - public AttachmentParent generateAttachmentParent(String entityId, Container c) - { - return new ListItemAttachmentParent(entityId, c); - } - } - - /** - * Get Domain from list definition, unless null then get from super - */ - @Override - protected Domain getDomain() - { - return _list != null? - _list.getDomain() : - super.getDomain(); - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.attachments.AttachmentParent; +import org.labkey.api.attachments.AttachmentParentFactory; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.LookupResolutionType; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.Selector.ForEachBatchBlock; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.exp.ObjectProperty; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListImportProgress; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.IPropertyValidator; +import org.labkey.api.exp.property.ValidatorContext; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.lists.permissions.ManagePicklistsPermission; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.PropertyValidationError; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.ValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reader.DataLoader; +import org.labkey.api.security.ElevatedUser; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.VirtualFile; +import org.labkey.list.view.ListItemAttachmentParent; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.labkey.api.util.IntegerUtils.isIntegral; + +/** + * Implementation of QueryUpdateService for Lists + */ +public class ListQueryUpdateService extends DefaultQueryUpdateService +{ + private final ListDefinitionImpl _list; + private static final String ID = "entityId"; + + public ListQueryUpdateService(ListTable queryTable, TableInfo dbTable, @NotNull ListDefinition list) + { + super(queryTable, dbTable, createMVMapping(queryTable.getList().getDomain())); + _list = (ListDefinitionImpl) list; + } + + @Override + public void configureDataIteratorContext(DataIteratorContext context) + { + if (context.getInsertOption().batch) + { + context.setMaxRowErrors(100); + context.setFailFast(false); + } + + context.putConfigParameter(ConfigParameters.TrimStringRight, Boolean.TRUE); + } + + @Override + protected @Nullable AttachmentParentFactory getAttachmentParentFactory() + { + return new ListItemAttachmentParentFactory(); + } + + @Override + protected Map getRow(User user, Container container, Map listRow) throws InvalidKeyException + { + Map ret = null; + + if (null != listRow) + { + SimpleFilter keyFilter = getKeyFilter(listRow); + + if (null != keyFilter) + { + TableInfo queryTable = getQueryTable(); + Map raw = new TableSelector(queryTable, keyFilter, null).getMap(); + + if (null != raw && !raw.isEmpty()) + { + ret = new CaseInsensitiveHashMap<>(); + + // EntityId + ret.put("EntityId", raw.get("entityid")); + + for (DomainProperty prop : _list.getDomainOrThrow().getProperties()) + { + String propName = prop.getName(); + ColumnInfo column = queryTable.getColumn(propName); + Object value = column.getValue(raw); + if (value != null) + ret.put(propName, value); + } + } + } + } + + return ret; + } + + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + for (Map row : rows) + { + aliasColumns(getColumnMapping(), row); + } + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + + if (null != result) + { + ListManager mgr = ListManager.get(); + + for (Map row : result) + { + if (null != row.get(ID)) + { + // Audit each row + String entityId = (String) row.get(ID); + String newRecord = mgr.formatAuditItem(_list, user, row); + + mgr.addAuditEvent(_list, user, container, "A new list record was inserted", entityId, null, newRecord); + } + } + + if (!result.isEmpty() && !errors.hasErrors()) + mgr.indexList(_list); + } + + return result; + } + + private User getListUser(User user, Container container) + { + if (_list.isPicklist() && container.hasPermission(user, ManagePicklistsPermission.class)) + { + // if the list is a picklist and you have permission to manage picklists, that equates + // to having editor permission. + return ElevatedUser.ensureContextualRoles(container, user, Pair.of(DeletePermission.class, EditorRole.class)); + } + return user; + } + + @Override + protected @Nullable List> _insertRowsUsingDIB(User user, Container container, List> rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + return super._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + @Override + protected @Nullable List> _updateRowsUsingDIB(User user, Container container, List> rows, + DataIteratorContext context, @Nullable Map extraScriptContext) + { + if (!hasPermission(user, UpdatePermission.class)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + return super._updateRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); + } + + public int insertUsingDataIterator(DataLoader loader, User user, Container container, BatchValidationException errors, @Nullable VirtualFile attachmentDir, + @Nullable ListImportProgress progress, boolean supportAutoIncrementKey, InsertOption insertOption, LookupResolutionType lookupResolutionType) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + User updatedUser = getListUser(user, container); + DataIteratorContext context = new DataIteratorContext(errors); + context.setFailFast(false); + context.setInsertOption(insertOption); // this method is used by ListImporter and BackgroundListImporter + context.setSupportAutoIncrementKey(supportAutoIncrementKey); + context.setLookupResolutionType(lookupResolutionType); + setAttachmentDirectory(attachmentDir); + TableInfo ti = _list.getTable(updatedUser); + + if (null != ti) + { + try (DbScope.Transaction transaction = ti.getSchema().getScope().ensureTransaction()) + { + int imported = _importRowsUsingDIB(updatedUser, container, loader, null, context, new HashMap<>()); + + if (!errors.hasErrors()) + { + //Make entry to audit log if anything was inserted + if (imported > 0) + ListManager.get().addAuditEvent(_list, updatedUser, "Bulk " + (insertOption.updateOnly ? "updated " : (insertOption.mergeRows ? "imported " : "inserted ")) + imported + " rows to list."); + + transaction.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + + return imported; + } + + return 0; + } + } + + return 0; + } + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + @Nullable Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data in this table."); + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + } + + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, + Map configParameters, Map extraScriptContext) + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to insert data into this table."); + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); + if (count > 0 && !errors.hasErrors()) + ListManager.get().indexList(_list); + return count; + } + + @Override + public List> updateRows(User user, Container container, List> rows, List> oldKeys, + BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to update data into this table."); + + List> result = super.updateRows(getListUser(user, container), container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (!result.isEmpty()) + ListManager.get().indexList(_list); + return result; + } + + @Override + protected Map updateRow(User user, Container container, Map row, @NotNull Map oldRow, @Nullable Map configParameters) + throws InvalidKeyException, ValidationException, QueryUpdateServiceException, SQLException + { + // TODO: Check for equivalency so that attachments can be deleted etc. + + Map dps = new HashMap<>(); + for (DomainProperty dp : _list.getDomainOrThrow().getProperties()) + { + dps.put(dp.getPropertyURI(), dp); + } + + ValidatorContext validatorCache = new ValidatorContext(container, user); + + ListItm itm = new ListItm(); + itm.setEntityId((String) oldRow.get(ID)); + itm.setListId(_list.getListId()); + itm.setKey(oldRow.get(_list.getKeyName())); + + ListItem item = new ListItemImpl(_list, itm); + + if (item.getProperties() != null) + { + List errors = new ArrayList<>(); + for (Map.Entry entry : dps.entrySet()) + { + Object value = row.get(entry.getValue().getName()); + validateProperty(entry.getValue(), value, row, errors, validatorCache); + } + + if (!errors.isEmpty()) + throw new ValidationException(errors); + } + + // MVIndicators + Map rowCopy = new CaseInsensitiveHashMap<>(); + ArrayList modifiedAttachmentColumns = new ArrayList<>(); + ArrayList attachmentFiles = new ArrayList<>(); + + TableInfo qt = getQueryTable(); + for (Map.Entry r : row.entrySet()) + { + ColumnInfo column = qt.getColumn(FieldKey.fromParts(r.getKey())); + rowCopy.put(r.getKey(), r.getValue()); + + // 22747: Attachment columns + if (null != column) + { + DomainProperty dp = _list.getDomainOrThrow().getPropertyByURI(column.getPropertyURI()); + if (null != dp && isAttachmentProperty(dp)) + { + modifiedAttachmentColumns.add(column); + + // setup any new attachments + if (r.getValue() instanceof AttachmentFile file) + { + if (null != file.getFilename()) + attachmentFiles.add(file); + } + else if (r.getValue() != null && !StringUtils.isEmpty(String.valueOf(r.getValue()))) + { + throw new ValidationException("Can't upload '" + r.getValue() + "' to field " + r.getKey() + " with type Attachment."); + } + } + } + } + + // Attempt to include key from oldRow if not found in row (As stated in the QUS Interface) + Object newRowKey = getField(rowCopy, _list.getKeyName()); + Object oldRowKey = getField(oldRow, _list.getKeyName()); + + if (null == newRowKey && null != oldRowKey) + rowCopy.put(_list.getKeyName(), oldRowKey); + + Map result = super.updateRow(getListUser(user, container), container, rowCopy, oldRow, true, false); + + if (null != result) + { + result = getRow(user, container, result); + + if (null != result && null != result.get(ID)) + { + ListManager mgr = ListManager.get(); + String entityId = (String) result.get(ID); + + try + { + // Remove prior attachment -- only includes columns which are modified in this update + for (ColumnInfo col : modifiedAttachmentColumns) + { + Object value = oldRow.get(col.getName()); + if (null != value) + { + AttachmentService.get().deleteAttachment(new ListItemAttachmentParent(entityId, _list.getContainer()), value.toString(), user); + } + } + + // Update attachments + if (!attachmentFiles.isEmpty()) + AttachmentService.get().addAttachments(new ListItemAttachmentParent(entityId, _list.getContainer()), attachmentFiles, user); + } + catch (AttachmentService.DuplicateFilenameException | AttachmentService.FileTooLargeException e) + { + // issues 21503, 28633: turn these into a validation exception to get a nicer error + throw new ValidationException(e.getMessage()); + } + catch (IOException e) + { + throw UnexpectedException.wrap(e); + } + finally + { + for (AttachmentFile attachmentFile : attachmentFiles) + { + try { attachmentFile.closeInputStream(); } catch (IOException ignored) {} + } + } + + String oldRecord = mgr.formatAuditItem(_list, user, oldRow); + String newRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was modified", entityId, oldRecord, newRecord); + } + } + + return result; + } + + // TODO: Consolidate with ColumnValidator and OntologyManager.validateProperty() + private boolean validateProperty(DomainProperty prop, Object value, Map newRow, List errors, ValidatorContext validatorCache) + { + //check for isRequired + if (prop.isRequired()) + { + // for mv indicator columns either an indicator or a field value is sufficient + boolean hasMvIndicator = prop.isMvEnabled() && (value instanceof ObjectProperty && ((ObjectProperty)value).getMvIndicator() != null); + if (!hasMvIndicator && (null == value || (value instanceof ObjectProperty && ((ObjectProperty)value).value() == null))) + { + if (newRow.containsKey(prop.getName()) && newRow.get(prop.getName()) == null) + { + errors.add(new PropertyValidationError("The field '" + prop.getName() + "' is required.", prop.getName())); + return false; + } + } + } + + if (null != value) + { + for (IPropertyValidator validator : prop.getValidators()) + { + if (!validator.validate(prop.getPropertyDescriptor(), value, errors, validatorCache)) + return false; + } + } + + return true; + } + + private record ListRecord(Object key, String entityId) { } + + @Override + public Map moveRows( + User _user, + Container container, + Container targetContainer, + List> rows, + BatchValidationException errors, + @Nullable Map configParameters, + @Nullable Map extraScriptContext + ) throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException + { + // Ensure the list is in scope for the target container + if (null == ListService.get().getList(targetContainer, _list.getName(), true)) + { + errors.addRowError(new ValidationException(String.format("List '%s' is not accessible from folder %s.", _list.getName(), targetContainer.getPath()))); + throw errors; + } + + User user = getListUser(_user, container); + Map> containerRows = getListRowsForMoveRows(container, user, targetContainer, rows, errors); + if (errors.hasErrors()) + throw errors; + + int fileAttachmentsMovedCount = 0; + int listAuditEventsCreatedCount = 0; + int listAuditEventsMovedCount = 0; + int listRecordsCount = 0; + int queryAuditEventsMovedCount = 0; + + if (containerRows.isEmpty()) + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + + AuditBehaviorType auditBehavior = configParameters != null ? (AuditBehaviorType) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior) : null; + String auditUserComment = configParameters != null ? (String) configParameters.get(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment) : null; + boolean hasAttachmentProperties = _list.getDomainOrThrow() + .getProperties() + .stream() + .anyMatch(prop -> PropertyType.ATTACHMENT.equals(prop.getPropertyType())); + + ListAuditProvider listAuditProvider = new ListAuditProvider(); + final int BATCH_SIZE = 5_000; + boolean isAuditEnabled = auditBehavior != null && AuditBehaviorType.NONE != auditBehavior; + + try (DbScope.Transaction tx = getDbTable().getSchema().getScope().ensureTransaction()) + { + if (isAuditEnabled && tx.getAuditEvent() == null) + { + TransactionAuditProvider.TransactionAuditEvent auditEvent = createTransactionAuditEvent(targetContainer, QueryService.AuditAction.UPDATE); + auditEvent.updateCommentRowCount(containerRows.values().stream().mapToInt(List::size).sum()); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, auditEvent); + } + + List listAuditEvents = new ArrayList<>(); + + for (GUID containerId : containerRows.keySet()) + { + Container sourceContainer = ContainerManager.getForId(containerId); + if (sourceContainer == null) + throw new InvalidKeyException("Container '" + containerId + "' does not exist."); + + if (!sourceContainer.hasPermission(user, MoveEntitiesPermission.class)) + throw new UnauthorizedException("You do not have permission to move list records out of '" + sourceContainer.getName() + "'."); + + TableInfo listTable = _list.getTable(user, sourceContainer); + if (listTable == null) + throw new QueryUpdateServiceException(String.format("Failed to retrieve table for list '%s' in folder %s.", _list.getName(), sourceContainer.getPath())); + + List records = containerRows.get(containerId); + int numRecords = records.size(); + + for (int start = 0; start < numRecords; start += BATCH_SIZE) + { + int end = Math.min(start + BATCH_SIZE, numRecords); + List batch = records.subList(start, end); + List rowPks = batch.stream().map(ListRecord::key).toList(); + + // Before trigger per batch + Map extraContext = Map.of("targetContainer", targetContainer, "keys", rowPks); + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, true, errors, extraContext); + if (errors.hasErrors()) + throw errors; + + listRecordsCount += Table.updateContainer(getDbTable(), _list.getKeyName(), rowPks, targetContainer, user, true); + if (errors.hasErrors()) + throw errors; + + if (hasAttachmentProperties) + { + fileAttachmentsMovedCount += moveAttachments(user, sourceContainer, targetContainer, batch, errors); + if (errors.hasErrors()) + throw errors; + } + + queryAuditEventsMovedCount += QueryService.get().moveAuditEvents(targetContainer, rowPks, ListQuerySchema.NAME, _list.getName()); + listAuditEventsMovedCount += listAuditProvider.moveEvents(targetContainer, batch.stream().map(ListRecord::entityId).toList()); + + // Detailed audit events per row + if (AuditBehaviorType.DETAILED == listTable.getEffectiveAuditBehavior(auditBehavior)) + listAuditEventsCreatedCount += addDetailedMoveAuditEvents(user, sourceContainer, targetContainer, batch); + + // After trigger per batch + listTable.fireBatchTrigger(sourceContainer, user, TableInfo.TriggerType.MOVE, false, errors, extraContext); + if (errors.hasErrors()) + throw errors; + } + + // Create a summary audit event for the source container + if (isAuditEnabled) + { + String comment = String.format("Moved %s to %s", StringUtilsLabKey.pluralize(numRecords, "row"), targetContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(sourceContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + + // Create a summary audit event for the target container + if (isAuditEnabled) + { + String comment = String.format("Moved %s from %s", StringUtilsLabKey.pluralize(numRecords, "row"), sourceContainer.getPath()); + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, comment, _list); + event.setUserComment(auditUserComment); + listAuditEvents.add(event); + } + } + + if (!listAuditEvents.isEmpty()) + { + AuditLogService.get().addEvents(user, listAuditEvents, true); + listAuditEventsCreatedCount += listAuditEvents.size(); + } + + tx.addCommitTask(() -> ListManager.get().indexList(_list), DbScope.CommitTaskOption.POSTCOMMIT); + + tx.commit(); + + SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); + } + + return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); + } + + private Map moveRowsCounts(int fileAttachmentsMovedCount, int listAuditEventsCreated, int listAuditEventsMoved, int listRecords, int queryAuditEventsMoved) + { + return Map.of( + "fileAttachmentsMoved", fileAttachmentsMovedCount, + "listAuditEventsCreated", listAuditEventsCreated, + "listAuditEventsMoved", listAuditEventsMoved, + "listRecords", listRecords, + "queryAuditEventsMoved", queryAuditEventsMoved + ); + } + + private Map> getListRowsForMoveRows( + Container container, + User user, + Container targetContainer, + List> rows, + BatchValidationException errors + ) throws QueryUpdateServiceException + { + if (rows.isEmpty()) + return Collections.emptyMap(); + + String keyName = _list.getKeyName(); + List keys = new ArrayList<>(); + for (var row : rows) + { + Object key = getField(row, keyName); + if (key == null) + { + errors.addRowError(new ValidationException("Key field value required for moving list rows.")); + return Collections.emptyMap(); + } + + keys.add(getKeyFilterValue(key)); + } + + SimpleFilter filter = new SimpleFilter(); + FieldKey fieldKey = FieldKey.fromParts(keyName); + filter.addInClause(fieldKey, keys); + filter.addCondition(FieldKey.fromParts("Container"), targetContainer.getId(), CompareType.NEQ); + + // Request all rows without a container filter so that rows are more easily resolved across the list scope. + // Read permissions are subsequently checked upon loading a row. + TableInfo table = _list.getTable(user, container, ContainerFilter.getUnsafeEverythingFilter()); + if (table == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve table for list %s in %s", _list.getName(), container.getPath())); + + Map> containerRows = new HashMap<>(); + try (var result = new TableSelector(table, PageFlowUtil.set(keyName, "Container", "EntityId"), filter, null).getResults()) + { + while (result.next()) + { + GUID containerId = new GUID(result.getString("Container")); + if (!containerRows.containsKey(containerId)) + { + var c = ContainerManager.getForId(containerId); + if (c == null) + throw new QueryUpdateServiceException(String.format("Failed to resolve container for row in list %s in %s.", _list.getName(), container.getPath())); + else if (!c.hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read list rows in all source containers."); + } + + containerRows.computeIfAbsent(containerId, k -> new ArrayList<>()); + containerRows.get(containerId).add(new ListRecord(result.getObject(fieldKey), result.getString("EntityId"))); + } + } + catch (SQLException e) + { + throw new RuntimeSQLException(e); + } + + return containerRows; + } + + private int moveAttachments(User user, Container sourceContainer, Container targetContainer, List records, BatchValidationException errors) + { + List parents = new ArrayList<>(); + for (ListRecord record : records) + parents.add(new ListItemAttachmentParent(record.entityId, sourceContainer)); + + int count = 0; + try + { + count = AttachmentService.get().moveAttachments(targetContainer, parents, user); + } + catch (IOException e) + { + errors.addRowError(new ValidationException("Failed to move attachments when moving list rows. Error: " + e.getMessage())); + } + + return count; + } + + private int addDetailedMoveAuditEvents(User user, Container sourceContainer, Container targetContainer, List records) + { + List auditEvents = new ArrayList<>(records.size()); + String keyName = _list.getKeyName(); + String sourcePath = sourceContainer.getPath(); + String targetPath = targetContainer.getPath(); + + for (ListRecord record : records) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(targetContainer, "An existing list record was moved", _list); + event.setListItemEntityId(record.entityId); + event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", sourcePath, keyName, record.key.toString()))); + event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of("Folder", targetPath, keyName, record.key.toString()))); + auditEvents.add(event); + } + + AuditLogService.get().addEvents(user, auditEvents, true); + + return auditEvents.size(); + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRowMap) throws InvalidKeyException, QueryUpdateServiceException, SQLException + { + if (!_list.isVisible(user)) + throw new UnauthorizedException("You do not have permission to delete data from this table."); + + Map result = super.deleteRow(getListUser(user, container), container, oldRowMap); + + if (null != result) + { + String entityId = (String) result.get(ID); + + if (null != entityId) + { + ListManager mgr = ListManager.get(); + String deletedRecord = mgr.formatAuditItem(_list, user, result); + + // Audit + mgr.addAuditEvent(_list, user, container, "An existing list record was deleted", entityId, deletedRecord, null); + + // Remove attachments + if (hasAttachmentProperties()) + AttachmentService.get().deleteAttachments(new ListItemAttachmentParent(entityId, container)); + + // Clean up Search indexer + if (!result.isEmpty()) + mgr.deleteItemIndex(_list, entityId); + } + } + + return result; + } + + + // Deletes attachments & discussions, and removes list documents from full-text search index. + public void deleteRelatedListData(final User user, final Container container) + { + // Unindex all item docs and the entire list doc + ListManager.get().deleteIndexedList(_list); + + // Delete attachments and discussions associated with a list in batches of 1,000 + new TableSelector(getDbTable(), Collections.singleton("entityId")).forEachBatch(String.class, 1000, new ForEachBatchBlock<>() + { + @Override + public boolean accept(String entityId) + { + return null != entityId; + } + + @Override + public void exec(List entityIds) + { + // delete the related list data for this block + deleteRelatedListData(user, container, entityIds); + } + }); + } + + // delete the related list data for this block of entityIds + private void deleteRelatedListData(User user, Container container, List entityIds) + { + // Build up set of entityIds and AttachmentParents + List attachmentParents = new ArrayList<>(); + + // Delete Attachments + if (hasAttachmentProperties()) + { + for (String entityId : entityIds) + { + attachmentParents.add(new ListItemAttachmentParent(entityId, container)); + } + AttachmentService.get().deleteAttachments(attachmentParents); + } + } + + @Override + protected int truncateRows(User user, Container container) throws QueryUpdateServiceException, SQLException + { + int result; + try (DbScope.Transaction transaction = getDbTable().getSchema().getScope().ensureTransaction()) + { + deleteRelatedListData(user, container); + result = super.truncateRows(getListUser(user, container), container); + transaction.addCommitTask(() -> ListManager.get().addAuditEvent(_list, user, "Deleted " + result + " rows from list."), DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } + + return result; + } + + @Nullable + public SimpleFilter getKeyFilter(Map map) throws InvalidKeyException + { + String keyName = _list.getKeyName(); + Object key = getField(map, keyName); + + if (null == key) + { + // Auto-increment lists might not provide a key so allow them to pass through + if (ListDefinition.KeyType.AutoIncrementInteger.equals(_list.getKeyType())) + return null; + throw new InvalidKeyException("No " + keyName + " provided for list \"" + _list.getName() + "\""); + } + + return new SimpleFilter(FieldKey.fromParts(keyName), getKeyFilterValue(key)); + } + + @NotNull + private Object getKeyFilterValue(@NotNull Object key) + { + ListDefinition.KeyType type = _list.getKeyType(); + + // Check the type of the list to ensure proper casting of the key type + if (ListDefinition.KeyType.Integer.equals(type) || ListDefinition.KeyType.AutoIncrementInteger.equals(type)) + return isIntegral(key) ? key : Integer.valueOf(key.toString()); + + return key.toString(); + } + + @Nullable + private Object getField(Map map, String key) + { + /* TODO: this is very strange, we have a TableInfo we should be using its ColumnInfo objects to figure out aliases, we don't need to guess */ + Object value = map.get(key); + + if (null == value) + value = map.get(key + "_"); + + if (null == value) + value = map.get(getDbTable().getSqlDialect().legalNameFromName(key)); + + return value; + } + + /** + * Delegate class to generate an AttachmentParent + */ + public static class ListItemAttachmentParentFactory implements AttachmentParentFactory + { + @Override + public AttachmentParent generateAttachmentParent(String entityId, Container c) + { + return new ListItemAttachmentParent(entityId, c); + } + } + + /** + * Get Domain from list definition, unless null then get from super + */ + @Override + protected Domain getDomain() + { + return _list != null? + _list.getDomain() : + super.getDomain(); + } +} diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 58d659275f1..ce48da2a02c 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -1,304 +1,304 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.query.audit; - -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.AuditTypeProvider; -import org.labkey.api.audit.DetailedAuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.UserSchema; -import org.labkey.api.view.ViewContext; -import org.labkey.query.controllers.QueryController; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; - - public static final String COLUMN_NAME_ROW_PK = "RowPk"; - public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; - public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; - - static final List defaultVisibleColumns = new ArrayList<>(); - - static { - - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); - defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); - } - - public QueryUpdateAuditProvider() - { - super(new QueryUpdateAuditDomainKind()); - } - - @Override - public String getEventName() - { - return QUERY_UPDATE_AUDIT_EVENT; - } - - @Override - public String getLabel() - { - return "Query update events"; - } - - @Override - public String getDescription() - { - return "Data about insert and update queries."; - } - - @Override - public Map legacyNameMap() - { - Map legacyMap = super.legacyNameMap(); - legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); - legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); - legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); - legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); - return legacyMap; - } - - @Override - public Class getEventClass() - { - return (Class)QueryUpdateAuditEvent.class; - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) - { - @Override - protected void initColumn(MutableColumnInfo col) - { - if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Schema Name"); - } - else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) - { - col.setLabel("Query Name"); - } - else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) - { - col.setLabel("User Comment"); - } - } - }; - appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); - - return table; - } - - @Override - public List getDefaultVisibleColumns() - { - return defaultVisibleColumns; - } - - - public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) - { - return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); - } - - public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) - { - UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); - if (schema != null) - { - QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); - - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); - filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); - - settings.setBaseFilter(filter); - settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); - return schema.createView(context, settings, errors); - } - return null; - } - - public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) - { - SimpleFilter filter = new SimpleFilter(); - filter.addInClause(FieldKey.fromParts(COLUMN_NAME_ROW_PK), rowPks.stream().map(Object::toString).toList()); - filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); - filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); - - return Table.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); - } - - public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent - { - private String _rowPk; - private String _schemaName; - private String _queryName; - - /** Important for reflection-based instantiation */ - @SuppressWarnings("unused") - public QueryUpdateAuditEvent() - { - super(); - } - - public QueryUpdateAuditEvent(Container container, String comment) - { - super(QUERY_UPDATE_AUDIT_EVENT, container, comment); - setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), QUERY_UPDATE_AUDIT_EVENT); - } - - public String getRowPk() - { - return _rowPk; - } - - public void setRowPk(String rowPk) - { - _rowPk = rowPk; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - @Override - public Map getAuditLogMessageElements() - { - Map elements = new LinkedHashMap<>(); - elements.put("rowPk", getRowPk()); - elements.put("schemaName", getSchemaName()); - elements.put("queryName", getQueryName()); - elements.put("transactionId", getTransactionId()); - elements.put("userComment", getUserComment()); - // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) - elements.putAll(super.getAuditLogMessageElements()); - return elements; - } - } - - public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind - { - public static final String NAME = "QueryUpdateAuditDomain"; - public static String NAMESPACE_PREFIX = "Audit-" + NAME; - - private final Set _fields; - - public QueryUpdateAuditDomainKind() - { - super(QUERY_UPDATE_AUDIT_EVENT); - - Set fields = new LinkedHashSet<>(); - fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); - fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); - fields.add(createOldDataMapPropertyDescriptor()); - fields.add(createNewDataMapPropertyDescriptor()); - fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); - fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); - _fields = Collections.unmodifiableSet(fields); - } - - @Override - public Set getProperties() - { - return _fields; - } - - @Override - protected String getNamespacePrefix() - { - return NAMESPACE_PREFIX; - } - - @Override - public String getKindName() - { - return NAME; - } - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.query.audit; + +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.AuditTypeProvider; +import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.UserSchema; +import org.labkey.api.view.ViewContext; +import org.labkey.query.controllers.QueryController; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class QueryUpdateAuditProvider extends AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_UPDATE_AUDIT_EVENT = "QueryUpdateAuditEvent"; + + public static final String COLUMN_NAME_ROW_PK = "RowPk"; + public static final String COLUMN_NAME_SCHEMA_NAME = "SchemaName"; + public static final String COLUMN_NAME_QUERY_NAME = "QueryName"; + + static final List defaultVisibleColumns = new ArrayList<>(); + + static { + + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CREATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_IMPERSONATED_BY)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_PROJECT_ID)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_CONTAINER)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_COMMENT)); + defaultVisibleColumns.add(FieldKey.fromParts(COLUMN_NAME_USER_COMMENT)); + } + + public QueryUpdateAuditProvider() + { + super(new QueryUpdateAuditDomainKind()); + } + + @Override + public String getEventName() + { + return QUERY_UPDATE_AUDIT_EVENT; + } + + @Override + public String getLabel() + { + return "Query update events"; + } + + @Override + public String getDescription() + { + return "Data about insert and update queries."; + } + + @Override + public Map legacyNameMap() + { + Map legacyMap = super.legacyNameMap(); + legacyMap.put(FieldKey.fromParts("key1"), COLUMN_NAME_ROW_PK); + legacyMap.put(FieldKey.fromParts("key2"), COLUMN_NAME_SCHEMA_NAME); + legacyMap.put(FieldKey.fromParts("key3"), COLUMN_NAME_QUERY_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.OLD_RECORD_PROP_NAME), AbstractAuditDomainKind.OLD_RECORD_PROP_NAME); + legacyMap.put(FieldKey.fromParts("Property", AbstractAuditDomainKind.NEW_RECORD_PROP_NAME), AbstractAuditDomainKind.NEW_RECORD_PROP_NAME); + return legacyMap; + } + + @Override + public Class getEventClass() + { + return (Class)QueryUpdateAuditEvent.class; + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + DefaultAuditTypeTable table = new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, defaultVisibleColumns) + { + @Override + protected void initColumn(MutableColumnInfo col) + { + if (COLUMN_NAME_SCHEMA_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Schema Name"); + } + else if (COLUMN_NAME_QUERY_NAME.equalsIgnoreCase(col.getName())) + { + col.setLabel("Query Name"); + } + else if (COLUMN_NAME_USER_COMMENT.equalsIgnoreCase(col.getName())) + { + col.setLabel("User Comment"); + } + } + }; + appendValueMapColumns(table, QUERY_UPDATE_AUDIT_EVENT); + + return table; + } + + @Override + public List getDefaultVisibleColumns() + { + return defaultVisibleColumns; + } + + + public static QueryView createHistoryQueryView(ViewContext context, QueryForm form, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), form.getSchemaName()); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), form.getQueryName()); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public static QueryView createDetailsQueryView(ViewContext context, QueryController.QueryDetailsForm form, BindException errors) + { + return createDetailsQueryView(context, form.getSchemaName(), form.getQueryName(), form.getKeyValue(), errors); + } + + public static QueryView createDetailsQueryView(ViewContext context, String schemaName, String queryName, String keyValue, BindException errors) + { + UserSchema schema = AuditLogService.getAuditLogSchema(context.getUser(), context.getContainer()); + if (schema != null) + { + QuerySettings settings = new QuerySettings(context, QueryView.DATAREGIONNAME_DEFAULT); + + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_QUERY_NAME), queryName); + filter.addCondition(FieldKey.fromParts(QueryUpdateAuditProvider.COLUMN_NAME_ROW_PK), keyValue); + + settings.setBaseFilter(filter); + settings.setQueryName(QUERY_UPDATE_AUDIT_EVENT); + return schema.createView(context, settings, errors); + } + return null; + } + + public int moveEvents(Container targetContainer, Collection rowPks, String schemaName, String queryName) + { + SimpleFilter filter = new SimpleFilter(); + filter.addInClause(FieldKey.fromParts(COLUMN_NAME_ROW_PK), rowPks.stream().map(Object::toString).toList()); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_SCHEMA_NAME), schemaName); + filter.addCondition(FieldKey.fromParts(COLUMN_NAME_QUERY_NAME), queryName); + + return Table.updateContainer(createStorageTableInfo(), targetContainer, filter, null, false); + } + + public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent + { + private String _rowPk; + private String _schemaName; + private String _queryName; + + /** Important for reflection-based instantiation */ + @SuppressWarnings("unused") + public QueryUpdateAuditEvent() + { + super(); + } + + public QueryUpdateAuditEvent(Container container, String comment) + { + super(QUERY_UPDATE_AUDIT_EVENT, container, comment); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), QUERY_UPDATE_AUDIT_EVENT); + } + + public String getRowPk() + { + return _rowPk; + } + + public void setRowPk(String rowPk) + { + _rowPk = rowPk; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + @Override + public Map getAuditLogMessageElements() + { + Map elements = new LinkedHashMap<>(); + elements.put("rowPk", getRowPk()); + elements.put("schemaName", getSchemaName()); + elements.put("queryName", getQueryName()); + elements.put("transactionId", getTransactionId()); + elements.put("userComment", getUserComment()); + // N.B. oldRecordMap and newRecordMap are potentially very large (and are not displayed in the default grid view) + elements.putAll(super.getAuditLogMessageElements()); + return elements; + } + } + + public static class QueryUpdateAuditDomainKind extends AbstractAuditDomainKind + { + public static final String NAME = "QueryUpdateAuditDomain"; + public static String NAMESPACE_PREFIX = "Audit-" + NAME; + + private final Set _fields; + + public QueryUpdateAuditDomainKind() + { + super(QUERY_UPDATE_AUDIT_EVENT); + + Set fields = new LinkedHashSet<>(); + fields.add(createPropertyDescriptor(COLUMN_NAME_ROW_PK, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_SCHEMA_NAME, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_QUERY_NAME, PropertyType.STRING)); + fields.add(createOldDataMapPropertyDescriptor()); + fields.add(createNewDataMapPropertyDescriptor()); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_ID, PropertyType.BIGINT)); + fields.add(createPropertyDescriptor(COLUMN_NAME_USER_COMMENT, PropertyType.STRING)); + _fields = Collections.unmodifiableSet(fields); + } + + @Override + public Set getProperties() + { + return _fields; + } + + @Override + protected String getNamespacePrefix() + { + return NAMESPACE_PREFIX; + } + + @Override + public String getKindName() + { + return NAME; + } + } +} diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 7b9e9c71908..0b8f41b0dca 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -1,8782 +1,8782 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.query.controllers; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.servlet.ServletException; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.servlet.http.HttpServletResponse; -import jakarta.servlet.http.HttpSession; -import org.antlr.runtime.tree.Tree; -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; -import org.apache.commons.collections4.multimap.HashSetValuedHashMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Strings; -import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.poi.ss.usermodel.Workbook; -import org.apache.xmlbeans.XmlError; -import org.apache.xmlbeans.XmlException; -import org.apache.xmlbeans.XmlOptions; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.json.JSONParserConfiguration; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.action.Action; -import org.labkey.api.action.ActionType; -import org.labkey.api.action.ApiJsonForm; -import org.labkey.api.action.ApiJsonWriter; -import org.labkey.api.action.ApiQueryResponse; -import org.labkey.api.action.ApiResponse; -import org.labkey.api.action.ApiResponseWriter; -import org.labkey.api.action.ApiSimpleResponse; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.action.ApiVersion; -import org.labkey.api.action.ConfirmAction; -import org.labkey.api.action.ExportAction; -import org.labkey.api.action.ExportException; -import org.labkey.api.action.ExtendedApiQueryResponse; -import org.labkey.api.action.FormHandlerAction; -import org.labkey.api.action.FormViewAction; -import org.labkey.api.action.HasBindParameters; -import org.labkey.api.action.JsonInputLimit; -import org.labkey.api.action.LabKeyError; -import org.labkey.api.action.Marshal; -import org.labkey.api.action.Marshaller; -import org.labkey.api.action.MutatingApiAction; -import org.labkey.api.action.NullSafeBindException; -import org.labkey.api.action.ReadOnlyApiAction; -import org.labkey.api.action.ReportingApiQueryResponse; -import org.labkey.api.action.SimpleApiJsonForm; -import org.labkey.api.action.SimpleErrorView; -import org.labkey.api.action.SimpleRedirectAction; -import org.labkey.api.action.SimpleViewAction; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.attachments.SpringAttachmentFile; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.AuditTypeEvent; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.audit.provider.ContainerAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.IntHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.collections.RowMapFactory; -import org.labkey.api.collections.Sets; -import org.labkey.api.data.AbstractTableInfo; -import org.labkey.api.data.ActionButton; -import org.labkey.api.data.Aggregate; -import org.labkey.api.data.AnalyticsProviderItem; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ButtonBar; -import org.labkey.api.data.CachedResultSets; -import org.labkey.api.data.ColumnHeaderType; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.DataRegion; -import org.labkey.api.data.DataRegionSelection; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.DisplayColumn; -import org.labkey.api.data.ExcelWriter; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.JdbcMetaDataSelector; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.JsonWriter; -import org.labkey.api.data.PHI; -import org.labkey.api.data.PropertyManager; -import org.labkey.api.data.PropertyManager.PropertyMap; -import org.labkey.api.data.PropertyManager.WritablePropertyMap; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.QueryLogging; -import org.labkey.api.data.ResultSetView; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.ShowRows; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TSVWriter; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.VirtualTable; -import org.labkey.api.data.dialect.JdbcMetaDataLocator; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DataIteratorUtil; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.ListofMapsDataIterator; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.api.ProvenanceRecordingParams; -import org.labkey.api.exp.api.ProvenanceService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainKind; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.files.FileContentService; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.module.ModuleHtmlView; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.RecordedAction; -import org.labkey.api.query.AbstractQueryImportAction; -import org.labkey.api.query.AbstractQueryUpdateService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.CustomView; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.DuplicateKeyException; -import org.labkey.api.query.ExportScriptModel; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryAction; -import org.labkey.api.query.QueryDefinition; -import org.labkey.api.query.QueryException; -import org.labkey.api.query.QueryForm; -import org.labkey.api.query.QueryParam; -import org.labkey.api.query.QueryParseException; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryUpdateForm; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.QueryUrls; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.RuntimeValidationException; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.query.SimpleSchemaTreeVisitor; -import org.labkey.api.query.TempQuerySettings; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.UserSchemaAction; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.report.ReportDescriptor; -import org.labkey.api.security.ActionNames; -import org.labkey.api.security.AdminConsoleAction; -import org.labkey.api.security.CSRF; -import org.labkey.api.security.IgnoresTermsOfUse; -import org.labkey.api.security.MutableSecurityPolicy; -import org.labkey.api.security.RequiresAllOf; -import org.labkey.api.security.RequiresAnyOf; -import org.labkey.api.security.RequiresNoPermission; -import org.labkey.api.security.RequiresPermission; -import org.labkey.api.security.SecurityManager; -import org.labkey.api.security.SecurityPolicyManager; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.security.ValidEmail; -import org.labkey.api.security.permissions.AbstractActionPermissionTest; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.security.permissions.DeletePermission; -import org.labkey.api.security.permissions.EditSharedViewPermission; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.MoveEntitiesPermission; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.security.permissions.PlatformDeveloperPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.security.permissions.UpdatePermission; -import org.labkey.api.security.roles.EditorRole; -import org.labkey.api.settings.AdminConsole; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.LookAndFeelProperties; -import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; -import org.labkey.api.stats.ColumnAnalyticsProvider; -import org.labkey.api.util.ButtonBuilder; -import org.labkey.api.util.DOM; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.JavaScriptFragment; -import org.labkey.api.util.JsonUtil; -import org.labkey.api.util.LinkBuilder; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.ResponseHelper; -import org.labkey.api.util.ReturnURLString; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URLHelper; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.util.XmlBeansUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.DetailsView; -import org.labkey.api.view.HtmlView; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.InsertView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.view.UpdateView; -import org.labkey.api.view.VBox; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.ViewServlet; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.PageConfig; -import org.labkey.api.writer.HtmlWriter; -import org.labkey.api.writer.ZipFile; -import org.labkey.data.xml.ColumnType; -import org.labkey.data.xml.ImportTemplateType; -import org.labkey.data.xml.TableType; -import org.labkey.data.xml.TablesDocument; -import org.labkey.data.xml.TablesType; -import org.labkey.data.xml.externalSchema.TemplateSchemaType; -import org.labkey.data.xml.queryCustomView.FilterType; -import org.labkey.query.AutoGeneratedDetailsCustomView; -import org.labkey.query.AutoGeneratedInsertCustomView; -import org.labkey.query.AutoGeneratedUpdateCustomView; -import org.labkey.query.CustomViewImpl; -import org.labkey.query.CustomViewUtil; -import org.labkey.query.EditQueriesPermission; -import org.labkey.query.EditableCustomView; -import org.labkey.query.LinkedTableInfo; -import org.labkey.query.MetadataTableJSON; -import org.labkey.query.ModuleCustomQueryDefinition; -import org.labkey.query.ModuleCustomView; -import org.labkey.query.QueryServiceImpl; -import org.labkey.query.TableXML; -import org.labkey.query.audit.QueryExportAuditProvider; -import org.labkey.query.audit.QueryUpdateAuditProvider; -import org.labkey.query.model.MetadataTableJSONMixin; -import org.labkey.query.persist.AbstractExternalSchemaDef; -import org.labkey.query.persist.CstmView; -import org.labkey.query.persist.ExternalSchemaDef; -import org.labkey.query.persist.ExternalSchemaDefCache; -import org.labkey.query.persist.LinkedSchemaDef; -import org.labkey.query.persist.QueryDef; -import org.labkey.query.persist.QueryManager; -import org.labkey.query.reports.ReportsController; -import org.labkey.query.reports.getdata.DataRequest; -import org.labkey.query.sql.QNode; -import org.labkey.query.sql.Query; -import org.labkey.query.sql.SqlParser; -import org.labkey.query.xml.ApiTestsDocument; -import org.labkey.query.xml.TestCaseType; -import org.labkey.remoteapi.RemoteConnections; -import org.labkey.remoteapi.SelectRowsStreamHack; -import org.labkey.remoteapi.query.SelectRowsCommand; -import org.labkey.vfs.FileLike; -import org.springframework.beans.MutablePropertyValues; -import org.springframework.beans.PropertyValue; -import org.springframework.beans.PropertyValues; -import org.springframework.dao.DataAccessException; -import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.mock.web.MockHttpServletResponse; -import org.springframework.validation.BindException; -import org.springframework.validation.Errors; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.ModelAndView; - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintWriter; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.trimToEmpty; -import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; -import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; -import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; -import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; -import static org.labkey.api.util.DOM.BR; -import static org.labkey.api.util.DOM.DIV; -import static org.labkey.api.util.DOM.FONT; -import static org.labkey.api.util.DOM.Renderable; -import static org.labkey.api.util.DOM.TABLE; -import static org.labkey.api.util.DOM.TD; -import static org.labkey.api.util.DOM.TR; -import static org.labkey.api.util.DOM.at; -import static org.labkey.api.util.DOM.cl; -import static org.labkey.query.MetadataTableJSON.getTableType; -import static org.labkey.query.MetadataTableJSON.parseDocument; - -@SuppressWarnings("DefaultAnnotationParam") - -public class QueryController extends SpringActionController -{ - private static final Logger LOG = LogManager.getLogger(QueryController.class); - private static final String ROW_ATTACHMENT_INDEX_DELIM = "::"; - - private static final Set RESERVED_VIEW_NAMES = CaseInsensitiveHashSet.of( - "Default", - AutoGeneratedDetailsCustomView.NAME, - AutoGeneratedInsertCustomView.NAME, - AutoGeneratedUpdateCustomView.NAME - ); - - private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(QueryController.class, - ValidateQueryAction.class, - ValidateQueriesAction.class, - GetSchemaQueryTreeAction.class, - GetQueryDetailsAction.class, - ViewQuerySourceAction.class - ); - - public QueryController() - { - setActionResolver(_actionResolver); - } - - public static void registerAdminConsoleLinks() - { - AdminConsole.addLink(AdminConsole.SettingsLinkType.Diagnostics, "data sources", new ActionURL(DataSourceAdminAction.class, ContainerManager.getRoot())); - } - - public static class RemoteQueryConnectionUrls - { - public static ActionURL urlManageRemoteConnection(Container c) - { - return new ActionURL(ManageRemoteConnectionsAction.class, c); - } - - public static ActionURL urlCreateRemoteConnection(Container c) - { - return new ActionURL(EditRemoteConnectionAction.class, c); - } - - public static ActionURL urlEditRemoteConnection(Container c, String connectionName) - { - ActionURL url = new ActionURL(EditRemoteConnectionAction.class, c); - url.addParameter("connectionName", connectionName); - return url; - } - - public static ActionURL urlSaveRemoteConnection(Container c) - { - return new ActionURL(EditRemoteConnectionAction.class, c); - } - - public static ActionURL urlDeleteRemoteConnection(Container c, @Nullable String connectionName) - { - ActionURL url = new ActionURL(DeleteRemoteConnectionAction.class, c); - if (connectionName != null) - url.addParameter("connectionName", connectionName); - return url; - } - - public static ActionURL urlTestRemoteConnection(Container c, String connectionName) - { - ActionURL url = new ActionURL(TestRemoteConnectionAction.class, c); - url.addParameter("connectionName", connectionName); - return url; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); - if (!errors.hasErrors()) - { - String name = remoteConnectionForm.getConnectionName(); - // package the remote-connection properties into the remoteConnectionForm and pass them along - Map map1 = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); - remoteConnectionForm.setUrl(map1.get("URL")); - remoteConnectionForm.setUserEmail(map1.get("user")); - remoteConnectionForm.setPassword(map1.get("password")); - remoteConnectionForm.setFolderPath(map1.get("container")); - } - setHelpTopic("remoteConnection"); - return new JspView<>("/org/labkey/query/view/createRemoteConnection.jsp", remoteConnectionForm, errors); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - return RemoteConnections.createOrEditRemoteConnection(remoteConnectionForm, getContainer(), errors); - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Create/Edit Remote Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class DeleteRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/query/view/confirmDeleteConnection.jsp", remoteConnectionForm, errors); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); - return RemoteConnections.deleteRemoteConnection(remoteConnectionForm, getContainer()); - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Confirm Delete Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestRemoteConnectionAction extends FormViewAction - { - @Override - public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) - { - String name = remoteConnectionForm.getConnectionName(); - String schemaName = "core"; // test Schema Name - String queryName = "Users"; // test Query Name - - // Extract the username, password, and container from the secure property store - Map singleConnectionMap = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); - if (singleConnectionMap.isEmpty()) - throw new NotFoundException(); - String url = singleConnectionMap.get(RemoteConnections.FIELD_URL); - String user = singleConnectionMap.get(RemoteConnections.FIELD_USER); - String password = singleConnectionMap.get(RemoteConnections.FIELD_PASSWORD); - String container = singleConnectionMap.get(RemoteConnections.FIELD_CONTAINER); - - // connect to the remote server and retrieve an input stream - org.labkey.remoteapi.Connection cn = new org.labkey.remoteapi.Connection(url, user, password); - final SelectRowsCommand cmd = new SelectRowsCommand(schemaName, queryName); - try - { - DataIteratorBuilder source = SelectRowsStreamHack.go(cn, container, cmd, getContainer()); - // immediately close the source after opening it, this is a test. - source.getDataIterator(new DataIteratorContext()).close(); - } - catch (Exception e) - { - errors.addError(new LabKeyError("The listed credentials for this remote connection failed to connect.")); - return new JspView<>("/org/labkey/query/view/testRemoteConnectionsFailure.jsp", remoteConnectionForm); - } - - return new JspView<>("/org/labkey/query/view/testRemoteConnectionsSuccess.jsp", remoteConnectionForm); - } - - @Override - public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) - { - return true; - } - - @Override - public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) - { - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - public static class QueryUrlsImpl implements QueryUrls - { - @Override - public ActionURL urlSchemaBrowser(Container c) - { - return new ActionURL(BeginAction.class, c); - } - - @Override - public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName) - { - ActionURL ret = urlSchemaBrowser(c); - if (schemaName != null) - { - ret.addParameter(QueryParam.schemaName.toString(), schemaName); - } - return ret; - } - - @Override - public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName, @Nullable String queryName) - { - if (StringUtils.isEmpty(queryName)) - return urlSchemaBrowser(c, schemaName); - ActionURL ret = urlSchemaBrowser(c); - ret.addParameter(QueryParam.schemaName.toString(), trimToEmpty(schemaName)); - ret.addParameter(QueryParam.queryName.toString(), trimToEmpty(queryName)); - return ret; - } - - public ActionURL urlExternalSchemaAdmin(Container c) - { - return urlExternalSchemaAdmin(c, null); - } - - public ActionURL urlExternalSchemaAdmin(Container c, @Nullable String message) - { - ActionURL url = new ActionURL(AdminAction.class, c); - - if (null != message) - url.addParameter("message", message); - - return url; - } - - public ActionURL urlInsertExternalSchema(Container c) - { - return new ActionURL(InsertExternalSchemaAction.class, c); - } - - public ActionURL urlNewQuery(Container c) - { - return new ActionURL(NewQueryAction.class, c); - } - - public ActionURL urlUpdateExternalSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(EditExternalSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - public ActionURL urlReloadExternalSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(ReloadExternalSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - public ActionURL urlDeleteSchema(Container c, AbstractExternalSchemaDef def) - { - ActionURL url = new ActionURL(DeleteSchemaAction.class, c); - url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); - return url; - } - - @Override - public ActionURL urlStartBackgroundRReport(@NotNull ActionURL baseURL, String reportId) - { - ActionURL result = baseURL.clone(); - result.setAction(ReportsController.StartBackgroundRReportAction.class); - result.replaceParameter(ReportDescriptor.Prop.reportId, reportId); - return result; - } - - @Override - public ActionURL urlExecuteQuery(@NotNull ActionURL baseURL) - { - ActionURL result = baseURL.clone(); - result.setAction(ExecuteQueryAction.class); - return result; - } - - @Override - public ActionURL urlExecuteQuery(Container c, String schemaName, String queryName) - { - return new ActionURL(ExecuteQueryAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter(QueryParam.queryName, queryName); - } - - @Override - public @NotNull ActionURL urlCreateExcelTemplate(Container c, String schemaName, String queryName) - { - return new ActionURL(ExportExcelTemplateAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter("query.queryName", queryName); - } - - @Override - public ActionURL urlMetadataQuery(Container c, String schemaName, String queryName) - { - return new ActionURL(MetadataQueryAction.class, c) - .addParameter(QueryParam.schemaName, schemaName) - .addParameter(QueryParam.queryName, queryName); - } - } - - @Override - public PageConfig defaultPageConfig() - { - // set default help topic for query controller - PageConfig config = super.defaultPageConfig(); - config.setHelpTopic("querySchemaBrowser"); - return config; - } - - @AdminConsoleAction(AdminOperationsPermission.class) - public static class DataSourceAdminAction extends SimpleViewAction - { - public DataSourceAdminAction() - { - } - - public DataSourceAdminAction(ViewContext viewContext) - { - setViewContext(viewContext); - } - - @Override - public ModelAndView getView(Object o, BindException errors) - { - // Site Admin or Troubleshooter? Troubleshooters can see all the information but can't test data sources. - boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); - List allDefs = QueryManager.get().getExternalSchemaDefs(null); - - MultiValuedMap byDataSourceName = new ArrayListValuedHashMap<>(); - - for (ExternalSchemaDef def : allDefs) - byDataSourceName.put(def.getDataSource(), def); - - MutableInt row = new MutableInt(); - - Renderable r = DOM.DIV( - DIV("This page lists all the data sources defined in your " + AppProps.getInstance().getWebappConfigurationFilename() + " file that were available when first referenced and the external schemas defined in each."), - BR(), - TABLE(cl("labkey-data-region"), - TR(cl("labkey-show-borders"), - hasAdminOpsPerms ? TD(cl("labkey-column-header"), "Test") : null, - TD(cl("labkey-column-header"), "Data Source"), - TD(cl("labkey-column-header"), "Current Status"), - TD(cl("labkey-column-header"), "URL"), - TD(cl("labkey-column-header"), "Database Name"), - TD(cl("labkey-column-header"), "Product Name"), - TD(cl("labkey-column-header"), "Product Version"), - TD(cl("labkey-column-header"), "Max Connections"), - TD(cl("labkey-column-header"), "Active Connections"), - TD(cl("labkey-column-header"), "Idle Connections"), - TD(cl("labkey-column-header"), "Max Wait (ms)") - ), - DbScope.getDbScopes().stream() - .flatMap(scope -> { - String rowStyle = row.getAndIncrement() % 2 == 0 ? "labkey-alternate-row labkey-show-borders" : "labkey-row labkey-show-borders"; - Object status; - boolean connected = false; - try (Connection ignore = scope.getConnection()) - { - status = "connected"; - connected = true; - } - catch (Exception e) - { - status = FONT(cl("labkey-error"), "disconnected"); - } - - return Stream.of( - TR( - cl(rowStyle), - hasAdminOpsPerms ? TD(connected ? new ButtonBuilder("Test").href(new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", scope.getDataSourceName())) : "") : null, - TD(HtmlString.NBSP, scope.getDisplayName()), - TD(status), - TD(scope.getDatabaseUrl()), - TD(scope.getDatabaseName()), - TD(scope.getDatabaseProductName()), - TD(scope.getDatabaseProductVersion()), - TD(scope.getDataSourceProperties().getMaxTotal()), - TD(scope.getDataSourceProperties().getNumActive()), - TD(scope.getDataSourceProperties().getNumIdle()), - TD(scope.getDataSourceProperties().getMaxWaitMillis()) - ), - TR( - cl(rowStyle), - TD(HtmlString.NBSP), - TD(at(DOM.Attribute.colspan, 10), getDataSourceTable(byDataSourceName.get(scope.getDataSourceName()))) - ) - ); - }) - ) - ); - - return new HtmlView(r); - } - - private Renderable getDataSourceTable(Collection dsDefs) - { - if (dsDefs.isEmpty()) - return TABLE(TR(TD(HtmlString.NBSP))); - - MultiValuedMap byContainerPath = new ArrayListValuedHashMap<>(); - - for (ExternalSchemaDef def : dsDefs) - byContainerPath.put(def.getContainerPath(), def); - - TreeSet paths = new TreeSet<>(byContainerPath.keySet()); - - return TABLE(paths.stream() - .map(path -> TR(TD(at(DOM.Attribute.colspan, 4), getDataSourcePath(path, byContainerPath.get(path))))) - ); - } - - private Renderable getDataSourcePath(String path, Collection unsorted) - { - List defs = new ArrayList<>(unsorted); - defs.sort(Comparator.comparing(AbstractExternalSchemaDef::getUserSchemaName, String.CASE_INSENSITIVE_ORDER)); - Container c = ContainerManager.getForPath(path); - - if (null == c) - return TD(); - - boolean hasRead = c.hasPermission(getUser(), ReadPermission.class); - QueryUrlsImpl urls = new QueryUrlsImpl(); - - return - TD(TABLE( - TR(TD( - at(DOM.Attribute.colspan, 3), - hasRead ? LinkBuilder.simpleLink(path, urls.urlExternalSchemaAdmin(c)) : path - )), - TR(TD(TABLE( - defs.stream() - .map(def -> TR(TD( - at(DOM.Attribute.style, "padding-left:20px"), - hasRead ? LinkBuilder.simpleLink(def.getUserSchemaName() + - (!Strings.CS.equals(def.getSourceSchemaName(), def.getUserSchemaName()) ? " (" + def.getSourceSchemaName() + ")" : ""), urls.urlUpdateExternalSchema(c, def)) - : def.getUserSchemaName() - ))) - ))) - )); - } - - @Override - public void addNavTrail(NavTree root) - { - urlProvider(AdminUrls.class).addAdminNavTrail(root, "Data Source Administration", getClass(), getContainer()); - } - } - - public static class TestDataSourceForm - { - private String _dataSource; - - public String getDataSource() - { - return _dataSource; - } - - @SuppressWarnings("unused") - public void setDataSource(String dataSource) - { - _dataSource = dataSource; - } - } - - public static class TestDataSourceConfirmForm extends TestDataSourceForm - { - private String _excludeSchemas; - private String _excludeTables; - - public String getExcludeSchemas() - { - return _excludeSchemas; - } - - @SuppressWarnings("unused") - public void setExcludeSchemas(String excludeSchemas) - { - _excludeSchemas = excludeSchemas; - } - - public String getExcludeTables() - { - return _excludeTables; - } - - @SuppressWarnings("unused") - public void setExcludeTables(String excludeTables) - { - _excludeTables = excludeTables; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestDataSourceConfirmAction extends FormViewAction - { - private DbScope _scope; - - @Override - public ModelAndView getView(TestDataSourceConfirmForm form, boolean reshow, BindException errors) throws Exception - { - validateCommand(form, errors); - return new JspView<>("/org/labkey/query/view/testDataSourceConfirm.jsp", _scope); - } - - @Override - public void validateCommand(TestDataSourceConfirmForm form, Errors errors) - { - _scope = DbScope.getDbScope(form.getDataSource()); - - if (null == _scope) - throw new NotFoundException("Could not resolve data source " + form.getDataSource()); - } - - @Override - public boolean handlePost(TestDataSourceConfirmForm form, BindException errors) throws Exception - { - saveTestDataSourceProperties(form); - return true; - } - - @Override - public URLHelper getSuccessURL(TestDataSourceConfirmForm form) - { - return new ActionURL(TestDataSourceAction.class, getContainer()).addParameter("dataSource", _scope.getDataSourceName()); - } - - @Override - public void addNavTrail(NavTree root) - { - new DataSourceAdminAction(getViewContext()).addNavTrail(root); - root.addChild("Prepare Test of " + _scope.getDataSourceName()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class TestDataSourceAction extends SimpleViewAction - { - private DbScope _scope; - - @Override - public ModelAndView getView(TestDataSourceForm form, BindException errors) - { - _scope = DbScope.getDbScope(form.getDataSource()); - - if (null == _scope) - throw new NotFoundException("Could not resolve data source " + form.getDataSource()); - - return new JspView<>("/org/labkey/query/view/testDataSource.jsp", _scope); - } - - @Override - public void addNavTrail(NavTree root) - { - new DataSourceAdminAction(getViewContext()).addNavTrail(root); - root.addChild("Test " + _scope.getDataSourceName()); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ResetDataSourcePropertiesAction extends FormHandlerAction - { - @Override - public void validateCommand(TestDataSourceForm target, Errors errors) - { - } - - @Override - public boolean handlePost(TestDataSourceForm form, BindException errors) throws Exception - { - WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), false); - if (map != null) - map.delete(); - return true; - } - - @Override - public URLHelper getSuccessURL(TestDataSourceForm form) - { - return new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", form.getDataSource()) ; - } - } - - private static final String TEST_DATA_SOURCE_CATEGORY = "testDataSourceProperties"; - private static final String TEST_DATA_SOURCE_SCHEMAS_PROPERTY = "excludeSchemas"; - private static final String TEST_DATA_SOURCE_TABLES_PROPERTY = "excludeTables"; - - private static String getCategory(String dataSourceName) - { - return TEST_DATA_SOURCE_CATEGORY + "|" + dataSourceName; - } - - public static void saveTestDataSourceProperties(TestDataSourceConfirmForm form) - { - WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), true); - // Save empty entries as empty string to distinguish from null (which results in default values) - map.put(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, StringUtils.trimToEmpty(form.getExcludeSchemas())); - map.put(TEST_DATA_SOURCE_TABLES_PROPERTY, StringUtils.trimToEmpty(form.getExcludeTables())); - map.save(); - } - - public static TestDataSourceConfirmForm getTestDataSourceProperties(DbScope scope) - { - TestDataSourceConfirmForm form = new TestDataSourceConfirmForm(); - PropertyMap map = PropertyManager.getProperties(getCategory(scope.getDataSourceName())); - form.setExcludeSchemas(map.getOrDefault(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, scope.getSqlDialect().getDefaultSchemasToExcludeFromTesting())); - form.setExcludeTables(map.getOrDefault(TEST_DATA_SOURCE_TABLES_PROPERTY, scope.getSqlDialect().getDefaultTablesToExcludeFromTesting())); - - return form; - } - - @RequiresPermission(ReadPermission.class) - public static class BrowseAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/query/view/browse.jsp", null); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Schema Browser"); - } - } - - @RequiresPermission(ReadPermission.class) - public static class BeginAction extends QueryViewAction - { - @SuppressWarnings("UnusedDeclaration") - public BeginAction() - { - } - - public BeginAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - JspView view = new JspView<>("/org/labkey/query/view/browse.jsp", form); - view.setFrame(WebPartView.FrameType.NONE); - return view; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Query Schema Browser", new QueryUrlsImpl().urlSchemaBrowser(getContainer())); - } - } - - @RequiresPermission(ReadPermission.class) - public class SchemaAction extends QueryViewAction - { - public SchemaAction() {} - - SchemaAction(QueryForm form) - { - _form = form; - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - _form = form; - return new JspView<>("/org/labkey/query/view/browse.jsp", form); - } - - @Override - public void addNavTrail(NavTree root) - { - if (_form != null && _form.getSchema() != null) - addSchemaActionNavTrail(root, _form.getSchema().getSchemaPath(), _form.getQueryName()); - } - } - - - void addSchemaActionNavTrail(NavTree root, SchemaKey schemaKey, String queryName) - { - if (getContainer().hasOneOf(getUser(), AdminPermission.class, PlatformDeveloperPermission.class)) - { - // Don't show the full query nav trail to non-admin/non-developer users as they almost certainly don't - // want it - try - { - String schemaName = schemaKey.toDisplayString(); - ActionURL url = new ActionURL(BeginAction.class, getContainer()); - url.addParameter("schemaName", schemaKey.toString()); - url.addParameter("queryName", queryName); - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild(schemaName + " Schema", url); - } - catch (NullPointerException e) - { - LOG.error("NullPointerException in addNavTrail", e); - } - } - } - - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.SelectData.class) - public class NewQueryAction extends FormViewAction - { - private NewQueryForm _form; - private ActionURL _successUrl; - - @Override - public void validateCommand(NewQueryForm target, org.springframework.validation.Errors errors) - { - target.ff_newQueryName = StringUtils.trimToNull(target.ff_newQueryName); - if (null == target.ff_newQueryName) - errors.reject(ERROR_MSG, "QueryName is required"); - } - - @Override - public ModelAndView getView(NewQueryForm form, boolean reshow, BindException errors) - { - form.ensureSchemaExists(); - - if (!form.getSchema().canCreate()) - { - throw new UnauthorizedException(); - } - - getPageConfig().setFocusId("ff_newQueryName"); - _form = form; - setHelpTopic("sqlTutorial"); - return new JspView<>("/org/labkey/query/view/newQuery.jsp", form, errors); - } - - @Override - public boolean handlePost(NewQueryForm form, BindException errors) - { - form.ensureSchemaExists(); - - if (!form.getSchema().canCreate()) - { - throw new UnauthorizedException(); - } - - try - { - if (StringUtils.isEmpty(form.ff_baseTableName)) - { - errors.reject(ERROR_MSG, "You must select a base table or query name."); - return false; - } - - UserSchema schema = form.getSchema(); - String newQueryName = form.ff_newQueryName; - QueryDef existing = QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), newQueryName, true); - if (existing != null) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); - return false; - } - TableInfo existingTable = form.getSchema().getTable(newQueryName, null); - if (existingTable != null) - { - errors.reject(ERROR_MSG, "A table with the name '" + newQueryName + "' already exists."); - return false; - } - // bug 6095 -- conflicting query and dataset names - if (form.getSchema().getTableNames().contains(newQueryName)) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists as a table"); - return false; - } - QueryDefinition newDef = QueryService.get().createQueryDef(getUser(), getContainer(), form.getSchemaKey(), form.ff_newQueryName); - Query query = new Query(schema); - query.setRootTable(FieldKey.fromParts(form.ff_baseTableName)); - String sql = query.getQueryText(); - if (null == sql) - sql = "SELECT * FROM \"" + form.ff_baseTableName + "\""; - newDef.setSql(sql); - - try - { - newDef.save(getUser(), getContainer()); - } - catch (SQLException x) - { - if (RuntimeSQLException.isConstraintException(x)) - { - errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); - return false; - } - else - { - throw x; - } - } - - _successUrl = newDef.urlFor(form.ff_redirect); - return true; - } - catch (Exception e) - { - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); - errors.reject(ERROR_MSG, Objects.toString(e.getMessage(), e.toString())); - return false; - } - } - - @Override - public ActionURL getSuccessURL(NewQueryForm newQueryForm) - { - return _successUrl; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - root.addChild("New Query", new QueryUrlsImpl().urlNewQuery(getContainer())); - } - } - - // CONSIDER : deleting this action after the SQL editor UI changes are finalized, keep in mind that built-in views - // use this view as well via the edit metadata page. - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) // Note: This action deals with just meta data; it AJAXes data into place using GetWebPartAction - public class SourceQueryAction extends SimpleViewAction - { - public SourceForm _form; - public UserSchema _schema; - public QueryDefinition _queryDef; - - - @Override - public void validate(SourceForm target, BindException errors) - { - _form = target; - if (StringUtils.isEmpty(target.getSchemaName())) - throw new NotFoundException("schema name not specified"); - if (StringUtils.isEmpty(target.getQueryName())) - throw new NotFoundException("query name not specified"); - - QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); - if (null == querySchema) - throw new NotFoundException("schema not found: " + _form.getSchemaKey().toDisplayString()); - if (!(querySchema instanceof UserSchema)) - throw new NotFoundException("Could not find the schema '" + _form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); - _schema = (UserSchema)querySchema; - } - - - @Override - public ModelAndView getView(SourceForm form, BindException errors) - { - _queryDef = _schema.getQueryDef(form.getQueryName()); - if (null == _queryDef) - _queryDef = _schema.getQueryDefForTable(form.getQueryName()); - if (null == _queryDef) - throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - try - { - if (form.ff_queryText == null) - { - form.ff_queryText = _queryDef.getSql(); - form.ff_metadataText = _queryDef.getMetadataXml(); - if (null == form.ff_metadataText) - form.ff_metadataText = form.getDefaultMetadataText(); - } - - for (QueryException qpe : _queryDef.getParseErrors(_schema)) - { - errors.reject(ERROR_MSG, Objects.toString(qpe.getMessage(), qpe.toString())); - } - } - catch (Exception e) - { - try - { - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); - } - catch (Throwable t) - { - // - } - errors.reject("ERROR_MSG", e.toString()); - LOG.error("Error", e); - } - - Renderable moduleWarning = null; - if (_queryDef instanceof ModuleCustomQueryDefinition mcqd && _queryDef.canEdit(getUser())) - { - moduleWarning = DIV(cl("labkey-warning-messages"), - "This SQL query is defined in the '" + mcqd.getModuleName() + "' module in directory '" + mcqd.getSqlFile().getParent() + "'.", - BR(), - "Changes to this query will be reflected in all usages across different folders on the server." - ); - } - - var sourceQueryView = new JspView<>("/org/labkey/query/view/sourceQuery.jsp", this, errors); - WebPartView ret = sourceQueryView; - if (null != moduleWarning) - ret = new VBox(new HtmlView(moduleWarning), sourceQueryView); - return ret; - } - - @Override - public void addNavTrail(NavTree root) - { - setHelpTopic("useSqlEditor"); - - addSchemaActionNavTrail(root, _form.getSchemaKey(), _form.getQueryName()); - - root.addChild("Edit " + _form.getQueryName()); - } - } - - - /** - * Ajax action to save a query. If the save is successful the request will return successfully. A query - * with SQL syntax errors can still be saved successfully. - * - * If the SQL contains parse errors, a parseErrors object will be returned which contains an array of - * JSON serialized error information. - */ - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.Configure.class) - public static class SaveSourceQueryAction extends MutatingApiAction - { - private UserSchema _schema; - - @Override - public void validateForm(SourceForm form, Errors errors) - { - if (StringUtils.isEmpty(form.getSchemaName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - if (StringUtils.isEmpty(form.getQueryName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - - QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), form.getSchemaKey()); - if (null == querySchema) - throw new NotFoundException("schema not found: " + form.getSchemaKey().toDisplayString()); - if (!(querySchema instanceof UserSchema)) - throw new NotFoundException("Could not find the schema '" + form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); - _schema = (UserSchema)querySchema; - - XmlOptions options = XmlBeansUtil.getDefaultParseOptions(); - List xmlErrors = new ArrayList<>(); - options.setErrorListener(xmlErrors); - try - { - // had a couple of real-world failures due to null pointers in this code, so it's time to be paranoid - if (form.ff_metadataText != null) - { - TablesDocument tablesDoc = TablesDocument.Factory.parse(form.ff_metadataText, options); - if (tablesDoc != null) - { - tablesDoc.validate(options); - TablesType tablesType = tablesDoc.getTables(); - if (tablesType != null) - { - for (TableType tableType : tablesType.getTableArray()) - { - if (null != tableType) - { - if (!Objects.equals(tableType.getTableName(), form.getQueryName())) - { - errors.reject(ERROR_MSG, "Table name in the XML metadata must match the table/query name: " + form.getQueryName()); - } - - TableType.Columns tableColumns = tableType.getColumns(); - if (null != tableColumns) - { - ColumnType[] tableColumnArray = tableColumns.getColumnArray(); - for (ColumnType column : tableColumnArray) - { - if (column.isSetPhi() || column.isSetProtected()) - { - throw new IllegalArgumentException("PHI/protected metadata must not be set here."); - } - - ColumnType.Fk fk = column.getFk(); - if (null != fk) - { - try - { - validateForeignKey(fk, column, errors); - validateLookupFilter(AbstractTableInfo.parseXMLLookupFilters(fk.getFilters()), errors); - } - catch (ValidationException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - } - } - } - } - } - } - } - } - } - catch (XmlException e) - { - throw new RuntimeValidationException(e); - } - - for (XmlError xmle : xmlErrors) - { - errors.reject(ERROR_MSG, XmlBeansUtil.getErrorMessage(xmle)); - } - } - - private void validateForeignKey(ColumnType.Fk fk, ColumnType column, Errors errors) - { - if (fk.isSetFkMultiValued()) - { - // issue 51695 : don't let users create unsupported MVFK types - String type = fk.getFkMultiValued(); - if (!AbstractTableInfo.MultiValuedFkType.junction.name().equals(type)) - { - errors.reject(ERROR_MSG, String.format("Column : \"%s\" has an invalid fkMultiValued value : \"%s\" is not supported.", column.getColumnName(), type)); - } - } - } - - private void validateLookupFilter(Map> filterMap, Errors errors) - { - filterMap.forEach((operation, filters) -> { - - String displayStr = "Filter for operation : " + operation.name(); - for (FilterType filter : filters) - { - if (isBlank(filter.getColumn())) - errors.reject(ERROR_MSG, displayStr + " requires columnName"); - - if (null == filter.getOperator()) - { - errors.reject(ERROR_MSG, displayStr + " requires operator"); - } - else - { - CompareType compareType = CompareType.getByURLKey(filter.getOperator().toString()); - if (null == compareType) - { - errors.reject(ERROR_MSG, displayStr + " operator is invalid"); - } - else - { - if (compareType.isDataValueRequired() && null == filter.getValue()) - errors.reject(ERROR_MSG, displayStr + " requires a value but none is specified"); - } - } - } - - try - { - // attempt to convert to something we can query against - SimpleFilter.fromXml(filters.toArray(new FilterType[0])); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - }); - } - - @Override - public ApiResponse execute(SourceForm form, BindException errors) - { - var queryDef = _schema.getQueryDef(form.getQueryName()); - if (null == queryDef) - queryDef = _schema.getQueryDefForTable(form.getQueryName()); - if (null == queryDef) - throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - ApiSimpleResponse response = new ApiSimpleResponse(); - - try - { - if (form.ff_queryText != null) - { - if (!queryDef.isSqlEditable()) - throw new UnauthorizedException("Query SQL is not editable."); - - if (!queryDef.canEdit(getUser())) - throw new UnauthorizedException("Edit permissions are required."); - - queryDef.setSql(form.ff_queryText); - } - - String metadataText = StringUtils.trimToNull(form.ff_metadataText); - if (!Objects.equals(metadataText, queryDef.getMetadataXml())) - { - if (queryDef.isMetadataEditable()) - { - if (!queryDef.canEditMetadata(getUser())) - throw new UnauthorizedException("Edit metadata permissions are required."); - - if (!getUser().isTrustedBrowserDev()) - { - JavaScriptFragment.ensureXMLMetadataNoJavaScript(metadataText); - } - - queryDef.setMetadataXml(metadataText); - } - else - { - if (metadataText != null) - throw new UnsupportedOperationException("Query metadata is not editable."); - } - } - - queryDef.save(getUser(), getContainer()); - - // the query was successfully saved, validate the query but return any errors in the success response - List parseErrors = new ArrayList<>(); - List parseWarnings = new ArrayList<>(); - queryDef.validateQuery(_schema, parseErrors, parseWarnings); - if (!parseErrors.isEmpty()) - { - JSONArray errorArray = new JSONArray(); - - for (QueryException e : parseErrors) - { - errorArray.put(e.toJSON(form.ff_queryText)); - } - response.put("parseErrors", errorArray); - } - else if (!parseWarnings.isEmpty()) - { - JSONArray errorArray = new JSONArray(); - - for (QueryException e : parseWarnings) - { - errorArray.put(e.toJSON(form.ff_queryText)); - } - response.put("parseWarnings", errorArray); - } - } - catch (SQLException e) - { - errors.reject(ERROR_MSG, "An exception occurred: " + e); - LOG.error("Error", e); - } - catch (RuntimeException e) - { - errors.reject(ERROR_MSG, "An exception occurred: " + e.getMessage()); - LOG.error("Error", e); - } - - if (errors.hasErrors()) - return null; - - //if we got here, the query is OK - response.put("success", true); - return response; - } - - } - - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, DeletePermission.class}) - @Action(ActionType.Configure.class) - public static class DeleteQueryAction extends ConfirmAction - { - public SourceForm _form; - public QuerySchema _baseSchema; - public QueryDefinition _queryDef; - - - @Override - public void validateCommand(SourceForm target, Errors errors) - { - _form = target; - if (StringUtils.isEmpty(target.getSchemaName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - if (StringUtils.isEmpty(target.getQueryName())) - throw new NotFoundException("Query definition not found, schemaName and queryName are required."); - - _baseSchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); - if (null == _baseSchema) - throw new NotFoundException("Schema not found: " + _form.getSchemaKey().toDisplayString()); - } - - - @Override - public ModelAndView getConfirmView(SourceForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Query"); - _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); - - if (null == _queryDef) - throw new NotFoundException("Query not found: " + form.getQueryName()); - - if (!_queryDef.canDelete(getUser())) - { - errors.reject(ERROR_MSG, "Sorry, this query can not be deleted"); - } - - return new JspView<>("/org/labkey/query/view/deleteQuery.jsp", this, errors); - } - - - @Override - public boolean handlePost(SourceForm form, BindException errors) throws Exception - { - _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); - - if (null == _queryDef) - return false; - try - { - _queryDef.delete(getUser()); - } - catch (OptimisticConflictException x) - { - /* reshow will throw NotFound, so just ignore */ - } - return true; - } - - @Override - @NotNull - public ActionURL getSuccessURL(SourceForm queryForm) - { - return ((UserSchema)_baseSchema).urlFor(QueryAction.schema); - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public class ExecuteQueryAction extends QueryViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _form = form; - - if (errors.hasErrors()) - return new SimpleErrorView(errors, true); - - QueryView queryView = Objects.requireNonNull(form.getQueryView()); - - var t = queryView.getTable(); - if (null != t && !t.allowRobotsIndex()) - { - getPageConfig().setRobotsNone(); - } - - if (isPrint()) - { - queryView.setPrintView(true); - getPageConfig().setTemplate(PageConfig.Template.Print); - getPageConfig().setShowPrintDialog(true); - } - - queryView.setShadeAlternatingRows(true); - queryView.setShowBorders(true); - setHelpTopic("customSQL"); - _queryView = queryView; - return queryView; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - TableInfo ti = null; - try - { - if (null != _queryView) - ti = _queryView.getTable(); - } - catch (QueryParseException x) - { - /* */ - } - String display = ti == null ? _form.getQueryName() : ti.getTitle(); - root.addChild(display); - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class RawTableMetaDataAction extends QueryViewAction - { - private String _dbSchemaName; - private String _dbTableName; - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _form = form; - - QueryView queryView = form.getQueryView(); - String userSchemaName = queryView.getSchema().getName(); - TableInfo ti = queryView.getTable(); - if (null == ti) - throw new NotFoundException(); - - DbScope scope = ti.getSchema().getScope(); - - // Test for provisioned table - if (ti.getDomain() != null) - { - Domain domain = ti.getDomain(); - if (domain.getStorageTableName() != null) - { - // Use the real table and schema names for getting the metadata - _dbTableName = domain.getStorageTableName(); - _dbSchemaName = domain.getDomainKind().getStorageSchemaName(); - } - } - - // No domain or domain with non-provisioned storage (e.g., core.Users) - if (null == _dbSchemaName || null == _dbTableName) - { - DbSchema dbSchema = ti.getSchema(); - _dbSchemaName = dbSchema.getName(); - - // Try to get the underlying schema table and use the meta data name, #12015 - if (ti instanceof FilteredTable fti) - ti = fti.getRealTable(); - - if (ti instanceof SchemaTableInfo) - _dbTableName = ti.getMetaDataIdentifier().getId(); - else if (ti instanceof LinkedTableInfo) - _dbTableName = ti.getName(); - - if (null == _dbTableName) - { - TableInfo tableInfo = dbSchema.getTable(ti.getName()); - if (null != tableInfo) - _dbTableName = tableInfo.getMetaDataIdentifier().getId(); - } - } - - if (null != _dbTableName) - { - VBox result = new VBox(); - - ActionURL url = null; - QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(userSchemaName); - if (qs != null) - { - url = new ActionURL(RawSchemaMetaDataAction.class, getContainer()); - url.addParameter("schemaName", userSchemaName); - } - - SqlDialect dialect = scope.getSqlDialect(); - ScopeView scopeInfo = new ScopeView("Scope and Schema Information", scope, _dbSchemaName, url, _dbTableName); - - result.addView(scopeInfo); - - try (JdbcMetaDataLocator locator = dialect.getTableResolver().getSingleTableLocator(scope, _dbSchemaName, _dbTableName)) - { - JdbcMetaDataSelector columnSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getColumns(l.getCatalogName(), l.getSchemaNamePattern(), l.getTableNamePattern(), null)); - result.addView(new ResultSetView(CachedResultSets.create(columnSelector.getResultSet(), true, Table.ALL_ROWS), "Table Meta Data")); - - JdbcMetaDataSelector pkSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getPrimaryKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(pkSelector.getResultSet(), true, Table.ALL_ROWS), "Primary Key Meta Data")); - - if (dialect.canCheckIndices(ti)) - { - JdbcMetaDataSelector indexSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getIndexInfo(l.getCatalogName(), l.getSchemaName(), l.getTableName(), false, false)); - result.addView(new ResultSetView(CachedResultSets.create(indexSelector.getResultSet(), true, Table.ALL_ROWS), "Other Index Meta Data")); - } - - JdbcMetaDataSelector ikSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getImportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(ikSelector.getResultSet(), true, Table.ALL_ROWS), "Imported Keys Meta Data")); - - JdbcMetaDataSelector ekSelector = new JdbcMetaDataSelector(locator, - (dbmd, l) -> dbmd.getExportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); - result.addView(new ResultSetView(CachedResultSets.create(ekSelector.getResultSet(), true, Table.ALL_ROWS), "Exported Keys Meta Data")); - } - return result; - } - else - { - errors.reject(ERROR_MSG, "Raw metadata not accessible for table " + ti.getName()); - return new SimpleErrorView(errors); - } - } - - @Override - public void addNavTrail(NavTree root) - { - (new SchemaAction(_form)).addNavTrail(root); - if (null != _dbTableName) - root.addChild("JDBC Meta Data For Table \"" + _dbSchemaName + "." + _dbTableName + "\""); - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public class RawSchemaMetaDataAction extends SimpleViewAction - { - private String _schemaName; - - @Override - public ModelAndView getView(Object form, BindException errors) throws Exception - { - _schemaName = getViewContext().getActionURL().getParameter("schemaName"); - if (null == _schemaName) - throw new NotFoundException(); - QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(_schemaName); - if (null == qs) - throw new NotFoundException(_schemaName); - DbSchema schema = qs.getDbSchema(); - String dbSchemaName = schema.getName(); - DbScope scope = schema.getScope(); - SqlDialect dialect = scope.getSqlDialect(); - - HttpView scopeInfo = new ScopeView("Scope Information", scope); - - ModelAndView tablesView; - - try (JdbcMetaDataLocator locator = dialect.getTableResolver().getAllTablesLocator(scope, dbSchemaName)) - { - JdbcMetaDataSelector selector = new JdbcMetaDataSelector(locator, - (dbmd, locator1) -> dbmd.getTables(locator1.getCatalogName(), locator1.getSchemaNamePattern(), locator1.getTableNamePattern(), null)); - Set tableNames = Sets.newCaseInsensitiveHashSet(qs.getTableNames()); - - ActionURL url = new ActionURL(RawTableMetaDataAction.class, getContainer()) - .addParameter("schemaName", _schemaName) - .addParameter("query.queryName", null); - tablesView = new ResultSetView(CachedResultSets.create(selector.getResultSet(), true, Table.ALL_ROWS), "Tables", "TABLE_NAME", url) - { - @Override - protected boolean shouldLink(ResultSet rs) throws SQLException - { - // Only link to tables and views (not indexes or sequences). And only if they're defined in the query schema. - String name = rs.getString("TABLE_NAME"); - String type = rs.getString("TABLE_TYPE"); - return ("TABLE".equalsIgnoreCase(type) || "VIEW".equalsIgnoreCase(type)) && tableNames.contains(name); - } - }; - } - - return new VBox(scopeInfo, tablesView); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("JDBC Meta Data For Schema \"" + _schemaName + "\""); - } - } - - - public static class ScopeView extends WebPartView - { - private final DbScope _scope; - private final String _schemaName; - private final String _tableName; - private final ActionURL _url; - - private ScopeView(String title, DbScope scope) - { - this(title, scope, null, null, null); - } - - private ScopeView(String title, DbScope scope, String schemaName, ActionURL url, String tableName) - { - super(title); - _scope = scope; - _schemaName = schemaName; - _tableName = tableName; - _url = url; - } - - @Override - protected void renderView(Object model, HtmlWriter out) - { - TABLE( - null != _schemaName ? getLabelAndContents("Schema", _url == null ? _schemaName : LinkBuilder.simpleLink(_schemaName, _url)) : null, - null != _tableName ? getLabelAndContents("Table", _tableName) : null, - getLabelAndContents("Scope", _scope.getDisplayName()), - getLabelAndContents("Dialect", _scope.getSqlDialect().getClass().getSimpleName()), - getLabelAndContents("URL", _scope.getDatabaseUrl()) - ).appendTo(out); - } - - // Return a single row (TR) with styled label and contents in separate TDs - private Renderable getLabelAndContents(String label, Object contents) - { - return TR( - TD( - cl("labkey-form-label"), - label - ), - TD( - contents - ) - ); - } - } - - // for backwards compat same as _executeQuery.view ?_print=1 - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public class PrintRowsAction extends ExecuteQueryAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - _print = true; - ModelAndView result = super.getView(form, errors); - String title = form.getQueryName(); - if (StringUtils.isEmpty(title)) - title = form.getSchemaName(); - getPageConfig().setTitle(title, true); - return result; - } - } - - - abstract static class _ExportQuery extends SimpleViewAction - { - @Override - public ModelAndView getView(K form, BindException errors) throws Exception - { - QueryView view = form.getQueryView(); - getPageConfig().setTemplate(PageConfig.Template.None); - HttpServletResponse response = getViewContext().getResponse(); - response.setHeader("X-Robots-Tag", "noindex"); - try - { - _export(form, view); - return null; - } - catch (QueryService.NamedParameterNotProvided | QueryParseException x) - { - ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - throw x; - } - } - - abstract void _export(K form, QueryView view) throws Exception; - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportScriptForm extends QueryForm - { - private String _type; - - public String getScriptType() - { - return _type; - } - - public void setScriptType(String type) - { - _type = type; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) // This is called "export" but it doesn't export any data - @CSRF(CSRF.Method.ALL) - public static class ExportScriptAction extends SimpleViewAction - { - @Override - public void validate(ExportScriptForm form, BindException errors) - { - // calling form.getQueryView() as a validation check as it will throw if schema/query missing - form.getQueryView(); - - if (StringUtils.isEmpty(form.getScriptType())) - throw new NotFoundException("Missing required parameter: scriptType."); - } - - @Override - public ModelAndView getView(ExportScriptForm form, BindException errors) - { - return ExportScriptModel.getExportScriptView(QueryView.create(form, errors), form.getScriptType(), getPageConfig(), getViewContext().getResponse()); - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsExcelAction extends _ExportQuery - { - @Override - void _export(ExportQueryForm form, QueryView view) throws Exception - { - view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xls, form.getRenameColumnMap()); - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsXLSXAction extends _ExportQuery - { - @Override - void _export(ExportQueryForm form, QueryView view) throws Exception - { - view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xlsx, form.getRenameColumnMap()); - } - } - - public static class ExportQueriesForm extends ExportQueryForm implements ApiJsonForm - { - private String filename; - private List queryForms; - - public void setFilename(String filename) - { - this.filename = filename; - } - - public String getFilename() - { - return filename; - } - - public void setQueryForms(List queryForms) - { - this.queryForms = queryForms; - } - - public List getQueryForms() - { - return queryForms; - } - - /** - * Map JSON to Spring PropertyValue objects. - * @param json the properties - */ - private MutablePropertyValues getPropertyValues(JSONObject json) - { - // Collecting mapped properties as a list because adding them to an existing MutablePropertyValues object replaces existing values - List properties = new ArrayList<>(); - - for (String key : json.keySet()) - { - Object value = json.get(key); - if (value instanceof JSONArray val) - { - // Split arrays into individual pairs to be bound (Issue #45452) - for (int i = 0; i < val.length(); i++) - { - properties.add(new PropertyValue(key, val.get(i).toString())); - } - } - else - { - properties.add(new PropertyValue(key, value)); - } - } - - return new MutablePropertyValues(properties); - } - - @Override - public void bindJson(JSONObject json) - { - setFilename(json.get("filename").toString()); - List forms = new ArrayList<>(); - - JSONArray models = json.optJSONArray("queryForms"); - if (models == null) - { - QueryController.LOG.error("No models to export; Form's `queryForms` property was null"); - throw new RuntimeValidationException("No queries to export; Form's `queryForms` property was null"); - } - - for (JSONObject queryModel : JsonUtil.toJSONObjectList(models)) - { - ExportQueryForm qf = new ExportQueryForm(); - qf.setViewContext(getViewContext()); - - qf.bindParameters(getPropertyValues(queryModel)); - forms.add(qf); - } - - setQueryForms(forms); - } - } - - /** - * Export multiple query forms - */ - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportQueriesXLSXAction extends ReadOnlyApiAction - { - @Override - public Object execute(ExportQueriesForm form, BindException errors) throws Exception - { - getPageConfig().setTemplate(PageConfig.Template.None); - HttpServletResponse response = getViewContext().getResponse(); - response.setHeader("X-Robots-Tag", "noindex"); - ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment); - ViewContext viewContext = getViewContext(); - - Map> nameFormMap = new CaseInsensitiveHashMap<>(); - Map sheetNames = new HashMap<>(); - form.getQueryForms().forEach(qf -> { - String sheetName = qf.getSheetName(); - QueryView qv = qf.getQueryView(); - // use the given sheet name if provided, otherwise try the query definition name - String name = StringUtils.isNotBlank(sheetName) ? sheetName : qv.getQueryDef().getName(); - // if there is no sheet name or queryDefinition name, use a data region name if provided. Otherwise, use "Data" - name = StringUtils.isNotBlank(name) ? name : StringUtils.isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; - // clean it to remove undesirable characters and make it of an acceptable length - name = ExcelWriter.cleanSheetName(name); - nameFormMap.computeIfAbsent(name, k -> new ArrayList<>()).add(qf); - }); - // Issue 53722: Need to assure unique names for the sheets in the presence of really long names - for (Map.Entry> entry : nameFormMap.entrySet()) { - String name = entry.getKey(); - if (entry.getValue().size() > 1) - { - List queryForms = entry.getValue(); - int countLength = String.valueOf(queryForms.size()).length() + 2; - if (countLength > name.length()) - throw new IllegalArgumentException("Cannot create sheet names from overlapping query names."); - for (int i = 0; i < queryForms.size(); i++) - { - sheetNames.put(entry.getValue().get(i), name.substring(0, name.length() - countLength) + "(" + i + ")"); - } - } - else - { - sheetNames.put(entry.getValue().get(0), name); - } - } - ExcelWriter writer = new ExcelWriter(ExcelWriter.ExcelDocumentType.xlsx) { - @Override - protected void renderSheets(Workbook workbook) - { - for (ExportQueryForm qf : form.getQueryForms()) - { - qf.setViewContext(viewContext); - qf.getSchema(); - - QueryView qv = qf.getQueryView(); - QueryView.ExcelExportConfig config = new QueryView.ExcelExportConfig(response, qf.getHeaderType()) - .setExcludeColumns(qf.getExcludeColumns()) - .setRenamedColumns(qf.getRenameColumnMap()); - qv.configureExcelWriter(this, config); - setSheetName(sheetNames.get(qf)); - setAutoSize(true); - renderNewSheet(workbook); - qv.logAuditEvent("Exported to Excel", getDataRowCount()); - } - - workbook.setActiveSheet(0); - } - }; - writer.setFilenamePrefix(form.getFilename()); - writer.renderWorkbook(response); - return null; //Returning anything here will cause error as excel writer will close the response stream - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class TemplateForm extends ExportQueryForm - { - boolean insertColumnsOnly = true; - String filenamePrefix; - FieldKey[] includeColumn; - String fileType; - - public TemplateForm() - { - _headerType = ColumnHeaderType.Caption; - } - - // "captionType" field backwards compatibility - public void setCaptionType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public ColumnHeaderType getCaptionType() - { - return _headerType; - } - - public List getIncludeColumns() - { - if (includeColumn == null || includeColumn.length == 0) - return Collections.emptyList(); - return Arrays.asList(includeColumn); - } - - public FieldKey[] getIncludeColumn() - { - return includeColumn; - } - - public void setIncludeColumn(FieldKey[] includeColumn) - { - this.includeColumn = includeColumn; - } - - @NotNull - public String getFilenamePrefix() - { - return filenamePrefix == null ? getQueryName() : filenamePrefix; - } - - public void setFilenamePrefix(String prefix) - { - filenamePrefix = prefix; - } - - public String getFileType() - { - return fileType; - } - - public void setFileType(String fileType) - { - this.fileType = fileType; - } - } - - - /** - * Can be used to generate an Excel template for import into a table. Supported URL params include: - *
- *
filenamePrefix
- *
the prefix of the excel file that is generated, defaults to '_data'
- * - *
query.viewName
- *
if provided, the resulting excel file will use the fields present in this view. - * Non-usereditable columns will be skipped. - * Non-existent columns (like a lookup) unless includeMissingColumns is true. - * Any required columns missing from this view will be appended to the end of the query. - *
- * - *
includeColumn
- *
List of column names to include, even if the column doesn't exist or is non-userEditable. - * For example, this can be used to add a fake column that is only supported during the import process. - *
- * - *
excludeColumn
- *
List of column names to exclude. - *
- * - *
exportAlias.columns
- *
Use alternative column name in excel: exportAlias.originalColumnName=aliasColumnName - *
- * - *
captionType
- *
determines which column property is used in the header, either Label or Name
- *
- */ - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportExcelTemplateAction extends _ExportQuery - { - public ExportExcelTemplateAction() - { - setCommandClass(TemplateForm.class); - } - - @Override - void _export(TemplateForm form, QueryView view) throws Exception - { - boolean respectView = form.getViewName() != null; - ExcelWriter.ExcelDocumentType fileType = ExcelWriter.ExcelDocumentType.xlsx; - if (form.getFileType() != null) - { - try - { - fileType = ExcelWriter.ExcelDocumentType.valueOf(form.getFileType().toLowerCase()); - } - catch (IllegalArgumentException ignored) {} - } - view.exportToExcel( new QueryView.ExcelExportConfig(getViewContext().getResponse(), form.getHeaderType()) - .setTemplateOnly(true) - .setInsertColumnsOnly(form.insertColumnsOnly) - .setDocType(fileType) - .setRespectView(respectView) - .setIncludeColumns(form.getIncludeColumns()) - .setExcludeColumns(form.getExcludeColumns()) - .setRenamedColumns(form.getRenameColumnMap()) - .setPrefix((StringUtils.isEmpty(form.getFilenamePrefix()) ? "Import" : form.getFilenamePrefix()) + "_Template") // Issue 48028: Change template file names - ); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportQueryForm extends QueryForm - { - protected ColumnHeaderType _headerType = null; // QueryView will provide a default header type if the user doesn't select one - FieldKey[] excludeColumn; - Map renameColumns = null; - private String sheetName; - - public void setSheetName(String sheetName) - { - this.sheetName = sheetName; - } - - public String getSheetName() - { - return sheetName; - } - - public ColumnHeaderType getHeaderType() - { - return _headerType; - } - - public void setHeaderType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public List getExcludeColumns() - { - if (excludeColumn == null || excludeColumn.length == 0) - return Collections.emptyList(); - return Arrays.asList(excludeColumn); - } - - public void setExcludeColumn(FieldKey[] excludeColumn) - { - this.excludeColumn = excludeColumn; - } - - public Map getRenameColumnMap() - { - if (renameColumns != null) - return renameColumns; - - renameColumns = new CaseInsensitiveHashMap<>(); - final String renameParamPrefix = "exportAlias."; - PropertyValue[] pvs = getInitParameters().getPropertyValues(); - for (PropertyValue pv : pvs) - { - String paramName = pv.getName(); - if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) - continue; - - renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); - } - - return renameColumns; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportRowsTsvForm extends ExportQueryForm - { - private TSVWriter.DELIM _delim = TSVWriter.DELIM.TAB; - private TSVWriter.QUOTE _quote = TSVWriter.QUOTE.DOUBLE; - - public TSVWriter.DELIM getDelim() - { - return _delim; - } - - public void setDelim(TSVWriter.DELIM delim) - { - _delim = delim; - } - - public TSVWriter.QUOTE getQuote() - { - return _quote; - } - - public void setQuote(TSVWriter.QUOTE quote) - { - _quote = quote; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportRowsTsvAction extends _ExportQuery - { - public ExportRowsTsvAction() - { - setCommandClass(ExportRowsTsvForm.class); - } - - @Override - void _export(ExportRowsTsvForm form, QueryView view) throws Exception - { - view.exportToTsv(getViewContext().getResponse(), form.getDelim(), form.getQuote(), form.getHeaderType(), form.getRenameColumnMap()); - } - } - - - @RequiresNoPermission - @IgnoresTermsOfUse - @Action(ActionType.Export.class) - public static class ExcelWebQueryAction extends ExportRowsTsvAction - { - @Override - public ModelAndView getView(ExportRowsTsvForm form, BindException errors) throws Exception - { - if (!getContainer().hasPermission(getUser(), ReadPermission.class)) - { - if (!getUser().isGuest()) - { - throw new UnauthorizedException(); - } - getViewContext().getResponse().setHeader("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); - getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); - return null; - } - - // Bug 5610. Excel web queries don't work over SSL if caching is disabled, - // so we need to allow caching so that Excel can read from IE on Windows. - HttpServletResponse response = getViewContext().getResponse(); - // Set the headers to allow the client to cache, but not proxies - ResponseHelper.setPrivate(response); - - QueryView view = form.getQueryView(); - getPageConfig().setTemplate(PageConfig.Template.None); - view.exportToExcelWebQuery(getViewContext().getResponse()); - return null; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExcelWebQueryDefinitionAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - getPageConfig().setTemplate(PageConfig.Template.None); - form.getQueryView(); - String queryViewActionURL = form.getQueryViewActionURL(); - ActionURL url; - if (queryViewActionURL != null) - { - url = new ActionURL(queryViewActionURL); - } - else - { - url = getViewContext().cloneActionURL(); - url.setAction(ExcelWebQueryAction.class); - } - getViewContext().getResponse().setContentType("text/x-ms-iqy"); - String filename = FileUtil.makeFileNameWithTimestamp(form.getQueryName(), "iqy"); - ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, filename); - PrintWriter writer = getViewContext().getResponse().getWriter(); - writer.println("WEB"); - writer.println("1"); - writer.println(url.getURIString()); - - QueryService.get().addAuditEvent(getUser(), getContainer(), form.getSchemaName(), form.getQueryName(), url, "Exported to Excel Web Query definition", null); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - // Trusted analysts who are editors can create and modify queries - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - @Action(ActionType.SelectMetaData.class) - public class MetadataQueryAction extends SimpleViewAction - { - QueryForm _form = null; - - @Override - public ModelAndView getView(QueryForm queryForm, BindException errors) throws Exception - { - String schemaName = queryForm.getSchemaName(); - String queryName = queryForm.getQueryName(); - - _form = queryForm; - - if (schemaName.isEmpty() && (null == queryName || queryName.isEmpty())) - { - throw new NotFoundException("Must provide schemaName and queryName."); - } - - if (schemaName.isEmpty()) - { - throw new NotFoundException("Must provide schemaName."); - } - - if (null == queryName || queryName.isEmpty()) - { - throw new NotFoundException("Must provide queryName."); - } - - if (!queryForm.getQueryDef().isMetadataEditable()) - throw new UnauthorizedException("Query metadata is not editable"); - - if (!queryForm.canEditMetadata()) - throw new UnauthorizedException("You do not have permission to edit the query metadata"); - - return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("queryMetadataEditor")); - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - var metadataQuery = _form.getQueryDef().getName(); - if (null != metadataQuery) - root.addChild("Edit Metadata: " + _form.getQueryName(), metadataQuery); - else - root.addChild("Edit Metadata: " + _form.getQueryName()); - } - } - - // Uck. Supports the old and new view designer. - protected JSONObject saveCustomView(Container container, QueryDefinition queryDef, - String regionName, String viewName, boolean replaceExisting, - boolean share, boolean inherit, - boolean session, boolean saveFilter, - boolean hidden, JSONObject jsonView, - ActionURL returnUrl, - BindException errors) - { - User owner = getUser(); - boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); - if (share && canSaveForAllUsers && !session) - { - owner = null; - } - String name = StringUtils.trimToNull(viewName); - - if (name != null && RESERVED_VIEW_NAMES.contains(name.toLowerCase())) - errors.reject(ERROR_MSG, "The grid view name '" + name + "' is not allowed."); - - boolean isHidden = hidden; - CustomView view; - if (owner == null) - view = queryDef.getSharedCustomView(name); - else - view = queryDef.getCustomView(owner, getViewContext().getRequest(), name); - - if (view != null && !replaceExisting && !StringUtils.isEmpty(name)) - errors.reject(ERROR_MSG, "A saved view by the name \"" + viewName + "\" already exists. "); - - // 11179: Allow editing the view if we're saving to session. - // NOTE: Check for session flag first otherwise the call to canEdit() will add errors to the errors collection. - boolean canEdit = view == null || session || view.canEdit(container, errors); - if (errors.hasErrors()) - return null; - - if (canEdit) - { - // Issue 13594: Disallow setting of the customview inherit bit for query views - // that have no available container filter types. Unfortunately, the only way - // to get the container filters is from the QueryView. Ideally, the query def - // would know if it was container filterable or not instead of using the QueryView. - if (inherit && canSaveForAllUsers && !session) - { - UserSchema schema = queryDef.getSchema(); - QueryView queryView = schema.createView(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, queryDef.getName(), errors); - if (queryView != null) - { - Set allowableContainerFilterTypes = queryView.getAllowableContainerFilterTypes(); - if (allowableContainerFilterTypes.size() <= 1) - { - errors.reject(ERROR_MSG, "QueryView doesn't support inherited custom views"); - return null; - } - } - } - - // Create a new view if none exists or the current view is a shared view - // and the user wants to override the shared view with a personal view. - if (view == null || (owner != null && view.isShared())) - { - if (owner == null) - view = queryDef.createSharedCustomView(name); - else - view = queryDef.createCustomView(owner, name); - - if (owner != null && session) - ((CustomViewImpl) view).isSession(true); - view.setIsHidden(hidden); - } - else if (session != view.isSession()) - { - if (session) - { - assert !view.isSession(); - if (owner == null) - { - errors.reject(ERROR_MSG, "Session views can't be saved for all users"); - return null; - } - - // The form is saving to session but the view is in the database. - // Make a copy in case it's a read-only version from an XML file - view = queryDef.createCustomView(owner, name); - ((CustomViewImpl) view).isSession(true); - } - else - { - // Remove the session view and call saveCustomView again to either create a new view or update an existing view. - assert view.isSession(); - boolean success = false; - try - { - view.delete(getUser(), getViewContext().getRequest()); - JSONObject ret = saveCustomView(container, queryDef, regionName, viewName, replaceExisting, share, inherit, session, saveFilter, hidden, jsonView, returnUrl, errors); - success = !errors.hasErrors() && ret != null; - return success ? ret : null; - } - finally - { - if (!success) - { - // dirty the view then save the deleted session view back in session state - view.setName(view.getName()); - view.save(getUser(), getViewContext().getRequest()); - } - } - } - } - - // NOTE: Updating, saving, and deleting the view may throw an exception - CustomViewImpl cview = null; - if (view instanceof EditableCustomView && view.isOverridable()) - { - cview = ((EditableCustomView)view).getEditableViewInfo(owner, session); - } - if (null == cview) - { - throw new IllegalArgumentException("View cannot be edited"); - } - - cview.update(jsonView, saveFilter); - if (canSaveForAllUsers && !session) - { - cview.setCanInherit(inherit); - } - isHidden = view.isHidden(); - cview.setContainer(container); - cview.save(getUser(), getViewContext().getRequest()); - if (owner == null) - { - // New view is shared so delete any previous custom view owned by the user with the same name. - CustomView personalView = queryDef.getCustomView(getUser(), getViewContext().getRequest(), name); - if (personalView != null && !personalView.isShared()) - { - personalView.delete(getUser(), getViewContext().getRequest()); - } - } - } - - if (null == returnUrl) - { - returnUrl = getViewContext().cloneActionURL().setAction(ExecuteQueryAction.class); - } - else - { - returnUrl = returnUrl.clone(); - if (name == null || !canEdit) - { - returnUrl.deleteParameter(regionName + "." + QueryParam.viewName); - } - else if (!isHidden) - { - returnUrl.replaceParameter(regionName + "." + QueryParam.viewName, name); - } - returnUrl.deleteParameter(regionName + "." + QueryParam.ignoreFilter); - if (saveFilter) - { - for (String key : returnUrl.getKeysByPrefix(regionName + ".")) - { - if (isFilterOrSort(regionName, key)) - returnUrl.deleteFilterParameters(key); - } - } - } - - JSONObject ret = new JSONObject(); - ret.put("redirect", returnUrl); - Map viewAsMap = CustomViewUtil.toMap(view, getUser(), true); - try - { - ret.put("view", new JSONObject(viewAsMap, new JSONParserConfiguration().withMaxNestingDepth(10))); - } - catch (JSONException e) - { - LOG.error("Failed to save view: {}", jsonView, e); - } - return ret; - } - - private boolean isFilterOrSort(String dataRegionName, String param) - { - assert param.startsWith(dataRegionName + "."); - String check = param.substring(dataRegionName.length() + 1); - if (check.contains("~")) - return true; - if ("sort".equals(check)) - return true; - if (check.equals("containerFilterName")) - return true; - return false; - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Configure.class) - @JsonInputLimit(100_000) - public class SaveQueryViewsAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SimpleApiJsonForm form, BindException errors) - { - JSONObject json = form.getJsonObject(); - if (json == null) - throw new NotFoundException("Empty request"); - - String schemaName = json.optString(QueryParam.schemaName.toString(), null); - String queryName = json.optString(QueryParam.queryName.toString(), null); - if (schemaName == null || queryName == null) - throw new NotFoundException("schemaName and queryName are required"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - if (schema == null) - throw new NotFoundException("schema not found"); - - QueryDefinition queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), schemaName, queryName); - if (queryDef == null) - queryDef = schema.getQueryDefForTable(queryName); - if (queryDef == null) - throw new NotFoundException("query not found"); - - JSONObject response = new JSONObject(); - response.put(QueryParam.schemaName.toString(), schemaName); - response.put(QueryParam.queryName.toString(), queryName); - JSONArray views = new JSONArray(); - response.put("views", views); - - ActionURL redirect = null; - JSONArray jsonViews = json.getJSONArray("views"); - for (int i = 0; i < jsonViews.length(); i++) - { - final JSONObject jsonView = jsonViews.getJSONObject(i); - String viewName = jsonView.optString("name", null); - if (viewName == null) - throw new NotFoundException("'name' is required all views'"); - - boolean shared = jsonView.optBoolean("shared", false); - boolean replace = jsonView.optBoolean("replace", true); // "replace" was the default before the flag is introduced - boolean inherit = jsonView.optBoolean("inherit", false); - boolean session = jsonView.optBoolean("session", false); - boolean hidden = jsonView.optBoolean("hidden", false); - // Users may save views to a location other than the current container - String containerPath = jsonView.optString("containerPath", getContainer().getPath()); - Container container; - if (inherit) - { - // Only respect this request if it's a view that is inheritable in subfolders - container = ContainerManager.getForPath(containerPath); - } - else - { - // Otherwise, save it in the current container - container = getContainer().getContainerFor(ContainerType.DataType.customQueryViews); - } - - if (container == null) - { - throw new NotFoundException("No such container: " + containerPath); - } - - JSONObject savedView = saveCustomView( - container, queryDef, QueryView.DATAREGIONNAME_DEFAULT, viewName, replace, - shared, inherit, session, true, hidden, jsonView, null, errors); - - if (savedView != null) - { - if (redirect == null) - redirect = (ActionURL)savedView.get("redirect"); - views.put(savedView.getJSONObject("view")); - } - } - - if (redirect != null) - response.put("redirect", redirect); - - if (errors.hasErrors()) - return null; - else - return new ApiSimpleResponse(response); - } - } - - public static class RenameQueryViewForm extends QueryForm - { - private String newName; - - public String getNewName() - { - return newName; - } - - public void setNewName(String newName) - { - this.newName = newName; - } - } - - @RequiresPermission(ReadPermission.class) - public class RenameQueryViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(RenameQueryViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - - Container container = getContainer(); - User user = getUser(); - - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - if (schemaName == null || queryName == null) - throw new NotFoundException("schemaName and queryName are required"); - - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); - if (schema == null) - throw new NotFoundException("schema not found"); - - QueryDefinition queryDef = QueryService.get().getQueryDef(user, container, schemaName, queryName); - if (queryDef == null) - queryDef = schema.getQueryDefForTable(queryName); - if (queryDef == null) - throw new NotFoundException("query not found"); - - renameCustomView(container, queryDef, view, form.getNewName(), errors); - - if (errors.hasErrors()) - return null; - else - return new ApiSimpleResponse("success", true); - } - } - - protected void renameCustomView(Container container, QueryDefinition queryDef, CustomView fromView, String newViewName, BindException errors) - { - if (newViewName != null && RESERVED_VIEW_NAMES.contains(newViewName.toLowerCase())) - errors.reject(ERROR_MSG, "The grid view name '" + newViewName + "' is not allowed."); - - String newName = StringUtils.trimToNull(newViewName); - if (StringUtils.isEmpty(newName)) - errors.reject(ERROR_MSG, "View name cannot be blank."); - - if (errors.hasErrors()) - return; - - User owner = getUser(); - boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); - - if (!fromView.canEdit(container, errors)) - return; - - if (fromView.isSession()) - { - errors.reject(ERROR_MSG, "Cannot rename a session view."); - return; - } - - CustomView duplicateView = queryDef.getCustomView(owner, getViewContext().getRequest(), newName); - if (duplicateView == null && canSaveForAllUsers) - duplicateView = queryDef.getSharedCustomView(newName); - if (duplicateView != null) - { - // only allow duplicate view name if creating a new private view to shadow an existing shared view - if (!(!fromView.isShared() && duplicateView.isShared())) - { - errors.reject(ERROR_MSG, "Another saved view by the name \"" + newName + "\" already exists. "); - return; - } - } - - fromView.setName(newViewName); - fromView.save(getUser(), getViewContext().getRequest()); - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Configure.class) - public class PropertiesQueryAction extends FormViewAction - { - PropertiesForm _form = null; - private String _queryName; - - @Override - public void validateCommand(PropertiesForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(PropertiesForm form, boolean reshow, BindException errors) - { - // assertQueryExists requires that it be well-formed - // assertQueryExists(form); - QueryDefinition queryDef = form.getQueryDef(); - _form = form; - _form.setDescription(queryDef.getDescription()); - _form.setInheritable(queryDef.canInherit()); - _form.setHidden(queryDef.isHidden()); - setHelpTopic("editQueryProperties"); - _queryName = form.getQueryName(); - - return new JspView<>("/org/labkey/query/view/propertiesQuery.jsp", form, errors); - } - - @Override - public boolean handlePost(PropertiesForm form, BindException errors) throws Exception - { - // assertQueryExists requires that it be well-formed - // assertQueryExists(form); - if (!form.canEdit()) - { - throw new UnauthorizedException(); - } - QueryDefinition queryDef = form.getQueryDef(); - _queryName = form.getQueryName(); - if (!queryDef.getDefinitionContainer().getId().equals(getContainer().getId())) - throw new NotFoundException("Query not found"); - - _form = form; - - if (!StringUtils.isEmpty(form.rename) && !form.rename.equalsIgnoreCase(queryDef.getName())) - { - // issue 17766: check if query or table exist with this name - if (null != QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), form.rename, true) - || null != form.getSchema().getTable(form.rename,null)) - { - errors.reject(ERROR_MSG, "A query or table with the name \"" + form.rename + "\" already exists."); - return false; - } - - // Issue 40895: update queryName in xml metadata - updateXmlMetadata(queryDef); - queryDef.setName(form.rename); - // update form so getSuccessURL() works - _form = new PropertiesForm(form.getSchemaName(), form.rename); - _form.setViewContext(form.getViewContext()); - _queryName = form.rename; - } - - queryDef.setDescription(form.description); - queryDef.setCanInherit(form.inheritable); - queryDef.setIsHidden(form.hidden); - queryDef.save(getUser(), getContainer()); - return true; - } - - private void updateXmlMetadata(QueryDefinition queryDef) throws XmlException - { - if (null != queryDef.getMetadataXml()) - { - TablesDocument doc = TablesDocument.Factory.parse(queryDef.getMetadataXml()); - if (null != doc) - { - for (TableType tableType : doc.getTables().getTableArray()) - { - if (tableType.getTableName().equalsIgnoreCase(queryDef.getName())) - { - // update tableName in xml - tableType.setTableName(_form.rename); - } - } - XmlOptions xmlOptions = new XmlOptions(); - xmlOptions.setSavePrettyPrint(); - // Don't use an explicit namespace, making the XML much more readable - xmlOptions.setUseDefaultNamespace(); - queryDef.setMetadataXml(doc.xmlText(xmlOptions)); - } - } - } - - @Override - public ActionURL getSuccessURL(PropertiesForm propertiesForm) - { - ActionURL url = new ActionURL(BeginAction.class, propertiesForm.getViewContext().getContainer()); - url.addParameter("schemaName", propertiesForm.getSchemaName()); - if (null != _queryName) - url.addParameter("queryName", _queryName); - return url; - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - root.addChild("Edit query properties"); - } - } - - @ActionNames("truncateTable") - @RequiresPermission(AdminPermission.class) - public static class TruncateTableAction extends MutatingApiAction - { - UserSchema schema; - TableInfo table; - - @Override - public void validateForm(QueryForm form, Errors errors) - { - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - - if (isBlank(schemaName) || isBlank(queryName)) - throw new NotFoundException("schemaName and queryName are required"); - - schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - if (null == schema) - throw new NotFoundException("The schema '" + schemaName + "' does not exist."); - - table = schema.getTable(queryName, null); - if (null == table) - throw new NotFoundException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); - } - - @Override - public ApiResponse execute(QueryForm form, BindException errors) throws Exception - { - int deletedRows; - QueryUpdateService qus = table.getUpdateService(); - - if (null == qus) - throw new IllegalArgumentException("The query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "' is not truncatable."); - - try (DbScope.Transaction transaction = table.getSchema().getScope().ensureTransaction()) - { - deletedRows = qus.truncateRows(getUser(), getContainer(), null, null); - transaction.commit(); - } - - ApiSimpleResponse response = new ApiSimpleResponse(); - - response.put("success", true); - response.put(BaseSaveRowsAction.PROP_SCHEMA_NAME, form.getSchemaName()); - response.put(BaseSaveRowsAction.PROP_QUERY_NAME, form.getQueryName()); - response.put("deletedRows", deletedRows); - - return response; - } - } - - - @RequiresPermission(DeletePermission.class) - public static class DeleteQueryRowsAction extends FormHandlerAction - { - @Override - public void validateCommand(QueryForm target, Errors errors) - { - } - - @Override - public boolean handlePost(QueryForm form, BindException errors) - { - TableInfo table = form.getQueryView().getTable(); - - if (!table.hasPermission(getUser(), DeletePermission.class)) - { - throw new UnauthorizedException(); - } - - QueryUpdateService updateService = table.getUpdateService(); - if (updateService == null) - throw new UnsupportedOperationException("Unable to delete - no QueryUpdateService registered for " + form.getSchemaName() + "." + form.getQueryName()); - - Set ids = DataRegionSelection.getSelected(form.getViewContext(), null, true); - List pks = table.getPkColumns(); - int numPks = pks.size(); - - //normalize the pks to arrays of correctly-typed objects - List> keyValues = new ArrayList<>(ids.size()); - for (String id : ids) - { - String[] stringValues; - if (numPks > 1) - { - stringValues = id.split(","); - if (stringValues.length != numPks) - throw new IllegalStateException("This table has " + numPks + " primary-key columns, but " + stringValues.length + " primary-key values were provided!"); - } - else - stringValues = new String[]{id}; - - Map rowKeyValues = new CaseInsensitiveHashMap<>(); - for (int idx = 0; idx < numPks; ++idx) - { - ColumnInfo keyColumn = pks.get(idx); - Object keyValue = keyColumn.getJavaClass() == String.class ? stringValues[idx] : ConvertUtils.convert(stringValues[idx], keyColumn.getJavaClass()); - rowKeyValues.put(keyColumn.getName(), keyValue); - } - keyValues.add(rowKeyValues); - } - - DbSchema dbSchema = table.getSchema(); - try - { - dbSchema.getScope().executeWithRetry(tx -> - { - try - { - updateService.deleteRows(getUser(), getContainer(), keyValues, null, null); - } - catch (SQLException x) - { - if (!RuntimeSQLException.isConstraintException(x)) - throw new RuntimeSQLException(x); - errors.reject(ERROR_MSG, getMessage(table.getSchema().getSqlDialect(), x)); - } - catch (DataIntegrityViolationException | OptimisticConflictException e) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - catch (BatchValidationException x) - { - x.addToErrors(errors); - } - catch (Exception x) - { - errors.reject(ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); - ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); - } - // need to throw here to avoid committing tx - if (errors.hasErrors()) - throw new DbScope.RetryPassthroughException(errors); - return true; - }); - } - catch (DbScope.RetryPassthroughException x) - { - if (x.getCause() != errors) - x.throwRuntimeException(); - } - return !errors.hasErrors(); - } - - @Override - public ActionURL getSuccessURL(QueryForm form) - { - return form.getReturnActionURL(); - } - } - - @RequiresPermission(ReadPermission.class) - public static class DetailsQueryRowAction extends UserSchemaAction - { - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - ButtonBar bb = new ButtonBar(); - bb.setStyle(ButtonBar.Style.separateButtons); - - if (_schema != null && _table != null) - { - if (_table.hasPermission(getUser(), UpdatePermission.class)) - { - StringExpression updateExpr = _form.getQueryDef().urlExpr(QueryAction.updateQueryRow, _schema.getContainer()); - if (updateExpr != null) - { - String url = updateExpr.eval(tableForm.getTypedValues()); - if (url != null) - { - ActionURL updateUrl = new ActionURL(url); - ActionButton editButton = new ActionButton("Edit", updateUrl); - bb.add(editButton); - } - } - } - - - ActionURL gridUrl; - if (_form.getReturnActionURL() != null) - { - // If we have a specific return URL requested, use that - gridUrl = _form.getReturnActionURL(); - } - else - { - // Otherwise go back to the default grid view - gridUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); - } - if (gridUrl != null) - { - ActionButton gridButton = new ActionButton("Show Grid", gridUrl); - bb.add(gridButton); - } - } - - DetailsView detailsView = new DetailsView(tableForm); - detailsView.setFrame(WebPartView.FrameType.PORTAL); - detailsView.getDataRegion().setButtonBar(bb); - - VBox view = new VBox(detailsView); - - DetailsURL detailsURL = QueryService.get().getAuditDetailsURL(getUser(), getContainer(), _table); - - if (detailsURL != null) - { - String url = detailsURL.eval(tableForm.getTypedValues()); - if (url != null) - { - ActionURL auditURL = new ActionURL(url); - - QueryView historyView = QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), - auditURL.getParameter(QueryParam.schemaName), - auditURL.getParameter(QueryParam.queryName), - auditURL.getParameter("keyValue"), errors); - - if (null != historyView) - { - historyView.setFrame(WebPartView.FrameType.PORTAL); - historyView.setTitle("History"); - - view.addView(historyView); - } - } - } - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - return false; - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Details"); - } - } - - @RequiresPermission(InsertPermission.class) - public static class InsertQueryRowAction extends UserSchemaAction - { - @Override - public BindException bindParameters(PropertyValues m) throws Exception - { - BindException bind = super.bindParameters(m); - - // what is going on with UserSchemaAction and form binding? Why doesn't successUrl bind? - QueryUpdateForm form = (QueryUpdateForm)bind.getTarget(); - if (null == form.getSuccessUrl() && null != m.getPropertyValue(ActionURL.Param.successUrl.name())) - form.setSuccessUrl(new ReturnURLString(m.getPropertyValue(ActionURL.Param.successUrl.name()).getValue().toString())); - return bind; - } - - Map insertedRow = null; - - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Insert Row"); - - InsertView view = new InsertView(tableForm, errors); - view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) - { - List> list = doInsertUpdate(tableForm, errors, true); - if (null != list && list.size() == 1) - insertedRow = list.get(0); - return 0 == errors.getErrorCount(); - } - - /** - * NOTE: UserSchemaAction.addNavTrail() uses this method getSuccessURL() for the nav trail link (form==null). - * It is used for where to go on success, and also as a "back" link in the nav trail - * If there is a setSuccessUrl specified, we will use that for successful submit - */ - @Override - public ActionURL getSuccessURL(QueryUpdateForm form) - { - if (null == form) - return super.getSuccessURL(null); - - String str = null; - if (form.getSuccessUrl() != null) - str = form.getSuccessUrl().toString(); - if (isBlank(str)) - str = form.getReturnUrl(); - - if ("details.view".equals(str)) - { - if (null == insertedRow) - return super.getSuccessURL(form); - StringExpression se = form.getTable().getDetailsURL(null, getContainer()); - if (null == se) - return super.getSuccessURL(form); - str = se.eval(insertedRow); - } - try - { - if (!isBlank(str)) - return new ActionURL(str); - } - catch (IllegalArgumentException x) - { - // pass - } - return super.getSuccessURL(form); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Insert " + _table.getName()); - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateQueryRowAction extends UserSchemaAction - { - @Override - public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) - { - ButtonBar bb = createSubmitCancelButtonBar(tableForm); - UpdateView view = new UpdateView(tableForm, errors); - view.getDataRegion().setButtonBar(bb); - return view; - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception - { - doInsertUpdate(tableForm, errors, false); - return 0 == errors.getErrorCount(); - } - - @Override - public void addNavTrail(NavTree root) - { - super.addNavTrail(root); - root.addChild("Edit " + _table.getName()); - } - } - - @RequiresPermission(UpdatePermission.class) - public static class UpdateQueryRowsAction extends UpdateQueryRowAction - { - @Override - public ModelAndView handleRequest(QueryUpdateForm tableForm, BindException errors) throws Exception - { - tableForm.setBulkUpdate(true); - return super.handleRequest(tableForm, errors); - } - - @Override - public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception - { - boolean ret; - - if (tableForm.isDataSubmit()) - { - ret = super.handlePost(tableForm, errors); - if (ret) - DataRegionSelection.clearAll(getViewContext(), null); // in case we altered primary keys, see issue #35055 - return ret; - } - - return false; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Edit Multiple " + _table.getName()); - } - } - - // alias - public static class DeleteAction extends DeleteQueryRowsAction - { - } - - public abstract static class QueryViewAction extends SimpleViewAction - { - QueryForm _form; - QueryView _queryView; - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class APIQueryForm extends ContainerFilterQueryForm - { - private Integer _start; - private Integer _limit; - private boolean _includeDetailsColumn = false; - private boolean _includeUpdateColumn = false; - private boolean _includeTotalCount = true; - private boolean _includeStyle = false; - private boolean _includeDisplayValues = false; - private boolean _minimalColumns = true; - private boolean _includeMetadata = true; - - public Integer getStart() - { - return _start; - } - - public void setStart(Integer start) - { - _start = start; - } - - public Integer getLimit() - { - return _limit; - } - - public void setLimit(Integer limit) - { - _limit = limit; - } - - public boolean isIncludeTotalCount() - { - return _includeTotalCount; - } - - public void setIncludeTotalCount(boolean includeTotalCount) - { - _includeTotalCount = includeTotalCount; - } - - public boolean isIncludeStyle() - { - return _includeStyle; - } - - public void setIncludeStyle(boolean includeStyle) - { - _includeStyle = includeStyle; - } - - public boolean isIncludeDetailsColumn() - { - return _includeDetailsColumn; - } - - public void setIncludeDetailsColumn(boolean includeDetailsColumn) - { - _includeDetailsColumn = includeDetailsColumn; - } - - public boolean isIncludeUpdateColumn() - { - return _includeUpdateColumn; - } - - public void setIncludeUpdateColumn(boolean includeUpdateColumn) - { - _includeUpdateColumn = includeUpdateColumn; - } - - public boolean isIncludeDisplayValues() - { - return _includeDisplayValues; - } - - public void setIncludeDisplayValues(boolean includeDisplayValues) - { - _includeDisplayValues = includeDisplayValues; - } - - public boolean isMinimalColumns() - { - return _minimalColumns; - } - - public void setMinimalColumns(boolean minimalColumns) - { - _minimalColumns = minimalColumns; - } - - public boolean isIncludeMetadata() - { - return _includeMetadata; - } - - public void setIncludeMetadata(boolean includeMetadata) - { - _includeMetadata = includeMetadata; - } - - @Override - protected QuerySettings createQuerySettings(UserSchema schema) - { - QuerySettings results = super.createQuerySettings(schema); - - // See dataintegration/202: The java client api / remote ETL calls selectRows with showRows=all. We need to test _initParameters to properly read this - boolean missingShowRows = null == getViewContext().getRequest().getParameter(getDataRegionName() + "." + QueryParam.showRows) && null == _initParameters.getPropertyValue(getDataRegionName() + "." + QueryParam.showRows); - if (null == getLimit() && !results.isMaxRowsSet() && missingShowRows) - { - results.setShowRows(ShowRows.PAGINATED); - results.setMaxRows(DEFAULT_API_MAX_ROWS); - } - - if (getLimit() != null) - { - results.setShowRows(ShowRows.PAGINATED); - results.setMaxRows(getLimit()); - } - if (getStart() != null) - results.setOffset(getStart()); - - return results; - } - } - - public static final int DEFAULT_API_MAX_ROWS = 100000; - - @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 - @ActionNames("selectRows, getQuery") - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.1) - @Action(ActionType.SelectData.class) - public class SelectRowsAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(APIQueryForm form, BindException errors) - { - // Issue 12233: add implicit maxRows=100k when using client API - QueryView view = form.getQueryView(); - - view.setShowPagination(form.isIncludeTotalCount()); - - //if viewName was specified, ensure that it was actually found and used - //QueryView.create() will happily ignore an invalid view name and just return the default view - if (null != StringUtils.trimToNull(form.getViewName()) && - null == view.getQueryDef().getCustomView(getUser(), getViewContext().getRequest(), form.getViewName())) - { - throw new NotFoundException("The requested view '" + form.getViewName() + "' does not exist for this user."); - } - - TableInfo t = view.getTable(); - if (null == t) - { - List qpes = view.getParseErrors(); - if (!qpes.isEmpty()) - throw qpes.get(0); - throw new NotFoundException(form.getQueryName()); - } - - boolean isEditable = isQueryEditable(view.getTable()); - boolean metaDataOnly = form.getQuerySettings().getMaxRows() == 0; - boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; - boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; - - ApiQueryResponse response; - - // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. - if (getRequestedApiVersion() >= 13.2) - { - ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, true, view.getQueryDef().getName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); - fancyResponse.includeFormattedValue(includeFormattedValue); - response = fancyResponse; - } - //if requested version is >= 9.1, use the extended api query response - else if (getRequestedApiVersion() >= 9.1) - { - response = new ExtendedApiQueryResponse(view, isEditable, true, - form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - } - else - { - response = new ApiQueryResponse(view, isEditable, true, - form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), - form.isIncludeDisplayValues(), form.isIncludeMetadata()); - } - response.includeStyle(form.isIncludeStyle()); - - // Issues 29515 and 32269 - force key and other non-requested columns to be sent back, but only if the client has - // requested minimal columns, as we now do for ExtJS stores - if (form.isMinimalColumns()) - { - // Be sure to use the settings from the view, as it may have swapped it out with a customized version. - // See issue 38747. - response.setColumnFilter(view.getSettings().getFieldKeys()); - } - - return response; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class GetDataAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception - { - ObjectMapper mapper = JsonUtil.createDefaultMapper(); - mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - JSONObject object = form.getJsonObject(); - if (object == null) - { - object = new JSONObject(); - } - DataRequest builder = mapper.readValue(object.toString(), DataRequest.class); - - return builder.render(getViewContext(), errors); - } - } - - protected boolean isQueryEditable(TableInfo table) - { - if (!getContainer().hasPermission("isQueryEditable", getUser(), DeletePermission.class)) - return false; - QueryUpdateService updateService = null; - try - { - updateService = table.getUpdateService(); - } - catch(Exception ignore) {} - return null != table && null != updateService; - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExecuteSqlForm extends APIQueryForm - { - private String _sql; - private Integer _maxRows; - private Integer _offset; - private boolean _saveInSession; - - public String getSql() - { - return _sql; - } - - public void setSql(String sql) - { - _sql = PageFlowUtil.wafDecode(StringUtils.trim(sql)); - } - - public Integer getMaxRows() - { - return _maxRows; - } - - public void setMaxRows(Integer maxRows) - { - _maxRows = maxRows; - } - - public Integer getOffset() - { - return _offset; - } - - public void setOffset(Integer offset) - { - _offset = offset; - } - - @Override - public void setLimit(Integer limit) - { - _maxRows = limit; - } - - @Override - public void setStart(Integer start) - { - _offset = start; - } - - public boolean isSaveInSession() - { - return _saveInSession; - } - - public void setSaveInSession(boolean saveInSession) - { - _saveInSession = saveInSession; - } - - @Override - public String getQueryName() - { - // ExecuteSqlAction doesn't allow setting query name parameter. - return null; - } - - @Override - public void setQueryName(String name) - { - // ExecuteSqlAction doesn't allow setting query name parameter. - } - } - - @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.1) - @Action(ActionType.SelectData.class) - public class ExecuteSqlAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(ExecuteSqlForm form, BindException errors) - { - form.ensureSchemaExists(); - - String schemaName = StringUtils.trimToNull(form.getQuerySettings().getSchemaName()); - if (null == schemaName) - throw new IllegalArgumentException("No value was supplied for the required parameter 'schemaName'."); - String sql = form.getSql(); - if (StringUtils.isBlank(sql)) - throw new IllegalArgumentException("No value was supplied for the required parameter 'sql'."); - - //create a temp query settings object initialized with the posted LabKey SQL - //this will provide a temporary QueryDefinition to Query - QuerySettings settings = form.getQuerySettings(); - if (form.isSaveInSession()) - { - HttpSession session = getViewContext().getSession(); - if (session == null) - throw new IllegalStateException("Session required"); - - QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), schemaName, sql); - settings.setDataRegionName("executeSql"); - settings.setQueryName(def.getName()); - } - else - { - settings = new TempQuerySettings(getViewContext(), sql, settings); - } - - //need to explicitly turn off various UI options that will try to refer to the - //current URL and query string - settings.setAllowChooseView(false); - settings.setAllowCustomizeView(false); - - // Issue 12233: add implicit maxRows=100k when using client API - settings.setShowRows(ShowRows.PAGINATED); - settings.setMaxRows(DEFAULT_API_MAX_ROWS); - - // 16961: ExecuteSql API without maxRows parameter defaults to returning 100 rows - //apply optional settings (maxRows, offset) - boolean metaDataOnly = false; - if (null != form.getMaxRows() && (form.getMaxRows() >= 0 || form.getMaxRows() == Table.ALL_ROWS)) - { - settings.setMaxRows(form.getMaxRows()); - metaDataOnly = Table.NO_ROWS == form.getMaxRows(); - } - - int offset = 0; - if (null != form.getOffset()) - { - settings.setOffset(form.getOffset().longValue()); - offset = form.getOffset(); - } - - //build a query view using the schema and settings - QueryView view = new QueryView(form.getSchema(), settings, errors); - view.setShowRecordSelectors(false); - view.setShowExportButtons(false); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - view.setShowPagination(form.isIncludeTotalCount()); - - TableInfo t = view.getTable(); - boolean isEditable = null != t && isQueryEditable(view.getTable()); - boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; - boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; - - ApiQueryResponse response; - - // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. - if (getRequestedApiVersion() >= 13.2) - { - ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, false, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); - fancyResponse.includeFormattedValue(includeFormattedValue); - response = fancyResponse; - } - else if (getRequestedApiVersion() >= 9.1) - { - response = new ExtendedApiQueryResponse(view, isEditable, - false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); - } - else - { - response = new ApiQueryResponse(view, isEditable, - false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, - metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), - form.isIncludeDisplayValues()); - } - response.includeStyle(form.isIncludeStyle()); - - return response; - } - } - - public static class ContainerFilterQueryForm extends QueryForm - { - private String _containerFilter; - - public String getContainerFilter() - { - return _containerFilter; - } - - public void setContainerFilter(String containerFilter) - { - _containerFilter = containerFilter; - } - - @Override - protected QuerySettings createQuerySettings(UserSchema schema) - { - var result = super.createQuerySettings(schema); - if (getContainerFilter() != null) - { - // If the user specified an incorrect filter, throw an IllegalArgumentException - try - { - ContainerFilter.Type containerFilterType = ContainerFilter.Type.valueOf(getContainerFilter()); - result.setContainerFilterName(containerFilterType.name()); - } - catch (IllegalArgumentException e) - { - // Remove bogus value from error message, Issue 45567 - throw new IllegalArgumentException("'containerFilter' parameter is not valid"); - } - } - return result; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public class SelectDistinctAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(ContainerFilterQueryForm form, BindException errors) throws Exception - { - TableInfo table = form.getQueryView().getTable(); - if (null == table) - throw new NotFoundException(); - SqlSelector sqlSelector = getDistinctSql(table, form, errors); - - if (errors.hasErrors() || null == sqlSelector) - return null; - - ApiResponseWriter writer = new ApiJsonWriter(getViewContext().getResponse()); - - try (ResultSet rs = sqlSelector.getResultSet()) - { - writer.startResponse(); - writer.writeProperty("schemaName", form.getSchemaName()); - writer.writeProperty("queryName", form.getQueryName()); - writer.startList("values"); - - while (rs.next()) - { - writer.writeListEntry(rs.getObject(1)); - } - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - catch (DataAccessException x) // Spring error translator can return various subclasses of this - { - throw new RuntimeException(x); - } - writer.endList(); - writer.endResponse(); - - return null; - } - - @Nullable - private SqlSelector getDistinctSql(TableInfo table, ContainerFilterQueryForm form, BindException errors) - { - QuerySettings settings = form.getQuerySettings(); - QueryService service = QueryService.get(); - - if (null == getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())) - { - settings.setMaxRows(DEFAULT_API_MAX_ROWS); - } - else - { - try - { - int maxRows = Integer.parseInt(getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())); - settings.setMaxRows(maxRows); - } - catch (NumberFormatException e) - { - // Standard exception message, Issue 45567 - QuerySettings.throwParameterParseException(QueryParam.maxRows); - } - } - - List fieldKeys = settings.getFieldKeys(); - if (null == fieldKeys || fieldKeys.size() != 1) - { - errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); - return null; - } - Map columns = service.getColumns(table, fieldKeys); - if (columns.size() != 1) - { - errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); - return null; - } - - ColumnInfo col = columns.get(settings.getFieldKeys().get(0)); - if (col == null) - { - errors.reject(ERROR_MSG, "\"" + settings.getFieldKeys().get(0).getName() + "\" is not a valid column."); - return null; - } - - try - { - SimpleFilter filter = getFilterFromQueryForm(form); - - // Strip out filters on columns that don't exist - issue 21669 - service.ensureRequiredColumns(table, columns.values(), filter, null, new HashSet<>()); - QueryLogging queryLogging = new QueryLogging(); - QueryService.SelectBuilder builder = service.getSelectBuilder(table) - .columns(columns.values()) - .filter(filter) - .queryLogging(queryLogging) - .distinct(true); - SQLFragment selectSql = builder.buildSqlFragment(); - - // TODO: queryLogging.isShouldAudit() is always false at this point. - // The only place that seems to set this is ComplianceQueryLoggingProfileListener.queryInvoked() - if (queryLogging.isShouldAudit() && null != queryLogging.getExceptionToThrowIfLoggingIsEnabled()) - { - // this is probably a more helpful message - errors.reject(ERROR_MSG, "Cannot choose values from a column that requires logging."); - return null; - } - - // Regenerate the column since the alias may have changed after call to getSelectSQL() - columns = service.getColumns(table, settings.getFieldKeys()); - var colGetAgain = columns.get(settings.getFieldKeys().get(0)); - // I don't believe the above comment, so here's an assert - assert(colGetAgain.getAlias().equals(col.getAlias())); - - SQLFragment sql = new SQLFragment("SELECT ").appendIdentifier(col.getAlias()).append(" AS value FROM ("); - sql.append(selectSql); - sql.append(") S ORDER BY value"); - - sql = table.getSqlDialect().limitRows(sql, settings.getMaxRows()); - - // 18875: Support Parameterized queries in Select Distinct - Map _namedParameters = settings.getQueryParameters(); - - service.bindNamedParameters(sql, _namedParameters); - service.validateNamedParameters(sql); - - return new SqlSelector(table.getSchema().getScope(), sql, queryLogging); - } - catch (ConversionException | QueryService.NamedParameterNotProvided e) - { - errors.reject(ERROR_MSG, e.getMessage()); - return null; - } - } - } - - private SimpleFilter getFilterFromQueryForm(QueryForm form) - { - QuerySettings settings = form.getQuerySettings(); - SimpleFilter filter = null; - - // 21032: Respect 'ignoreFilter' - if (settings != null && !settings.getIgnoreUserFilter()) - { - // Attach any URL-based filters. This would apply to 'filterArray' from the JavaScript API. - filter = new SimpleFilter(settings.getBaseFilter()); - - String dataRegionName = form.getDataRegionName(); - if (StringUtils.trimToNull(dataRegionName) == null) - dataRegionName = QueryView.DATAREGIONNAME_DEFAULT; - - // Support for 'viewName' - CustomView view = settings.getCustomView(getViewContext(), form.getQueryDef()); - if (null != view && view.hasFilterOrSort() && !settings.getIgnoreViewFilter()) - { - ActionURL url = new ActionURL(SelectDistinctAction.class, getContainer()); - view.applyFilterAndSortToURL(url, dataRegionName); - filter.addAllClauses(new SimpleFilter(url, dataRegionName)); - } - - filter.addUrlFilters(settings.getSortFilterURL(), dataRegionName, Collections.emptyList(), getUser(), getContainer()); - } - - return filter; - } - - @RequiresPermission(ReadPermission.class) - public class GetColumnSummaryStatsAction extends ReadOnlyApiAction - { - private FieldKey _colFieldKey; - - @Override - public void validateForm(QueryForm form, Errors errors) - { - QuerySettings settings = form.getQuerySettings(); - List fieldKeys = settings != null ? settings.getFieldKeys() : null; - if (null == fieldKeys || fieldKeys.size() != 1) - errors.reject(ERROR_MSG, "GetColumnSummaryStats requires that only one column be requested."); - else - _colFieldKey = fieldKeys.get(0); - } - - @Override - public ApiResponse execute(QueryForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - QueryView view = form.getQueryView(); - DisplayColumn displayColumn = null; - - for (DisplayColumn dc : view.getDisplayColumns()) - { - if (dc.getColumnInfo() != null && _colFieldKey.equals(dc.getColumnInfo().getFieldKey())) - { - displayColumn = dc; - break; - } - } - - if (displayColumn != null && displayColumn.getColumnInfo() != null) - { - // get the map of the analytics providers to their relevant aggregates and add the information to the response - Map> analyticsProviders = new LinkedHashMap<>(); - Set colAggregates = new HashSet<>(); - for (ColumnAnalyticsProvider analyticsProvider : displayColumn.getAnalyticsProviders()) - { - if (analyticsProvider instanceof BaseAggregatesAnalyticsProvider baseAggProvider) - { - Map props = new HashMap<>(); - props.put("label", baseAggProvider.getLabel()); - - List aggregateNames = new ArrayList<>(); - for (Aggregate aggregate : AnalyticsProviderItem.createAggregates(baseAggProvider, _colFieldKey, null)) - { - aggregateNames.add(aggregate.getType().getName()); - colAggregates.add(aggregate); - } - props.put("aggregates", aggregateNames); - - analyticsProviders.put(baseAggProvider.getName(), props); - } - } - - // get the filter set from the queryform and verify that they resolve - SimpleFilter filter = getFilterFromQueryForm(form); - if (filter != null) - { - Map resolvedCols = QueryService.get().getColumns(view.getTable(), filter.getAllFieldKeys()); - for (FieldKey filterFieldKey : filter.getAllFieldKeys()) - { - if (!resolvedCols.containsKey(filterFieldKey)) - filter.deleteConditions(filterFieldKey); - } - } - - // query the table/view for the aggregate results - Collection columns = Collections.singleton(displayColumn.getColumnInfo()); - TableSelector selector = new TableSelector(view.getTable(), columns, filter, null).setNamedParameters(form.getQuerySettings().getQueryParameters()); - Map> aggResults = selector.getAggregates(new ArrayList<>(colAggregates)); - - // create a response object mapping the analytics providers to their relevant aggregate results - Map> aggregateResults = new HashMap<>(); - if (aggResults.containsKey(_colFieldKey.toString())) - { - for (Aggregate.Result r : aggResults.get(_colFieldKey.toString())) - { - Map props = new HashMap<>(); - Aggregate.Type type = r.getAggregate().getType(); - props.put("label", type.getFullLabel()); - props.put("description", type.getDescription()); - props.put("value", r.getFormattedValue(displayColumn, getContainer()).value()); - aggregateResults.put(type.getName(), props); - } - - response.put("success", true); - response.put("analyticsProviders", analyticsProviders); - response.put("aggregateResults", aggregateResults); - } - else - { - response.put("success", false); - response.put("message", "Unable to get aggregate results for " + _colFieldKey); - } - } - else - { - response.put("success", false); - response.put("message", "Unable to find ColumnInfo for " + _colFieldKey); - } - - return response; - } - } - - @RequiresPermission(ReadPermission.class) - public class ImportAction extends AbstractQueryImportAction - { - private QueryForm _form; - - @Override - protected void initRequest(QueryForm form) throws ServletException - { - _form = form; - - _insertOption = form.getInsertOption(); - QueryDefinition query = form.getQueryDef(); - List qpe = new ArrayList<>(); - TableInfo t = query.getTable(form.getSchema(), qpe, true); - if (!qpe.isEmpty()) - throw qpe.get(0); - if (null != t) - setTarget(t); - _auditBehaviorType = form.getAuditBehavior(); - _auditUserComment = form.getAuditUserComment(); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) throws Exception - { - initRequest(form); - return super.getDefaultImportView(form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new SchemaAction(_form).addNavTrail(root); - var executeQuery = _form.urlFor(QueryAction.executeQuery); - if (null == executeQuery) - root.addChild(_form.getQueryName()); - else - root.addChild(_form.getQueryName(), executeQuery); - root.addChild("Import Data"); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportSqlForm - { - private String _sql; - private String _schemaName; - private String _containerFilter; - private String _format = "excel"; - - public String getSql() - { - return _sql; - } - - public void setSql(String sql) - { - _sql = PageFlowUtil.wafDecode(sql); - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getContainerFilter() - { - return _containerFilter; - } - - public void setContainerFilter(String containerFilter) - { - _containerFilter = containerFilter; - } - - public String getFormat() - { - return _format; - } - - public void setFormat(String format) - { - _format = format; - } - } - - @RequiresPermission(ReadPermission.class) - @ApiVersion(9.2) - @Action(ActionType.Export.class) - public static class ExportSqlAction extends ExportAction - { - @Override - public void export(ExportSqlForm form, HttpServletResponse response, BindException errors) throws IOException, ExportException - { - String schemaName = StringUtils.trimToNull(form.getSchemaName()); - if (null == schemaName) - throw new NotFoundException("No value was supplied for the required parameter 'schemaName'"); - String sql = StringUtils.trimToNull(form.getSql()); - if (null == sql) - throw new NotFoundException("No value was supplied for the required parameter 'sql'"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); - - if (null == schema) - throw new NotFoundException("Schema '" + schemaName + "' not found in this folder"); - - //create a temp query settings object initialized with the posted LabKey SQL - //this will provide a temporary QueryDefinition to Query - TempQuerySettings settings = new TempQuerySettings(getViewContext(), sql); - - //need to explicitly turn off various UI options that will try to refer to the - //current URL and query string - settings.setAllowChooseView(false); - settings.setAllowCustomizeView(false); - - //return all rows - settings.setShowRows(ShowRows.ALL); - - //add container filter if supplied - if (form.getContainerFilter() != null && !form.getContainerFilter().isEmpty()) - { - ContainerFilter.Type containerFilterType = - ContainerFilter.Type.valueOf(form.getContainerFilter()); - settings.setContainerFilterName(containerFilterType.name()); - } - - //build a query view using the schema and settings - QueryView view = new QueryView(schema, settings, errors); - view.setShowRecordSelectors(false); - view.setShowExportButtons(false); - view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); - - //export it - ResponseHelper.setPrivate(response); - response.setHeader("X-Robots-Tag", "noindex"); - - if ("excel".equalsIgnoreCase(form.getFormat())) - view.exportToExcel(response); - else if ("tsv".equalsIgnoreCase(form.getFormat())) - view.exportToTsv(response); - else - errors.reject(null, "Invalid format specified; must be 'excel' or 'tsv'"); - - for (QueryException qe : view.getParseErrors()) - errors.reject(null, qe.getMessage()); - - if (errors.hasErrors()) - throw new ExportException(new SimpleErrorView(errors, false)); - } - } - - public static class ApiSaveRowsForm extends SimpleApiJsonForm - { - } - - private enum CommandType - { - insert(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException - { - BatchValidationException errors = new BatchValidationException(); - List> insertedRows = qus.insertRows(user, container, rows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - // Issue 42519: Submitter role not able to insert - // as per the definition of submitter, should allow insert without read - if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) - { - return qus.getRows(user, container, insertedRows); - } - else - { - return insertedRows; - } - } - }, - insertWithKeys(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException - { - List> newRows = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - for (Map row : rows) - { - //issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null - CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); - newRows.add(newMap); - - CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); - oldKeys.add(oldMap); - } - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.insertRows(user, container, newRows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - // Issue 42519: Submitter role not able to insert - // as per the definition of submitter, should allow insert without read - if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) - { - updatedRows = qus.getRows(user, container, updatedRows); - } - List> results = new ArrayList<>(); - for (int i = 0; i < updatedRows.size(); i++) - { - Map result = new HashMap<>(); - result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); - result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); - results.add(result); - } - return results; - } - }, - importRows(InsertPermission.class, QueryService.AuditAction.INSERT) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - DataIteratorBuilder it = new ListofMapsDataIterator.Builder(rows.get(0).keySet(), rows); - qus.importRows(user, container, it, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return Collections.emptyList(); - } - }, - moveRows(MoveEntitiesPermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - - Container targetContainer = (Container) configParameters.get(QueryUpdateService.ConfigParameters.TargetContainer); - Map updatedCounts = qus.moveRows(user, container, targetContainer, rows, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return Collections.singletonList(updatedCounts); - } - }, - update(UpdatePermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.updateRows(user, container, rows, null, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - return shouldReselect(configParameters) ? qus.getRows(user, container, updatedRows) : updatedRows; - } - }, - updateChangingKeys(UpdatePermission.class, QueryService.AuditAction.UPDATE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - List> newRows = new ArrayList<>(); - List> oldKeys = new ArrayList<>(); - for (Map row : rows) - { - // issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null. - // this should never happen on an update, but we will let it fail later with a better error message instead of the NPE here - CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); - newRows.add(newMap); - - CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); - oldKeys.add(oldMap); - } - BatchValidationException errors = new BatchValidationException(); - List> updatedRows = qus.updateRows(user, container, newRows, oldKeys, errors, configParameters, extraContext); - if (errors.hasErrors()) - throw errors; - if (shouldReselect(configParameters)) - updatedRows = qus.getRows(user, container, updatedRows); - List> results = new ArrayList<>(); - for (int i = 0; i < updatedRows.size(); i++) - { - Map result = new HashMap<>(); - result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); - result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); - results.add(result); - } - return results; - } - }, - delete(DeletePermission.class, QueryService.AuditAction.DELETE) - { - @Override - public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException - { - return qus.deleteRows(user, container, rows, configParameters, extraContext); - } - }; - - private final Class _permission; - private final QueryService.AuditAction _auditAction; - - CommandType(Class permission, QueryService.AuditAction auditAction) - { - _permission = permission; - _auditAction = auditAction; - } - - public Class getPermission() - { - return _permission; - } - - public QueryService.AuditAction getAuditAction() - { - return _auditAction; - } - - public static boolean shouldReselect(Map configParameters) - { - if (configParameters == null || !configParameters.containsKey(QueryUpdateService.ConfigParameters.SkipReselectRows)) - return true; - - return Boolean.TRUE != configParameters.get(QueryUpdateService.ConfigParameters.SkipReselectRows); - } - - public abstract List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) - throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException; - } - - /** - * Base action class for insert/update/delete actions - */ - protected abstract static class BaseSaveRowsAction extends MutatingApiAction - { - public static final String PROP_SCHEMA_NAME = "schemaName"; - public static final String PROP_QUERY_NAME = "queryName"; - public static final String PROP_CONTAINER_PATH = "containerPath"; - public static final String PROP_TARGET_CONTAINER_PATH = "targetContainerPath"; - public static final String PROP_COMMAND = "command"; - public static final String PROP_ROWS = "rows"; - - private JSONObject _json; - - @Override - public void validateForm(FORM apiSaveRowsForm, Errors errors) - { - _json = apiSaveRowsForm.getJsonObject(); - - // if the POST was done using FormData, the apiSaveRowsForm would not have bound the json data, so - // we'll instead look for that data in the request param directly - if (_json == null && getViewContext().getRequest() != null && getViewContext().getRequest().getParameter("json") != null) - _json = new JSONObject(getViewContext().getRequest().getParameter("json")); - } - - protected JSONObject getJsonObject() - { - return _json; - } - - protected Container getContainerForCommand(JSONObject json) - { - return getContainerForCommand(json, PROP_CONTAINER_PATH, getContainer()); - } - - protected Container getContainerForCommand(JSONObject json, String containerPathProp, @Nullable Container defaultContainer) - { - Container container; - String containerPath = StringUtils.trimToNull(json.optString(containerPathProp)); - if (containerPath == null) - { - if (defaultContainer != null) - container = defaultContainer; - else - throw new IllegalArgumentException(containerPathProp + " is required but was not provided."); - } - else - { - container = ContainerManager.getForPath(containerPath); - if (container == null) - { - throw new IllegalArgumentException("Unknown container: " + containerPath); - } - } - - // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more downstream - if (!container.hasPermission(getUser(), ReadPermission.class) && - !container.hasPermission(getUser(), DeletePermission.class) && - !container.hasPermission(getUser(), InsertPermission.class) && - !container.hasPermission(getUser(), UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - return container; - } - - protected String getTargetContainerProp() - { - JSONObject json = getJsonObject(); - return json.optString(PROP_TARGET_CONTAINER_PATH, null); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors) throws Exception - { - return executeJson(json, commandType, allowTransaction, errors, false); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction) throws Exception - { - return executeJson(json, commandType, allowTransaction, errors, isNestedTransaction, null); - } - - protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction, @Nullable Integer commandIndex) throws Exception - { - JSONObject response = new JSONObject(); - Container container = getContainerForCommand(json); - User user = getUser(); - - if (json == null) - throw new ValidationException("Empty request"); - - JSONArray rows; - try - { - rows = json.getJSONArray(PROP_ROWS); - if (rows.isEmpty()) - throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); - } - catch (JSONException x) - { - throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); - } - - String schemaName = json.getString(PROP_SCHEMA_NAME); - String queryName = json.getString(PROP_QUERY_NAME); - TableInfo table = getTableInfo(container, user, schemaName, queryName); - - if (!table.hasPermission(user, commandType.getPermission())) - throw new UnauthorizedException(); - - if (commandType != CommandType.insert && table.getPkColumns().isEmpty()) - throw new IllegalArgumentException("The table '" + table.getPublicSchemaName() + "." + - table.getPublicName() + "' cannot be updated because it has no primary key defined!"); - - QueryUpdateService qus = table.getUpdateService(); - if (null == qus) - throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + - "' is not updatable via the HTTP-based APIs."); - - int rowsAffected = 0; - - List> rowsToProcess = new ArrayList<>(); - - // NOTE RowMapFactory is faster, but for update it's important to preserve missing v explicit NULL values - // Do we need to support some sort of UNDEFINED and NULL instance of MvFieldWrapper? - RowMapFactory f = null; - if (commandType == CommandType.insert || commandType == CommandType.insertWithKeys || commandType == CommandType.delete) - f = new RowMapFactory<>(); - CaseInsensitiveHashMap referenceCasing = new CaseInsensitiveHashMap<>(); - - for (int idx = 0; idx < rows.length(); ++idx) - { - JSONObject jsonObj; - try - { - jsonObj = rows.getJSONObject(idx); - } - catch (JSONException x) - { - throw new IllegalArgumentException("rows[" + idx + "] is not an object."); - } - if (null != jsonObj) - { - Map rowMap = null == f ? new CaseInsensitiveHashMap<>(new HashMap<>(), referenceCasing) : f.getRowMap(); - // Use shallow copy since jsonObj.toMap() will translate contained JSONObjects into Maps, which we don't want - boolean conflictingCasing = JsonUtil.fillMapShallow(jsonObj, rowMap); - if (conflictingCasing) - { - // Issue 52616 - LOG.error("Row contained conflicting casing for key names in the incoming row: {}", jsonObj); - } - if (allowRowAttachments()) - addRowAttachments(table, rowMap, idx, commandIndex); - - rowsToProcess.add(rowMap); - rowsAffected++; - } - } - - Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); - - Map auditDetails = json.has("auditDetails") ? json.getJSONObject("auditDetails").toMap() : new CaseInsensitiveHashMap<>(); - - Map configParameters = new HashMap<>(); - - // Check first if the audit behavior has been defined for the table either in code or through XML. - // If not defined there, check for the audit behavior defined in the action form (json). - AuditBehaviorType behaviorType = table.getEffectiveAuditBehavior(json.optString("auditBehavior", null)); - if (behaviorType != null) - { - configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, behaviorType); - String auditComment = json.optString("auditUserComment", null); - if (!StringUtils.isEmpty(auditComment)) - configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment, auditComment); - } - - boolean skipReselectRows = json.optBoolean("skipReselectRows", false); - if (skipReselectRows) - configParameters.put(QueryUpdateService.ConfigParameters.SkipReselectRows, true); - - if (getTargetContainerProp() != null) - { - Container targetContainer = getContainerForCommand(json, PROP_TARGET_CONTAINER_PATH, null); - configParameters.put(QueryUpdateService.ConfigParameters.TargetContainer, targetContainer); - } - - //set up the response, providing the schema name, query name, and operation - //so that the client can sort out which request this response belongs to - //(clients often submit these async) - response.put(PROP_SCHEMA_NAME, schemaName); - response.put(PROP_QUERY_NAME, queryName); - response.put("command", commandType.name()); - response.put("containerPath", container.getPath()); - - //we will transact operations by default, but the user may - //override this by sending a "transacted" property set to false - // 11741: A transaction may already be active if we're trying to - // insert/update/delete from within a transformation/validation script. - boolean transacted = allowTransaction && json.optBoolean("transacted", true); - TransactionAuditProvider.TransactionAuditEvent auditEvent = null; - try (DbScope.Transaction transaction = transacted ? table.getSchema().getScope().ensureTransaction() : NO_OP_TRANSACTION) - { - if (behaviorType != null && behaviorType != AuditBehaviorType.NONE) - { - DbScope.Transaction auditTransaction = !transacted && isNestedTransaction ? table.getSchema().getScope().getCurrentTransaction() : transaction; - if (auditTransaction == null) - auditTransaction = NO_OP_TRANSACTION; - - if (auditTransaction.getAuditEvent() != null) - { - auditEvent = auditTransaction.getAuditEvent(); - // detect trigger event? - } - else - { - Map transactionDetails = getTransactionAuditDetails(); - TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetails); - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), transactionDetails); - AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); - } - auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.QueryCommand, commandType.name()); - } - - QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); - List> responseRows = - commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); - if (auditEvent != null) - { - auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); - if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.BatchAction))) - auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true); - } - - if (commandType == CommandType.moveRows) - { - // moveRows returns a single map of updateCounts - response.put("updateCounts", responseRows.get(0)); - } - else if (commandType != CommandType.importRows) - { - response.put("rows", responseRows.stream() - .map(JsonUtil::toMapPreserveNonFinite) - .map(JsonUtil::toJsonPreserveNulls) - .collect(LabKeyCollectors.toJSONArray())); - } - - // if there is any provenance information, save it here - ProvenanceService svc = ProvenanceService.get(); - if (json.has("provenance")) - { - JSONObject provenanceJSON = json.getJSONObject("provenance"); - ProvenanceRecordingParams params = svc.createRecordingParams(getViewContext(), provenanceJSON, ProvenanceService.ADD_RECORDING); - RecordedAction action = svc.createRecordedAction(getViewContext(), params); - if (action != null && params.getRecordingId() != null) - { - // check for any row level provenance information - if (json.has("rows")) - { - Object rowObject = json.get("rows"); - if (rowObject instanceof JSONArray jsonArray) - { - // we need to match any provenance object inputs to the object outputs from the response rows, this typically would - // be the row lsid but it configurable in the provenance recording params - // - List> provenanceMap = svc.createProvenanceMapFromRows(getViewContext(), params, jsonArray, responseRows); - if (!provenanceMap.isEmpty()) - { - action.getProvenanceMap().addAll(provenanceMap); - } - svc.addRecordingStep(getViewContext().getRequest(), params.getRecordingId(), action); - } - else - { - errors.reject(SpringActionController.ERROR_MSG, "Unable to process provenance information, the rows object was not an array"); - } - } - } - } - transaction.commit(); - } - catch (OptimisticConflictException e) - { - //issue 13967: provide better message for OptimisticConflictException - errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); - } - catch (QueryUpdateServiceException | ConversionException | DuplicateKeyException | DataIntegrityViolationException e) - { - //Issue 14294: improve handling of ConversionException (and DuplicateKeyException (Issue 28037), and DataIntegrity (uniqueness) (Issue 22779) - errors.reject(SpringActionController.ERROR_MSG, e.getMessage() == null ? e.toString() : e.getMessage()); - } - catch (BatchValidationException e) - { - if (isSuccessOnValidationError()) - { - response.put("errors", createResponseWriter().toJSON(e)); - } - else - { - ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); - throw e; - } - } - if (auditEvent != null) - { - response.put("transactionAuditId", auditEvent.getRowId()); - response.put("reselectRowCount", auditEvent.hasMultiActions()); - } - - response.put("rowsAffected", rowsAffected); - - return response; - } - - protected boolean allowRowAttachments() - { - return false; - } - - private void addRowAttachments(TableInfo tableInfo, Map rowMap, int rowIndex, @Nullable Integer commandIndex) - { - if (getFileMap() != null) - { - for (Map.Entry fileEntry : getFileMap().entrySet()) - { - // Allow for the fileMap key to include the row index, and optionally command index, for defining - // which row to attach this file to - String fullKey = fileEntry.getKey(); - String fieldKey = fullKey; - // Issue 52827: Cannot attach a file if the field name contains :: - // use lastIndexOf instead of split to get the proper parts - int lastDelimIndex = fullKey.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); - if (lastDelimIndex > -1) - { - String fieldKeyExcludeIndex = fullKey.substring(0, lastDelimIndex); - String fieldRowIndex = fullKey.substring(lastDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); - if (!fieldRowIndex.equals(rowIndex+"")) continue; - - if (commandIndex == null) - { - // Single command, so we're parsing file names in the format of: FileField::0 - fieldKey = fieldKeyExcludeIndex; - } - else - { - // Multi-command, so we're parsing file names in the format of: FileField::0::1 - int subDelimIndex = fieldKeyExcludeIndex.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); - if (subDelimIndex > -1) - { - fieldKey = fieldKeyExcludeIndex.substring(0, subDelimIndex); - String fieldCommandIndex = fieldKeyExcludeIndex.substring(subDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); - if (!fieldCommandIndex.equals(commandIndex+"")) - continue; - } - else - continue; - } - } - - SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); - rowMap.put(fieldKey, file.isEmpty() ? null : file); - } - } - - for (ColumnInfo col : tableInfo.getColumns()) - DataIteratorUtil.MatchType.multiPartFormData.updateRowMap(col, rowMap); - } - - protected boolean isSuccessOnValidationError() - { - return getRequestedApiVersion() >= 13.2; - } - - @NotNull - protected TableInfo getTableInfo(Container container, User user, String schemaName, String queryName) - { - if (null == schemaName || null == queryName) - throw new IllegalArgumentException("You must supply a schemaName and queryName!"); - - UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); - if (null == schema) - throw new IllegalArgumentException("The schema '" + schemaName + "' does not exist."); - - TableInfo table = schema.getTableForInsert(queryName); - if (table == null) - throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); - return table; - } - } - - // Issue: 20522 - require read access to the action but executeJson will check for update privileges from the table - // - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class UpdateRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.update, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - } - - @RequiresAnyOf({ReadPermission.class, InsertPermission.class}) //will check below - @ApiVersion(8.3) - public static class InsertRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.insert, true, errors); - if (response == null || errors.hasErrors()) - return null; - - return new ApiSimpleResponse(response); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - } - - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class ImportRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.importRows, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - } - - @ActionNames("deleteRows, delRows") - @RequiresPermission(ReadPermission.class) //will check below - @ApiVersion(8.3) - public static class DeleteRowsAction extends BaseSaveRowsAction - { - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - JSONObject response = executeJson(getJsonObject(), CommandType.delete, true, errors); - if (response == null || errors.hasErrors()) - return null; - return new ApiSimpleResponse(response); - } - } - - @RequiresPermission(ReadPermission.class) //will check below - public static class MoveRowsAction extends BaseSaveRowsAction - { - private Container _targetContainer; - - @Override - public void validateForm(MoveRowsForm form, Errors errors) - { - super.validateForm(form, errors); - - JSONObject json = getJsonObject(); - if (json == null) - { - errors.reject(ERROR_GENERIC, "Empty request"); - } - else - { - // Since we are moving between containers, we know we have product folders enabled - if (getContainer().getProject().getAuditCommentsRequired() && StringUtils.isBlank(json.optString("auditUserComment"))) - errors.reject(ERROR_GENERIC, "A reason for the move of data is required."); - else - { - String queryName = json.optString(PROP_QUERY_NAME, null); - _targetContainer = ContainerManager.getMoveTargetContainer(queryName, getContainer(), getUser(), getTargetContainerProp(), errors); - } - } - } - - @Override - public ApiResponse execute(MoveRowsForm form, BindException errors) throws Exception - { - // if JSON does not have rows array, see if they were provided via selectionKey - if (!getJsonObject().has(PROP_ROWS)) - setRowsFromSelectionKey(form); - - JSONObject response = executeJson(getJsonObject(), CommandType.moveRows, true, errors); - if (response == null || errors.hasErrors()) - return null; - - updateSelections(form); - - response.put("success", true); - response.put("containerPath", _targetContainer.getPath()); - return new ApiSimpleResponse(response); - } - - private void updateSelections(MoveRowsForm form) - { - String selectionKey = form.getDataRegionSelectionKey(); - if (selectionKey != null) - { - Set rowIds = form.getIds(getViewContext(), false) - .stream().map(Object::toString).collect(Collectors.toSet()); - DataRegionSelection.setSelected(getViewContext(), selectionKey, rowIds, false); - - // if moving entities from a type, the selections from other selectionKeys in that container will - // possibly be holding onto invalid keys after the move, so clear them based on the containerPath and selectionKey suffix - String[] keyParts = selectionKey.split("|"); - if (keyParts.length > 1) - DataRegionSelection.clearRelatedByContainerPath(getViewContext(), keyParts[keyParts.length - 1]); - } - } - - private void setRowsFromSelectionKey(MoveRowsForm form) - { - Set rowIds = form.getIds(getViewContext(), false); // handle clear of selectionKey after move complete - - // convert rowIds to a JSONArray of JSONObjects with a single property "RowId" - JSONArray rows = new JSONArray(); - for (Long rowId : rowIds) - { - JSONObject row = new JSONObject(); - row.put("RowId", rowId); - rows.put(row); - } - getJsonObject().put(PROP_ROWS, rows); - } - } - - public static class MoveRowsForm extends ApiSaveRowsForm - { - private String _dataRegionSelectionKey; - private boolean _useSnapshotSelection; - - public String getDataRegionSelectionKey() - { - return _dataRegionSelectionKey; - } - - public void setDataRegionSelectionKey(String dataRegionSelectionKey) - { - _dataRegionSelectionKey = dataRegionSelectionKey; - } - - public boolean isUseSnapshotSelection() - { - return _useSnapshotSelection; - } - - public void setUseSnapshotSelection(boolean useSnapshotSelection) - { - _useSnapshotSelection = useSnapshotSelection; - } - - @Override - public void bindJson(JSONObject json) - { - super.bindJson(json); - _dataRegionSelectionKey = json.optString("dataRegionSelectionKey", null); - _useSnapshotSelection = json.optBoolean("useSnapshotSelection", false); - } - - public Set getIds(ViewContext context, boolean clear) - { - if (_useSnapshotSelection) - return new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(context, getDataRegionSelectionKey())); - else - return DataRegionSelection.getSelectedIntegers(context, getDataRegionSelectionKey(), clear); - } - } - - @RequiresNoPermission //will check below - public static class SaveRowsAction extends BaseSaveRowsAction - { - public static final String PROP_VALUES = "values"; - public static final String PROP_OLD_KEYS = "oldKeys"; - - @Override - protected boolean isFailure(BindException errors) - { - return !isSuccessOnValidationError() && super.isFailure(errors); - } - - @Override - protected boolean allowRowAttachments() - { - return true; - } - - @Override - public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception - { - // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more - // specific permissions later once we've figured out exactly what they're trying to do. This helps us - // give a better HTTP response code when they're trying to access a resource that's not available to guests - if (!getContainer().hasPermission(getUser(), ReadPermission.class) && - !getContainer().hasPermission(getUser(), DeletePermission.class) && - !getContainer().hasPermission(getUser(), InsertPermission.class) && - !getContainer().hasPermission(getUser(), UpdatePermission.class)) - { - throw new UnauthorizedException(); - } - - JSONObject json = getJsonObject(); - if (json == null) - throw new IllegalArgumentException("Empty request"); - - JSONArray commands = json.optJSONArray("commands"); - if (commands == null || commands.isEmpty()) - { - throw new NotFoundException("Empty request"); - } - - boolean validateOnly = json.optBoolean("validateOnly", false); - // If we are going to validate and not commit, we need to be sure we're transacted as well. Otherwise, - // respect the client's request. - boolean transacted = validateOnly || json.optBoolean("transacted", true); - - // Keep track of whether we end up committing or not - boolean committed = false; - - DbScope scope = null; - if (transacted) - { - for (int i = 0; i < commands.length(); i++) - { - JSONObject commandJSON = commands.getJSONObject(i); - String schemaName = commandJSON.getString(PROP_SCHEMA_NAME); - String queryName = commandJSON.getString(PROP_QUERY_NAME); - Container container = getContainerForCommand(commandJSON); - TableInfo tableInfo = getTableInfo(container, getUser(), schemaName, queryName); - if (scope == null) - { - scope = tableInfo.getSchema().getScope(); - } - else if (scope != tableInfo.getSchema().getScope()) - { - throw new IllegalArgumentException("All queries must be from the same source database"); - } - } - assert scope != null; - } - - JSONArray resultArray = new JSONArray(); - JSONObject extraContext = json.optJSONObject("extraContext"); - - int startingErrorIndex = 0; - int errorCount = 0; - // 11741: A transaction may already be active if we're trying to - // insert/update/delete from within a transformation/validation script. - - try (DbScope.Transaction transaction = transacted ? scope.ensureTransaction() : NO_OP_TRANSACTION) - { - for (int i = 0; i < commands.length(); i++) - { - JSONObject commandObject = commands.getJSONObject(i); - String commandName = commandObject.getString(PROP_COMMAND); - if (commandName == null) - { - throw new ApiUsageException(PROP_COMMAND + " is required but was missing"); - } - CommandType command = CommandType.valueOf(commandName); - - // Copy the top-level 'extraContext' and merge in the command-level extraContext. - Map commandExtraContext = new HashMap<>(); - if (extraContext != null) - commandExtraContext.putAll(extraContext.toMap()); - if (commandObject.has("extraContext")) - { - commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); - } - commandObject.put("extraContext", commandExtraContext); - - JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); - // Bail out immediately if we're going to return a failure-type response message - if (commandResponse == null || (errors.hasErrors() && !isSuccessOnValidationError())) - return null; - - //this would be populated in executeJson when a BatchValidationException is thrown - if (commandResponse.has("errors")) - { - errorCount += commandResponse.getJSONObject("errors").getInt("errorCount"); - } - - // If we encountered errors with this particular command and the client requested that don't treat - // the whole request as a failure (non-200 HTTP status code), stash the errors for this particular - // command in its response section. - // NOTE: executeJson should handle and serialize BatchValidationException - // these errors upstream - if (errors.getErrorCount() > startingErrorIndex && isSuccessOnValidationError()) - { - commandResponse.put("errors", ApiResponseWriter.convertToJSON(errors, startingErrorIndex).getValue()); - startingErrorIndex = errors.getErrorCount(); - } - - resultArray.put(commandResponse); - } - - // Don't commit if we had errors or if the client requested that we only validate (and not commit) - if (!errors.hasErrors() && !validateOnly && errorCount == 0) - { - transaction.commit(); - committed = true; - } - } - - errorCount += errors.getErrorCount(); - JSONObject result = new JSONObject(); - result.put("result", resultArray); - result.put("committed", committed); - result.put("errorCount", errorCount); - - return new ApiSimpleResponse(result); - } - } - - @RequiresPermission(ReadPermission.class) - public static class ApiTestAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - return new JspView<>("/org/labkey/query/view/apitest.jsp"); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("API Test"); - } - } - - - @RequiresPermission(AdminPermission.class) - public static class AdminAction extends SimpleViewAction - { - @SuppressWarnings("UnusedDeclaration") - public AdminAction() - { - } - - public AdminAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/admin.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Schema Administration", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ResetRemoteConnectionsForm - { - private boolean _reset; - - public boolean isReset() - { - return _reset; - } - - public void setReset(boolean reset) - { - _reset = reset; - } - } - - - @RequiresPermission(AdminPermission.class) - public static class ManageRemoteConnectionsAction extends FormViewAction - { - @Override - public void validateCommand(ResetRemoteConnectionsForm target, Errors errors) {} - - @Override - public boolean handlePost(ResetRemoteConnectionsForm form, BindException errors) - { - if (form.isReset()) - { - PropertyManager.getEncryptedStore().deletePropertySet(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); - } - return true; - } - - @Override - public URLHelper getSuccessURL(ResetRemoteConnectionsForm queryForm) - { - return new ActionURL(ManageRemoteConnectionsAction.class, getContainer()); - } - - @Override - public ModelAndView getView(ResetRemoteConnectionsForm queryForm, boolean reshow, BindException errors) - { - Map connectionMap; - try - { - // if the encrypted property store is configured but no values have yet been set, and empty map is returned - connectionMap = PropertyManager.getEncryptedStore().getProperties(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); - } - catch (Exception e) - { - connectionMap = null; // render the failure page - } - setHelpTopic("remoteConnection"); - return new JspView<>("/org/labkey/query/view/manageRemoteConnections.jsp", connectionMap, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); - } - } - - private abstract static class BaseInsertExternalSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction - { - protected BaseInsertExternalSchemaAction(Class commandClass) - { - super(commandClass); - } - - @Override - public void validateCommand(F form, Errors errors) - { - form.validate(errors); - } - - @Override - public boolean handlePost(F form, BindException errors) throws Exception - { - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - form.doInsert(); - auditSchemaAdminActivity(form.getBean(), "created", getContainer(), getUser()); - QueryManager.get().updateExternalSchemas(getContainer()); - - t.commit(); - } - catch (RuntimeSQLException e) - { - if (e.isConstraintException()) - { - errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); - return false; - } - - throw e; - } - - return true; - } - - @Override - public ActionURL getSuccessURL(F form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext()).addNavTrail(root); - root.addChild("Define Schema", new ActionURL(getClass(), getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class InsertLinkedSchemaAction extends BaseInsertExternalSchemaAction - { - public InsertLinkedSchemaAction() - { - super(LinkedSchemaForm.class); - } - - @Override - public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) - { - setHelpTopic("filterSchema"); - return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), form.getBean(), true), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class InsertExternalSchemaAction extends BaseInsertExternalSchemaAction - { - public InsertExternalSchemaAction() - { - super(ExternalSchemaForm.class); - } - - @Override - public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) - { - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), form.getBean(), true), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class DeleteSchemaAction extends ConfirmAction - { - @Override - public String getConfirmText() - { - return "Delete"; - } - - @Override - public ModelAndView getConfirmView(SchemaForm form, BindException errors) - { - if (getPageConfig().getTitle() == null) - setTitle("Delete Schema"); - - AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - String schemaName = isBlank(def.getUserSchemaName()) ? "this schema" : "the schema '" + def.getUserSchemaName() + "'"; - return new HtmlView(HtmlString.of("Are you sure you want to delete " + schemaName + "? The tables and queries defined in this schema will no longer be accessible.")); - } - - @Override - public boolean handlePost(SchemaForm form, BindException errors) - { - AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - auditSchemaAdminActivity(def, "deleted", getContainer(), getUser()); - QueryManager.get().delete(def); - t.commit(); - } - return true; - } - - @Override - public void validateCommand(SchemaForm form, Errors errors) - { - } - - @Override - @NotNull - public ActionURL getSuccessURL(SchemaForm form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - } - - private static void auditSchemaAdminActivity(AbstractExternalSchemaDef def, String action, Container container, User user) - { - String comment = StringUtils.capitalize(def.getSchemaType().toString()) + " schema '" + def.getUserSchemaName() + "' " + action; - AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, comment); - AuditLogService.get().addEvent(user, event); - } - - - private abstract static class BaseEditSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction - { - protected BaseEditSchemaAction(Class commandClass) - { - super(commandClass); - } - - @Override - public void validateCommand(F form, Errors errors) - { - form.validate(errors); - } - - @Nullable - protected abstract T getCurrent(int externalSchemaId); - - @NotNull - protected T getDef(F form, boolean reshow) - { - T def; - Container defContainer; - - if (reshow) - { - def = form.getBean(); - T current = getCurrent(def.getExternalSchemaId()); - if (current == null) - throw new NotFoundException(); - - defContainer = current.lookupContainer(); - } - else - { - form.refreshFromDb(); - if (!form.isDataLoaded()) - throw new NotFoundException(); - - def = form.getBean(); - if (def == null) - throw new NotFoundException(); - - defContainer = def.lookupContainer(); - } - - if (!getContainer().equals(defContainer)) - throw new UnauthorizedException(); - - return def; - } - - @Override - public boolean handlePost(F form, BindException errors) throws Exception - { - T def = form.getBean(); - T fromDb = getCurrent(def.getExternalSchemaId()); - - // Unauthorized if def in the database reports a different container - if (!getContainer().equals(fromDb.lookupContainer())) - throw new UnauthorizedException(); - - try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) - { - form.doUpdate(); - auditSchemaAdminActivity(def, "updated", getContainer(), getUser()); - QueryManager.get().updateExternalSchemas(getContainer()); - t.commit(); - } - catch (RuntimeSQLException e) - { - if (e.isConstraintException()) - { - errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); - return false; - } - - throw e; - } - return true; - } - - @Override - public ActionURL getSuccessURL(F externalSchemaForm) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new AdminAction(getViewContext()).addNavTrail(root); - root.addChild("Edit Schema", new ActionURL(getClass(), getContainer())); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditLinkedSchemaAction extends BaseEditSchemaAction - { - public EditLinkedSchemaAction() - { - super(LinkedSchemaForm.class); - } - - @Nullable - @Override - protected LinkedSchemaDef getCurrent(int externalId) - { - return QueryManager.get().getLinkedSchemaDef(getContainer(), externalId); - } - - @Override - public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) - { - LinkedSchemaDef def = getDef(form, reshow); - - setHelpTopic("filterSchema"); - return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), def, false), errors); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class EditExternalSchemaAction extends BaseEditSchemaAction - { - public EditExternalSchemaAction() - { - super(ExternalSchemaForm.class); - } - - @Nullable - @Override - protected ExternalSchemaDef getCurrent(int externalId) - { - return QueryManager.get().getExternalSchemaDef(getContainer(), externalId); - } - - @Override - public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) - { - ExternalSchemaDef def = getDef(form, reshow); - - setHelpTopic("externalSchemas"); - return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), def, false), errors); - } - } - - - public static class DataSourceInfo - { - public final String sourceName; - public final String displayName; - public final boolean editable; - - public DataSourceInfo(DbScope scope) - { - this(scope.getDataSourceName(), scope.getDisplayName(), scope.getSqlDialect().isEditable()); - } - - public DataSourceInfo(Container c) - { - this(c.getId(), c.getName(), false); - } - - public DataSourceInfo(String sourceName, String displayName, boolean editable) - { - this.sourceName = sourceName; - this.displayName = displayName; - this.editable = editable; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - DataSourceInfo that = (DataSourceInfo) o; - return sourceName != null ? sourceName.equals(that.sourceName) : that.sourceName == null; - } - - @Override - public int hashCode() - { - return sourceName != null ? sourceName.hashCode() : 0; - } - } - - public static abstract class BaseExternalSchemaBean - { - protected final Container _c; - protected final T _def; - protected final boolean _insert; - protected final Map _help = new HashMap<>(); - - public BaseExternalSchemaBean(Container c, T def, boolean insert) - { - _c = c; - _def = def; - _insert = insert; - - TableInfo ti = QueryManager.get().getTableInfoExternalSchema(); - - ti.getColumns() - .stream() - .filter(ci -> null != ci.getDescription()) - .forEach(ci -> _help.put(ci.getName(), ci.getDescription())); - } - - public abstract DataSourceInfo getInitialSource(); - - public T getSchemaDef() - { - return _def; - } - - public boolean isInsert() - { - return _insert; - } - - public ActionURL getReturnURL() - { - return new ActionURL(AdminAction.class, _c); - } - - public ActionURL getDeleteURL() - { - return new QueryUrlsImpl().urlDeleteSchema(_c, _def); - } - - public String getHelpHTML(String fieldName) - { - return _help.get(fieldName); - } - } - - public static class LinkedSchemaBean extends BaseExternalSchemaBean - { - public LinkedSchemaBean(Container c, LinkedSchemaDef def, boolean insert) - { - super(c, def, insert); - } - - @Override - public DataSourceInfo getInitialSource() - { - Container sourceContainer = getInitialContainer(); - return new DataSourceInfo(sourceContainer); - } - - private @NotNull Container getInitialContainer() - { - LinkedSchemaDef def = getSchemaDef(); - Container sourceContainer = def.lookupSourceContainer(); - if (sourceContainer == null) - sourceContainer = def.lookupContainer(); - if (sourceContainer == null) - sourceContainer = _c; - return sourceContainer; - } - } - - public static class ExternalSchemaBean extends BaseExternalSchemaBean - { - protected final Map> _sourcesAndSchemas = new LinkedHashMap<>(); - protected final Map> _sourcesAndSchemasIncludingSystem = new LinkedHashMap<>(); - - public ExternalSchemaBean(Container c, ExternalSchemaDef def, boolean insert) - { - super(c, def, insert); - initSources(); - } - - public Collection getSources() - { - return _sourcesAndSchemas.keySet(); - } - - public Collection getSchemaNames(DataSourceInfo source, boolean includeSystem) - { - if (includeSystem) - return _sourcesAndSchemasIncludingSystem.get(source); - else - return _sourcesAndSchemas.get(source); - } - - @Override - public DataSourceInfo getInitialSource() - { - ExternalSchemaDef def = getSchemaDef(); - DbScope scope = def.lookupDbScope(); - if (scope == null) - scope = DbScope.getLabKeyScope(); - return new DataSourceInfo(scope); - } - - protected void initSources() - { - ModuleLoader moduleLoader = ModuleLoader.getInstance(); - - for (DbScope scope : DbScope.getDbScopes()) - { - SqlDialect dialect = scope.getSqlDialect(); - - Collection schemaNames = new LinkedList<>(); - Collection schemaNamesIncludingSystem = new LinkedList<>(); - - for (String schemaName : scope.getSchemaNames()) - { - schemaNamesIncludingSystem.add(schemaName); - - if (dialect.isSystemSchema(schemaName)) - continue; - - if (null != moduleLoader.getModule(scope, schemaName)) - continue; - - schemaNames.add(schemaName); - } - - DataSourceInfo source = new DataSourceInfo(scope); - _sourcesAndSchemas.put(source, schemaNames); - _sourcesAndSchemasIncludingSystem.put(source, schemaNamesIncludingSystem); - } - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetTablesForm - { - private String _dataSource; - private String _schemaName; - private boolean _sorted; - - public String getDataSource() - { - return _dataSource; - } - - public void setDataSource(String dataSource) - { - _dataSource = dataSource; - } - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isSorted() - { - return _sorted; - } - - public void setSorted(boolean sorted) - { - _sorted = sorted; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class GetTablesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(GetTablesForm form, BindException errors) - { - List> rows = new LinkedList<>(); - List tableNames = new ArrayList<>(); - - if (null != form.getSchemaName()) - { - DbScope scope = DbScope.getDbScope(form.getDataSource()); - if (null != scope) - { - DbSchema schema = scope.getSchema(form.getSchemaName(), DbSchemaType.Bare); - tableNames.addAll(schema.getTableNames()); - } - else - { - Container c = ContainerManager.getForId(form.getDataSource()); - if (null != c) - { - UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); - if (null != schema) - { - if (form.isSorted()) - for (TableInfo table : schema.getSortedTables()) - tableNames.add(table.getName()); - else - tableNames.addAll(schema.getTableAndQueryNames(true)); - } - } - } - } - - Collections.sort(tableNames); - - for (String tableName : tableNames) - { - Map row = new LinkedHashMap<>(); - row.put("table", tableName); - rows.add(row); - } - - Map properties = new HashMap<>(); - properties.put("rows", rows); - - return new ApiSimpleResponse(properties); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SchemaTemplateForm - { - private String _name; - - public String getName() - { - return _name; - } - - public void setName(String name) - { - _name = name; - } - } - - - @RequiresPermission(AdminOperationsPermission.class) - public static class SchemaTemplateAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(SchemaTemplateForm form, BindException errors) - { - String name = form.getName(); - if (name == null) - throw new IllegalArgumentException("name required"); - - Container c = getContainer(); - TemplateSchemaType template = QueryServiceImpl.get().getSchemaTemplate(c, name); - if (template == null) - throw new NotFoundException("template not found"); - - JSONObject templateJson = QueryServiceImpl.get().schemaTemplateJson(name, template); - - return new ApiSimpleResponse("template", templateJson); - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class SchemaTemplatesAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object form, BindException errors) - { - Container c = getContainer(); - QueryServiceImpl svc = QueryServiceImpl.get(); - Map templates = svc.getSchemaTemplates(c); - - JSONArray ret = new JSONArray(); - for (String key : templates.keySet()) - { - TemplateSchemaType template = templates.get(key); - JSONObject templateJson = svc.schemaTemplateJson(key, template); - ret.put(templateJson); - } - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("templates", ret); - resp.put("success", true); - return resp; - } - } - - @RequiresPermission(AdminPermission.class) - public static class ReloadExternalSchemaAction extends FormHandlerAction - { - private String _userSchemaName; - - @Override - public void validateCommand(SchemaForm form, Errors errors) - { - } - - @Override - public boolean handlePost(SchemaForm form, BindException errors) - { - ExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), ExternalSchemaDef.class); - if (def == null) - throw new NotFoundException(); - - QueryManager.get().reloadExternalSchema(def); - _userSchemaName = def.getUserSchemaName(); - - return true; - } - - @Override - public ActionURL getSuccessURL(SchemaForm form) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Schema " + _userSchemaName + " was reloaded successfully."); - } - } - - - @RequiresPermission(AdminPermission.class) - public static class ReloadAllUserSchemas extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - QueryManager.get().reloadAllExternalSchemas(getContainer()); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "All schemas in this folder were reloaded successfully."); - } - } - - @RequiresPermission(AdminPermission.class) - public static class ReloadFailedConnectionsAction extends FormHandlerAction - { - @Override - public void validateCommand(Object target, Errors errors) - { - } - - @Override - public boolean handlePost(Object o, BindException errors) - { - DbScope.clearFailedDbScopes(); - return true; - } - - @Override - public URLHelper getSuccessURL(Object o) - { - return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Reconnection was attempted on all data sources that failed previous connection attempts."); - } - } - - @RequiresPermission(ReadPermission.class) - public static class TableInfoAction extends SimpleViewAction - { - @Override - public ModelAndView getView(TableInfoForm form, BindException errors) throws Exception - { - TablesDocument ret = TablesDocument.Factory.newInstance(); - TablesType tables = ret.addNewTables(); - - FieldKey[] fields = form.getFieldKeys(); - if (fields.length != 0) - { - TableInfo tinfo = QueryView.create(form, errors).getTable(); - Map columnMap = CustomViewImpl.getColumnInfos(tinfo, Arrays.asList(fields)); - TableXML.initTable(tables.addNewTable(), tinfo, null, columnMap.values()); - } - - for (FieldKey tableKey : form.getTableKeys()) - { - TableInfo tableInfo = form.getTableInfo(tableKey); - TableType xbTable = tables.addNewTable(); - TableXML.initTable(xbTable, tableInfo, tableKey); - } - getViewContext().getResponse().setContentType("text/xml"); - getViewContext().getResponse().getWriter().write(ret.toString()); - return null; - } - - @Override - public void addNavTrail(NavTree root) - { - } - } - - - // Issue 18870: Guest user can't revert unsaved custom view changes - // Permission will be checked inline (guests are allowed to delete their session custom views) - @RequiresNoPermission - @Action(ActionType.Configure.class) - public static class DeleteViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(DeleteViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - - if (getUser().isGuest()) - { - // Guests can only delete session custom views. - if (!view.isSession()) - throw new UnauthorizedException(); - } - else - { - // Logged in users must have read permission - if (!getContainer().hasPermission(getUser(), ReadPermission.class)) - throw new UnauthorizedException(); - } - - if (view.isShared()) - { - if (!getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) - throw new UnauthorizedException(); - } - - view.delete(getUser(), getViewContext().getRequest()); - - // Delete the first shadowed custom view, if available. - if (form.isComplete()) - { - form.reset(); - CustomView shadowed = form.getCustomView(); - if (shadowed != null && shadowed.isEditable() && !(shadowed instanceof ModuleCustomView)) - { - if (!shadowed.isShared() || getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) - shadowed.delete(getUser(), getViewContext().getRequest()); - } - } - - // Try to get a custom view of the same name as the view we just deleted. - // The deleted view may have been a session view or a personal view masking shared view with the same name. - form.reset(); - view = form.getCustomView(); - String nextViewName = null; - if (view != null) - nextViewName = view.getName(); - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("viewName", nextViewName); - return response; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SaveSessionViewForm extends QueryForm - { - private String newName; - private boolean inherit; - private boolean shared; - private boolean hidden; - private boolean replace; - private String containerPath; - - public String getNewName() - { - return newName; - } - - public void setNewName(String newName) - { - this.newName = newName; - } - - public boolean isInherit() - { - return inherit; - } - - public void setInherit(boolean inherit) - { - this.inherit = inherit; - } - - public boolean isShared() - { - return shared; - } - - public void setShared(boolean shared) - { - this.shared = shared; - } - - public String getContainerPath() - { - return containerPath; - } - - public void setContainerPath(String containerPath) - { - this.containerPath = containerPath; - } - - public boolean isHidden() - { - return hidden; - } - - public void setHidden(boolean hidden) - { - this.hidden = hidden; - } - - public boolean isReplace() - { - return replace; - } - - public void setReplace(boolean replace) - { - this.replace = replace; - } - } - - // Moves a session view into the database. - @RequiresPermission(ReadPermission.class) - public static class SaveSessionViewAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SaveSessionViewForm form, BindException errors) - { - CustomView view = form.getCustomView(); - if (view == null) - { - throw new NotFoundException(); - } - if (!view.isSession()) - throw new IllegalArgumentException("This action only supports saving session views."); - - //if (!getContainer().getId().equals(view.getContainer().getId())) - // throw new IllegalArgumentException("View may only be saved from container it was created in."); - - assert !view.canInherit() && !view.isShared() && view.isEditable(): "Session view should never be inheritable or shared and always be editable"; - - // Users may save views to a location other than the current container - String containerPath = form.getContainerPath(); - Container container; - if (form.isInherit() && containerPath != null) - { - // Only respect this request if it's a view that is inheritable in subfolders - container = ContainerManager.getForPath(containerPath); - } - else - { - // Otherwise, save it in the current container - container = getContainer(); - } - - if (container == null) - throw new NotFoundException("No such container: " + containerPath); - - if (form.isShared() || form.isInherit()) - { - if (!container.hasPermission(getUser(), EditSharedViewPermission.class)) - throw new UnauthorizedException(); - } - - DbScope scope = QueryManager.get().getDbSchema().getScope(); - try (DbScope.Transaction tx = scope.ensureTransaction()) - { - // Delete the session view. The view will be restored if an exception is thrown. - view.delete(getUser(), getViewContext().getRequest()); - - // Get any previously existing non-session view. - // The session custom view and the view-to-be-saved may have different names. - // If they do have different names, we may need to delete an existing session view with that name. - // UNDONE: If the view has a different name, we will clobber it without asking. - CustomView existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); - if (existingView != null && existingView.isSession()) - { - // Delete any session view we are overwriting. - existingView.delete(getUser(), getViewContext().getRequest()); - existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); - } - - // save a new private view if shared is false but existing view is shared - if (existingView != null && !form.isShared() && existingView.getOwner() == null) - { - existingView = null; - } - - if (existingView != null && !form.isReplace() && !StringUtils.isEmpty(form.getNewName())) - throw new IllegalArgumentException("A saved view by the name \"" + form.getNewName() + "\" already exists. "); - - if (existingView == null || (existingView instanceof ModuleCustomView && existingView.isEditable())) - { - User owner = form.isShared() ? null : getUser(); - - CustomViewImpl viewCopy = new CustomViewImpl(form.getQueryDef(), owner, form.getNewName()); - viewCopy.setColumns(view.getColumns()); - viewCopy.setCanInherit(form.isInherit()); - viewCopy.setFilterAndSort(view.getFilterAndSort()); - viewCopy.setColumnProperties(view.getColumnProperties()); - viewCopy.setIsHidden(form.isHidden()); - if (form.isInherit()) - viewCopy.setContainer(container); - - viewCopy.save(getUser(), getViewContext().getRequest()); - } - else if (!existingView.isEditable()) - { - throw new IllegalArgumentException("Existing view '" + form.getNewName() + "' is not editable. You may save this view with a different name."); - } - else - { - // UNDONE: changing shared property of an existing view is unimplemented. Not sure if it makes sense from a usability point of view. - existingView.setColumns(view.getColumns()); - existingView.setFilterAndSort(view.getFilterAndSort()); - existingView.setColumnProperties(view.getColumnProperties()); - existingView.setCanInherit(form.isInherit()); - if (form.isInherit()) - ((CustomViewImpl)existingView).setContainer(container); - existingView.setIsHidden(form.isHidden()); - - existingView.save(getUser(), getViewContext().getRequest()); - } - - tx.commit(); - return new ApiSimpleResponse("success", true); - } - catch (Exception e) - { - // dirty the view then save the deleted session view back in session state - view.setName(view.getName()); - view.save(getUser(), getViewContext().getRequest()); - - throw e; - } - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class ManageViewsAction extends SimpleViewAction - { - @SuppressWarnings("UnusedDeclaration") - public ManageViewsAction() - { - } - - public ManageViewsAction(ViewContext ctx) - { - setViewContext(ctx); - } - - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return new JspView<>("/org/labkey/query/view/manageViews.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - new BeginAction(getViewContext()).addNavTrail(root); - root.addChild("Manage Views", QueryController.this.getViewContext().getActionURL()); - } - } - - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalDeleteView extends ConfirmAction - { - @Override - public ModelAndView getConfirmView(InternalViewForm form, BindException errors) - { - return new JspView<>("/org/labkey/query/view/internalDeleteView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalViewForm form, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - QueryManager.get().delete(view); - return true; - } - - @Override - public void validateCommand(InternalViewForm internalViewForm, Errors errors) - { - } - - @Override - @NotNull - public ActionURL getSuccessURL(InternalViewForm internalViewForm) - { - return new ActionURL(ManageViewsAction.class, getContainer()); - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalSourceViewAction extends FormViewAction - { - @Override - public void validateCommand(InternalSourceViewForm target, Errors errors) - { - } - - @Override - public ModelAndView getView(InternalSourceViewForm form, boolean reshow, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - form.ff_inherit = QueryManager.get().canInherit(view.getFlags()); - form.ff_hidden = QueryManager.get().isHidden(view.getFlags()); - form.ff_columnList = view.getColumns(); - form.ff_filter = view.getFilter(); - return new JspView<>("/org/labkey/query/view/internalSourceView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalSourceViewForm form, BindException errors) - { - CstmView view = form.getViewAndCheckPermission(); - int flags = view.getFlags(); - flags = QueryManager.get().setCanInherit(flags, form.ff_inherit); - flags = QueryManager.get().setIsHidden(flags, form.ff_hidden); - view.setFlags(flags); - view.setColumns(form.ff_columnList); - view.setFilter(form.ff_filter); - QueryManager.get().update(getUser(), view); - return true; - } - - @Override - public ActionURL getSuccessURL(InternalSourceViewForm form) - { - return new ActionURL(ManageViewsAction.class, getContainer()); - } - - @Override - public void addNavTrail(NavTree root) - { - new ManageViewsAction(getViewContext()).addNavTrail(root); - root.addChild("Edit source of Grid View"); - } - } - - /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ - @RequiresPermission(AdminPermission.class) - public class InternalNewViewAction extends FormViewAction - { - int _customViewId = 0; - - @Override - public void validateCommand(InternalNewViewForm form, Errors errors) - { - if (StringUtils.trimToNull(form.ff_schemaName) == null) - { - errors.reject(ERROR_MSG, "Schema name cannot be blank."); - } - if (StringUtils.trimToNull(form.ff_queryName) == null) - { - errors.reject(ERROR_MSG, "Query name cannot be blank"); - } - } - - @Override - public ModelAndView getView(InternalNewViewForm form, boolean reshow, BindException errors) - { - return new JspView<>("/org/labkey/query/view/internalNewView.jsp", form, errors); - } - - @Override - public boolean handlePost(InternalNewViewForm form, BindException errors) - { - if (form.ff_share) - { - if (!getContainer().hasPermission(getUser(), AdminPermission.class)) - throw new UnauthorizedException(); - } - List existing = QueryManager.get().getCstmViews(getContainer(), form.ff_schemaName, form.ff_queryName, form.ff_viewName, form.ff_share ? null : getUser(), false, false); - CstmView view; - if (!existing.isEmpty()) - { - } - else - { - view = new CstmView(); - view.setSchema(form.ff_schemaName); - view.setQueryName(form.ff_queryName); - view.setName(form.ff_viewName); - view.setContainerId(getContainer().getId()); - if (form.ff_share) - { - view.setCustomViewOwner(null); - } - else - { - view.setCustomViewOwner(getUser().getUserId()); - } - if (form.ff_inherit) - { - view.setFlags(QueryManager.get().setCanInherit(view.getFlags(), form.ff_inherit)); - } - InternalViewForm.checkEdit(getViewContext(), view); - try - { - view = QueryManager.get().insert(getUser(), view); - } - catch (Exception e) - { - LogManager.getLogger(QueryController.class).error("Error", e); - errors.reject(ERROR_MSG, "An exception occurred: " + e); - return false; - } - _customViewId = view.getCustomViewId(); - } - return true; - } - - @Override - public ActionURL getSuccessURL(InternalNewViewForm form) - { - ActionURL forward = new ActionURL(InternalSourceViewAction.class, getContainer()); - forward.addParameter("customViewId", Integer.toString(_customViewId)); - return forward; - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Create New Grid View"); - } - } - - - @ActionNames("clearSelected, selectNone") - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class SelectNoneAction extends MutatingApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) - { - errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - if (form.getQueryName() == null) - { - DataRegionSelection.clearAll(getViewContext(), form.getKey()); - return new DataRegionSelection.SelectionResponse(0); - } - - int count = DataRegionSelection.setSelectedFromForm(form); - return new DataRegionSelection.SelectionResponse(count); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SelectForm extends QueryForm - { - protected boolean clearSelected; - protected String key; - - public boolean isClearSelected() - { - return clearSelected; - } - - public void setClearSelected(boolean clearSelected) - { - this.clearSelected = clearSelected; - } - - public String getKey() - { - return key; - } - - public void setKey(String key) - { - this.key = key; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectData.class) - public static class SelectAllAction extends MutatingApiAction - { - @Override - public void validateForm(QueryForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() || form.getQueryName() == null) - { - errors.reject(ERROR_MSG, "schemaName and queryName required"); - } - } - - @Override - public ApiResponse execute(final QueryForm form, BindException errors) throws Exception - { - int count = DataRegionSelection.setSelectionForAll(form, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetSelectedAction extends ReadOnlyApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) - { - errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - getViewContext().getResponse().setHeader("Content-Type", CONTENT_TYPE_JSON); - Set selected; - - if (form.getQueryName() == null) - selected = DataRegionSelection.getSelected(getViewContext(), form.getKey(), form.isClearSelected()); - else - selected = DataRegionSelection.getSelected(form, form.isClearSelected()); - - return new ApiSimpleResponse("selected", selected); - } - } - - @ActionNames("setSelected, setCheck") - @RequiresPermission(ReadPermission.class) - public static class SetCheckAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) throws Exception - { - String[] ids = form.getId(getViewContext().getRequest()); - Set selection = new LinkedHashSet<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - int count; - if (form.getQueryName() != null && form.isValidateIds() && form.isChecked()) - { - selection = DataRegionSelection.getValidatedIds(selection, form); - } - - count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, form.isChecked()); - - return new DataRegionSelection.SelectionResponse(count); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class SetCheckForm extends SelectForm - { - protected String[] ids; - protected boolean checked; - protected boolean validateIds; - - public String[] getId(HttpServletRequest request) - { - // 5025 : DataRegion checkbox names may contain comma - // Beehive parses a single parameter value with commas into an array - // which is not what we want. - String[] paramIds = request.getParameterValues("id"); - return paramIds == null ? ids: paramIds; - } - - public void setId(String[] ids) - { - this.ids = ids; - } - - public boolean isChecked() - { - return checked; - } - - public void setChecked(boolean checked) - { - this.checked = checked; - } - - public boolean isValidateIds() - { - return validateIds; - } - - public void setValidateIds(boolean validateIds) - { - this.validateIds = validateIds; - } - } - - @RequiresPermission(ReadPermission.class) - public static class ReplaceSelectedAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) - { - String[] ids = form.getId(getViewContext().getRequest()); - List selection = new ArrayList<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - - DataRegionSelection.clearAll(getViewContext(), form.getKey()); - int count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class SetSnapshotSelectionAction extends MutatingApiAction - { - @Override - public ApiResponse execute(final SetCheckForm form, BindException errors) - { - String[] ids = form.getId(getViewContext().getRequest()); - List selection = new ArrayList<>(); - if (ids != null) - { - for (String id : ids) - { - if (StringUtils.isNotBlank(id)) - selection.add(id); - } - } - - DataRegionSelection.clearAll(getViewContext(), form.getKey(), true); - int count = DataRegionSelection.setSelected( - getViewContext(), form.getKey(), - selection, true, true); - return new DataRegionSelection.SelectionResponse(count); - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetSnapshotSelectionAction extends ReadOnlyApiAction - { - @Override - public void validateForm(SelectForm form, Errors errors) - { - if (StringUtils.isEmpty(form.getKey())) - { - errors.reject(ERROR_MSG, "Selection key is required"); - } - } - - @Override - public ApiResponse execute(final SelectForm form, BindException errors) throws Exception - { - List selected = DataRegionSelection.getSnapshotSelected(getViewContext(), form.getKey()); - return new ApiSimpleResponse("selected", selected); - } - } - - public static String getMessage(SqlDialect d, SQLException x) - { - return x.getMessage(); - } - - - public static class GetSchemasForm - { - private boolean _includeHidden = true; - private SchemaKey _schemaName; - - public SchemaKey getSchemaName() - { - return _schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(SchemaKey schemaName) - { - _schemaName = schemaName; - } - - public boolean isIncludeHidden() - { - return _includeHidden; - } - - @SuppressWarnings("unused") - public void setIncludeHidden(boolean includeHidden) - { - _includeHidden = includeHidden; - } - } - - - @RequiresPermission(ReadPermission.class) - @ApiVersion(12.3) - public static class GetSchemasAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetSchemasForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetSchemasForm form, BindException errors) - { - final Container container = getContainer(); - final User user = getUser(); - - final boolean includeHidden = form.isIncludeHidden(); - if (getRequestedApiVersion() >= 9.3) - { - SimpleSchemaTreeVisitor visitor = new SimpleSchemaTreeVisitor<>(includeHidden) - { - @Override - public Void visitUserSchema(UserSchema schema, Path path, JSONObject json) - { - JSONObject schemaProps = new JSONObject(); - - schemaProps.put("schemaName", schema.getName()); - schemaProps.put("fullyQualifiedName", schema.getSchemaName()); - schemaProps.put("description", schema.getDescription()); - schemaProps.put("hidden", schema.isHidden()); - NavTree tree = schema.getSchemaBrowserLinks(user); - if (tree != null && tree.hasChildren()) - schemaProps.put("menu", tree.toJSON()); - - // Collect children schemas - JSONObject children = new JSONObject(); - visit(schema.getSchemas(_includeHidden), path, children); - if (!children.isEmpty()) - schemaProps.put("schemas", children); - - // Add node's schemaProps to the parent's json. - json.put(schema.getName(), schemaProps); - return null; - } - }; - - // By default, start from the root. - QuerySchema schema; - if (form.getSchemaName() != null) - schema = DefaultSchema.get(user, container, form.getSchemaName()); - else - schema = DefaultSchema.get(user, container); - - // Ensure consistent exception as other query actions - QueryForm.ensureSchemaNotNull(schema); - - // Create the JSON response by visiting the schema children. The parent schema information isn't included. - JSONObject ret = new JSONObject(); - visitor.visitTop(schema.getSchemas(includeHidden), ret); - - return new ApiSimpleResponse(ret); - } - else - { - return new ApiSimpleResponse("schemas", DefaultSchema.get(user, container).getUserSchemaPaths(includeHidden)); - } - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetQueriesForm - { - private String _schemaName; - private boolean _includeUserQueries = true; - private boolean _includeSystemQueries = true; - private boolean _includeColumns = true; - private boolean _includeViewDataUrl = true; - private boolean _includeTitle = true; - private boolean _queryDetailColumns = false; - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isIncludeUserQueries() - { - return _includeUserQueries; - } - - public void setIncludeUserQueries(boolean includeUserQueries) - { - _includeUserQueries = includeUserQueries; - } - - public boolean isIncludeSystemQueries() - { - return _includeSystemQueries; - } - - public void setIncludeSystemQueries(boolean includeSystemQueries) - { - _includeSystemQueries = includeSystemQueries; - } - - public boolean isIncludeColumns() - { - return _includeColumns; - } - - public void setIncludeColumns(boolean includeColumns) - { - _includeColumns = includeColumns; - } - - public boolean isQueryDetailColumns() - { - return _queryDetailColumns; - } - - public void setQueryDetailColumns(boolean queryDetailColumns) - { - _queryDetailColumns = queryDetailColumns; - } - - public boolean isIncludeViewDataUrl() - { - return _includeViewDataUrl; - } - - public void setIncludeViewDataUrl(boolean includeViewDataUrl) - { - _includeViewDataUrl = includeViewDataUrl; - } - - public boolean isIncludeTitle() - { - return _includeTitle; - } - - public void setIncludeTitle(boolean includeTitle) - { - _includeTitle = includeTitle; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class GetQueriesAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetQueriesForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetQueriesForm form, BindException errors) - { - if (null == StringUtils.trimToNull(form.getSchemaName())) - throw new IllegalArgumentException("You must supply a value for the 'schemaName' parameter!"); - - ApiSimpleResponse response = new ApiSimpleResponse(); - UserSchema uschema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); - if (null == uschema) - throw new NotFoundException("The schema name '" + form.getSchemaName() - + "' was not found within the folder '" + getContainer().getPath() + "'"); - - response.put("schemaName", form.getSchemaName()); - - List> qinfos = new ArrayList<>(); - - //user-defined queries - if (form.isIncludeUserQueries()) - { - for (QueryDefinition qdef : uschema.getQueryDefs().values()) - { - if (!qdef.isTemporary()) - { - ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; - qinfos.add(getQueryProps(qdef, viewDataUrl, true, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); - } - } - } - - //built-in tables - if (form.isIncludeSystemQueries()) - { - for (String qname : uschema.getVisibleTableNames()) - { - // Go direct against the UserSchema instead of calling into QueryService, which takes a schema and - // query name as strings and therefore has to create new instances - QueryDefinition qdef = uschema.getQueryDefForTable(qname); - if (qdef != null) - { - ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; - qinfos.add(getQueryProps(qdef, viewDataUrl, false, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); - } - } - } - response.put("queries", qinfos); - - return response; - } - - protected Map getQueryProps(QueryDefinition qdef, ActionURL viewDataUrl, boolean isUserDefined, UserSchema schema, boolean includeColumns, boolean useQueryDetailColumns, boolean includeTitle) - { - Map qinfo = new HashMap<>(); - qinfo.put("hidden", qdef.isHidden()); - qinfo.put("snapshot", qdef.isSnapshot()); - qinfo.put("inherit", qdef.canInherit()); - qinfo.put("isUserDefined", isUserDefined); - boolean canEdit = qdef.canEdit(getUser()); - qinfo.put("canEdit", canEdit); - qinfo.put("canEditSharedViews", getContainer().hasPermission(getUser(), EditSharedViewPermission.class)); - // CONSIDER: do we want to separate the 'canEditMetadata' property and 'isMetadataOverridable' properties to differentiate between capability and the permission check? - qinfo.put("isMetadataOverrideable", qdef.isMetadataEditable() && qdef.canEditMetadata(getUser())); - - if (isUserDefined) - qinfo.put("moduleName", qdef.getModuleName()); - boolean isInherited = qdef.canInherit() && !getContainer().equals(qdef.getDefinitionContainer()); - qinfo.put("isInherited", isInherited); - if (isInherited) - qinfo.put("containerPath", qdef.getDefinitionContainer().getPath()); - qinfo.put("isIncludedForLookups", qdef.isIncludedForLookups()); - - if (null != qdef.getDescription()) - qinfo.put("description", qdef.getDescription()); - if (viewDataUrl != null) - qinfo.put("viewDataUrl", viewDataUrl); - - String title = qdef.getName(); - String name = qdef.getName(); - try - { - // get the TableInfo if the user requested column info or title, otherwise skip (it can be expensive) - if (includeColumns || includeTitle) - { - TableInfo table = qdef.getTable(schema, null, true); - - if (null != table) - { - if (includeColumns) - { - Collection> columns; - - if (useQueryDetailColumns) - { - columns = JsonWriter - .getNativeColProps(table, Collections.emptyList(), null, false, false) - .values(); - } - else - { - columns = new ArrayList<>(); - for (ColumnInfo col : table.getColumns()) - { - Map cinfo = new HashMap<>(); - cinfo.put("name", col.getName()); - if (null != col.getLabel()) - cinfo.put("caption", col.getLabel()); - if (null != col.getShortLabel()) - cinfo.put("shortCaption", col.getShortLabel()); - if (null != col.getDescription()) - cinfo.put("description", col.getDescription()); - - columns.add(cinfo); - } - } - - if (!columns.isEmpty()) - qinfo.put("columns", columns); - } - - if (includeTitle) - { - name = table.getPublicName(); - title = table.getTitle(); - } - } - } - } - catch(Exception e) - { - //may happen due to query failing parse - } - - qinfo.put("title", title); - qinfo.put("name", name); - return qinfo; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class GetQueryViewsForm - { - private String _schemaName; - private String _queryName; - private String _viewName; - private boolean _metadata; - private boolean _excludeSessionView; - - public String getSchemaName() - { - return _schemaName; - } - - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public String getQueryName() - { - return _queryName; - } - - public void setQueryName(String queryName) - { - _queryName = queryName; - } - - public String getViewName() - { - return _viewName; - } - - public void setViewName(String viewName) - { - _viewName = viewName; - } - - public boolean isMetadata() - { - return _metadata; - } - - public void setMetadata(boolean metadata) - { - _metadata = metadata; - } - - public boolean isExcludeSessionView() - { - return _excludeSessionView; - } - - public void setExcludeSessionView(boolean excludeSessionView) - { - _excludeSessionView = excludeSessionView; - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class GetQueryViewsAction extends ReadOnlyApiAction - { - @Override - protected long getLastModified(GetQueryViewsForm form) - { - return QueryService.get().metadataLastModified(); - } - - @Override - public ApiResponse execute(GetQueryViewsForm form, BindException errors) - { - if (null == StringUtils.trimToNull(form.getSchemaName())) - throw new IllegalArgumentException("You must pass a value for the 'schemaName' parameter!"); - if (null == StringUtils.trimToNull(form.getQueryName())) - throw new IllegalArgumentException("You must pass a value for the 'queryName' parameter!"); - - UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); - if (null == schema) - throw new NotFoundException("The schema name '" + form.getSchemaName() - + "' was not found within the folder '" + getContainer().getPath() + "'"); - - QueryDefinition querydef = QueryService.get().createQueryDefForTable(schema, form.getQueryName()); - if (null == querydef || querydef.getTable(null, true) == null) - throw new NotFoundException("The query '" + form.getQueryName() + "' was not found within the '" - + form.getSchemaName() + "' schema in the container '" - + getContainer().getPath() + "'!"); - - Map views = querydef.getCustomViews(getUser(), getViewContext().getRequest(), true, false, form.isExcludeSessionView()); - if (null == views) - views = Collections.emptyMap(); - - Map> columnMetadata = new HashMap<>(); - - List> viewInfos = Collections.emptyList(); - if (getViewContext().getBindPropertyValues().contains("viewName")) - { - // Get info for a named view or the default view (null) - String viewName = StringUtils.trimToNull(form.getViewName()); - CustomView view = views.get(viewName); - if (view != null) - { - viewInfos = Collections.singletonList(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); - } - else if (viewName == null) - { - // The default view was requested but it hasn't been customized yet. Create the 'default default' view. - viewInfos = Collections.singletonList(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); - } - } - else - { - boolean foundDefault = false; - viewInfos = new ArrayList<>(views.size()); - for (CustomView view : views.values()) - { - if (view.getName() == null) - foundDefault = true; - viewInfos.add(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); - } - - if (!foundDefault) - { - // The default view hasn't been customized yet. Create the 'default default' view. - viewInfos.add(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); - } - } - - ApiSimpleResponse response = new ApiSimpleResponse(); - response.put("schemaName", form.getSchemaName()); - response.put("queryName", form.getQueryName()); - response.put("views", viewInfos); - - return response; - } - } - - @RequiresNoPermission - public static class GetServerDateAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(Object o, BindException errors) - { - return new ApiSimpleResponse("date", new Date()); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - private static class SaveApiTestForm - { - private String _getUrl; - private String _postUrl; - private String _postData; - private String _response; - - public String getGetUrl() - { - return _getUrl; - } - - public void setGetUrl(String getUrl) - { - _getUrl = getUrl; - } - - public String getPostUrl() - { - return _postUrl; - } - - public void setPostUrl(String postUrl) - { - _postUrl = postUrl; - } - - public String getResponse() - { - return _response; - } - - public void setResponse(String response) - { - _response = response; - } - - public String getPostData() - { - return _postData; - } - - public void setPostData(String postData) - { - _postData = postData; - } - } - - - @RequiresPermission(ReadPermission.class) - public static class SaveApiTestAction extends MutatingApiAction - { - @Override - public ApiResponse execute(SaveApiTestForm form, BindException errors) - { - ApiSimpleResponse response = new ApiSimpleResponse(); - - ApiTestsDocument doc = ApiTestsDocument.Factory.newInstance(); - - TestCaseType test = doc.addNewApiTests().addNewTest(); - test.setName("recorded test case"); - ActionURL url = null; - - if (!StringUtils.isEmpty(form.getGetUrl())) - { - test.setType("get"); - url = new ActionURL(form.getGetUrl()); - } - else if (!StringUtils.isEmpty(form.getPostUrl())) - { - test.setType("post"); - test.setFormData(form.getPostData()); - url = new ActionURL(form.getPostUrl()); - } - - if (url != null) - { - String uri = url.getLocalURIString(); - if (uri.startsWith(url.getContextPath())) - uri = uri.substring(url.getContextPath().length() + 1); - - test.setUrl(uri); - } - test.setResponse(form.getResponse()); - - XmlOptions opts = new XmlOptions(); - opts.setSaveCDataEntityCountThreshold(0); - opts.setSaveCDataLengthThreshold(0); - opts.setSavePrettyPrint(); - opts.setUseDefaultNamespace(); - - response.put("xml", doc.xmlText(opts)); - - return response; - } - } - - - private abstract static class ParseAction extends SimpleViewAction - { - @Override - public ModelAndView getView(Object o, BindException errors) - { - List qpe = new ArrayList<>(); - String expr = getViewContext().getRequest().getParameter("q"); - ArrayList html = new ArrayList<>(); - PageConfig config = getPageConfig(); - var inputId = config.makeId("submit_"); - config.addHandler(inputId, "click", "Ext.getBody().mask();"); - html.add("
\n" + - "" - ); - - QNode e = null; - if (null != expr) - { - try - { - e = _parse(expr,qpe); - } - catch (RuntimeException x) - { - qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); - } - } - - Tree tree = null; - if (null != expr) - { - try - { - tree = _tree(expr); - } catch (Exception x) - { - qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); - } - } - - for (Throwable x : qpe) - { - if (null != x.getCause() && x != x.getCause()) - x = x.getCause(); - html.add("
" + PageFlowUtil.filter(x.toString())); - LogManager.getLogger(QueryController.class).debug(expr,x); - } - if (null != e) - { - String prefix = SqlParser.toPrefixString(e); - html.add("
"); - html.add(PageFlowUtil.filter(prefix)); - } - if (null != tree) - { - String prefix = SqlParser.toPrefixString(tree); - html.add("
"); - html.add(PageFlowUtil.filter(prefix)); - } - html.add(""); - return HtmlView.unsafe(StringUtils.join(html,"")); - } - - @Override - public void addNavTrail(NavTree root) - { - } - - abstract QNode _parse(String e, List errors); - abstract Tree _tree(String e) throws Exception; - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ParseExpressionAction extends ParseAction - { - @Override - QNode _parse(String s, List errors) - { - return new SqlParser().parseExpr(s, true, errors); - } - - @Override - Tree _tree(String e) - { - return null; - } - } - - @RequiresPermission(AdminOperationsPermission.class) - public static class ParseQueryAction extends ParseAction - { - @Override - QNode _parse(String s, List errors) - { - return new SqlParser().parseQuery(s, errors, null); - } - - @Override - Tree _tree(String s) throws Exception - { - return new SqlParser().rawQuery(s); - } - } - - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.SelectMetaData.class) - public static class ValidateQueryMetadataAction extends ReadOnlyApiAction - { - @Override - public ApiResponse execute(QueryForm form, BindException errors) - { - UserSchema schema = form.getSchema(); - - if (null == schema) - { - errors.reject(ERROR_MSG, "could not resolve schema: " + form.getSchemaName()); - return null; - } - - List parseErrors = new ArrayList<>(); - List parseWarnings = new ArrayList<>(); - ApiSimpleResponse response = new ApiSimpleResponse(); - - try - { - TableInfo table = schema.getTable(form.getQueryName(), null); - - if (null == table) - { - errors.reject(ERROR_MSG, "could not resolve table: " + form.getQueryName()); - return null; - } - - if (!QueryManager.get().validateQuery(table, true, parseErrors, parseWarnings)) - { - for (QueryParseException e : parseErrors) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - return response; - } - - SchemaKey schemaKey = SchemaKey.fromString(form.getSchemaName()); - QueryManager.get().validateQueryMetadata(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); - QueryManager.get().validateQueryViews(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); - } - catch (QueryParseException e) - { - parseErrors.add(e); - } - - for (QueryParseException e : parseErrors) - { - errors.reject(ERROR_MSG, e.getMessage()); - } - - for (QueryParseException e : parseWarnings) - { - errors.reject(ERROR_MSG, "WARNING: " + e.getMessage()); - } - - return response; - } - - @Override - protected ApiResponseWriter createResponseWriter() throws IOException - { - ApiResponseWriter result = super.createResponseWriter(); - // Issue 44875 - don't send a 400 or 500 response code when there's a bogus query or metadata - result.setErrorResponseStatus(HttpServletResponse.SC_OK); - return result; - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class QueryExportAuditForm - { - private int rowId; - - public int getRowId() - { - return rowId; - } - - public void setRowId(int rowId) - { - this.rowId = rowId; - } - } - - /** - * Action used to redirect QueryAuditProvider [details] column to the exported table's grid view. - */ - @RequiresPermission(AdminPermission.class) - public static class QueryExportAuditRedirectAction extends SimpleRedirectAction - { - @Override - public URLHelper getRedirectURL(QueryExportAuditForm form) - { - if (form.getRowId() == 0) - throw new NotFoundException("Query export audit rowid required"); - - UserSchema auditSchema = QueryService.get().getUserSchema(getUser(), getContainer(), AbstractAuditTypeProvider.QUERY_SCHEMA_NAME); - TableInfo queryExportAuditTable = auditSchema.getTable(QueryExportAuditProvider.QUERY_AUDIT_EVENT, null); - if (null == queryExportAuditTable) - throw new NotFoundException(); - - TableSelector selector = new TableSelector(queryExportAuditTable, - PageFlowUtil.set( - QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME, - QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME, - QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL), - new SimpleFilter(FieldKey.fromParts(AbstractAuditTypeProvider.COLUMN_NAME_ROW_ID), form.getRowId()), null); - - Map result = selector.getMap(); - if (result == null) - throw new NotFoundException("Query export audit event not found for rowId"); - - String schemaName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME); - String queryName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME); - String detailsURL = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL); - - if (schemaName == null || queryName == null) - throw new NotFoundException("Query export audit event has not schemaName or queryName"); - - ActionURL url = new ActionURL(ExecuteQueryAction.class, getContainer()); - - // Apply the sorts and filters - if (detailsURL != null) - { - ActionURL sortFilterURL = new ActionURL(detailsURL); - url.setPropertyValues(sortFilterURL.getPropertyValues()); - } - - if (url.getParameter(QueryParam.schemaName) == null) - url.addParameter(QueryParam.schemaName, schemaName); - if (url.getParameter(QueryParam.queryName) == null && url.getParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName) == null) - url.addParameter(QueryParam.queryName, queryName); - - return url; - } - } - - @RequiresPermission(ReadPermission.class) - public static class AuditHistoryAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryForm form, BindException errors) - { - return QueryUpdateAuditProvider.createHistoryQueryView(getViewContext(), form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Audit History"); - } - } - - @RequiresPermission(ReadPermission.class) - public static class AuditDetailsAction extends SimpleViewAction - { - @Override - public ModelAndView getView(QueryDetailsForm form, BindException errors) - { - return QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Audit History"); - } - } - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class QueryDetailsForm extends QueryForm - { - String _keyValue; - - public String getKeyValue() - { - return _keyValue; - } - - public void setKeyValue(String keyValue) - { - _keyValue = keyValue; - } - } - - @RequiresPermission(ReadPermission.class) - @Action(ActionType.Export.class) - public static class ExportTablesAction extends FormViewAction - { - private ActionURL _successUrl; - - @Override - public void validateCommand(ExportTablesForm form, Errors errors) - { - } - - @Override - public boolean handlePost(ExportTablesForm form, BindException errors) - { - HttpServletResponse httpResponse = getViewContext().getResponse(); - Container container = getContainer(); - QueryServiceImpl svc = (QueryServiceImpl)QueryService.get(); - - try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream outputStream = new BufferedOutputStream(baos)) - { - try (ZipFile zip = new ZipFile(outputStream, true)) - { - svc.writeTables(container, getUser(), zip, form.getSchemas(), form.getHeaderType()); - } - - PageFlowUtil.streamFileBytes(httpResponse, FileUtil.makeFileNameWithTimestamp(container.getName(), "tables.zip"), baos.toByteArray(), false); - } - catch (Exception e) - { - errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : e.getClass().getName()); - LOG.error("Errror exporting tables", e); - } - - if (errors.hasErrors()) - { - _successUrl = new ActionURL(ExportTablesAction.class, getContainer()); - } - - return !errors.hasErrors(); - } - - @Override - public ModelAndView getView(ExportTablesForm form, boolean reshow, BindException errors) - { - // When exporting the zip to the browser, the base action will attempt to reshow the view since we returned - // null as the success URL; returning null here causes the base action to stop pestering the action. - if (reshow && !errors.hasErrors()) - return null; - - return new JspView<>("/org/labkey/query/view/exportTables.jsp", form, errors); - } - - @Override - public void addNavTrail(NavTree root) - { - root.addChild("Export Tables"); - } - - @Override - public ActionURL getSuccessURL(ExportTablesForm form) - { - return _successUrl; - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class ExportTablesForm implements HasBindParameters - { - ColumnHeaderType _headerType = ColumnHeaderType.DisplayFieldKey; - Map>> _schemas = new HashMap<>(); - - public ColumnHeaderType getHeaderType() - { - return _headerType; - } - - public void setHeaderType(ColumnHeaderType headerType) - { - _headerType = headerType; - } - - public Map>> getSchemas() - { - return _schemas; - } - - public void setSchemas(Map>> schemas) - { - _schemas = schemas; - } - - @Override - public @NotNull BindException bindParameters(PropertyValues values) - { - BindException errors = new NullSafeBindException(this, "form"); - - PropertyValue schemasProperty = values.getPropertyValue("schemas"); - if (schemasProperty != null && schemasProperty.getValue() != null) - { - try - { - _schemas = JsonUtil.DEFAULT_MAPPER.readValue((String)schemasProperty.getValue(), _schemas.getClass()); - } - catch (IOException e) - { - errors.rejectValue("schemas", ERROR_MSG, e.getMessage()); - } - } - - PropertyValue headerTypeProperty = values.getPropertyValue("headerType"); - if (headerTypeProperty != null && headerTypeProperty.getValue() != null) - { - try - { - _headerType = ColumnHeaderType.valueOf(String.valueOf(headerTypeProperty.getValue())); - } - catch (IllegalArgumentException ex) - { - // ignore - } - } - - return errors; - } - } - - - @RequiresPermission(ReadPermission.class) - public static class SaveNamedSetAction extends MutatingApiAction - { - @Override - public Object execute(NamedSetForm namedSetForm, BindException errors) - { - QueryService.get().saveNamedSet(namedSetForm.getSetName(), namedSetForm.parseSetList()); - return new ApiSimpleResponse("success", true); - } - } - - - @SuppressWarnings({"unused", "WeakerAccess"}) - public static class NamedSetForm - { - String setName; - String[] setList; - - public String getSetName() - { - return setName; - } - - public void setSetName(String setName) - { - this.setName = setName; - } - - public String[] getSetList() - { - return setList; - } - - public void setSetList(String[] setList) - { - this.setList = setList; - } - - public List parseSetList() - { - return Arrays.asList(setList); - } - } - - - @RequiresPermission(ReadPermission.class) - public static class DeleteNamedSetAction extends MutatingApiAction - { - - @Override - public Object execute(NamedSetForm namedSetForm, BindException errors) - { - QueryService.get().deleteNamedSet(namedSetForm.getSetName()); - return new ApiSimpleResponse("success", true); - } - } - - @RequiresPermission(ReadPermission.class) - public static class AnalyzeQueriesAction extends ReadOnlyApiAction - { - @Override - public Object execute(Object o, BindException errors) throws Exception - { - JSONObject ret = new JSONObject(); - - try - { - QueryService.QueryAnalysisService analysisService = QueryService.get().getQueryAnalysisService(); - if (analysisService != null) - { - DefaultSchema start = DefaultSchema.get(getUser(), getContainer()); - var deps = new HashSetValuedHashMap(); - - analysisService.analyzeFolder(start, deps); - ret.put("success", true); - - JSONObject objects = new JSONObject(); - for (var from : deps.keySet()) - { - objects.put(from.getKey(), from.toJSON()); - for (var to : deps.get(from)) - objects.put(to.getKey(), to.toJSON()); - } - ret.put("objects", objects); - - JSONArray dependants = new JSONArray(); - for (var from : deps.keySet()) - { - for (var to : deps.get(from)) - dependants.put(new String[] {from.getKey(), to.getKey()}); - } - ret.put("graph", dependants); - } - else - { - ret.put("success", false); - } - return ret; - } - catch (Throwable e) - { - LOG.error(e); - throw UnexpectedException.wrap(e); - } - } - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) - public static class GetQueryEditorMetadataAction extends ReadOnlyApiAction - { - @Override - protected ObjectMapper createRequestObjectMapper() - { - PropertyService propertyService = PropertyService.get(); - if (null != propertyService) - { - ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); - mapper.addMixIn(GWTPropertyDescriptor.class, MetadataTableJSONMixin.class); - return mapper; - } - else - { - throw new RuntimeException("Could not serialize request object"); - } - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return createRequestObjectMapper(); - } - - @Override - public Object execute(QueryForm queryForm, BindException errors) throws Exception - { - QueryDefinition queryDef = queryForm.getQueryDef(); - return MetadataTableJSON.getMetadata(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); - } - } - - @Marshal(Marshaller.Jackson) - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - public static class SaveQueryMetadataAction extends MutatingApiAction - { - @Override - protected ObjectMapper createRequestObjectMapper() - { - PropertyService propertyService = PropertyService.get(); - if (null != propertyService) - { - ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); - propertyService.configureObjectMapper(mapper, null); - return mapper; - } - else - { - throw new RuntimeException("Could not serialize request object"); - } - } - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return createRequestObjectMapper(); - } - - @Override - public Object execute(QueryMetadataApiForm queryMetadataApiForm, BindException errors) throws Exception - { - String schemaName = queryMetadataApiForm.getSchemaName(); - MetadataTableJSON domain = queryMetadataApiForm.getDomain(); - MetadataTableJSON.saveMetadata(schemaName, domain.getName(), null, domain.getFields(true), queryMetadataApiForm.isUserDefinedQuery(), false, getUser(), getContainer()); - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - resp.put("domain", MetadataTableJSON.getMetadata(schemaName, domain.getName(), getUser(), getContainer())); - return resp; - } - } - - @Marshal(Marshaller.Jackson) - @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) - public static class ResetQueryMetadataAction extends MutatingApiAction - { - @Override - public Object execute(QueryForm queryForm, BindException errors) throws Exception - { - QueryDefinition queryDef = queryForm.getQueryDef(); - return MetadataTableJSON.resetToDefault(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); - } - } - - private static class QueryMetadataApiForm - { - private MetadataTableJSON _domain; - private String _schemaName; - private boolean _userDefinedQuery; - - public MetadataTableJSON getDomain() - { - return _domain; - } - - @SuppressWarnings("unused") - public void setDomain(MetadataTableJSON domain) - { - _domain = domain; - } - - public String getSchemaName() - { - return _schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(String schemaName) - { - _schemaName = schemaName; - } - - public boolean isUserDefinedQuery() - { - return _userDefinedQuery; - } - - @SuppressWarnings("unused") - public void setUserDefinedQuery(boolean userDefinedQuery) - { - _userDefinedQuery = userDefinedQuery; - } - } - - @RequiresPermission(ReadPermission.class) - public static class GetDefaultVisibleColumnsAction extends ReadOnlyApiAction - { - @Override - public Object execute(GetQueryDetailsAction.Form form, BindException errors) throws Exception - { - ApiSimpleResponse resp = new ApiSimpleResponse(); - - Container container = getContainer(); - User user = getUser(); - - if (StringUtils.isEmpty(form.getSchemaName())) - throw new NotFoundException("SchemaName not specified"); - - QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); - if (!(querySchema instanceof UserSchema schema)) - throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'"); - - QuerySettings settings = schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); - QueryDefinition queryDef = settings.getQueryDef(schema); - if (null == queryDef) - // Don't echo the provided query name, but schema name is legit since it was found. See #44528. - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'"); - - TableInfo tinfo = queryDef.getTable(null, true); - if (null == tinfo) - throw new NotFoundException("Could not find the specified query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); - - List fields = tinfo.getDefaultVisibleColumns(); - - List displayColumns = QueryService.get().getColumns(tinfo, fields) - .values() - .stream() - .filter(cinfo -> fields.contains(cinfo.getFieldKey())) - .map(cinfo -> cinfo.getDisplayColumnFactory().createRenderer(cinfo)) - .collect(Collectors.toList()); - - resp.put("columns", JsonWriter.getNativeColProps(displayColumns, null, false)); - - return resp; - } - } - - public static class ParseForm implements ApiJsonForm - { - String expression = ""; - Map columnMap = new HashMap<>(); - List phiColumns = new ArrayList<>(); - - Map getColumnMap() - { - return columnMap; - } - - public String getExpression() - { - return expression; - } - - public void setExpression(String expression) - { - this.expression = expression; - } - - public List getPhiColumns() - { - return phiColumns; - } - - public void setPhiColumns(List phiColumns) - { - this.phiColumns = phiColumns; - } - - @Override - public void bindJson(JSONObject json) - { - if (json.has("expression")) - setExpression(json.getString("expression")); - if (json.has("phiColumns")) - setPhiColumns(json.getJSONArray("phiColumns").toList().stream().map(s -> FieldKey.fromParts(s.toString())).collect(Collectors.toList())); - if (json.has("columnMap")) - { - JSONObject columnMap = json.getJSONObject("columnMap"); - for (String key : columnMap.keySet()) - { - try - { - getColumnMap().put(FieldKey.fromParts(key), JdbcType.valueOf(String.valueOf(columnMap.get(key)))); - } - catch (IllegalArgumentException iae) - { - getColumnMap().put(FieldKey.fromParts(key), JdbcType.OTHER); - } - } - } - } - } - - - /** - * Since this api purpose is to return parse errors, it does not generally return success:false. - *
- * The API expects JSON like this, note that column names should be in FieldKey.toString() encoded to match the response JSON format. - * 
-     *     { "expression": "A$ + B", "columnMap":{"A$D":"VARCHAR", "X":"VARCHAR"}}
-     *
- * and returns a response like this - * 
-     *     {
-     *       "jdbcType" : "OTHER",
-     *       "success" : true,
-     *       "columnMap" : {"A$D":"VARCHAR", "B":"OTHER"}
-     *       "errors" : [ { "msg" : "\"B\" not found.", "type" : "sql" } ]
-     *     }
-     *
- * The columnMap object keys are the names of columns found in the expression. Names are returned - * in FieldKey.toString() formatting e.g. dollar-sign encoded. The object structure - * is compatible with the columnMap input parameter, so it can be used as a template to make a second request - * with types filled in. If provided, the type will be copied from the input columnMap, otherwise it will be "OTHER". - *
- * Parse exceptions may contain a line (usually 1) and col location e.g. - * 
-     * {
-     *     "msg" : "Error on line 1: Syntax error near 'error', expected 'EOF'
-     *     "col" : 2,
-     *     "line" : 1,
-     *     "type" : "sql",
-     *     "errorStr" : "A error B"
-     *   }
-     *
- */ - @RequiresNoPermission - @CSRF(CSRF.Method.NONE) - public static class ParseCalculatedColumnAction extends ReadOnlyApiAction - { - @Override - public Object execute(ParseForm form, BindException errors) throws Exception - { - if (errors.hasErrors()) - return errors; - JSONObject result = new JSONObject(Map.of("success",true)); - var requiredColumns = new HashSet(); - JdbcType jdbcType = JdbcType.OTHER; - try - { - var schema = DefaultSchema.get(getViewContext().getUser(), getViewContext().getContainer()).getUserSchema("core"); - var table = new VirtualTable<>(schema.getDbSchema(), "EXPR", schema){}; - ColumnInfo calculatedCol = QueryServiceImpl.get().createQueryExpressionColumn(table, new FieldKey(null, "expr"), form.getExpression(), null); - Map columns = new HashMap<>(); - for (var entry : form.getColumnMap().entrySet()) - { - BaseColumnInfo entryCol = new BaseColumnInfo(entry.getKey(), entry.getValue()); - // bindQueryExpressionColumn has a check that restricts PHI columns from being used in expressions - // so we need to set the PHI level to something other than NotPHI on these fake BaseColumnInfo objects - if (form.getPhiColumns().contains(entry.getKey())) - entryCol.setPHI(PHI.PHI); - columns.put(entry.getKey(), entryCol); - table.addColumn(entryCol); - } - // TODO: calculating jdbcType still uses calculatedCol.getParentTable().getColumns() - QueryServiceImpl.get().bindQueryExpressionColumn(calculatedCol, columns, false, requiredColumns); - jdbcType = calculatedCol.getJdbcType(); - } - catch (QueryException x) - { - JSONArray parseErrors = new JSONArray(); - parseErrors.put(x.toJSON(form.getExpression())); - result.put("errors", parseErrors); - } - finally - { - if (!requiredColumns.isEmpty()) - { - JSONObject columnMap = new JSONObject(); - for (FieldKey fk : requiredColumns) - { - JdbcType type = Objects.requireNonNullElse(form.getColumnMap().get(fk), JdbcType.OTHER); - columnMap.put(fk.toString(), type); - } - result.put("columnMap", columnMap); - } - } - result.put("jdbcType", jdbcType.name()); - return result; - } - } - - @JsonIgnoreProperties(ignoreUnknown = true) - public static class QueryImportTemplateForm - { - private String schemaName; - private String queryName; - private String auditUserComment; - private List templateLabels; - private List templateUrls; - private Long _lastKnownModified; - - public void setQueryName(String queryName) - { - this.queryName = queryName; - } - - public List getTemplateLabels() - { - return templateLabels == null ? Collections.emptyList() : templateLabels; - } - - public void setTemplateLabels(List templateLabels) - { - this.templateLabels = templateLabels; - } - - public List getTemplateUrls() - { - return templateUrls == null ? Collections.emptyList() : templateUrls; - } - - public void setTemplateUrls(List templateUrls) - { - this.templateUrls = templateUrls; - } - - public String getSchemaName() - { - return schemaName; - } - - @SuppressWarnings("unused") - public void setSchemaName(String schemaName) - { - this.schemaName = schemaName; - } - - public String getQueryName() - { - return queryName; - } - - public Long getLastKnownModified() - { - return _lastKnownModified; - } - - public void setLastKnownModified(Long lastKnownModified) - { - _lastKnownModified = lastKnownModified; - } - - public String getAuditUserComment() - { - return auditUserComment; - } - - public void setAuditUserComment(String auditUserComment) - { - this.auditUserComment = auditUserComment; - } - - } - - @Marshal(Marshaller.Jackson) - @RequiresPermission(ReadPermission.class) //Real permissions will be enforced later on by the DomainKind - public static class UpdateQueryImportTemplateAction extends MutatingApiAction - { - private DomainKind _kind; - private UserSchema _schema; - private TableInfo _tInfo; - private QueryDefinition _queryDef; - private Domain _domain; - - @Override - protected ObjectMapper createResponseObjectMapper() - { - return this.createRequestObjectMapper(); - } - - @Override - public void validateForm(QueryImportTemplateForm form, Errors errors) - { - User user = getUser(); - Container container = getContainer(); - String domainURI = PropertyService.get().getDomainURI(form.getSchemaName(), form.getQueryName(), container, user); - _kind = PropertyService.get().getDomainKind(domainURI); - _domain = PropertyService.get().getDomain(container, domainURI); - if (_domain == null) - throw new IllegalArgumentException("Domain '" + domainURI + "' not found."); - - if (!_kind.canEditDefinition(user, _domain)) - throw new UnauthorizedException("You don't have permission to update import templates for this domain."); - - QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); - if (!(querySchema instanceof UserSchema _schema)) - throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'."); - QuerySettings settings = _schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); - _queryDef = settings.getQueryDef(_schema); - if (null == _queryDef) - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); - if (!_queryDef.isMetadataEditable()) - throw new UnsupportedOperationException("Query metadata is not editable."); - _tInfo = _queryDef.getTable(_schema, new ArrayList<>(), true, true); - if (_tInfo == null) - throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); - - } - - private Map getRowFiles() - { - Map rowFiles = new IntHashMap<>(); - if (getFileMap() != null) - { - for (Map.Entry fileEntry : getFileMap().entrySet()) - { - // allow for the fileMap key to include the row index for defining which row to attach this file to - // ex: "templateFile::0", "templateFile::1" - String fieldKey = fileEntry.getKey(); - int delimIndex = fieldKey.lastIndexOf("::"); - if (delimIndex > -1) - { - Integer fieldRowIndex = Integer.parseInt(fieldKey.substring(delimIndex + 2)); - SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); - rowFiles.put(fieldRowIndex, file.isEmpty() ? null : file); - } - } - } - return rowFiles; - } - - private List> getUploadedTemplates(QueryImportTemplateForm form, DomainKind kind) throws ValidationException, QueryUpdateServiceException, ExperimentException - { - FileContentService fcs = FileContentService.get(); - if (fcs == null) - throw new IllegalStateException("Unable to load file service."); - - User user = getUser(); - Container container = getContainer(); - - Map rowFiles = getRowFiles(); - List templateLabels = form.getTemplateLabels(); - Set labels = new HashSet<>(templateLabels); - if (labels.size() < templateLabels.size()) - throw new IllegalArgumentException("Duplicate template name is not allowed."); - - List templateUrls = form.getTemplateUrls(); - List> uploadedTemplates = new ArrayList<>(); - for (int rowIndex = 0; rowIndex < form.getTemplateLabels().size(); rowIndex++) - { - String templateLabel = templateLabels.get(rowIndex); - if (StringUtils.isBlank(templateLabel.trim())) - throw new IllegalArgumentException("Template name cannot be blank."); - String templateUrl = templateUrls.get(rowIndex); - Object file = rowFiles.get(rowIndex); - if (StringUtils.isEmpty(templateUrl) && file == null) - throw new IllegalArgumentException("Template file is not provided."); - - if (file instanceof MultipartFile || file instanceof SpringAttachmentFile) - { - String fileName; - if (file instanceof MultipartFile f) - fileName = f.getName(); - else - { - SpringAttachmentFile f = (SpringAttachmentFile) file; - fileName = f.getFilename(); - } - String fileNameValidation = FileUtil.validateFileName(fileName); - if (!StringUtils.isEmpty(fileNameValidation)) - throw new IllegalArgumentException(fileNameValidation); - - FileLike uploadDir = ensureUploadDirectory(container, kind.getDomainFileDirectory()); - uploadDir = uploadDir.resolveChild("_templates"); - Object savedFile = saveFile(user, container, "template file", file, uploadDir); - Path savedFilePath; - - if (savedFile instanceof File ioFile) - savedFilePath = ioFile.toPath(); - else if (savedFile instanceof FileLike fl) - savedFilePath = fl.toNioPathForRead(); - else - throw UnexpectedException.wrap(null,"Unable to upload template file."); - - templateUrl = fcs.getWebDavUrl(savedFilePath, container, FileContentService.PathType.serverRelative).toString(); - } - - uploadedTemplates.add(Pair.of(templateLabel, templateUrl)); - } - return uploadedTemplates; - } - - @Override - public Object execute(QueryImportTemplateForm form, BindException errors) throws ValidationException, QueryUpdateServiceException, SQLException, ExperimentException, MetadataUnavailableException - { - User user = getUser(); - Container container = getContainer(); - String schemaName = form.getSchemaName(); - String queryName = form.getQueryName(); - QueryDef queryDef = QueryManager.get().getQueryDef(container, schemaName, queryName, false); - if (queryDef != null && queryDef.getQueryDefId() != 0) - { - Long lastKnownModified = form.getLastKnownModified(); - if (lastKnownModified == null || lastKnownModified != queryDef.getModified().getTime()) - throw new ApiUsageException("Unable to save import templates. The templates appear out of date, reload the page and try again."); - } - - List> updatedTemplates = getUploadedTemplates(form, _kind); - - List> existingTemplates = _tInfo.getImportTemplates(getViewContext()); - List> existingCustomTemplates = new ArrayList<>(); - for (Pair template_ : existingTemplates) - { - if (!template_.second.toLowerCase().contains("exportexceltemplate")) - existingCustomTemplates.add(template_); - } - if (!updatedTemplates.equals(existingCustomTemplates)) - { - TablesDocument doc = null; - TableType xmlTable = null; - TableType.ImportTemplates xmlImportTemplates; - - if (queryDef != null) - { - try - { - doc = parseDocument(queryDef.getMetaData()); - } - catch (XmlException e) - { - throw new MetadataUnavailableException(e.getMessage()); - } - xmlTable = getTableType(form.getQueryName(), doc); - // when there is a queryDef but xmlTable is null it means the xmlMetaData contains tableName which does not - // match with actual queryName then reconstruct the xml table metadata : See Issue 43523 - if (xmlTable == null) - { - doc = null; - } - } - else - { - queryDef = new QueryDef(); - queryDef.setSchema(schemaName); - queryDef.setContainer(container.getId()); - queryDef.setName(queryName); - } - - if (doc == null) - { - doc = TablesDocument.Factory.newInstance(); - } - - if (xmlTable == null) - { - TablesType tables = doc.addNewTables(); - xmlTable = tables.addNewTable(); - xmlTable.setTableName(queryName); - } - - if (xmlTable.getTableDbType() == null) - { - xmlTable.setTableDbType("NOT_IN_DB"); - } - - // remove existing templates - if (xmlTable.isSetImportTemplates()) - xmlTable.unsetImportTemplates(); - xmlImportTemplates = xmlTable.addNewImportTemplates(); - - // set new templates - if (!updatedTemplates.isEmpty()) - { - for (Pair template_ : updatedTemplates) - { - ImportTemplateType importTemplateType = xmlImportTemplates.addNewTemplate(); - importTemplateType.setLabel(template_.first); - importTemplateType.setUrl(template_.second); - } - } - - XmlOptions xmlOptions = new XmlOptions(); - xmlOptions.setSavePrettyPrint(); - // Don't use an explicit namespace, making the XML much more readable - xmlOptions.setUseDefaultNamespace(); - queryDef.setMetaData(doc.xmlText(xmlOptions)); - if (queryDef.getQueryDefId() == 0) - { - QueryManager.get().insert(user, queryDef); - } - else - { - QueryManager.get().update(user, queryDef); - } - - DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "Import templates updated."); - event.setUserComment(form.getAuditUserComment()); - event.setDomainUri(_domain.getTypeURI()); - event.setDomainName(_domain.getName()); - AuditLogService.get().addEvent(user, event); - } - - ApiSimpleResponse resp = new ApiSimpleResponse(); - resp.put("success", true); - return resp; - } - } - - - public static class TestCase extends AbstractActionPermissionTest - { - @Override - public void testActionPermissions() - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - QueryController controller = new QueryController(); - - // @RequiresPermission(ReadPermission.class) - assertForReadPermission(user, false, - new BrowseAction(), - new BeginAction(), - controller.new SchemaAction(), - controller.new SourceQueryAction(), - controller.new ExecuteQueryAction(), - controller.new PrintRowsAction(), - new ExportScriptAction(), - new ExportRowsExcelAction(), - new ExportRowsXLSXAction(), - new ExportQueriesXLSXAction(), - new ExportExcelTemplateAction(), - new ExportRowsTsvAction(), - new ExcelWebQueryDefinitionAction(), - controller.new SaveQueryViewsAction(), - controller.new PropertiesQueryAction(), - controller.new SelectRowsAction(), - new GetDataAction(), - controller.new ExecuteSqlAction(), - controller.new SelectDistinctAction(), - controller.new GetColumnSummaryStatsAction(), - controller.new ImportAction(), - new ExportSqlAction(), - new UpdateRowsAction(), - new ImportRowsAction(), - new DeleteRowsAction(), - new TableInfoAction(), - new SaveSessionViewAction(), - new GetSchemasAction(), - new GetQueriesAction(), - new GetQueryViewsAction(), - new SaveApiTestAction(), - new ValidateQueryMetadataAction(), - new AuditHistoryAction(), - new AuditDetailsAction(), - new ExportTablesAction(), - new SaveNamedSetAction(), - new DeleteNamedSetAction(), - new ApiTestAction(), - new GetDefaultVisibleColumnsAction() - ); - - - // submitter should be allowed for InsertRows - assertForReadPermission(user, true, new InsertRowsAction()); - - // @RequiresPermission(DeletePermission.class) - assertForUpdateOrDeletePermission(user, - new DeleteQueryRowsAction() - ); - - // @RequiresPermission(AdminPermission.class) - assertForAdminPermission(user, - new DeleteQueryAction(), - controller.new MetadataQueryAction(), - controller.new NewQueryAction(), - new SaveSourceQueryAction(), - - new TruncateTableAction(), - new AdminAction(), - new ManageRemoteConnectionsAction(), - new ReloadExternalSchemaAction(), - new ReloadAllUserSchemas(), - controller.new ManageViewsAction(), - controller.new InternalDeleteView(), - controller.new InternalSourceViewAction(), - controller.new InternalNewViewAction(), - new QueryExportAuditRedirectAction() - ); - - // @RequiresPermission(AdminOperationsPermission.class) - assertForAdminOperationsPermission(user, - new EditRemoteConnectionAction(), - new DeleteRemoteConnectionAction(), - new TestRemoteConnectionAction(), - controller.new RawTableMetaDataAction(), - controller.new RawSchemaMetaDataAction(), - new InsertLinkedSchemaAction(), - new InsertExternalSchemaAction(), - new DeleteSchemaAction(), - new EditLinkedSchemaAction(), - new EditExternalSchemaAction(), - new GetTablesAction(), - new SchemaTemplateAction(), - new SchemaTemplatesAction(), - new ParseExpressionAction(), - new ParseQueryAction() - ); - - // @AdminConsoleAction - assertForAdminPermission(ContainerManager.getRoot(), user, - new DataSourceAdminAction() - ); - - // In addition to administrators (tested above), trusted analysts who are editors can create and edit queries - assertTrustedEditorPermission( - new DeleteQueryAction(), - controller.new MetadataQueryAction(), - controller.new NewQueryAction(), - new SaveSourceQueryAction() - ); - } - } - - public static class SaveRowsTestCase extends Assert - { - private static final String PROJECT_NAME1 = "SaveRowsTestProject1"; - private static final String PROJECT_NAME2 = "SaveRowsTestProject2"; - - private static final String USER_EMAIL = "saveRows@action.test"; - - private static final String LIST1 = "List1"; - private static final String LIST2 = "List2"; - - @Before - public void doSetup() throws Exception - { - doCleanup(); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME1, TestContext.get().getUser()); - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME2, TestContext.get().getUser()); - - //disable search so we dont get conflicts when deleting folder quickly - ContainerManager.updateSearchable(project1, false, TestContext.get().getUser()); - ContainerManager.updateSearchable(project2, false, TestContext.get().getUser()); - - ListDefinition ld1 = ListService.get().createList(project1, LIST1, ListDefinition.KeyType.Varchar); - ld1.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); - ld1.setKeyName("TextField"); - ld1.save(TestContext.get().getUser()); - - ListDefinition ld2 = ListService.get().createList(project2, LIST2, ListDefinition.KeyType.Varchar); - ld2.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); - ld2.setKeyName("TextField"); - ld2.save(TestContext.get().getUser()); - } - - @After - public void doCleanup() throws Exception - { - Container project = ContainerManager.getForPath(PROJECT_NAME1); - if (project != null) - { - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - if (project2 != null) - { - ContainerManager.deleteAll(project2, TestContext.get().getUser()); - } - - User u = UserManager.getUser(new ValidEmail(USER_EMAIL)); - if (u != null) - { - UserManager.deleteUser(u.getUserId()); - } - } - - private JSONObject getCommand(String val1, String val2) - { - JSONObject command1 = new JSONObject(); - command1.put("containerPath", ContainerManager.getForPath(PROJECT_NAME1).getPath()); - command1.put("command", "insert"); - command1.put("schemaName", "lists"); - command1.put("queryName", LIST1); - command1.put("rows", getTestRows(val1)); - - JSONObject command2 = new JSONObject(); - command2.put("containerPath", ContainerManager.getForPath(PROJECT_NAME2).getPath()); - command2.put("command", "insert"); - command2.put("schemaName", "lists"); - command2.put("queryName", LIST2); - command2.put("rows", getTestRows(val2)); - - JSONObject json = new JSONObject(); - json.put("commands", Arrays.asList(command1, command2)); - - return json; - } - - private MockHttpServletResponse makeRequest(JSONObject json, User user) throws Exception - { - Map headers = new HashMap<>(); - headers.put("Content-Type", "application/json"); - - HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), DetailsURL.fromString("/query/saveRows.view").copy(ContainerManager.getForPath(PROJECT_NAME1)).getActionURL(), user, headers, json.toString()); - return ViewServlet.mockDispatch(request, null); - } - - @Test - public void testCrossFolderSaveRows() throws Exception - { - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - JSONObject json = getCommand(PROJECT_NAME1, PROJECT_NAME2); - MockHttpServletResponse response = makeRequest(json, TestContext.get().getUser()); - if (response.getStatus() != HttpServletResponse.SC_OK) - { - JSONObject responseJson = new JSONObject(response.getContentAsString()); - throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); - } - - Container project1 = ContainerManager.getForPath(PROJECT_NAME1); - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - - TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); - TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); - - assertEquals("Incorrect row count, list1", 1L, new TableSelector(list1).getRowCount()); - assertEquals("Incorrect row count, list2", 1L, new TableSelector(list2).getRowCount()); - - assertEquals("Incorrect value", PROJECT_NAME1, new TableSelector(list1, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME1, String.class)); - assertEquals("Incorrect value", PROJECT_NAME2, new TableSelector(list2, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME2, String.class)); - - list1.getUpdateService().truncateRows(TestContext.get().getUser(), project1, null, null); - list2.getUpdateService().truncateRows(TestContext.get().getUser(), project2, null, null); - } - - @Test - public void testWithoutPermissions() throws Exception - { - // Now test failure without appropriate permissions: - User withoutPermissions = SecurityManager.addUser(new ValidEmail(USER_EMAIL), TestContext.get().getUser()).getUser(); - - User user = TestContext.get().getUser(); - assertTrue(user.hasSiteAdminPermission()); - - Container project1 = ContainerManager.getForPath(PROJECT_NAME1); - Container project2 = ContainerManager.getForPath(PROJECT_NAME2); - - MutableSecurityPolicy securityPolicy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(project1)); - securityPolicy.addRoleAssignment(withoutPermissions, EditorRole.class); - SecurityPolicyManager.savePolicyForTests(securityPolicy, TestContext.get().getUser()); - - assertTrue("Should have insert permission", project1.hasPermission(withoutPermissions, InsertPermission.class)); - assertFalse("Should not have insert permission", project2.hasPermission(withoutPermissions, InsertPermission.class)); - - // repeat insert: - JSONObject json = getCommand("ShouldFail1", "ShouldFail2"); - MockHttpServletResponse response = makeRequest(json, withoutPermissions); - if (response.getStatus() != HttpServletResponse.SC_FORBIDDEN) - { - JSONObject responseJson = new JSONObject(response.getContentAsString()); - throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); - } - - TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); - TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); - - // The insert should have failed - assertEquals("Incorrect row count, list1", 0L, new TableSelector(list1).getRowCount()); - assertEquals("Incorrect row count, list2", 0L, new TableSelector(list2).getRowCount()); - } - - private JSONArray getTestRows(String val) - { - JSONArray rows = new JSONArray(); - rows.put(Map.of("TextField", val)); - - return rows; - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.query.controllers; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpSession; +import org.antlr.runtime.tree.Tree; +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.collections4.multimap.ArrayListValuedHashMap; +import org.apache.commons.collections4.multimap.HashSetValuedHashMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.xmlbeans.XmlError; +import org.apache.xmlbeans.XmlException; +import org.apache.xmlbeans.XmlOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONParserConfiguration; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.action.Action; +import org.labkey.api.action.ActionType; +import org.labkey.api.action.ApiJsonForm; +import org.labkey.api.action.ApiJsonWriter; +import org.labkey.api.action.ApiQueryResponse; +import org.labkey.api.action.ApiResponse; +import org.labkey.api.action.ApiResponseWriter; +import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.action.ApiVersion; +import org.labkey.api.action.ConfirmAction; +import org.labkey.api.action.ExportAction; +import org.labkey.api.action.ExportException; +import org.labkey.api.action.ExtendedApiQueryResponse; +import org.labkey.api.action.FormHandlerAction; +import org.labkey.api.action.FormViewAction; +import org.labkey.api.action.HasBindParameters; +import org.labkey.api.action.JsonInputLimit; +import org.labkey.api.action.LabKeyError; +import org.labkey.api.action.Marshal; +import org.labkey.api.action.Marshaller; +import org.labkey.api.action.MutatingApiAction; +import org.labkey.api.action.NullSafeBindException; +import org.labkey.api.action.ReadOnlyApiAction; +import org.labkey.api.action.ReportingApiQueryResponse; +import org.labkey.api.action.SimpleApiJsonForm; +import org.labkey.api.action.SimpleErrorView; +import org.labkey.api.action.SimpleRedirectAction; +import org.labkey.api.action.SimpleViewAction; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.audit.provider.ContainerAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.IntHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.collections.RowMapFactory; +import org.labkey.api.collections.Sets; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.ActionButton; +import org.labkey.api.data.Aggregate; +import org.labkey.api.data.AnalyticsProviderItem; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ButtonBar; +import org.labkey.api.data.CachedResultSets; +import org.labkey.api.data.ColumnHeaderType; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.DataRegion; +import org.labkey.api.data.DataRegionSelection; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.DisplayColumn; +import org.labkey.api.data.ExcelWriter; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.JdbcMetaDataSelector; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.JsonWriter; +import org.labkey.api.data.PHI; +import org.labkey.api.data.PropertyManager; +import org.labkey.api.data.PropertyManager.PropertyMap; +import org.labkey.api.data.PropertyManager.WritablePropertyMap; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.QueryLogging; +import org.labkey.api.data.ResultSetView; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.ShowRows; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TSVWriter; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.VirtualTable; +import org.labkey.api.data.dialect.JdbcMetaDataLocator; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DataIteratorUtil; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.ListofMapsDataIterator; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.api.ProvenanceRecordingParams; +import org.labkey.api.exp.api.ProvenanceService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainKind; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.module.ModuleHtmlView; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.RecordedAction; +import org.labkey.api.query.AbstractQueryImportAction; +import org.labkey.api.query.AbstractQueryUpdateService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.CustomView; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.DuplicateKeyException; +import org.labkey.api.query.ExportScriptModel; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryAction; +import org.labkey.api.query.QueryDefinition; +import org.labkey.api.query.QueryException; +import org.labkey.api.query.QueryForm; +import org.labkey.api.query.QueryParam; +import org.labkey.api.query.QueryParseException; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryUpdateForm; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.QueryUrls; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.RuntimeValidationException; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.SimpleSchemaTreeVisitor; +import org.labkey.api.query.TempQuerySettings; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.UserSchemaAction; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.report.ReportDescriptor; +import org.labkey.api.security.ActionNames; +import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.CSRF; +import org.labkey.api.security.IgnoresTermsOfUse; +import org.labkey.api.security.MutableSecurityPolicy; +import org.labkey.api.security.RequiresAllOf; +import org.labkey.api.security.RequiresAnyOf; +import org.labkey.api.security.RequiresNoPermission; +import org.labkey.api.security.RequiresPermission; +import org.labkey.api.security.SecurityManager; +import org.labkey.api.security.SecurityPolicyManager; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.security.ValidEmail; +import org.labkey.api.security.permissions.AbstractActionPermissionTest; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.security.permissions.DeletePermission; +import org.labkey.api.security.permissions.EditSharedViewPermission; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.MoveEntitiesPermission; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.security.permissions.PlatformDeveloperPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.security.roles.EditorRole; +import org.labkey.api.settings.AdminConsole; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.LookAndFeelProperties; +import org.labkey.api.stats.BaseAggregatesAnalyticsProvider; +import org.labkey.api.stats.ColumnAnalyticsProvider; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.DOM; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.JavaScriptFragment; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.LinkBuilder; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.ResponseHelper; +import org.labkey.api.util.ReturnURLString; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URLHelper; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.util.XmlBeansUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.DetailsView; +import org.labkey.api.view.HtmlView; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.InsertView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.view.UpdateView; +import org.labkey.api.view.VBox; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.ViewServlet; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.PageConfig; +import org.labkey.api.writer.HtmlWriter; +import org.labkey.api.writer.ZipFile; +import org.labkey.data.xml.ColumnType; +import org.labkey.data.xml.ImportTemplateType; +import org.labkey.data.xml.TableType; +import org.labkey.data.xml.TablesDocument; +import org.labkey.data.xml.TablesType; +import org.labkey.data.xml.externalSchema.TemplateSchemaType; +import org.labkey.data.xml.queryCustomView.FilterType; +import org.labkey.query.AutoGeneratedDetailsCustomView; +import org.labkey.query.AutoGeneratedInsertCustomView; +import org.labkey.query.AutoGeneratedUpdateCustomView; +import org.labkey.query.CustomViewImpl; +import org.labkey.query.CustomViewUtil; +import org.labkey.query.EditQueriesPermission; +import org.labkey.query.EditableCustomView; +import org.labkey.query.LinkedTableInfo; +import org.labkey.query.MetadataTableJSON; +import org.labkey.query.ModuleCustomQueryDefinition; +import org.labkey.query.ModuleCustomView; +import org.labkey.query.QueryServiceImpl; +import org.labkey.query.TableXML; +import org.labkey.query.audit.QueryExportAuditProvider; +import org.labkey.query.audit.QueryUpdateAuditProvider; +import org.labkey.query.model.MetadataTableJSONMixin; +import org.labkey.query.persist.AbstractExternalSchemaDef; +import org.labkey.query.persist.CstmView; +import org.labkey.query.persist.ExternalSchemaDef; +import org.labkey.query.persist.ExternalSchemaDefCache; +import org.labkey.query.persist.LinkedSchemaDef; +import org.labkey.query.persist.QueryDef; +import org.labkey.query.persist.QueryManager; +import org.labkey.query.reports.ReportsController; +import org.labkey.query.reports.getdata.DataRequest; +import org.labkey.query.sql.QNode; +import org.labkey.query.sql.Query; +import org.labkey.query.sql.SqlParser; +import org.labkey.query.xml.ApiTestsDocument; +import org.labkey.query.xml.TestCaseType; +import org.labkey.remoteapi.RemoteConnections; +import org.labkey.remoteapi.SelectRowsStreamHack; +import org.labkey.remoteapi.query.SelectRowsCommand; +import org.labkey.vfs.FileLike; +import org.springframework.beans.MutablePropertyValues; +import org.springframework.beans.PropertyValue; +import org.springframework.beans.PropertyValues; +import org.springframework.dao.DataAccessException; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.validation.BindException; +import org.springframework.validation.Errors; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.ModelAndView; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; +import static org.labkey.api.action.ApiJsonWriter.CONTENT_TYPE_JSON; +import static org.labkey.api.assay.AssayFileWriter.ensureUploadDirectory; +import static org.labkey.api.data.DbScope.NO_OP_TRANSACTION; +import static org.labkey.api.query.AbstractQueryUpdateService.saveFile; +import static org.labkey.api.util.DOM.BR; +import static org.labkey.api.util.DOM.DIV; +import static org.labkey.api.util.DOM.FONT; +import static org.labkey.api.util.DOM.Renderable; +import static org.labkey.api.util.DOM.TABLE; +import static org.labkey.api.util.DOM.TD; +import static org.labkey.api.util.DOM.TR; +import static org.labkey.api.util.DOM.at; +import static org.labkey.api.util.DOM.cl; +import static org.labkey.query.MetadataTableJSON.getTableType; +import static org.labkey.query.MetadataTableJSON.parseDocument; + +@SuppressWarnings("DefaultAnnotationParam") + +public class QueryController extends SpringActionController +{ + private static final Logger LOG = LogManager.getLogger(QueryController.class); + private static final String ROW_ATTACHMENT_INDEX_DELIM = "::"; + + private static final Set RESERVED_VIEW_NAMES = CaseInsensitiveHashSet.of( + "Default", + AutoGeneratedDetailsCustomView.NAME, + AutoGeneratedInsertCustomView.NAME, + AutoGeneratedUpdateCustomView.NAME + ); + + private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(QueryController.class, + ValidateQueryAction.class, + ValidateQueriesAction.class, + GetSchemaQueryTreeAction.class, + GetQueryDetailsAction.class, + ViewQuerySourceAction.class + ); + + public QueryController() + { + setActionResolver(_actionResolver); + } + + public static void registerAdminConsoleLinks() + { + AdminConsole.addLink(AdminConsole.SettingsLinkType.Diagnostics, "data sources", new ActionURL(DataSourceAdminAction.class, ContainerManager.getRoot())); + } + + public static class RemoteQueryConnectionUrls + { + public static ActionURL urlManageRemoteConnection(Container c) + { + return new ActionURL(ManageRemoteConnectionsAction.class, c); + } + + public static ActionURL urlCreateRemoteConnection(Container c) + { + return new ActionURL(EditRemoteConnectionAction.class, c); + } + + public static ActionURL urlEditRemoteConnection(Container c, String connectionName) + { + ActionURL url = new ActionURL(EditRemoteConnectionAction.class, c); + url.addParameter("connectionName", connectionName); + return url; + } + + public static ActionURL urlSaveRemoteConnection(Container c) + { + return new ActionURL(EditRemoteConnectionAction.class, c); + } + + public static ActionURL urlDeleteRemoteConnection(Container c, @Nullable String connectionName) + { + ActionURL url = new ActionURL(DeleteRemoteConnectionAction.class, c); + if (connectionName != null) + url.addParameter("connectionName", connectionName); + return url; + } + + public static ActionURL urlTestRemoteConnection(Container c, String connectionName) + { + ActionURL url = new ActionURL(TestRemoteConnectionAction.class, c); + url.addParameter("connectionName", connectionName); + return url; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); + if (!errors.hasErrors()) + { + String name = remoteConnectionForm.getConnectionName(); + // package the remote-connection properties into the remoteConnectionForm and pass them along + Map map1 = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); + remoteConnectionForm.setUrl(map1.get("URL")); + remoteConnectionForm.setUserEmail(map1.get("user")); + remoteConnectionForm.setPassword(map1.get("password")); + remoteConnectionForm.setFolderPath(map1.get("container")); + } + setHelpTopic("remoteConnection"); + return new JspView<>("/org/labkey/query/view/createRemoteConnection.jsp", remoteConnectionForm, errors); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + return RemoteConnections.createOrEditRemoteConnection(remoteConnectionForm, getContainer(), errors); + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Create/Edit Remote Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class DeleteRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/query/view/confirmDeleteConnection.jsp", remoteConnectionForm, errors); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + remoteConnectionForm.setConnectionKind(RemoteConnections.CONNECTION_KIND_QUERY); + return RemoteConnections.deleteRemoteConnection(remoteConnectionForm, getContainer()); + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return RemoteQueryConnectionUrls.urlManageRemoteConnection(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Confirm Delete Connection", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestRemoteConnectionAction extends FormViewAction + { + @Override + public void validateCommand(RemoteConnections.RemoteConnectionForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(RemoteConnections.RemoteConnectionForm remoteConnectionForm, boolean reshow, BindException errors) + { + String name = remoteConnectionForm.getConnectionName(); + String schemaName = "core"; // test Schema Name + String queryName = "Users"; // test Query Name + + // Extract the username, password, and container from the secure property store + Map singleConnectionMap = RemoteConnections.getRemoteConnection(RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY, name, getContainer()); + if (singleConnectionMap.isEmpty()) + throw new NotFoundException(); + String url = singleConnectionMap.get(RemoteConnections.FIELD_URL); + String user = singleConnectionMap.get(RemoteConnections.FIELD_USER); + String password = singleConnectionMap.get(RemoteConnections.FIELD_PASSWORD); + String container = singleConnectionMap.get(RemoteConnections.FIELD_CONTAINER); + + // connect to the remote server and retrieve an input stream + org.labkey.remoteapi.Connection cn = new org.labkey.remoteapi.Connection(url, user, password); + final SelectRowsCommand cmd = new SelectRowsCommand(schemaName, queryName); + try + { + DataIteratorBuilder source = SelectRowsStreamHack.go(cn, container, cmd, getContainer()); + // immediately close the source after opening it, this is a test. + source.getDataIterator(new DataIteratorContext()).close(); + } + catch (Exception e) + { + errors.addError(new LabKeyError("The listed credentials for this remote connection failed to connect.")); + return new JspView<>("/org/labkey/query/view/testRemoteConnectionsFailure.jsp", remoteConnectionForm); + } + + return new JspView<>("/org/labkey/query/view/testRemoteConnectionsSuccess.jsp", remoteConnectionForm); + } + + @Override + public boolean handlePost(RemoteConnections.RemoteConnectionForm remoteConnectionForm, BindException errors) + { + return true; + } + + @Override + public URLHelper getSuccessURL(RemoteConnections.RemoteConnectionForm remoteConnectionForm) + { + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + public static class QueryUrlsImpl implements QueryUrls + { + @Override + public ActionURL urlSchemaBrowser(Container c) + { + return new ActionURL(BeginAction.class, c); + } + + @Override + public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName) + { + ActionURL ret = urlSchemaBrowser(c); + if (schemaName != null) + { + ret.addParameter(QueryParam.schemaName.toString(), schemaName); + } + return ret; + } + + @Override + public ActionURL urlSchemaBrowser(Container c, @Nullable String schemaName, @Nullable String queryName) + { + if (StringUtils.isEmpty(queryName)) + return urlSchemaBrowser(c, schemaName); + ActionURL ret = urlSchemaBrowser(c); + ret.addParameter(QueryParam.schemaName.toString(), trimToEmpty(schemaName)); + ret.addParameter(QueryParam.queryName.toString(), trimToEmpty(queryName)); + return ret; + } + + public ActionURL urlExternalSchemaAdmin(Container c) + { + return urlExternalSchemaAdmin(c, null); + } + + public ActionURL urlExternalSchemaAdmin(Container c, @Nullable String message) + { + ActionURL url = new ActionURL(AdminAction.class, c); + + if (null != message) + url.addParameter("message", message); + + return url; + } + + public ActionURL urlInsertExternalSchema(Container c) + { + return new ActionURL(InsertExternalSchemaAction.class, c); + } + + public ActionURL urlNewQuery(Container c) + { + return new ActionURL(NewQueryAction.class, c); + } + + public ActionURL urlUpdateExternalSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(EditExternalSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + public ActionURL urlReloadExternalSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(ReloadExternalSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + public ActionURL urlDeleteSchema(Container c, AbstractExternalSchemaDef def) + { + ActionURL url = new ActionURL(DeleteSchemaAction.class, c); + url.addParameter("externalSchemaId", Integer.toString(def.getExternalSchemaId())); + return url; + } + + @Override + public ActionURL urlStartBackgroundRReport(@NotNull ActionURL baseURL, String reportId) + { + ActionURL result = baseURL.clone(); + result.setAction(ReportsController.StartBackgroundRReportAction.class); + result.replaceParameter(ReportDescriptor.Prop.reportId, reportId); + return result; + } + + @Override + public ActionURL urlExecuteQuery(@NotNull ActionURL baseURL) + { + ActionURL result = baseURL.clone(); + result.setAction(ExecuteQueryAction.class); + return result; + } + + @Override + public ActionURL urlExecuteQuery(Container c, String schemaName, String queryName) + { + return new ActionURL(ExecuteQueryAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter(QueryParam.queryName, queryName); + } + + @Override + public @NotNull ActionURL urlCreateExcelTemplate(Container c, String schemaName, String queryName) + { + return new ActionURL(ExportExcelTemplateAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter("query.queryName", queryName); + } + + @Override + public ActionURL urlMetadataQuery(Container c, String schemaName, String queryName) + { + return new ActionURL(MetadataQueryAction.class, c) + .addParameter(QueryParam.schemaName, schemaName) + .addParameter(QueryParam.queryName, queryName); + } + } + + @Override + public PageConfig defaultPageConfig() + { + // set default help topic for query controller + PageConfig config = super.defaultPageConfig(); + config.setHelpTopic("querySchemaBrowser"); + return config; + } + + @AdminConsoleAction(AdminOperationsPermission.class) + public static class DataSourceAdminAction extends SimpleViewAction + { + public DataSourceAdminAction() + { + } + + public DataSourceAdminAction(ViewContext viewContext) + { + setViewContext(viewContext); + } + + @Override + public ModelAndView getView(Object o, BindException errors) + { + // Site Admin or Troubleshooter? Troubleshooters can see all the information but can't test data sources. + boolean hasAdminOpsPerms = getContainer().hasPermission(getUser(), AdminOperationsPermission.class); + List allDefs = QueryManager.get().getExternalSchemaDefs(null); + + MultiValuedMap byDataSourceName = new ArrayListValuedHashMap<>(); + + for (ExternalSchemaDef def : allDefs) + byDataSourceName.put(def.getDataSource(), def); + + MutableInt row = new MutableInt(); + + Renderable r = DOM.DIV( + DIV("This page lists all the data sources defined in your " + AppProps.getInstance().getWebappConfigurationFilename() + " file that were available when first referenced and the external schemas defined in each."), + BR(), + TABLE(cl("labkey-data-region"), + TR(cl("labkey-show-borders"), + hasAdminOpsPerms ? TD(cl("labkey-column-header"), "Test") : null, + TD(cl("labkey-column-header"), "Data Source"), + TD(cl("labkey-column-header"), "Current Status"), + TD(cl("labkey-column-header"), "URL"), + TD(cl("labkey-column-header"), "Database Name"), + TD(cl("labkey-column-header"), "Product Name"), + TD(cl("labkey-column-header"), "Product Version"), + TD(cl("labkey-column-header"), "Max Connections"), + TD(cl("labkey-column-header"), "Active Connections"), + TD(cl("labkey-column-header"), "Idle Connections"), + TD(cl("labkey-column-header"), "Max Wait (ms)") + ), + DbScope.getDbScopes().stream() + .flatMap(scope -> { + String rowStyle = row.getAndIncrement() % 2 == 0 ? "labkey-alternate-row labkey-show-borders" : "labkey-row labkey-show-borders"; + Object status; + boolean connected = false; + try (Connection ignore = scope.getConnection()) + { + status = "connected"; + connected = true; + } + catch (Exception e) + { + status = FONT(cl("labkey-error"), "disconnected"); + } + + return Stream.of( + TR( + cl(rowStyle), + hasAdminOpsPerms ? TD(connected ? new ButtonBuilder("Test").href(new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", scope.getDataSourceName())) : "") : null, + TD(HtmlString.NBSP, scope.getDisplayName()), + TD(status), + TD(scope.getDatabaseUrl()), + TD(scope.getDatabaseName()), + TD(scope.getDatabaseProductName()), + TD(scope.getDatabaseProductVersion()), + TD(scope.getDataSourceProperties().getMaxTotal()), + TD(scope.getDataSourceProperties().getNumActive()), + TD(scope.getDataSourceProperties().getNumIdle()), + TD(scope.getDataSourceProperties().getMaxWaitMillis()) + ), + TR( + cl(rowStyle), + TD(HtmlString.NBSP), + TD(at(DOM.Attribute.colspan, 10), getDataSourceTable(byDataSourceName.get(scope.getDataSourceName()))) + ) + ); + }) + ) + ); + + return new HtmlView(r); + } + + private Renderable getDataSourceTable(Collection dsDefs) + { + if (dsDefs.isEmpty()) + return TABLE(TR(TD(HtmlString.NBSP))); + + MultiValuedMap byContainerPath = new ArrayListValuedHashMap<>(); + + for (ExternalSchemaDef def : dsDefs) + byContainerPath.put(def.getContainerPath(), def); + + TreeSet paths = new TreeSet<>(byContainerPath.keySet()); + + return TABLE(paths.stream() + .map(path -> TR(TD(at(DOM.Attribute.colspan, 4), getDataSourcePath(path, byContainerPath.get(path))))) + ); + } + + private Renderable getDataSourcePath(String path, Collection unsorted) + { + List defs = new ArrayList<>(unsorted); + defs.sort(Comparator.comparing(AbstractExternalSchemaDef::getUserSchemaName, String.CASE_INSENSITIVE_ORDER)); + Container c = ContainerManager.getForPath(path); + + if (null == c) + return TD(); + + boolean hasRead = c.hasPermission(getUser(), ReadPermission.class); + QueryUrlsImpl urls = new QueryUrlsImpl(); + + return + TD(TABLE( + TR(TD( + at(DOM.Attribute.colspan, 3), + hasRead ? LinkBuilder.simpleLink(path, urls.urlExternalSchemaAdmin(c)) : path + )), + TR(TD(TABLE( + defs.stream() + .map(def -> TR(TD( + at(DOM.Attribute.style, "padding-left:20px"), + hasRead ? LinkBuilder.simpleLink(def.getUserSchemaName() + + (!Strings.CS.equals(def.getSourceSchemaName(), def.getUserSchemaName()) ? " (" + def.getSourceSchemaName() + ")" : ""), urls.urlUpdateExternalSchema(c, def)) + : def.getUserSchemaName() + ))) + ))) + )); + } + + @Override + public void addNavTrail(NavTree root) + { + urlProvider(AdminUrls.class).addAdminNavTrail(root, "Data Source Administration", getClass(), getContainer()); + } + } + + public static class TestDataSourceForm + { + private String _dataSource; + + public String getDataSource() + { + return _dataSource; + } + + @SuppressWarnings("unused") + public void setDataSource(String dataSource) + { + _dataSource = dataSource; + } + } + + public static class TestDataSourceConfirmForm extends TestDataSourceForm + { + private String _excludeSchemas; + private String _excludeTables; + + public String getExcludeSchemas() + { + return _excludeSchemas; + } + + @SuppressWarnings("unused") + public void setExcludeSchemas(String excludeSchemas) + { + _excludeSchemas = excludeSchemas; + } + + public String getExcludeTables() + { + return _excludeTables; + } + + @SuppressWarnings("unused") + public void setExcludeTables(String excludeTables) + { + _excludeTables = excludeTables; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestDataSourceConfirmAction extends FormViewAction + { + private DbScope _scope; + + @Override + public ModelAndView getView(TestDataSourceConfirmForm form, boolean reshow, BindException errors) throws Exception + { + validateCommand(form, errors); + return new JspView<>("/org/labkey/query/view/testDataSourceConfirm.jsp", _scope); + } + + @Override + public void validateCommand(TestDataSourceConfirmForm form, Errors errors) + { + _scope = DbScope.getDbScope(form.getDataSource()); + + if (null == _scope) + throw new NotFoundException("Could not resolve data source " + form.getDataSource()); + } + + @Override + public boolean handlePost(TestDataSourceConfirmForm form, BindException errors) throws Exception + { + saveTestDataSourceProperties(form); + return true; + } + + @Override + public URLHelper getSuccessURL(TestDataSourceConfirmForm form) + { + return new ActionURL(TestDataSourceAction.class, getContainer()).addParameter("dataSource", _scope.getDataSourceName()); + } + + @Override + public void addNavTrail(NavTree root) + { + new DataSourceAdminAction(getViewContext()).addNavTrail(root); + root.addChild("Prepare Test of " + _scope.getDataSourceName()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class TestDataSourceAction extends SimpleViewAction + { + private DbScope _scope; + + @Override + public ModelAndView getView(TestDataSourceForm form, BindException errors) + { + _scope = DbScope.getDbScope(form.getDataSource()); + + if (null == _scope) + throw new NotFoundException("Could not resolve data source " + form.getDataSource()); + + return new JspView<>("/org/labkey/query/view/testDataSource.jsp", _scope); + } + + @Override + public void addNavTrail(NavTree root) + { + new DataSourceAdminAction(getViewContext()).addNavTrail(root); + root.addChild("Test " + _scope.getDataSourceName()); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ResetDataSourcePropertiesAction extends FormHandlerAction + { + @Override + public void validateCommand(TestDataSourceForm target, Errors errors) + { + } + + @Override + public boolean handlePost(TestDataSourceForm form, BindException errors) throws Exception + { + WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), false); + if (map != null) + map.delete(); + return true; + } + + @Override + public URLHelper getSuccessURL(TestDataSourceForm form) + { + return new ActionURL(TestDataSourceConfirmAction.class, getContainer()).addParameter("dataSource", form.getDataSource()) ; + } + } + + private static final String TEST_DATA_SOURCE_CATEGORY = "testDataSourceProperties"; + private static final String TEST_DATA_SOURCE_SCHEMAS_PROPERTY = "excludeSchemas"; + private static final String TEST_DATA_SOURCE_TABLES_PROPERTY = "excludeTables"; + + private static String getCategory(String dataSourceName) + { + return TEST_DATA_SOURCE_CATEGORY + "|" + dataSourceName; + } + + public static void saveTestDataSourceProperties(TestDataSourceConfirmForm form) + { + WritablePropertyMap map = PropertyManager.getWritableProperties(getCategory(form.getDataSource()), true); + // Save empty entries as empty string to distinguish from null (which results in default values) + map.put(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, StringUtils.trimToEmpty(form.getExcludeSchemas())); + map.put(TEST_DATA_SOURCE_TABLES_PROPERTY, StringUtils.trimToEmpty(form.getExcludeTables())); + map.save(); + } + + public static TestDataSourceConfirmForm getTestDataSourceProperties(DbScope scope) + { + TestDataSourceConfirmForm form = new TestDataSourceConfirmForm(); + PropertyMap map = PropertyManager.getProperties(getCategory(scope.getDataSourceName())); + form.setExcludeSchemas(map.getOrDefault(TEST_DATA_SOURCE_SCHEMAS_PROPERTY, scope.getSqlDialect().getDefaultSchemasToExcludeFromTesting())); + form.setExcludeTables(map.getOrDefault(TEST_DATA_SOURCE_TABLES_PROPERTY, scope.getSqlDialect().getDefaultTablesToExcludeFromTesting())); + + return form; + } + + @RequiresPermission(ReadPermission.class) + public static class BrowseAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/query/view/browse.jsp", null); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Schema Browser"); + } + } + + @RequiresPermission(ReadPermission.class) + public static class BeginAction extends QueryViewAction + { + @SuppressWarnings("UnusedDeclaration") + public BeginAction() + { + } + + public BeginAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + JspView view = new JspView<>("/org/labkey/query/view/browse.jsp", form); + view.setFrame(WebPartView.FrameType.NONE); + return view; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Query Schema Browser", new QueryUrlsImpl().urlSchemaBrowser(getContainer())); + } + } + + @RequiresPermission(ReadPermission.class) + public class SchemaAction extends QueryViewAction + { + public SchemaAction() {} + + SchemaAction(QueryForm form) + { + _form = form; + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + _form = form; + return new JspView<>("/org/labkey/query/view/browse.jsp", form); + } + + @Override + public void addNavTrail(NavTree root) + { + if (_form != null && _form.getSchema() != null) + addSchemaActionNavTrail(root, _form.getSchema().getSchemaPath(), _form.getQueryName()); + } + } + + + void addSchemaActionNavTrail(NavTree root, SchemaKey schemaKey, String queryName) + { + if (getContainer().hasOneOf(getUser(), AdminPermission.class, PlatformDeveloperPermission.class)) + { + // Don't show the full query nav trail to non-admin/non-developer users as they almost certainly don't + // want it + try + { + String schemaName = schemaKey.toDisplayString(); + ActionURL url = new ActionURL(BeginAction.class, getContainer()); + url.addParameter("schemaName", schemaKey.toString()); + url.addParameter("queryName", queryName); + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild(schemaName + " Schema", url); + } + catch (NullPointerException e) + { + LOG.error("NullPointerException in addNavTrail", e); + } + } + } + + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.SelectData.class) + public class NewQueryAction extends FormViewAction + { + private NewQueryForm _form; + private ActionURL _successUrl; + + @Override + public void validateCommand(NewQueryForm target, org.springframework.validation.Errors errors) + { + target.ff_newQueryName = StringUtils.trimToNull(target.ff_newQueryName); + if (null == target.ff_newQueryName) + errors.reject(ERROR_MSG, "QueryName is required"); + } + + @Override + public ModelAndView getView(NewQueryForm form, boolean reshow, BindException errors) + { + form.ensureSchemaExists(); + + if (!form.getSchema().canCreate()) + { + throw new UnauthorizedException(); + } + + getPageConfig().setFocusId("ff_newQueryName"); + _form = form; + setHelpTopic("sqlTutorial"); + return new JspView<>("/org/labkey/query/view/newQuery.jsp", form, errors); + } + + @Override + public boolean handlePost(NewQueryForm form, BindException errors) + { + form.ensureSchemaExists(); + + if (!form.getSchema().canCreate()) + { + throw new UnauthorizedException(); + } + + try + { + if (StringUtils.isEmpty(form.ff_baseTableName)) + { + errors.reject(ERROR_MSG, "You must select a base table or query name."); + return false; + } + + UserSchema schema = form.getSchema(); + String newQueryName = form.ff_newQueryName; + QueryDef existing = QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), newQueryName, true); + if (existing != null) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); + return false; + } + TableInfo existingTable = form.getSchema().getTable(newQueryName, null); + if (existingTable != null) + { + errors.reject(ERROR_MSG, "A table with the name '" + newQueryName + "' already exists."); + return false; + } + // bug 6095 -- conflicting query and dataset names + if (form.getSchema().getTableNames().contains(newQueryName)) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists as a table"); + return false; + } + QueryDefinition newDef = QueryService.get().createQueryDef(getUser(), getContainer(), form.getSchemaKey(), form.ff_newQueryName); + Query query = new Query(schema); + query.setRootTable(FieldKey.fromParts(form.ff_baseTableName)); + String sql = query.getQueryText(); + if (null == sql) + sql = "SELECT * FROM \"" + form.ff_baseTableName + "\""; + newDef.setSql(sql); + + try + { + newDef.save(getUser(), getContainer()); + } + catch (SQLException x) + { + if (RuntimeSQLException.isConstraintException(x)) + { + errors.reject(ERROR_MSG, "The query '" + newQueryName + "' already exists."); + return false; + } + else + { + throw x; + } + } + + _successUrl = newDef.urlFor(form.ff_redirect); + return true; + } + catch (Exception e) + { + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); + errors.reject(ERROR_MSG, Objects.toString(e.getMessage(), e.toString())); + return false; + } + } + + @Override + public ActionURL getSuccessURL(NewQueryForm newQueryForm) + { + return _successUrl; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + root.addChild("New Query", new QueryUrlsImpl().urlNewQuery(getContainer())); + } + } + + // CONSIDER : deleting this action after the SQL editor UI changes are finalized, keep in mind that built-in views + // use this view as well via the edit metadata page. + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) // Note: This action deals with just meta data; it AJAXes data into place using GetWebPartAction + public class SourceQueryAction extends SimpleViewAction + { + public SourceForm _form; + public UserSchema _schema; + public QueryDefinition _queryDef; + + + @Override + public void validate(SourceForm target, BindException errors) + { + _form = target; + if (StringUtils.isEmpty(target.getSchemaName())) + throw new NotFoundException("schema name not specified"); + if (StringUtils.isEmpty(target.getQueryName())) + throw new NotFoundException("query name not specified"); + + QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); + if (null == querySchema) + throw new NotFoundException("schema not found: " + _form.getSchemaKey().toDisplayString()); + if (!(querySchema instanceof UserSchema)) + throw new NotFoundException("Could not find the schema '" + _form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); + _schema = (UserSchema)querySchema; + } + + + @Override + public ModelAndView getView(SourceForm form, BindException errors) + { + _queryDef = _schema.getQueryDef(form.getQueryName()); + if (null == _queryDef) + _queryDef = _schema.getQueryDefForTable(form.getQueryName()); + if (null == _queryDef) + throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + try + { + if (form.ff_queryText == null) + { + form.ff_queryText = _queryDef.getSql(); + form.ff_metadataText = _queryDef.getMetadataXml(); + if (null == form.ff_metadataText) + form.ff_metadataText = form.getDefaultMetadataText(); + } + + for (QueryException qpe : _queryDef.getParseErrors(_schema)) + { + errors.reject(ERROR_MSG, Objects.toString(qpe.getMessage(), qpe.toString())); + } + } + catch (Exception e) + { + try + { + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), e); + } + catch (Throwable t) + { + // + } + errors.reject("ERROR_MSG", e.toString()); + LOG.error("Error", e); + } + + Renderable moduleWarning = null; + if (_queryDef instanceof ModuleCustomQueryDefinition mcqd && _queryDef.canEdit(getUser())) + { + moduleWarning = DIV(cl("labkey-warning-messages"), + "This SQL query is defined in the '" + mcqd.getModuleName() + "' module in directory '" + mcqd.getSqlFile().getParent() + "'.", + BR(), + "Changes to this query will be reflected in all usages across different folders on the server." + ); + } + + var sourceQueryView = new JspView<>("/org/labkey/query/view/sourceQuery.jsp", this, errors); + WebPartView ret = sourceQueryView; + if (null != moduleWarning) + ret = new VBox(new HtmlView(moduleWarning), sourceQueryView); + return ret; + } + + @Override + public void addNavTrail(NavTree root) + { + setHelpTopic("useSqlEditor"); + + addSchemaActionNavTrail(root, _form.getSchemaKey(), _form.getQueryName()); + + root.addChild("Edit " + _form.getQueryName()); + } + } + + + /** + * Ajax action to save a query. If the save is successful the request will return successfully. A query + * with SQL syntax errors can still be saved successfully. + * + * If the SQL contains parse errors, a parseErrors object will be returned which contains an array of + * JSON serialized error information. + */ + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.Configure.class) + public static class SaveSourceQueryAction extends MutatingApiAction + { + private UserSchema _schema; + + @Override + public void validateForm(SourceForm form, Errors errors) + { + if (StringUtils.isEmpty(form.getSchemaName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + if (StringUtils.isEmpty(form.getQueryName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + + QuerySchema querySchema = DefaultSchema.get(getUser(), getContainer(), form.getSchemaKey()); + if (null == querySchema) + throw new NotFoundException("schema not found: " + form.getSchemaKey().toDisplayString()); + if (!(querySchema instanceof UserSchema)) + throw new NotFoundException("Could not find the schema '" + form.getSchemaName() + "' in the folder '" + getContainer().getPath() + "'"); + _schema = (UserSchema)querySchema; + + XmlOptions options = XmlBeansUtil.getDefaultParseOptions(); + List xmlErrors = new ArrayList<>(); + options.setErrorListener(xmlErrors); + try + { + // had a couple of real-world failures due to null pointers in this code, so it's time to be paranoid + if (form.ff_metadataText != null) + { + TablesDocument tablesDoc = TablesDocument.Factory.parse(form.ff_metadataText, options); + if (tablesDoc != null) + { + tablesDoc.validate(options); + TablesType tablesType = tablesDoc.getTables(); + if (tablesType != null) + { + for (TableType tableType : tablesType.getTableArray()) + { + if (null != tableType) + { + if (!Objects.equals(tableType.getTableName(), form.getQueryName())) + { + errors.reject(ERROR_MSG, "Table name in the XML metadata must match the table/query name: " + form.getQueryName()); + } + + TableType.Columns tableColumns = tableType.getColumns(); + if (null != tableColumns) + { + ColumnType[] tableColumnArray = tableColumns.getColumnArray(); + for (ColumnType column : tableColumnArray) + { + if (column.isSetPhi() || column.isSetProtected()) + { + throw new IllegalArgumentException("PHI/protected metadata must not be set here."); + } + + ColumnType.Fk fk = column.getFk(); + if (null != fk) + { + try + { + validateForeignKey(fk, column, errors); + validateLookupFilter(AbstractTableInfo.parseXMLLookupFilters(fk.getFilters()), errors); + } + catch (ValidationException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + } + } + } + } + } + } + } + } + } + catch (XmlException e) + { + throw new RuntimeValidationException(e); + } + + for (XmlError xmle : xmlErrors) + { + errors.reject(ERROR_MSG, XmlBeansUtil.getErrorMessage(xmle)); + } + } + + private void validateForeignKey(ColumnType.Fk fk, ColumnType column, Errors errors) + { + if (fk.isSetFkMultiValued()) + { + // issue 51695 : don't let users create unsupported MVFK types + String type = fk.getFkMultiValued(); + if (!AbstractTableInfo.MultiValuedFkType.junction.name().equals(type)) + { + errors.reject(ERROR_MSG, String.format("Column : \"%s\" has an invalid fkMultiValued value : \"%s\" is not supported.", column.getColumnName(), type)); + } + } + } + + private void validateLookupFilter(Map> filterMap, Errors errors) + { + filterMap.forEach((operation, filters) -> { + + String displayStr = "Filter for operation : " + operation.name(); + for (FilterType filter : filters) + { + if (isBlank(filter.getColumn())) + errors.reject(ERROR_MSG, displayStr + " requires columnName"); + + if (null == filter.getOperator()) + { + errors.reject(ERROR_MSG, displayStr + " requires operator"); + } + else + { + CompareType compareType = CompareType.getByURLKey(filter.getOperator().toString()); + if (null == compareType) + { + errors.reject(ERROR_MSG, displayStr + " operator is invalid"); + } + else + { + if (compareType.isDataValueRequired() && null == filter.getValue()) + errors.reject(ERROR_MSG, displayStr + " requires a value but none is specified"); + } + } + } + + try + { + // attempt to convert to something we can query against + SimpleFilter.fromXml(filters.toArray(new FilterType[0])); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + }); + } + + @Override + public ApiResponse execute(SourceForm form, BindException errors) + { + var queryDef = _schema.getQueryDef(form.getQueryName()); + if (null == queryDef) + queryDef = _schema.getQueryDefForTable(form.getQueryName()); + if (null == queryDef) + throw new NotFoundException("Could not find the query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + if (form.ff_queryText != null) + { + if (!queryDef.isSqlEditable()) + throw new UnauthorizedException("Query SQL is not editable."); + + if (!queryDef.canEdit(getUser())) + throw new UnauthorizedException("Edit permissions are required."); + + queryDef.setSql(form.ff_queryText); + } + + String metadataText = StringUtils.trimToNull(form.ff_metadataText); + if (!Objects.equals(metadataText, queryDef.getMetadataXml())) + { + if (queryDef.isMetadataEditable()) + { + if (!queryDef.canEditMetadata(getUser())) + throw new UnauthorizedException("Edit metadata permissions are required."); + + if (!getUser().isTrustedBrowserDev()) + { + JavaScriptFragment.ensureXMLMetadataNoJavaScript(metadataText); + } + + queryDef.setMetadataXml(metadataText); + } + else + { + if (metadataText != null) + throw new UnsupportedOperationException("Query metadata is not editable."); + } + } + + queryDef.save(getUser(), getContainer()); + + // the query was successfully saved, validate the query but return any errors in the success response + List parseErrors = new ArrayList<>(); + List parseWarnings = new ArrayList<>(); + queryDef.validateQuery(_schema, parseErrors, parseWarnings); + if (!parseErrors.isEmpty()) + { + JSONArray errorArray = new JSONArray(); + + for (QueryException e : parseErrors) + { + errorArray.put(e.toJSON(form.ff_queryText)); + } + response.put("parseErrors", errorArray); + } + else if (!parseWarnings.isEmpty()) + { + JSONArray errorArray = new JSONArray(); + + for (QueryException e : parseWarnings) + { + errorArray.put(e.toJSON(form.ff_queryText)); + } + response.put("parseWarnings", errorArray); + } + } + catch (SQLException e) + { + errors.reject(ERROR_MSG, "An exception occurred: " + e); + LOG.error("Error", e); + } + catch (RuntimeException e) + { + errors.reject(ERROR_MSG, "An exception occurred: " + e.getMessage()); + LOG.error("Error", e); + } + + if (errors.hasErrors()) + return null; + + //if we got here, the query is OK + response.put("success", true); + return response; + } + + } + + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, DeletePermission.class}) + @Action(ActionType.Configure.class) + public static class DeleteQueryAction extends ConfirmAction + { + public SourceForm _form; + public QuerySchema _baseSchema; + public QueryDefinition _queryDef; + + + @Override + public void validateCommand(SourceForm target, Errors errors) + { + _form = target; + if (StringUtils.isEmpty(target.getSchemaName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + if (StringUtils.isEmpty(target.getQueryName())) + throw new NotFoundException("Query definition not found, schemaName and queryName are required."); + + _baseSchema = DefaultSchema.get(getUser(), getContainer(), _form.getSchemaKey()); + if (null == _baseSchema) + throw new NotFoundException("Schema not found: " + _form.getSchemaKey().toDisplayString()); + } + + + @Override + public ModelAndView getConfirmView(SourceForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Query"); + _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); + + if (null == _queryDef) + throw new NotFoundException("Query not found: " + form.getQueryName()); + + if (!_queryDef.canDelete(getUser())) + { + errors.reject(ERROR_MSG, "Sorry, this query can not be deleted"); + } + + return new JspView<>("/org/labkey/query/view/deleteQuery.jsp", this, errors); + } + + + @Override + public boolean handlePost(SourceForm form, BindException errors) throws Exception + { + _queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), _baseSchema.getSchemaName(), form.getQueryName()); + + if (null == _queryDef) + return false; + try + { + _queryDef.delete(getUser()); + } + catch (OptimisticConflictException x) + { + /* reshow will throw NotFound, so just ignore */ + } + return true; + } + + @Override + @NotNull + public ActionURL getSuccessURL(SourceForm queryForm) + { + return ((UserSchema)_baseSchema).urlFor(QueryAction.schema); + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public class ExecuteQueryAction extends QueryViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _form = form; + + if (errors.hasErrors()) + return new SimpleErrorView(errors, true); + + QueryView queryView = Objects.requireNonNull(form.getQueryView()); + + var t = queryView.getTable(); + if (null != t && !t.allowRobotsIndex()) + { + getPageConfig().setRobotsNone(); + } + + if (isPrint()) + { + queryView.setPrintView(true); + getPageConfig().setTemplate(PageConfig.Template.Print); + getPageConfig().setShowPrintDialog(true); + } + + queryView.setShadeAlternatingRows(true); + queryView.setShowBorders(true); + setHelpTopic("customSQL"); + _queryView = queryView; + return queryView; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + TableInfo ti = null; + try + { + if (null != _queryView) + ti = _queryView.getTable(); + } + catch (QueryParseException x) + { + /* */ + } + String display = ti == null ? _form.getQueryName() : ti.getTitle(); + root.addChild(display); + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class RawTableMetaDataAction extends QueryViewAction + { + private String _dbSchemaName; + private String _dbTableName; + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _form = form; + + QueryView queryView = form.getQueryView(); + String userSchemaName = queryView.getSchema().getName(); + TableInfo ti = queryView.getTable(); + if (null == ti) + throw new NotFoundException(); + + DbScope scope = ti.getSchema().getScope(); + + // Test for provisioned table + if (ti.getDomain() != null) + { + Domain domain = ti.getDomain(); + if (domain.getStorageTableName() != null) + { + // Use the real table and schema names for getting the metadata + _dbTableName = domain.getStorageTableName(); + _dbSchemaName = domain.getDomainKind().getStorageSchemaName(); + } + } + + // No domain or domain with non-provisioned storage (e.g., core.Users) + if (null == _dbSchemaName || null == _dbTableName) + { + DbSchema dbSchema = ti.getSchema(); + _dbSchemaName = dbSchema.getName(); + + // Try to get the underlying schema table and use the meta data name, #12015 + if (ti instanceof FilteredTable fti) + ti = fti.getRealTable(); + + if (ti instanceof SchemaTableInfo) + _dbTableName = ti.getMetaDataIdentifier().getId(); + else if (ti instanceof LinkedTableInfo) + _dbTableName = ti.getName(); + + if (null == _dbTableName) + { + TableInfo tableInfo = dbSchema.getTable(ti.getName()); + if (null != tableInfo) + _dbTableName = tableInfo.getMetaDataIdentifier().getId(); + } + } + + if (null != _dbTableName) + { + VBox result = new VBox(); + + ActionURL url = null; + QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(userSchemaName); + if (qs != null) + { + url = new ActionURL(RawSchemaMetaDataAction.class, getContainer()); + url.addParameter("schemaName", userSchemaName); + } + + SqlDialect dialect = scope.getSqlDialect(); + ScopeView scopeInfo = new ScopeView("Scope and Schema Information", scope, _dbSchemaName, url, _dbTableName); + + result.addView(scopeInfo); + + try (JdbcMetaDataLocator locator = dialect.getTableResolver().getSingleTableLocator(scope, _dbSchemaName, _dbTableName)) + { + JdbcMetaDataSelector columnSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getColumns(l.getCatalogName(), l.getSchemaNamePattern(), l.getTableNamePattern(), null)); + result.addView(new ResultSetView(CachedResultSets.create(columnSelector.getResultSet(), true, Table.ALL_ROWS), "Table Meta Data")); + + JdbcMetaDataSelector pkSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getPrimaryKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(pkSelector.getResultSet(), true, Table.ALL_ROWS), "Primary Key Meta Data")); + + if (dialect.canCheckIndices(ti)) + { + JdbcMetaDataSelector indexSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getIndexInfo(l.getCatalogName(), l.getSchemaName(), l.getTableName(), false, false)); + result.addView(new ResultSetView(CachedResultSets.create(indexSelector.getResultSet(), true, Table.ALL_ROWS), "Other Index Meta Data")); + } + + JdbcMetaDataSelector ikSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getImportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(ikSelector.getResultSet(), true, Table.ALL_ROWS), "Imported Keys Meta Data")); + + JdbcMetaDataSelector ekSelector = new JdbcMetaDataSelector(locator, + (dbmd, l) -> dbmd.getExportedKeys(l.getCatalogName(), l.getSchemaName(), l.getTableName())); + result.addView(new ResultSetView(CachedResultSets.create(ekSelector.getResultSet(), true, Table.ALL_ROWS), "Exported Keys Meta Data")); + } + return result; + } + else + { + errors.reject(ERROR_MSG, "Raw metadata not accessible for table " + ti.getName()); + return new SimpleErrorView(errors); + } + } + + @Override + public void addNavTrail(NavTree root) + { + (new SchemaAction(_form)).addNavTrail(root); + if (null != _dbTableName) + root.addChild("JDBC Meta Data For Table \"" + _dbSchemaName + "." + _dbTableName + "\""); + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public class RawSchemaMetaDataAction extends SimpleViewAction + { + private String _schemaName; + + @Override + public ModelAndView getView(Object form, BindException errors) throws Exception + { + _schemaName = getViewContext().getActionURL().getParameter("schemaName"); + if (null == _schemaName) + throw new NotFoundException(); + QuerySchema qs = DefaultSchema.get(getUser(), getContainer()).getSchema(_schemaName); + if (null == qs) + throw new NotFoundException(_schemaName); + DbSchema schema = qs.getDbSchema(); + String dbSchemaName = schema.getName(); + DbScope scope = schema.getScope(); + SqlDialect dialect = scope.getSqlDialect(); + + HttpView scopeInfo = new ScopeView("Scope Information", scope); + + ModelAndView tablesView; + + try (JdbcMetaDataLocator locator = dialect.getTableResolver().getAllTablesLocator(scope, dbSchemaName)) + { + JdbcMetaDataSelector selector = new JdbcMetaDataSelector(locator, + (dbmd, locator1) -> dbmd.getTables(locator1.getCatalogName(), locator1.getSchemaNamePattern(), locator1.getTableNamePattern(), null)); + Set tableNames = Sets.newCaseInsensitiveHashSet(qs.getTableNames()); + + ActionURL url = new ActionURL(RawTableMetaDataAction.class, getContainer()) + .addParameter("schemaName", _schemaName) + .addParameter("query.queryName", null); + tablesView = new ResultSetView(CachedResultSets.create(selector.getResultSet(), true, Table.ALL_ROWS), "Tables", "TABLE_NAME", url) + { + @Override + protected boolean shouldLink(ResultSet rs) throws SQLException + { + // Only link to tables and views (not indexes or sequences). And only if they're defined in the query schema. + String name = rs.getString("TABLE_NAME"); + String type = rs.getString("TABLE_TYPE"); + return ("TABLE".equalsIgnoreCase(type) || "VIEW".equalsIgnoreCase(type)) && tableNames.contains(name); + } + }; + } + + return new VBox(scopeInfo, tablesView); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("JDBC Meta Data For Schema \"" + _schemaName + "\""); + } + } + + + public static class ScopeView extends WebPartView + { + private final DbScope _scope; + private final String _schemaName; + private final String _tableName; + private final ActionURL _url; + + private ScopeView(String title, DbScope scope) + { + this(title, scope, null, null, null); + } + + private ScopeView(String title, DbScope scope, String schemaName, ActionURL url, String tableName) + { + super(title); + _scope = scope; + _schemaName = schemaName; + _tableName = tableName; + _url = url; + } + + @Override + protected void renderView(Object model, HtmlWriter out) + { + TABLE( + null != _schemaName ? getLabelAndContents("Schema", _url == null ? _schemaName : LinkBuilder.simpleLink(_schemaName, _url)) : null, + null != _tableName ? getLabelAndContents("Table", _tableName) : null, + getLabelAndContents("Scope", _scope.getDisplayName()), + getLabelAndContents("Dialect", _scope.getSqlDialect().getClass().getSimpleName()), + getLabelAndContents("URL", _scope.getDatabaseUrl()) + ).appendTo(out); + } + + // Return a single row (TR) with styled label and contents in separate TDs + private Renderable getLabelAndContents(String label, Object contents) + { + return TR( + TD( + cl("labkey-form-label"), + label + ), + TD( + contents + ) + ); + } + } + + // for backwards compat same as _executeQuery.view ?_print=1 + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public class PrintRowsAction extends ExecuteQueryAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + _print = true; + ModelAndView result = super.getView(form, errors); + String title = form.getQueryName(); + if (StringUtils.isEmpty(title)) + title = form.getSchemaName(); + getPageConfig().setTitle(title, true); + return result; + } + } + + + abstract static class _ExportQuery extends SimpleViewAction + { + @Override + public ModelAndView getView(K form, BindException errors) throws Exception + { + QueryView view = form.getQueryView(); + getPageConfig().setTemplate(PageConfig.Template.None); + HttpServletResponse response = getViewContext().getResponse(); + response.setHeader("X-Robots-Tag", "noindex"); + try + { + _export(form, view); + return null; + } + catch (QueryService.NamedParameterNotProvided | QueryParseException x) + { + ExceptionUtil.decorateException(x, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + throw x; + } + } + + abstract void _export(K form, QueryView view) throws Exception; + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportScriptForm extends QueryForm + { + private String _type; + + public String getScriptType() + { + return _type; + } + + public void setScriptType(String type) + { + _type = type; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) // This is called "export" but it doesn't export any data + @CSRF(CSRF.Method.ALL) + public static class ExportScriptAction extends SimpleViewAction + { + @Override + public void validate(ExportScriptForm form, BindException errors) + { + // calling form.getQueryView() as a validation check as it will throw if schema/query missing + form.getQueryView(); + + if (StringUtils.isEmpty(form.getScriptType())) + throw new NotFoundException("Missing required parameter: scriptType."); + } + + @Override + public ModelAndView getView(ExportScriptForm form, BindException errors) + { + return ExportScriptModel.getExportScriptView(QueryView.create(form, errors), form.getScriptType(), getPageConfig(), getViewContext().getResponse()); + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsExcelAction extends _ExportQuery + { + @Override + void _export(ExportQueryForm form, QueryView view) throws Exception + { + view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xls, form.getRenameColumnMap()); + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsXLSXAction extends _ExportQuery + { + @Override + void _export(ExportQueryForm form, QueryView view) throws Exception + { + view.exportToExcel(getViewContext().getResponse(), form.getHeaderType(), ExcelWriter.ExcelDocumentType.xlsx, form.getRenameColumnMap()); + } + } + + public static class ExportQueriesForm extends ExportQueryForm implements ApiJsonForm + { + private String filename; + private List queryForms; + + public void setFilename(String filename) + { + this.filename = filename; + } + + public String getFilename() + { + return filename; + } + + public void setQueryForms(List queryForms) + { + this.queryForms = queryForms; + } + + public List getQueryForms() + { + return queryForms; + } + + /** + * Map JSON to Spring PropertyValue objects. + * @param json the properties + */ + private MutablePropertyValues getPropertyValues(JSONObject json) + { + // Collecting mapped properties as a list because adding them to an existing MutablePropertyValues object replaces existing values + List properties = new ArrayList<>(); + + for (String key : json.keySet()) + { + Object value = json.get(key); + if (value instanceof JSONArray val) + { + // Split arrays into individual pairs to be bound (Issue #45452) + for (int i = 0; i < val.length(); i++) + { + properties.add(new PropertyValue(key, val.get(i).toString())); + } + } + else + { + properties.add(new PropertyValue(key, value)); + } + } + + return new MutablePropertyValues(properties); + } + + @Override + public void bindJson(JSONObject json) + { + setFilename(json.get("filename").toString()); + List forms = new ArrayList<>(); + + JSONArray models = json.optJSONArray("queryForms"); + if (models == null) + { + QueryController.LOG.error("No models to export; Form's `queryForms` property was null"); + throw new RuntimeValidationException("No queries to export; Form's `queryForms` property was null"); + } + + for (JSONObject queryModel : JsonUtil.toJSONObjectList(models)) + { + ExportQueryForm qf = new ExportQueryForm(); + qf.setViewContext(getViewContext()); + + qf.bindParameters(getPropertyValues(queryModel)); + forms.add(qf); + } + + setQueryForms(forms); + } + } + + /** + * Export multiple query forms + */ + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportQueriesXLSXAction extends ReadOnlyApiAction + { + @Override + public Object execute(ExportQueriesForm form, BindException errors) throws Exception + { + getPageConfig().setTemplate(PageConfig.Template.None); + HttpServletResponse response = getViewContext().getResponse(); + response.setHeader("X-Robots-Tag", "noindex"); + ResponseHelper.setContentDisposition(response, ResponseHelper.ContentDispositionType.attachment); + ViewContext viewContext = getViewContext(); + + Map> nameFormMap = new CaseInsensitiveHashMap<>(); + Map sheetNames = new HashMap<>(); + form.getQueryForms().forEach(qf -> { + String sheetName = qf.getSheetName(); + QueryView qv = qf.getQueryView(); + // use the given sheet name if provided, otherwise try the query definition name + String name = StringUtils.isNotBlank(sheetName) ? sheetName : qv.getQueryDef().getName(); + // if there is no sheet name or queryDefinition name, use a data region name if provided. Otherwise, use "Data" + name = StringUtils.isNotBlank(name) ? name : StringUtils.isNotBlank(qv.getDataRegionName()) ? qv.getDataRegionName() : "Data"; + // clean it to remove undesirable characters and make it of an acceptable length + name = ExcelWriter.cleanSheetName(name); + nameFormMap.computeIfAbsent(name, k -> new ArrayList<>()).add(qf); + }); + // Issue 53722: Need to assure unique names for the sheets in the presence of really long names + for (Map.Entry> entry : nameFormMap.entrySet()) { + String name = entry.getKey(); + if (entry.getValue().size() > 1) + { + List queryForms = entry.getValue(); + int countLength = String.valueOf(queryForms.size()).length() + 2; + if (countLength > name.length()) + throw new IllegalArgumentException("Cannot create sheet names from overlapping query names."); + for (int i = 0; i < queryForms.size(); i++) + { + sheetNames.put(entry.getValue().get(i), name.substring(0, name.length() - countLength) + "(" + i + ")"); + } + } + else + { + sheetNames.put(entry.getValue().get(0), name); + } + } + ExcelWriter writer = new ExcelWriter(ExcelWriter.ExcelDocumentType.xlsx) { + @Override + protected void renderSheets(Workbook workbook) + { + for (ExportQueryForm qf : form.getQueryForms()) + { + qf.setViewContext(viewContext); + qf.getSchema(); + + QueryView qv = qf.getQueryView(); + QueryView.ExcelExportConfig config = new QueryView.ExcelExportConfig(response, qf.getHeaderType()) + .setExcludeColumns(qf.getExcludeColumns()) + .setRenamedColumns(qf.getRenameColumnMap()); + qv.configureExcelWriter(this, config); + setSheetName(sheetNames.get(qf)); + setAutoSize(true); + renderNewSheet(workbook); + qv.logAuditEvent("Exported to Excel", getDataRowCount()); + } + + workbook.setActiveSheet(0); + } + }; + writer.setFilenamePrefix(form.getFilename()); + writer.renderWorkbook(response); + return null; //Returning anything here will cause error as excel writer will close the response stream + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class TemplateForm extends ExportQueryForm + { + boolean insertColumnsOnly = true; + String filenamePrefix; + FieldKey[] includeColumn; + String fileType; + + public TemplateForm() + { + _headerType = ColumnHeaderType.Caption; + } + + // "captionType" field backwards compatibility + public void setCaptionType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public ColumnHeaderType getCaptionType() + { + return _headerType; + } + + public List getIncludeColumns() + { + if (includeColumn == null || includeColumn.length == 0) + return Collections.emptyList(); + return Arrays.asList(includeColumn); + } + + public FieldKey[] getIncludeColumn() + { + return includeColumn; + } + + public void setIncludeColumn(FieldKey[] includeColumn) + { + this.includeColumn = includeColumn; + } + + @NotNull + public String getFilenamePrefix() + { + return filenamePrefix == null ? getQueryName() : filenamePrefix; + } + + public void setFilenamePrefix(String prefix) + { + filenamePrefix = prefix; + } + + public String getFileType() + { + return fileType; + } + + public void setFileType(String fileType) + { + this.fileType = fileType; + } + } + + + /** + * Can be used to generate an Excel template for import into a table. Supported URL params include: + *
+ *
filenamePrefix
+ *
the prefix of the excel file that is generated, defaults to '_data'
+ * + *
query.viewName
+ *
if provided, the resulting excel file will use the fields present in this view. + * Non-usereditable columns will be skipped. + * Non-existent columns (like a lookup) unless includeMissingColumns is true. + * Any required columns missing from this view will be appended to the end of the query. + *
+ * + *
includeColumn
+ *
List of column names to include, even if the column doesn't exist or is non-userEditable. + * For example, this can be used to add a fake column that is only supported during the import process. + *
+ * + *
excludeColumn
+ *
List of column names to exclude. + *
+ * + *
exportAlias.columns
+ *
Use alternative column name in excel: exportAlias.originalColumnName=aliasColumnName + *
+ * + *
captionType
+ *
determines which column property is used in the header, either Label or Name
+ *
+ */ + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportExcelTemplateAction extends _ExportQuery + { + public ExportExcelTemplateAction() + { + setCommandClass(TemplateForm.class); + } + + @Override + void _export(TemplateForm form, QueryView view) throws Exception + { + boolean respectView = form.getViewName() != null; + ExcelWriter.ExcelDocumentType fileType = ExcelWriter.ExcelDocumentType.xlsx; + if (form.getFileType() != null) + { + try + { + fileType = ExcelWriter.ExcelDocumentType.valueOf(form.getFileType().toLowerCase()); + } + catch (IllegalArgumentException ignored) {} + } + view.exportToExcel( new QueryView.ExcelExportConfig(getViewContext().getResponse(), form.getHeaderType()) + .setTemplateOnly(true) + .setInsertColumnsOnly(form.insertColumnsOnly) + .setDocType(fileType) + .setRespectView(respectView) + .setIncludeColumns(form.getIncludeColumns()) + .setExcludeColumns(form.getExcludeColumns()) + .setRenamedColumns(form.getRenameColumnMap()) + .setPrefix((StringUtils.isEmpty(form.getFilenamePrefix()) ? "Import" : form.getFilenamePrefix()) + "_Template") // Issue 48028: Change template file names + ); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportQueryForm extends QueryForm + { + protected ColumnHeaderType _headerType = null; // QueryView will provide a default header type if the user doesn't select one + FieldKey[] excludeColumn; + Map renameColumns = null; + private String sheetName; + + public void setSheetName(String sheetName) + { + this.sheetName = sheetName; + } + + public String getSheetName() + { + return sheetName; + } + + public ColumnHeaderType getHeaderType() + { + return _headerType; + } + + public void setHeaderType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public List getExcludeColumns() + { + if (excludeColumn == null || excludeColumn.length == 0) + return Collections.emptyList(); + return Arrays.asList(excludeColumn); + } + + public void setExcludeColumn(FieldKey[] excludeColumn) + { + this.excludeColumn = excludeColumn; + } + + public Map getRenameColumnMap() + { + if (renameColumns != null) + return renameColumns; + + renameColumns = new CaseInsensitiveHashMap<>(); + final String renameParamPrefix = "exportAlias."; + PropertyValue[] pvs = getInitParameters().getPropertyValues(); + for (PropertyValue pv : pvs) + { + String paramName = pv.getName(); + if (!paramName.startsWith(renameParamPrefix) || pv.getValue() == null) + continue; + + renameColumns.put(paramName.substring(renameParamPrefix.length()), (String) pv.getValue()); + } + + return renameColumns; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportRowsTsvForm extends ExportQueryForm + { + private TSVWriter.DELIM _delim = TSVWriter.DELIM.TAB; + private TSVWriter.QUOTE _quote = TSVWriter.QUOTE.DOUBLE; + + public TSVWriter.DELIM getDelim() + { + return _delim; + } + + public void setDelim(TSVWriter.DELIM delim) + { + _delim = delim; + } + + public TSVWriter.QUOTE getQuote() + { + return _quote; + } + + public void setQuote(TSVWriter.QUOTE quote) + { + _quote = quote; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportRowsTsvAction extends _ExportQuery + { + public ExportRowsTsvAction() + { + setCommandClass(ExportRowsTsvForm.class); + } + + @Override + void _export(ExportRowsTsvForm form, QueryView view) throws Exception + { + view.exportToTsv(getViewContext().getResponse(), form.getDelim(), form.getQuote(), form.getHeaderType(), form.getRenameColumnMap()); + } + } + + + @RequiresNoPermission + @IgnoresTermsOfUse + @Action(ActionType.Export.class) + public static class ExcelWebQueryAction extends ExportRowsTsvAction + { + @Override + public ModelAndView getView(ExportRowsTsvForm form, BindException errors) throws Exception + { + if (!getContainer().hasPermission(getUser(), ReadPermission.class)) + { + if (!getUser().isGuest()) + { + throw new UnauthorizedException(); + } + getViewContext().getResponse().setHeader("WWW-Authenticate", "Basic realm=\"" + LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getDescription() + "\""); + getViewContext().getResponse().setStatus(HttpServletResponse.SC_UNAUTHORIZED); + return null; + } + + // Bug 5610. Excel web queries don't work over SSL if caching is disabled, + // so we need to allow caching so that Excel can read from IE on Windows. + HttpServletResponse response = getViewContext().getResponse(); + // Set the headers to allow the client to cache, but not proxies + ResponseHelper.setPrivate(response); + + QueryView view = form.getQueryView(); + getPageConfig().setTemplate(PageConfig.Template.None); + view.exportToExcelWebQuery(getViewContext().getResponse()); + return null; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExcelWebQueryDefinitionAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + getPageConfig().setTemplate(PageConfig.Template.None); + form.getQueryView(); + String queryViewActionURL = form.getQueryViewActionURL(); + ActionURL url; + if (queryViewActionURL != null) + { + url = new ActionURL(queryViewActionURL); + } + else + { + url = getViewContext().cloneActionURL(); + url.setAction(ExcelWebQueryAction.class); + } + getViewContext().getResponse().setContentType("text/x-ms-iqy"); + String filename = FileUtil.makeFileNameWithTimestamp(form.getQueryName(), "iqy"); + ResponseHelper.setContentDisposition(getViewContext().getResponse(), ResponseHelper.ContentDispositionType.attachment, filename); + PrintWriter writer = getViewContext().getResponse().getWriter(); + writer.println("WEB"); + writer.println("1"); + writer.println(url.getURIString()); + + QueryService.get().addAuditEvent(getUser(), getContainer(), form.getSchemaName(), form.getQueryName(), url, "Exported to Excel Web Query definition", null); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + // Trusted analysts who are editors can create and modify queries + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + @Action(ActionType.SelectMetaData.class) + public class MetadataQueryAction extends SimpleViewAction + { + QueryForm _form = null; + + @Override + public ModelAndView getView(QueryForm queryForm, BindException errors) throws Exception + { + String schemaName = queryForm.getSchemaName(); + String queryName = queryForm.getQueryName(); + + _form = queryForm; + + if (schemaName.isEmpty() && (null == queryName || queryName.isEmpty())) + { + throw new NotFoundException("Must provide schemaName and queryName."); + } + + if (schemaName.isEmpty()) + { + throw new NotFoundException("Must provide schemaName."); + } + + if (null == queryName || queryName.isEmpty()) + { + throw new NotFoundException("Must provide queryName."); + } + + if (!queryForm.getQueryDef().isMetadataEditable()) + throw new UnauthorizedException("Query metadata is not editable"); + + if (!queryForm.canEditMetadata()) + throw new UnauthorizedException("You do not have permission to edit the query metadata"); + + return ModuleHtmlView.get(ModuleLoader.getInstance().getModule("core"), ModuleHtmlView.getGeneratedViewPath("queryMetadataEditor")); + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + var metadataQuery = _form.getQueryDef().getName(); + if (null != metadataQuery) + root.addChild("Edit Metadata: " + _form.getQueryName(), metadataQuery); + else + root.addChild("Edit Metadata: " + _form.getQueryName()); + } + } + + // Uck. Supports the old and new view designer. + protected JSONObject saveCustomView(Container container, QueryDefinition queryDef, + String regionName, String viewName, boolean replaceExisting, + boolean share, boolean inherit, + boolean session, boolean saveFilter, + boolean hidden, JSONObject jsonView, + ActionURL returnUrl, + BindException errors) + { + User owner = getUser(); + boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); + if (share && canSaveForAllUsers && !session) + { + owner = null; + } + String name = StringUtils.trimToNull(viewName); + + if (name != null && RESERVED_VIEW_NAMES.contains(name.toLowerCase())) + errors.reject(ERROR_MSG, "The grid view name '" + name + "' is not allowed."); + + boolean isHidden = hidden; + CustomView view; + if (owner == null) + view = queryDef.getSharedCustomView(name); + else + view = queryDef.getCustomView(owner, getViewContext().getRequest(), name); + + if (view != null && !replaceExisting && !StringUtils.isEmpty(name)) + errors.reject(ERROR_MSG, "A saved view by the name \"" + viewName + "\" already exists. "); + + // 11179: Allow editing the view if we're saving to session. + // NOTE: Check for session flag first otherwise the call to canEdit() will add errors to the errors collection. + boolean canEdit = view == null || session || view.canEdit(container, errors); + if (errors.hasErrors()) + return null; + + if (canEdit) + { + // Issue 13594: Disallow setting of the customview inherit bit for query views + // that have no available container filter types. Unfortunately, the only way + // to get the container filters is from the QueryView. Ideally, the query def + // would know if it was container filterable or not instead of using the QueryView. + if (inherit && canSaveForAllUsers && !session) + { + UserSchema schema = queryDef.getSchema(); + QueryView queryView = schema.createView(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, queryDef.getName(), errors); + if (queryView != null) + { + Set allowableContainerFilterTypes = queryView.getAllowableContainerFilterTypes(); + if (allowableContainerFilterTypes.size() <= 1) + { + errors.reject(ERROR_MSG, "QueryView doesn't support inherited custom views"); + return null; + } + } + } + + // Create a new view if none exists or the current view is a shared view + // and the user wants to override the shared view with a personal view. + if (view == null || (owner != null && view.isShared())) + { + if (owner == null) + view = queryDef.createSharedCustomView(name); + else + view = queryDef.createCustomView(owner, name); + + if (owner != null && session) + ((CustomViewImpl) view).isSession(true); + view.setIsHidden(hidden); + } + else if (session != view.isSession()) + { + if (session) + { + assert !view.isSession(); + if (owner == null) + { + errors.reject(ERROR_MSG, "Session views can't be saved for all users"); + return null; + } + + // The form is saving to session but the view is in the database. + // Make a copy in case it's a read-only version from an XML file + view = queryDef.createCustomView(owner, name); + ((CustomViewImpl) view).isSession(true); + } + else + { + // Remove the session view and call saveCustomView again to either create a new view or update an existing view. + assert view.isSession(); + boolean success = false; + try + { + view.delete(getUser(), getViewContext().getRequest()); + JSONObject ret = saveCustomView(container, queryDef, regionName, viewName, replaceExisting, share, inherit, session, saveFilter, hidden, jsonView, returnUrl, errors); + success = !errors.hasErrors() && ret != null; + return success ? ret : null; + } + finally + { + if (!success) + { + // dirty the view then save the deleted session view back in session state + view.setName(view.getName()); + view.save(getUser(), getViewContext().getRequest()); + } + } + } + } + + // NOTE: Updating, saving, and deleting the view may throw an exception + CustomViewImpl cview = null; + if (view instanceof EditableCustomView && view.isOverridable()) + { + cview = ((EditableCustomView)view).getEditableViewInfo(owner, session); + } + if (null == cview) + { + throw new IllegalArgumentException("View cannot be edited"); + } + + cview.update(jsonView, saveFilter); + if (canSaveForAllUsers && !session) + { + cview.setCanInherit(inherit); + } + isHidden = view.isHidden(); + cview.setContainer(container); + cview.save(getUser(), getViewContext().getRequest()); + if (owner == null) + { + // New view is shared so delete any previous custom view owned by the user with the same name. + CustomView personalView = queryDef.getCustomView(getUser(), getViewContext().getRequest(), name); + if (personalView != null && !personalView.isShared()) + { + personalView.delete(getUser(), getViewContext().getRequest()); + } + } + } + + if (null == returnUrl) + { + returnUrl = getViewContext().cloneActionURL().setAction(ExecuteQueryAction.class); + } + else + { + returnUrl = returnUrl.clone(); + if (name == null || !canEdit) + { + returnUrl.deleteParameter(regionName + "." + QueryParam.viewName); + } + else if (!isHidden) + { + returnUrl.replaceParameter(regionName + "." + QueryParam.viewName, name); + } + returnUrl.deleteParameter(regionName + "." + QueryParam.ignoreFilter); + if (saveFilter) + { + for (String key : returnUrl.getKeysByPrefix(regionName + ".")) + { + if (isFilterOrSort(regionName, key)) + returnUrl.deleteFilterParameters(key); + } + } + } + + JSONObject ret = new JSONObject(); + ret.put("redirect", returnUrl); + Map viewAsMap = CustomViewUtil.toMap(view, getUser(), true); + try + { + ret.put("view", new JSONObject(viewAsMap, new JSONParserConfiguration().withMaxNestingDepth(10))); + } + catch (JSONException e) + { + LOG.error("Failed to save view: {}", jsonView, e); + } + return ret; + } + + private boolean isFilterOrSort(String dataRegionName, String param) + { + assert param.startsWith(dataRegionName + "."); + String check = param.substring(dataRegionName.length() + 1); + if (check.contains("~")) + return true; + if ("sort".equals(check)) + return true; + if (check.equals("containerFilterName")) + return true; + return false; + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Configure.class) + @JsonInputLimit(100_000) + public class SaveQueryViewsAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SimpleApiJsonForm form, BindException errors) + { + JSONObject json = form.getJsonObject(); + if (json == null) + throw new NotFoundException("Empty request"); + + String schemaName = json.optString(QueryParam.schemaName.toString(), null); + String queryName = json.optString(QueryParam.queryName.toString(), null); + if (schemaName == null || queryName == null) + throw new NotFoundException("schemaName and queryName are required"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + if (schema == null) + throw new NotFoundException("schema not found"); + + QueryDefinition queryDef = QueryService.get().getQueryDef(getUser(), getContainer(), schemaName, queryName); + if (queryDef == null) + queryDef = schema.getQueryDefForTable(queryName); + if (queryDef == null) + throw new NotFoundException("query not found"); + + JSONObject response = new JSONObject(); + response.put(QueryParam.schemaName.toString(), schemaName); + response.put(QueryParam.queryName.toString(), queryName); + JSONArray views = new JSONArray(); + response.put("views", views); + + ActionURL redirect = null; + JSONArray jsonViews = json.getJSONArray("views"); + for (int i = 0; i < jsonViews.length(); i++) + { + final JSONObject jsonView = jsonViews.getJSONObject(i); + String viewName = jsonView.optString("name", null); + if (viewName == null) + throw new NotFoundException("'name' is required all views'"); + + boolean shared = jsonView.optBoolean("shared", false); + boolean replace = jsonView.optBoolean("replace", true); // "replace" was the default before the flag is introduced + boolean inherit = jsonView.optBoolean("inherit", false); + boolean session = jsonView.optBoolean("session", false); + boolean hidden = jsonView.optBoolean("hidden", false); + // Users may save views to a location other than the current container + String containerPath = jsonView.optString("containerPath", getContainer().getPath()); + Container container; + if (inherit) + { + // Only respect this request if it's a view that is inheritable in subfolders + container = ContainerManager.getForPath(containerPath); + } + else + { + // Otherwise, save it in the current container + container = getContainer().getContainerFor(ContainerType.DataType.customQueryViews); + } + + if (container == null) + { + throw new NotFoundException("No such container: " + containerPath); + } + + JSONObject savedView = saveCustomView( + container, queryDef, QueryView.DATAREGIONNAME_DEFAULT, viewName, replace, + shared, inherit, session, true, hidden, jsonView, null, errors); + + if (savedView != null) + { + if (redirect == null) + redirect = (ActionURL)savedView.get("redirect"); + views.put(savedView.getJSONObject("view")); + } + } + + if (redirect != null) + response.put("redirect", redirect); + + if (errors.hasErrors()) + return null; + else + return new ApiSimpleResponse(response); + } + } + + public static class RenameQueryViewForm extends QueryForm + { + private String newName; + + public String getNewName() + { + return newName; + } + + public void setNewName(String newName) + { + this.newName = newName; + } + } + + @RequiresPermission(ReadPermission.class) + public class RenameQueryViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(RenameQueryViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + + Container container = getContainer(); + User user = getUser(); + + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + if (schemaName == null || queryName == null) + throw new NotFoundException("schemaName and queryName are required"); + + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); + if (schema == null) + throw new NotFoundException("schema not found"); + + QueryDefinition queryDef = QueryService.get().getQueryDef(user, container, schemaName, queryName); + if (queryDef == null) + queryDef = schema.getQueryDefForTable(queryName); + if (queryDef == null) + throw new NotFoundException("query not found"); + + renameCustomView(container, queryDef, view, form.getNewName(), errors); + + if (errors.hasErrors()) + return null; + else + return new ApiSimpleResponse("success", true); + } + } + + protected void renameCustomView(Container container, QueryDefinition queryDef, CustomView fromView, String newViewName, BindException errors) + { + if (newViewName != null && RESERVED_VIEW_NAMES.contains(newViewName.toLowerCase())) + errors.reject(ERROR_MSG, "The grid view name '" + newViewName + "' is not allowed."); + + String newName = StringUtils.trimToNull(newViewName); + if (StringUtils.isEmpty(newName)) + errors.reject(ERROR_MSG, "View name cannot be blank."); + + if (errors.hasErrors()) + return; + + User owner = getUser(); + boolean canSaveForAllUsers = container.hasPermission(getUser(), EditSharedViewPermission.class); + + if (!fromView.canEdit(container, errors)) + return; + + if (fromView.isSession()) + { + errors.reject(ERROR_MSG, "Cannot rename a session view."); + return; + } + + CustomView duplicateView = queryDef.getCustomView(owner, getViewContext().getRequest(), newName); + if (duplicateView == null && canSaveForAllUsers) + duplicateView = queryDef.getSharedCustomView(newName); + if (duplicateView != null) + { + // only allow duplicate view name if creating a new private view to shadow an existing shared view + if (!(!fromView.isShared() && duplicateView.isShared())) + { + errors.reject(ERROR_MSG, "Another saved view by the name \"" + newName + "\" already exists. "); + return; + } + } + + fromView.setName(newViewName); + fromView.save(getUser(), getViewContext().getRequest()); + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Configure.class) + public class PropertiesQueryAction extends FormViewAction + { + PropertiesForm _form = null; + private String _queryName; + + @Override + public void validateCommand(PropertiesForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(PropertiesForm form, boolean reshow, BindException errors) + { + // assertQueryExists requires that it be well-formed + // assertQueryExists(form); + QueryDefinition queryDef = form.getQueryDef(); + _form = form; + _form.setDescription(queryDef.getDescription()); + _form.setInheritable(queryDef.canInherit()); + _form.setHidden(queryDef.isHidden()); + setHelpTopic("editQueryProperties"); + _queryName = form.getQueryName(); + + return new JspView<>("/org/labkey/query/view/propertiesQuery.jsp", form, errors); + } + + @Override + public boolean handlePost(PropertiesForm form, BindException errors) throws Exception + { + // assertQueryExists requires that it be well-formed + // assertQueryExists(form); + if (!form.canEdit()) + { + throw new UnauthorizedException(); + } + QueryDefinition queryDef = form.getQueryDef(); + _queryName = form.getQueryName(); + if (!queryDef.getDefinitionContainer().getId().equals(getContainer().getId())) + throw new NotFoundException("Query not found"); + + _form = form; + + if (!StringUtils.isEmpty(form.rename) && !form.rename.equalsIgnoreCase(queryDef.getName())) + { + // issue 17766: check if query or table exist with this name + if (null != QueryManager.get().getQueryDef(getContainer(), form.getSchemaName(), form.rename, true) + || null != form.getSchema().getTable(form.rename,null)) + { + errors.reject(ERROR_MSG, "A query or table with the name \"" + form.rename + "\" already exists."); + return false; + } + + // Issue 40895: update queryName in xml metadata + updateXmlMetadata(queryDef); + queryDef.setName(form.rename); + // update form so getSuccessURL() works + _form = new PropertiesForm(form.getSchemaName(), form.rename); + _form.setViewContext(form.getViewContext()); + _queryName = form.rename; + } + + queryDef.setDescription(form.description); + queryDef.setCanInherit(form.inheritable); + queryDef.setIsHidden(form.hidden); + queryDef.save(getUser(), getContainer()); + return true; + } + + private void updateXmlMetadata(QueryDefinition queryDef) throws XmlException + { + if (null != queryDef.getMetadataXml()) + { + TablesDocument doc = TablesDocument.Factory.parse(queryDef.getMetadataXml()); + if (null != doc) + { + for (TableType tableType : doc.getTables().getTableArray()) + { + if (tableType.getTableName().equalsIgnoreCase(queryDef.getName())) + { + // update tableName in xml + tableType.setTableName(_form.rename); + } + } + XmlOptions xmlOptions = new XmlOptions(); + xmlOptions.setSavePrettyPrint(); + // Don't use an explicit namespace, making the XML much more readable + xmlOptions.setUseDefaultNamespace(); + queryDef.setMetadataXml(doc.xmlText(xmlOptions)); + } + } + } + + @Override + public ActionURL getSuccessURL(PropertiesForm propertiesForm) + { + ActionURL url = new ActionURL(BeginAction.class, propertiesForm.getViewContext().getContainer()); + url.addParameter("schemaName", propertiesForm.getSchemaName()); + if (null != _queryName) + url.addParameter("queryName", _queryName); + return url; + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + root.addChild("Edit query properties"); + } + } + + @ActionNames("truncateTable") + @RequiresPermission(AdminPermission.class) + public static class TruncateTableAction extends MutatingApiAction + { + UserSchema schema; + TableInfo table; + + @Override + public void validateForm(QueryForm form, Errors errors) + { + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + + if (isBlank(schemaName) || isBlank(queryName)) + throw new NotFoundException("schemaName and queryName are required"); + + schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + if (null == schema) + throw new NotFoundException("The schema '" + schemaName + "' does not exist."); + + table = schema.getTable(queryName, null); + if (null == table) + throw new NotFoundException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); + } + + @Override + public ApiResponse execute(QueryForm form, BindException errors) throws Exception + { + int deletedRows; + QueryUpdateService qus = table.getUpdateService(); + + if (null == qus) + throw new IllegalArgumentException("The query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "' is not truncatable."); + + try (DbScope.Transaction transaction = table.getSchema().getScope().ensureTransaction()) + { + deletedRows = qus.truncateRows(getUser(), getContainer(), null, null); + transaction.commit(); + } + + ApiSimpleResponse response = new ApiSimpleResponse(); + + response.put("success", true); + response.put(BaseSaveRowsAction.PROP_SCHEMA_NAME, form.getSchemaName()); + response.put(BaseSaveRowsAction.PROP_QUERY_NAME, form.getQueryName()); + response.put("deletedRows", deletedRows); + + return response; + } + } + + + @RequiresPermission(DeletePermission.class) + public static class DeleteQueryRowsAction extends FormHandlerAction + { + @Override + public void validateCommand(QueryForm target, Errors errors) + { + } + + @Override + public boolean handlePost(QueryForm form, BindException errors) + { + TableInfo table = form.getQueryView().getTable(); + + if (!table.hasPermission(getUser(), DeletePermission.class)) + { + throw new UnauthorizedException(); + } + + QueryUpdateService updateService = table.getUpdateService(); + if (updateService == null) + throw new UnsupportedOperationException("Unable to delete - no QueryUpdateService registered for " + form.getSchemaName() + "." + form.getQueryName()); + + Set ids = DataRegionSelection.getSelected(form.getViewContext(), null, true); + List pks = table.getPkColumns(); + int numPks = pks.size(); + + //normalize the pks to arrays of correctly-typed objects + List> keyValues = new ArrayList<>(ids.size()); + for (String id : ids) + { + String[] stringValues; + if (numPks > 1) + { + stringValues = id.split(","); + if (stringValues.length != numPks) + throw new IllegalStateException("This table has " + numPks + " primary-key columns, but " + stringValues.length + " primary-key values were provided!"); + } + else + stringValues = new String[]{id}; + + Map rowKeyValues = new CaseInsensitiveHashMap<>(); + for (int idx = 0; idx < numPks; ++idx) + { + ColumnInfo keyColumn = pks.get(idx); + Object keyValue = keyColumn.getJavaClass() == String.class ? stringValues[idx] : ConvertUtils.convert(stringValues[idx], keyColumn.getJavaClass()); + rowKeyValues.put(keyColumn.getName(), keyValue); + } + keyValues.add(rowKeyValues); + } + + DbSchema dbSchema = table.getSchema(); + try + { + dbSchema.getScope().executeWithRetry(tx -> + { + try + { + updateService.deleteRows(getUser(), getContainer(), keyValues, null, null); + } + catch (SQLException x) + { + if (!RuntimeSQLException.isConstraintException(x)) + throw new RuntimeSQLException(x); + errors.reject(ERROR_MSG, getMessage(table.getSchema().getSqlDialect(), x)); + } + catch (DataIntegrityViolationException | OptimisticConflictException e) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + catch (BatchValidationException x) + { + x.addToErrors(errors); + } + catch (Exception x) + { + errors.reject(ERROR_MSG, null == x.getMessage() ? x.toString() : x.getMessage()); + ExceptionUtil.logExceptionToMothership(getViewContext().getRequest(), x); + } + // need to throw here to avoid committing tx + if (errors.hasErrors()) + throw new DbScope.RetryPassthroughException(errors); + return true; + }); + } + catch (DbScope.RetryPassthroughException x) + { + if (x.getCause() != errors) + x.throwRuntimeException(); + } + return !errors.hasErrors(); + } + + @Override + public ActionURL getSuccessURL(QueryForm form) + { + return form.getReturnActionURL(); + } + } + + @RequiresPermission(ReadPermission.class) + public static class DetailsQueryRowAction extends UserSchemaAction + { + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + ButtonBar bb = new ButtonBar(); + bb.setStyle(ButtonBar.Style.separateButtons); + + if (_schema != null && _table != null) + { + if (_table.hasPermission(getUser(), UpdatePermission.class)) + { + StringExpression updateExpr = _form.getQueryDef().urlExpr(QueryAction.updateQueryRow, _schema.getContainer()); + if (updateExpr != null) + { + String url = updateExpr.eval(tableForm.getTypedValues()); + if (url != null) + { + ActionURL updateUrl = new ActionURL(url); + ActionButton editButton = new ActionButton("Edit", updateUrl); + bb.add(editButton); + } + } + } + + + ActionURL gridUrl; + if (_form.getReturnActionURL() != null) + { + // If we have a specific return URL requested, use that + gridUrl = _form.getReturnActionURL(); + } + else + { + // Otherwise go back to the default grid view + gridUrl = _schema.urlFor(QueryAction.executeQuery, _form.getQueryDef()); + } + if (gridUrl != null) + { + ActionButton gridButton = new ActionButton("Show Grid", gridUrl); + bb.add(gridButton); + } + } + + DetailsView detailsView = new DetailsView(tableForm); + detailsView.setFrame(WebPartView.FrameType.PORTAL); + detailsView.getDataRegion().setButtonBar(bb); + + VBox view = new VBox(detailsView); + + DetailsURL detailsURL = QueryService.get().getAuditDetailsURL(getUser(), getContainer(), _table); + + if (detailsURL != null) + { + String url = detailsURL.eval(tableForm.getTypedValues()); + if (url != null) + { + ActionURL auditURL = new ActionURL(url); + + QueryView historyView = QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), + auditURL.getParameter(QueryParam.schemaName), + auditURL.getParameter(QueryParam.queryName), + auditURL.getParameter("keyValue"), errors); + + if (null != historyView) + { + historyView.setFrame(WebPartView.FrameType.PORTAL); + historyView.setTitle("History"); + + view.addView(historyView); + } + } + } + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + return false; + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Details"); + } + } + + @RequiresPermission(InsertPermission.class) + public static class InsertQueryRowAction extends UserSchemaAction + { + @Override + public BindException bindParameters(PropertyValues m) throws Exception + { + BindException bind = super.bindParameters(m); + + // what is going on with UserSchemaAction and form binding? Why doesn't successUrl bind? + QueryUpdateForm form = (QueryUpdateForm)bind.getTarget(); + if (null == form.getSuccessUrl() && null != m.getPropertyValue(ActionURL.Param.successUrl.name())) + form.setSuccessUrl(new ReturnURLString(m.getPropertyValue(ActionURL.Param.successUrl.name()).getValue().toString())); + return bind; + } + + Map insertedRow = null; + + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Insert Row"); + + InsertView view = new InsertView(tableForm, errors); + view.getDataRegion().setButtonBar(createSubmitCancelButtonBar(tableForm)); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) + { + List> list = doInsertUpdate(tableForm, errors, true); + if (null != list && list.size() == 1) + insertedRow = list.get(0); + return 0 == errors.getErrorCount(); + } + + /** + * NOTE: UserSchemaAction.addNavTrail() uses this method getSuccessURL() for the nav trail link (form==null). + * It is used for where to go on success, and also as a "back" link in the nav trail + * If there is a setSuccessUrl specified, we will use that for successful submit + */ + @Override + public ActionURL getSuccessURL(QueryUpdateForm form) + { + if (null == form) + return super.getSuccessURL(null); + + String str = null; + if (form.getSuccessUrl() != null) + str = form.getSuccessUrl().toString(); + if (isBlank(str)) + str = form.getReturnUrl(); + + if ("details.view".equals(str)) + { + if (null == insertedRow) + return super.getSuccessURL(form); + StringExpression se = form.getTable().getDetailsURL(null, getContainer()); + if (null == se) + return super.getSuccessURL(form); + str = se.eval(insertedRow); + } + try + { + if (!isBlank(str)) + return new ActionURL(str); + } + catch (IllegalArgumentException x) + { + // pass + } + return super.getSuccessURL(form); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Insert " + _table.getName()); + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateQueryRowAction extends UserSchemaAction + { + @Override + public ModelAndView getView(QueryUpdateForm tableForm, boolean reshow, BindException errors) + { + ButtonBar bb = createSubmitCancelButtonBar(tableForm); + UpdateView view = new UpdateView(tableForm, errors); + view.getDataRegion().setButtonBar(bb); + return view; + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception + { + doInsertUpdate(tableForm, errors, false); + return 0 == errors.getErrorCount(); + } + + @Override + public void addNavTrail(NavTree root) + { + super.addNavTrail(root); + root.addChild("Edit " + _table.getName()); + } + } + + @RequiresPermission(UpdatePermission.class) + public static class UpdateQueryRowsAction extends UpdateQueryRowAction + { + @Override + public ModelAndView handleRequest(QueryUpdateForm tableForm, BindException errors) throws Exception + { + tableForm.setBulkUpdate(true); + return super.handleRequest(tableForm, errors); + } + + @Override + public boolean handlePost(QueryUpdateForm tableForm, BindException errors) throws Exception + { + boolean ret; + + if (tableForm.isDataSubmit()) + { + ret = super.handlePost(tableForm, errors); + if (ret) + DataRegionSelection.clearAll(getViewContext(), null); // in case we altered primary keys, see issue #35055 + return ret; + } + + return false; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Edit Multiple " + _table.getName()); + } + } + + // alias + public static class DeleteAction extends DeleteQueryRowsAction + { + } + + public abstract static class QueryViewAction extends SimpleViewAction + { + QueryForm _form; + QueryView _queryView; + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class APIQueryForm extends ContainerFilterQueryForm + { + private Integer _start; + private Integer _limit; + private boolean _includeDetailsColumn = false; + private boolean _includeUpdateColumn = false; + private boolean _includeTotalCount = true; + private boolean _includeStyle = false; + private boolean _includeDisplayValues = false; + private boolean _minimalColumns = true; + private boolean _includeMetadata = true; + + public Integer getStart() + { + return _start; + } + + public void setStart(Integer start) + { + _start = start; + } + + public Integer getLimit() + { + return _limit; + } + + public void setLimit(Integer limit) + { + _limit = limit; + } + + public boolean isIncludeTotalCount() + { + return _includeTotalCount; + } + + public void setIncludeTotalCount(boolean includeTotalCount) + { + _includeTotalCount = includeTotalCount; + } + + public boolean isIncludeStyle() + { + return _includeStyle; + } + + public void setIncludeStyle(boolean includeStyle) + { + _includeStyle = includeStyle; + } + + public boolean isIncludeDetailsColumn() + { + return _includeDetailsColumn; + } + + public void setIncludeDetailsColumn(boolean includeDetailsColumn) + { + _includeDetailsColumn = includeDetailsColumn; + } + + public boolean isIncludeUpdateColumn() + { + return _includeUpdateColumn; + } + + public void setIncludeUpdateColumn(boolean includeUpdateColumn) + { + _includeUpdateColumn = includeUpdateColumn; + } + + public boolean isIncludeDisplayValues() + { + return _includeDisplayValues; + } + + public void setIncludeDisplayValues(boolean includeDisplayValues) + { + _includeDisplayValues = includeDisplayValues; + } + + public boolean isMinimalColumns() + { + return _minimalColumns; + } + + public void setMinimalColumns(boolean minimalColumns) + { + _minimalColumns = minimalColumns; + } + + public boolean isIncludeMetadata() + { + return _includeMetadata; + } + + public void setIncludeMetadata(boolean includeMetadata) + { + _includeMetadata = includeMetadata; + } + + @Override + protected QuerySettings createQuerySettings(UserSchema schema) + { + QuerySettings results = super.createQuerySettings(schema); + + // See dataintegration/202: The java client api / remote ETL calls selectRows with showRows=all. We need to test _initParameters to properly read this + boolean missingShowRows = null == getViewContext().getRequest().getParameter(getDataRegionName() + "." + QueryParam.showRows) && null == _initParameters.getPropertyValue(getDataRegionName() + "." + QueryParam.showRows); + if (null == getLimit() && !results.isMaxRowsSet() && missingShowRows) + { + results.setShowRows(ShowRows.PAGINATED); + results.setMaxRows(DEFAULT_API_MAX_ROWS); + } + + if (getLimit() != null) + { + results.setShowRows(ShowRows.PAGINATED); + results.setMaxRows(getLimit()); + } + if (getStart() != null) + results.setOffset(getStart()); + + return results; + } + } + + public static final int DEFAULT_API_MAX_ROWS = 100000; + + @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 + @ActionNames("selectRows, getQuery") + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.1) + @Action(ActionType.SelectData.class) + public class SelectRowsAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(APIQueryForm form, BindException errors) + { + // Issue 12233: add implicit maxRows=100k when using client API + QueryView view = form.getQueryView(); + + view.setShowPagination(form.isIncludeTotalCount()); + + //if viewName was specified, ensure that it was actually found and used + //QueryView.create() will happily ignore an invalid view name and just return the default view + if (null != StringUtils.trimToNull(form.getViewName()) && + null == view.getQueryDef().getCustomView(getUser(), getViewContext().getRequest(), form.getViewName())) + { + throw new NotFoundException("The requested view '" + form.getViewName() + "' does not exist for this user."); + } + + TableInfo t = view.getTable(); + if (null == t) + { + List qpes = view.getParseErrors(); + if (!qpes.isEmpty()) + throw qpes.get(0); + throw new NotFoundException(form.getQueryName()); + } + + boolean isEditable = isQueryEditable(view.getTable()); + boolean metaDataOnly = form.getQuerySettings().getMaxRows() == 0; + boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; + boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; + + ApiQueryResponse response; + + // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. + if (getRequestedApiVersion() >= 13.2) + { + ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, true, view.getQueryDef().getName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); + fancyResponse.includeFormattedValue(includeFormattedValue); + response = fancyResponse; + } + //if requested version is >= 9.1, use the extended api query response + else if (getRequestedApiVersion() >= 9.1) + { + response = new ExtendedApiQueryResponse(view, isEditable, true, + form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + } + else + { + response = new ApiQueryResponse(view, isEditable, true, + form.getSchemaName(), form.getQueryName(), form.getQuerySettings().getOffset(), null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), + form.isIncludeDisplayValues(), form.isIncludeMetadata()); + } + response.includeStyle(form.isIncludeStyle()); + + // Issues 29515 and 32269 - force key and other non-requested columns to be sent back, but only if the client has + // requested minimal columns, as we now do for ExtJS stores + if (form.isMinimalColumns()) + { + // Be sure to use the settings from the view, as it may have swapped it out with a customized version. + // See issue 38747. + response.setColumnFilter(view.getSettings().getFieldKeys()); + } + + return response; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class GetDataAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SimpleApiJsonForm form, BindException errors) throws Exception + { + ObjectMapper mapper = JsonUtil.createDefaultMapper(); + mapper.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + JSONObject object = form.getJsonObject(); + if (object == null) + { + object = new JSONObject(); + } + DataRequest builder = mapper.readValue(object.toString(), DataRequest.class); + + return builder.render(getViewContext(), errors); + } + } + + protected boolean isQueryEditable(TableInfo table) + { + if (!getContainer().hasPermission("isQueryEditable", getUser(), DeletePermission.class)) + return false; + QueryUpdateService updateService = null; + try + { + updateService = table.getUpdateService(); + } + catch(Exception ignore) {} + return null != table && null != updateService; + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExecuteSqlForm extends APIQueryForm + { + private String _sql; + private Integer _maxRows; + private Integer _offset; + private boolean _saveInSession; + + public String getSql() + { + return _sql; + } + + public void setSql(String sql) + { + _sql = PageFlowUtil.wafDecode(StringUtils.trim(sql)); + } + + public Integer getMaxRows() + { + return _maxRows; + } + + public void setMaxRows(Integer maxRows) + { + _maxRows = maxRows; + } + + public Integer getOffset() + { + return _offset; + } + + public void setOffset(Integer offset) + { + _offset = offset; + } + + @Override + public void setLimit(Integer limit) + { + _maxRows = limit; + } + + @Override + public void setStart(Integer start) + { + _offset = start; + } + + public boolean isSaveInSession() + { + return _saveInSession; + } + + public void setSaveInSession(boolean saveInSession) + { + _saveInSession = saveInSession; + } + + @Override + public String getQueryName() + { + // ExecuteSqlAction doesn't allow setting query name parameter. + return null; + } + + @Override + public void setQueryName(String name) + { + // ExecuteSqlAction doesn't allow setting query name parameter. + } + } + + @CSRF(CSRF.Method.NONE) // No need for CSRF token --- this is a non-mutating action that supports POST to allow for large payloads, see #36056 + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.1) + @Action(ActionType.SelectData.class) + public class ExecuteSqlAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(ExecuteSqlForm form, BindException errors) + { + form.ensureSchemaExists(); + + String schemaName = StringUtils.trimToNull(form.getQuerySettings().getSchemaName()); + if (null == schemaName) + throw new IllegalArgumentException("No value was supplied for the required parameter 'schemaName'."); + String sql = form.getSql(); + if (StringUtils.isBlank(sql)) + throw new IllegalArgumentException("No value was supplied for the required parameter 'sql'."); + + //create a temp query settings object initialized with the posted LabKey SQL + //this will provide a temporary QueryDefinition to Query + QuerySettings settings = form.getQuerySettings(); + if (form.isSaveInSession()) + { + HttpSession session = getViewContext().getSession(); + if (session == null) + throw new IllegalStateException("Session required"); + + QueryDefinition def = QueryService.get().saveSessionQuery(getViewContext(), getContainer(), schemaName, sql); + settings.setDataRegionName("executeSql"); + settings.setQueryName(def.getName()); + } + else + { + settings = new TempQuerySettings(getViewContext(), sql, settings); + } + + //need to explicitly turn off various UI options that will try to refer to the + //current URL and query string + settings.setAllowChooseView(false); + settings.setAllowCustomizeView(false); + + // Issue 12233: add implicit maxRows=100k when using client API + settings.setShowRows(ShowRows.PAGINATED); + settings.setMaxRows(DEFAULT_API_MAX_ROWS); + + // 16961: ExecuteSql API without maxRows parameter defaults to returning 100 rows + //apply optional settings (maxRows, offset) + boolean metaDataOnly = false; + if (null != form.getMaxRows() && (form.getMaxRows() >= 0 || form.getMaxRows() == Table.ALL_ROWS)) + { + settings.setMaxRows(form.getMaxRows()); + metaDataOnly = Table.NO_ROWS == form.getMaxRows(); + } + + int offset = 0; + if (null != form.getOffset()) + { + settings.setOffset(form.getOffset().longValue()); + offset = form.getOffset(); + } + + //build a query view using the schema and settings + QueryView view = new QueryView(form.getSchema(), settings, errors); + view.setShowRecordSelectors(false); + view.setShowExportButtons(false); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + view.setShowPagination(form.isIncludeTotalCount()); + + TableInfo t = view.getTable(); + boolean isEditable = null != t && isQueryEditable(view.getTable()); + boolean arrayMultiValueColumns = getRequestedApiVersion() >= 16.2; + boolean includeFormattedValue = getRequestedApiVersion() >= 17.1; + + ApiQueryResponse response; + + // 13.2 introduced the getData API action, a condensed response wire format, and a js wrapper to consume the wire format. Support this as an option for legacy APIs. + if (getRequestedApiVersion() >= 13.2) + { + ReportingApiQueryResponse fancyResponse = new ReportingApiQueryResponse(view, isEditable, false, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + fancyResponse.arrayMultiValueColumns(arrayMultiValueColumns); + fancyResponse.includeFormattedValue(includeFormattedValue); + response = fancyResponse; + } + else if (getRequestedApiVersion() >= 9.1) + { + response = new ExtendedApiQueryResponse(view, isEditable, + false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), form.isIncludeMetadata()); + } + else + { + response = new ApiQueryResponse(view, isEditable, + false, schemaName, form.isSaveInSession() ? settings.getQueryName() : "sql", offset, null, + metaDataOnly, form.isIncludeDetailsColumn(), form.isIncludeUpdateColumn(), + form.isIncludeDisplayValues()); + } + response.includeStyle(form.isIncludeStyle()); + + return response; + } + } + + public static class ContainerFilterQueryForm extends QueryForm + { + private String _containerFilter; + + public String getContainerFilter() + { + return _containerFilter; + } + + public void setContainerFilter(String containerFilter) + { + _containerFilter = containerFilter; + } + + @Override + protected QuerySettings createQuerySettings(UserSchema schema) + { + var result = super.createQuerySettings(schema); + if (getContainerFilter() != null) + { + // If the user specified an incorrect filter, throw an IllegalArgumentException + try + { + ContainerFilter.Type containerFilterType = ContainerFilter.Type.valueOf(getContainerFilter()); + result.setContainerFilterName(containerFilterType.name()); + } + catch (IllegalArgumentException e) + { + // Remove bogus value from error message, Issue 45567 + throw new IllegalArgumentException("'containerFilter' parameter is not valid"); + } + } + return result; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public class SelectDistinctAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(ContainerFilterQueryForm form, BindException errors) throws Exception + { + TableInfo table = form.getQueryView().getTable(); + if (null == table) + throw new NotFoundException(); + SqlSelector sqlSelector = getDistinctSql(table, form, errors); + + if (errors.hasErrors() || null == sqlSelector) + return null; + + ApiResponseWriter writer = new ApiJsonWriter(getViewContext().getResponse()); + + try (ResultSet rs = sqlSelector.getResultSet()) + { + writer.startResponse(); + writer.writeProperty("schemaName", form.getSchemaName()); + writer.writeProperty("queryName", form.getQueryName()); + writer.startList("values"); + + while (rs.next()) + { + writer.writeListEntry(rs.getObject(1)); + } + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + catch (DataAccessException x) // Spring error translator can return various subclasses of this + { + throw new RuntimeException(x); + } + writer.endList(); + writer.endResponse(); + + return null; + } + + @Nullable + private SqlSelector getDistinctSql(TableInfo table, ContainerFilterQueryForm form, BindException errors) + { + QuerySettings settings = form.getQuerySettings(); + QueryService service = QueryService.get(); + + if (null == getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())) + { + settings.setMaxRows(DEFAULT_API_MAX_ROWS); + } + else + { + try + { + int maxRows = Integer.parseInt(getViewContext().getRequest().getParameter(QueryParam.maxRows.toString())); + settings.setMaxRows(maxRows); + } + catch (NumberFormatException e) + { + // Standard exception message, Issue 45567 + QuerySettings.throwParameterParseException(QueryParam.maxRows); + } + } + + List fieldKeys = settings.getFieldKeys(); + if (null == fieldKeys || fieldKeys.size() != 1) + { + errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); + return null; + } + Map columns = service.getColumns(table, fieldKeys); + if (columns.size() != 1) + { + errors.reject(ERROR_MSG, "Select Distinct requires that only one column be requested."); + return null; + } + + ColumnInfo col = columns.get(settings.getFieldKeys().get(0)); + if (col == null) + { + errors.reject(ERROR_MSG, "\"" + settings.getFieldKeys().get(0).getName() + "\" is not a valid column."); + return null; + } + + try + { + SimpleFilter filter = getFilterFromQueryForm(form); + + // Strip out filters on columns that don't exist - issue 21669 + service.ensureRequiredColumns(table, columns.values(), filter, null, new HashSet<>()); + QueryLogging queryLogging = new QueryLogging(); + QueryService.SelectBuilder builder = service.getSelectBuilder(table) + .columns(columns.values()) + .filter(filter) + .queryLogging(queryLogging) + .distinct(true); + SQLFragment selectSql = builder.buildSqlFragment(); + + // TODO: queryLogging.isShouldAudit() is always false at this point. + // The only place that seems to set this is ComplianceQueryLoggingProfileListener.queryInvoked() + if (queryLogging.isShouldAudit() && null != queryLogging.getExceptionToThrowIfLoggingIsEnabled()) + { + // this is probably a more helpful message + errors.reject(ERROR_MSG, "Cannot choose values from a column that requires logging."); + return null; + } + + // Regenerate the column since the alias may have changed after call to getSelectSQL() + columns = service.getColumns(table, settings.getFieldKeys()); + var colGetAgain = columns.get(settings.getFieldKeys().get(0)); + // I don't believe the above comment, so here's an assert + assert(colGetAgain.getAlias().equals(col.getAlias())); + + SQLFragment sql = new SQLFragment("SELECT ").appendIdentifier(col.getAlias()).append(" AS value FROM ("); + sql.append(selectSql); + sql.append(") S ORDER BY value"); + + sql = table.getSqlDialect().limitRows(sql, settings.getMaxRows()); + + // 18875: Support Parameterized queries in Select Distinct + Map _namedParameters = settings.getQueryParameters(); + + service.bindNamedParameters(sql, _namedParameters); + service.validateNamedParameters(sql); + + return new SqlSelector(table.getSchema().getScope(), sql, queryLogging); + } + catch (ConversionException | QueryService.NamedParameterNotProvided e) + { + errors.reject(ERROR_MSG, e.getMessage()); + return null; + } + } + } + + private SimpleFilter getFilterFromQueryForm(QueryForm form) + { + QuerySettings settings = form.getQuerySettings(); + SimpleFilter filter = null; + + // 21032: Respect 'ignoreFilter' + if (settings != null && !settings.getIgnoreUserFilter()) + { + // Attach any URL-based filters. This would apply to 'filterArray' from the JavaScript API. + filter = new SimpleFilter(settings.getBaseFilter()); + + String dataRegionName = form.getDataRegionName(); + if (StringUtils.trimToNull(dataRegionName) == null) + dataRegionName = QueryView.DATAREGIONNAME_DEFAULT; + + // Support for 'viewName' + CustomView view = settings.getCustomView(getViewContext(), form.getQueryDef()); + if (null != view && view.hasFilterOrSort() && !settings.getIgnoreViewFilter()) + { + ActionURL url = new ActionURL(SelectDistinctAction.class, getContainer()); + view.applyFilterAndSortToURL(url, dataRegionName); + filter.addAllClauses(new SimpleFilter(url, dataRegionName)); + } + + filter.addUrlFilters(settings.getSortFilterURL(), dataRegionName, Collections.emptyList(), getUser(), getContainer()); + } + + return filter; + } + + @RequiresPermission(ReadPermission.class) + public class GetColumnSummaryStatsAction extends ReadOnlyApiAction + { + private FieldKey _colFieldKey; + + @Override + public void validateForm(QueryForm form, Errors errors) + { + QuerySettings settings = form.getQuerySettings(); + List fieldKeys = settings != null ? settings.getFieldKeys() : null; + if (null == fieldKeys || fieldKeys.size() != 1) + errors.reject(ERROR_MSG, "GetColumnSummaryStats requires that only one column be requested."); + else + _colFieldKey = fieldKeys.get(0); + } + + @Override + public ApiResponse execute(QueryForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + QueryView view = form.getQueryView(); + DisplayColumn displayColumn = null; + + for (DisplayColumn dc : view.getDisplayColumns()) + { + if (dc.getColumnInfo() != null && _colFieldKey.equals(dc.getColumnInfo().getFieldKey())) + { + displayColumn = dc; + break; + } + } + + if (displayColumn != null && displayColumn.getColumnInfo() != null) + { + // get the map of the analytics providers to their relevant aggregates and add the information to the response + Map> analyticsProviders = new LinkedHashMap<>(); + Set colAggregates = new HashSet<>(); + for (ColumnAnalyticsProvider analyticsProvider : displayColumn.getAnalyticsProviders()) + { + if (analyticsProvider instanceof BaseAggregatesAnalyticsProvider baseAggProvider) + { + Map props = new HashMap<>(); + props.put("label", baseAggProvider.getLabel()); + + List aggregateNames = new ArrayList<>(); + for (Aggregate aggregate : AnalyticsProviderItem.createAggregates(baseAggProvider, _colFieldKey, null)) + { + aggregateNames.add(aggregate.getType().getName()); + colAggregates.add(aggregate); + } + props.put("aggregates", aggregateNames); + + analyticsProviders.put(baseAggProvider.getName(), props); + } + } + + // get the filter set from the queryform and verify that they resolve + SimpleFilter filter = getFilterFromQueryForm(form); + if (filter != null) + { + Map resolvedCols = QueryService.get().getColumns(view.getTable(), filter.getAllFieldKeys()); + for (FieldKey filterFieldKey : filter.getAllFieldKeys()) + { + if (!resolvedCols.containsKey(filterFieldKey)) + filter.deleteConditions(filterFieldKey); + } + } + + // query the table/view for the aggregate results + Collection columns = Collections.singleton(displayColumn.getColumnInfo()); + TableSelector selector = new TableSelector(view.getTable(), columns, filter, null).setNamedParameters(form.getQuerySettings().getQueryParameters()); + Map> aggResults = selector.getAggregates(new ArrayList<>(colAggregates)); + + // create a response object mapping the analytics providers to their relevant aggregate results + Map> aggregateResults = new HashMap<>(); + if (aggResults.containsKey(_colFieldKey.toString())) + { + for (Aggregate.Result r : aggResults.get(_colFieldKey.toString())) + { + Map props = new HashMap<>(); + Aggregate.Type type = r.getAggregate().getType(); + props.put("label", type.getFullLabel()); + props.put("description", type.getDescription()); + props.put("value", r.getFormattedValue(displayColumn, getContainer()).value()); + aggregateResults.put(type.getName(), props); + } + + response.put("success", true); + response.put("analyticsProviders", analyticsProviders); + response.put("aggregateResults", aggregateResults); + } + else + { + response.put("success", false); + response.put("message", "Unable to get aggregate results for " + _colFieldKey); + } + } + else + { + response.put("success", false); + response.put("message", "Unable to find ColumnInfo for " + _colFieldKey); + } + + return response; + } + } + + @RequiresPermission(ReadPermission.class) + public class ImportAction extends AbstractQueryImportAction + { + private QueryForm _form; + + @Override + protected void initRequest(QueryForm form) throws ServletException + { + _form = form; + + _insertOption = form.getInsertOption(); + QueryDefinition query = form.getQueryDef(); + List qpe = new ArrayList<>(); + TableInfo t = query.getTable(form.getSchema(), qpe, true); + if (!qpe.isEmpty()) + throw qpe.get(0); + if (null != t) + setTarget(t); + _auditBehaviorType = form.getAuditBehavior(); + _auditUserComment = form.getAuditUserComment(); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) throws Exception + { + initRequest(form); + return super.getDefaultImportView(form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new SchemaAction(_form).addNavTrail(root); + var executeQuery = _form.urlFor(QueryAction.executeQuery); + if (null == executeQuery) + root.addChild(_form.getQueryName()); + else + root.addChild(_form.getQueryName(), executeQuery); + root.addChild("Import Data"); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportSqlForm + { + private String _sql; + private String _schemaName; + private String _containerFilter; + private String _format = "excel"; + + public String getSql() + { + return _sql; + } + + public void setSql(String sql) + { + _sql = PageFlowUtil.wafDecode(sql); + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getContainerFilter() + { + return _containerFilter; + } + + public void setContainerFilter(String containerFilter) + { + _containerFilter = containerFilter; + } + + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + } + + @RequiresPermission(ReadPermission.class) + @ApiVersion(9.2) + @Action(ActionType.Export.class) + public static class ExportSqlAction extends ExportAction + { + @Override + public void export(ExportSqlForm form, HttpServletResponse response, BindException errors) throws IOException, ExportException + { + String schemaName = StringUtils.trimToNull(form.getSchemaName()); + if (null == schemaName) + throw new NotFoundException("No value was supplied for the required parameter 'schemaName'"); + String sql = StringUtils.trimToNull(form.getSql()); + if (null == sql) + throw new NotFoundException("No value was supplied for the required parameter 'sql'"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), schemaName); + + if (null == schema) + throw new NotFoundException("Schema '" + schemaName + "' not found in this folder"); + + //create a temp query settings object initialized with the posted LabKey SQL + //this will provide a temporary QueryDefinition to Query + TempQuerySettings settings = new TempQuerySettings(getViewContext(), sql); + + //need to explicitly turn off various UI options that will try to refer to the + //current URL and query string + settings.setAllowChooseView(false); + settings.setAllowCustomizeView(false); + + //return all rows + settings.setShowRows(ShowRows.ALL); + + //add container filter if supplied + if (form.getContainerFilter() != null && !form.getContainerFilter().isEmpty()) + { + ContainerFilter.Type containerFilterType = + ContainerFilter.Type.valueOf(form.getContainerFilter()); + settings.setContainerFilterName(containerFilterType.name()); + } + + //build a query view using the schema and settings + QueryView view = new QueryView(schema, settings, errors); + view.setShowRecordSelectors(false); + view.setShowExportButtons(false); + view.setButtonBarPosition(DataRegion.ButtonBarPosition.NONE); + + //export it + ResponseHelper.setPrivate(response); + response.setHeader("X-Robots-Tag", "noindex"); + + if ("excel".equalsIgnoreCase(form.getFormat())) + view.exportToExcel(response); + else if ("tsv".equalsIgnoreCase(form.getFormat())) + view.exportToTsv(response); + else + errors.reject(null, "Invalid format specified; must be 'excel' or 'tsv'"); + + for (QueryException qe : view.getParseErrors()) + errors.reject(null, qe.getMessage()); + + if (errors.hasErrors()) + throw new ExportException(new SimpleErrorView(errors, false)); + } + } + + public static class ApiSaveRowsForm extends SimpleApiJsonForm + { + } + + private enum CommandType + { + insert(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException + { + BatchValidationException errors = new BatchValidationException(); + List> insertedRows = qus.insertRows(user, container, rows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + // Issue 42519: Submitter role not able to insert + // as per the definition of submitter, should allow insert without read + if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) + { + return qus.getRows(user, container, insertedRows); + } + else + { + return insertedRows; + } + } + }, + insertWithKeys(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException + { + List> newRows = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + for (Map row : rows) + { + //issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null + CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); + newRows.add(newMap); + + CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); + oldKeys.add(oldMap); + } + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.insertRows(user, container, newRows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + // Issue 42519: Submitter role not able to insert + // as per the definition of submitter, should allow insert without read + if (qus.hasPermission(user, ReadPermission.class) && shouldReselect(configParameters)) + { + updatedRows = qus.getRows(user, container, updatedRows); + } + List> results = new ArrayList<>(); + for (int i = 0; i < updatedRows.size(); i++) + { + Map result = new HashMap<>(); + result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); + result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); + results.add(result); + } + return results; + } + }, + importRows(InsertPermission.class, QueryService.AuditAction.INSERT) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + DataIteratorBuilder it = new ListofMapsDataIterator.Builder(rows.get(0).keySet(), rows); + qus.importRows(user, container, it, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return Collections.emptyList(); + } + }, + moveRows(MoveEntitiesPermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + + Container targetContainer = (Container) configParameters.get(QueryUpdateService.ConfigParameters.TargetContainer); + Map updatedCounts = qus.moveRows(user, container, targetContainer, rows, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return Collections.singletonList(updatedCounts); + } + }, + update(UpdatePermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.updateRows(user, container, rows, null, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + return shouldReselect(configParameters) ? qus.getRows(user, container, updatedRows) : updatedRows; + } + }, + updateChangingKeys(UpdatePermission.class, QueryService.AuditAction.UPDATE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + List> newRows = new ArrayList<>(); + List> oldKeys = new ArrayList<>(); + for (Map row : rows) + { + // issue 13719: use CaseInsensitiveHashMaps. Also allow either values or oldKeys to be null. + // this should never happen on an update, but we will let it fail later with a better error message instead of the NPE here + CaseInsensitiveHashMap newMap = row.get(SaveRowsAction.PROP_VALUES) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_VALUES)).toMap()) : new CaseInsensitiveHashMap<>(); + newRows.add(newMap); + + CaseInsensitiveHashMap oldMap = row.get(SaveRowsAction.PROP_OLD_KEYS) != null ? new CaseInsensitiveHashMap<>(((JSONObject)row.get(SaveRowsAction.PROP_OLD_KEYS)).toMap()) : new CaseInsensitiveHashMap<>(); + oldKeys.add(oldMap); + } + BatchValidationException errors = new BatchValidationException(); + List> updatedRows = qus.updateRows(user, container, newRows, oldKeys, errors, configParameters, extraContext); + if (errors.hasErrors()) + throw errors; + if (shouldReselect(configParameters)) + updatedRows = qus.getRows(user, container, updatedRows); + List> results = new ArrayList<>(); + for (int i = 0; i < updatedRows.size(); i++) + { + Map result = new HashMap<>(); + result.put(SaveRowsAction.PROP_VALUES, updatedRows.get(i)); + result.put(SaveRowsAction.PROP_OLD_KEYS, oldKeys.get(i)); + results.add(result); + } + return results; + } + }, + delete(DeletePermission.class, QueryService.AuditAction.DELETE) + { + @Override + public List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException + { + return qus.deleteRows(user, container, rows, configParameters, extraContext); + } + }; + + private final Class _permission; + private final QueryService.AuditAction _auditAction; + + CommandType(Class permission, QueryService.AuditAction auditAction) + { + _permission = permission; + _auditAction = auditAction; + } + + public Class getPermission() + { + return _permission; + } + + public QueryService.AuditAction getAuditAction() + { + return _auditAction; + } + + public static boolean shouldReselect(Map configParameters) + { + if (configParameters == null || !configParameters.containsKey(QueryUpdateService.ConfigParameters.SkipReselectRows)) + return true; + + return Boolean.TRUE != configParameters.get(QueryUpdateService.ConfigParameters.SkipReselectRows); + } + + public abstract List> saveRows(QueryUpdateService qus, List> rows, User user, Container container, Map configParameters, Map extraContext) + throws SQLException, InvalidKeyException, QueryUpdateServiceException, BatchValidationException, DuplicateKeyException; + } + + /** + * Base action class for insert/update/delete actions + */ + protected abstract static class BaseSaveRowsAction
extends MutatingApiAction + { + public static final String PROP_SCHEMA_NAME = "schemaName"; + public static final String PROP_QUERY_NAME = "queryName"; + public static final String PROP_CONTAINER_PATH = "containerPath"; + public static final String PROP_TARGET_CONTAINER_PATH = "targetContainerPath"; + public static final String PROP_COMMAND = "command"; + public static final String PROP_ROWS = "rows"; + + private JSONObject _json; + + @Override + public void validateForm(FORM apiSaveRowsForm, Errors errors) + { + _json = apiSaveRowsForm.getJsonObject(); + + // if the POST was done using FormData, the apiSaveRowsForm would not have bound the json data, so + // we'll instead look for that data in the request param directly + if (_json == null && getViewContext().getRequest() != null && getViewContext().getRequest().getParameter("json") != null) + _json = new JSONObject(getViewContext().getRequest().getParameter("json")); + } + + protected JSONObject getJsonObject() + { + return _json; + } + + protected Container getContainerForCommand(JSONObject json) + { + return getContainerForCommand(json, PROP_CONTAINER_PATH, getContainer()); + } + + protected Container getContainerForCommand(JSONObject json, String containerPathProp, @Nullable Container defaultContainer) + { + Container container; + String containerPath = StringUtils.trimToNull(json.optString(containerPathProp)); + if (containerPath == null) + { + if (defaultContainer != null) + container = defaultContainer; + else + throw new IllegalArgumentException(containerPathProp + " is required but was not provided."); + } + else + { + container = ContainerManager.getForPath(containerPath); + if (container == null) + { + throw new IllegalArgumentException("Unknown container: " + containerPath); + } + } + + // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more downstream + if (!container.hasPermission(getUser(), ReadPermission.class) && + !container.hasPermission(getUser(), DeletePermission.class) && + !container.hasPermission(getUser(), InsertPermission.class) && + !container.hasPermission(getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + return container; + } + + protected String getTargetContainerProp() + { + JSONObject json = getJsonObject(); + return json.optString(PROP_TARGET_CONTAINER_PATH, null); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors) throws Exception + { + return executeJson(json, commandType, allowTransaction, errors, false); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction) throws Exception + { + return executeJson(json, commandType, allowTransaction, errors, isNestedTransaction, null); + } + + protected JSONObject executeJson(JSONObject json, CommandType commandType, boolean allowTransaction, Errors errors, boolean isNestedTransaction, @Nullable Integer commandIndex) throws Exception + { + JSONObject response = new JSONObject(); + Container container = getContainerForCommand(json); + User user = getUser(); + + if (json == null) + throw new ValidationException("Empty request"); + + JSONArray rows; + try + { + rows = json.getJSONArray(PROP_ROWS); + if (rows.isEmpty()) + throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); + } + catch (JSONException x) + { + throw new ValidationException("No '" + PROP_ROWS + "' array supplied."); + } + + String schemaName = json.getString(PROP_SCHEMA_NAME); + String queryName = json.getString(PROP_QUERY_NAME); + TableInfo table = getTableInfo(container, user, schemaName, queryName); + + if (!table.hasPermission(user, commandType.getPermission())) + throw new UnauthorizedException(); + + if (commandType != CommandType.insert && table.getPkColumns().isEmpty()) + throw new IllegalArgumentException("The table '" + table.getPublicSchemaName() + "." + + table.getPublicName() + "' cannot be updated because it has no primary key defined!"); + + QueryUpdateService qus = table.getUpdateService(); + if (null == qus) + throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + + "' is not updatable via the HTTP-based APIs."); + + int rowsAffected = 0; + + List> rowsToProcess = new ArrayList<>(); + + // NOTE RowMapFactory is faster, but for update it's important to preserve missing v explicit NULL values + // Do we need to support some sort of UNDEFINED and NULL instance of MvFieldWrapper? + RowMapFactory f = null; + if (commandType == CommandType.insert || commandType == CommandType.insertWithKeys || commandType == CommandType.delete) + f = new RowMapFactory<>(); + CaseInsensitiveHashMap referenceCasing = new CaseInsensitiveHashMap<>(); + + for (int idx = 0; idx < rows.length(); ++idx) + { + JSONObject jsonObj; + try + { + jsonObj = rows.getJSONObject(idx); + } + catch (JSONException x) + { + throw new IllegalArgumentException("rows[" + idx + "] is not an object."); + } + if (null != jsonObj) + { + Map rowMap = null == f ? new CaseInsensitiveHashMap<>(new HashMap<>(), referenceCasing) : f.getRowMap(); + // Use shallow copy since jsonObj.toMap() will translate contained JSONObjects into Maps, which we don't want + boolean conflictingCasing = JsonUtil.fillMapShallow(jsonObj, rowMap); + if (conflictingCasing) + { + // Issue 52616 + LOG.error("Row contained conflicting casing for key names in the incoming row: {}", jsonObj); + } + if (allowRowAttachments()) + addRowAttachments(table, rowMap, idx, commandIndex); + + rowsToProcess.add(rowMap); + rowsAffected++; + } + } + + Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); + + Map auditDetails = json.has("auditDetails") ? json.getJSONObject("auditDetails").toMap() : new CaseInsensitiveHashMap<>(); + + Map configParameters = new HashMap<>(); + + // Check first if the audit behavior has been defined for the table either in code or through XML. + // If not defined there, check for the audit behavior defined in the action form (json). + AuditBehaviorType behaviorType = table.getEffectiveAuditBehavior(json.optString("auditBehavior", null)); + if (behaviorType != null) + { + configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, behaviorType); + String auditComment = json.optString("auditUserComment", null); + if (!StringUtils.isEmpty(auditComment)) + configParameters.put(DetailedAuditLogDataIterator.AuditConfigs.AuditUserComment, auditComment); + } + + boolean skipReselectRows = json.optBoolean("skipReselectRows", false); + if (skipReselectRows) + configParameters.put(QueryUpdateService.ConfigParameters.SkipReselectRows, true); + + if (getTargetContainerProp() != null) + { + Container targetContainer = getContainerForCommand(json, PROP_TARGET_CONTAINER_PATH, null); + configParameters.put(QueryUpdateService.ConfigParameters.TargetContainer, targetContainer); + } + + //set up the response, providing the schema name, query name, and operation + //so that the client can sort out which request this response belongs to + //(clients often submit these async) + response.put(PROP_SCHEMA_NAME, schemaName); + response.put(PROP_QUERY_NAME, queryName); + response.put("command", commandType.name()); + response.put("containerPath", container.getPath()); + + //we will transact operations by default, but the user may + //override this by sending a "transacted" property set to false + // 11741: A transaction may already be active if we're trying to + // insert/update/delete from within a transformation/validation script. + boolean transacted = allowTransaction && json.optBoolean("transacted", true); + TransactionAuditProvider.TransactionAuditEvent auditEvent = null; + try (DbScope.Transaction transaction = transacted ? table.getSchema().getScope().ensureTransaction() : NO_OP_TRANSACTION) + { + if (behaviorType != null && behaviorType != AuditBehaviorType.NONE) + { + DbScope.Transaction auditTransaction = !transacted && isNestedTransaction ? table.getSchema().getScope().getCurrentTransaction() : transaction; + if (auditTransaction == null) + auditTransaction = NO_OP_TRANSACTION; + + if (auditTransaction.getAuditEvent() != null) + { + auditEvent = auditTransaction.getAuditEvent(); + // detect trigger event? + } + else + { + Map transactionDetails = getTransactionAuditDetails(); + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetails); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), transactionDetails); + AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); + } + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.QueryCommand, commandType.name()); + } + + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); + List> responseRows = + commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); + if (auditEvent != null) + { + auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); + if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.BatchAction))) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true); + } + + if (commandType == CommandType.moveRows) + { + // moveRows returns a single map of updateCounts + response.put("updateCounts", responseRows.get(0)); + } + else if (commandType != CommandType.importRows) + { + response.put("rows", responseRows.stream() + .map(JsonUtil::toMapPreserveNonFinite) + .map(JsonUtil::toJsonPreserveNulls) + .collect(LabKeyCollectors.toJSONArray())); + } + + // if there is any provenance information, save it here + ProvenanceService svc = ProvenanceService.get(); + if (json.has("provenance")) + { + JSONObject provenanceJSON = json.getJSONObject("provenance"); + ProvenanceRecordingParams params = svc.createRecordingParams(getViewContext(), provenanceJSON, ProvenanceService.ADD_RECORDING); + RecordedAction action = svc.createRecordedAction(getViewContext(), params); + if (action != null && params.getRecordingId() != null) + { + // check for any row level provenance information + if (json.has("rows")) + { + Object rowObject = json.get("rows"); + if (rowObject instanceof JSONArray jsonArray) + { + // we need to match any provenance object inputs to the object outputs from the response rows, this typically would + // be the row lsid but it configurable in the provenance recording params + // + List> provenanceMap = svc.createProvenanceMapFromRows(getViewContext(), params, jsonArray, responseRows); + if (!provenanceMap.isEmpty()) + { + action.getProvenanceMap().addAll(provenanceMap); + } + svc.addRecordingStep(getViewContext().getRequest(), params.getRecordingId(), action); + } + else + { + errors.reject(SpringActionController.ERROR_MSG, "Unable to process provenance information, the rows object was not an array"); + } + } + } + } + transaction.commit(); + } + catch (OptimisticConflictException e) + { + //issue 13967: provide better message for OptimisticConflictException + errors.reject(SpringActionController.ERROR_MSG, e.getMessage()); + } + catch (QueryUpdateServiceException | ConversionException | DuplicateKeyException | DataIntegrityViolationException e) + { + //Issue 14294: improve handling of ConversionException (and DuplicateKeyException (Issue 28037), and DataIntegrity (uniqueness) (Issue 22779) + errors.reject(SpringActionController.ERROR_MSG, e.getMessage() == null ? e.toString() : e.getMessage()); + } + catch (BatchValidationException e) + { + if (isSuccessOnValidationError()) + { + response.put("errors", createResponseWriter().toJSON(e)); + } + else + { + ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.SkipMothershipLogging, "true", true); + throw e; + } + } + if (auditEvent != null) + { + response.put("transactionAuditId", auditEvent.getRowId()); + response.put("reselectRowCount", auditEvent.hasMultiActions()); + } + + response.put("rowsAffected", rowsAffected); + + return response; + } + + protected boolean allowRowAttachments() + { + return false; + } + + private void addRowAttachments(TableInfo tableInfo, Map rowMap, int rowIndex, @Nullable Integer commandIndex) + { + if (getFileMap() != null) + { + for (Map.Entry fileEntry : getFileMap().entrySet()) + { + // Allow for the fileMap key to include the row index, and optionally command index, for defining + // which row to attach this file to + String fullKey = fileEntry.getKey(); + String fieldKey = fullKey; + // Issue 52827: Cannot attach a file if the field name contains :: + // use lastIndexOf instead of split to get the proper parts + int lastDelimIndex = fullKey.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); + if (lastDelimIndex > -1) + { + String fieldKeyExcludeIndex = fullKey.substring(0, lastDelimIndex); + String fieldRowIndex = fullKey.substring(lastDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); + if (!fieldRowIndex.equals(rowIndex+"")) continue; + + if (commandIndex == null) + { + // Single command, so we're parsing file names in the format of: FileField::0 + fieldKey = fieldKeyExcludeIndex; + } + else + { + // Multi-command, so we're parsing file names in the format of: FileField::0::1 + int subDelimIndex = fieldKeyExcludeIndex.lastIndexOf(ROW_ATTACHMENT_INDEX_DELIM); + if (subDelimIndex > -1) + { + fieldKey = fieldKeyExcludeIndex.substring(0, subDelimIndex); + String fieldCommandIndex = fieldKeyExcludeIndex.substring(subDelimIndex + ROW_ATTACHMENT_INDEX_DELIM.length()); + if (!fieldCommandIndex.equals(commandIndex+"")) + continue; + } + else + continue; + } + } + + SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); + rowMap.put(fieldKey, file.isEmpty() ? null : file); + } + } + + for (ColumnInfo col : tableInfo.getColumns()) + DataIteratorUtil.MatchType.multiPartFormData.updateRowMap(col, rowMap); + } + + protected boolean isSuccessOnValidationError() + { + return getRequestedApiVersion() >= 13.2; + } + + @NotNull + protected TableInfo getTableInfo(Container container, User user, String schemaName, String queryName) + { + if (null == schemaName || null == queryName) + throw new IllegalArgumentException("You must supply a schemaName and queryName!"); + + UserSchema schema = QueryService.get().getUserSchema(user, container, schemaName); + if (null == schema) + throw new IllegalArgumentException("The schema '" + schemaName + "' does not exist."); + + TableInfo table = schema.getTableForInsert(queryName); + if (table == null) + throw new IllegalArgumentException("The query '" + queryName + "' in the schema '" + schemaName + "' does not exist."); + return table; + } + } + + // Issue: 20522 - require read access to the action but executeJson will check for update privileges from the table + // + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class UpdateRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.update, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + } + + @RequiresAnyOf({ReadPermission.class, InsertPermission.class}) //will check below + @ApiVersion(8.3) + public static class InsertRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.insert, true, errors); + if (response == null || errors.hasErrors()) + return null; + + return new ApiSimpleResponse(response); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + } + + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class ImportRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.importRows, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + } + + @ActionNames("deleteRows, delRows") + @RequiresPermission(ReadPermission.class) //will check below + @ApiVersion(8.3) + public static class DeleteRowsAction extends BaseSaveRowsAction + { + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + JSONObject response = executeJson(getJsonObject(), CommandType.delete, true, errors); + if (response == null || errors.hasErrors()) + return null; + return new ApiSimpleResponse(response); + } + } + + @RequiresPermission(ReadPermission.class) //will check below + public static class MoveRowsAction extends BaseSaveRowsAction + { + private Container _targetContainer; + + @Override + public void validateForm(MoveRowsForm form, Errors errors) + { + super.validateForm(form, errors); + + JSONObject json = getJsonObject(); + if (json == null) + { + errors.reject(ERROR_GENERIC, "Empty request"); + } + else + { + // Since we are moving between containers, we know we have product folders enabled + if (getContainer().getProject().getAuditCommentsRequired() && StringUtils.isBlank(json.optString("auditUserComment"))) + errors.reject(ERROR_GENERIC, "A reason for the move of data is required."); + else + { + String queryName = json.optString(PROP_QUERY_NAME, null); + _targetContainer = ContainerManager.getMoveTargetContainer(queryName, getContainer(), getUser(), getTargetContainerProp(), errors); + } + } + } + + @Override + public ApiResponse execute(MoveRowsForm form, BindException errors) throws Exception + { + // if JSON does not have rows array, see if they were provided via selectionKey + if (!getJsonObject().has(PROP_ROWS)) + setRowsFromSelectionKey(form); + + JSONObject response = executeJson(getJsonObject(), CommandType.moveRows, true, errors); + if (response == null || errors.hasErrors()) + return null; + + updateSelections(form); + + response.put("success", true); + response.put("containerPath", _targetContainer.getPath()); + return new ApiSimpleResponse(response); + } + + private void updateSelections(MoveRowsForm form) + { + String selectionKey = form.getDataRegionSelectionKey(); + if (selectionKey != null) + { + Set rowIds = form.getIds(getViewContext(), false) + .stream().map(Object::toString).collect(Collectors.toSet()); + DataRegionSelection.setSelected(getViewContext(), selectionKey, rowIds, false); + + // if moving entities from a type, the selections from other selectionKeys in that container will + // possibly be holding onto invalid keys after the move, so clear them based on the containerPath and selectionKey suffix + String[] keyParts = selectionKey.split("|"); + if (keyParts.length > 1) + DataRegionSelection.clearRelatedByContainerPath(getViewContext(), keyParts[keyParts.length - 1]); + } + } + + private void setRowsFromSelectionKey(MoveRowsForm form) + { + Set rowIds = form.getIds(getViewContext(), false); // handle clear of selectionKey after move complete + + // convert rowIds to a JSONArray of JSONObjects with a single property "RowId" + JSONArray rows = new JSONArray(); + for (Long rowId : rowIds) + { + JSONObject row = new JSONObject(); + row.put("RowId", rowId); + rows.put(row); + } + getJsonObject().put(PROP_ROWS, rows); + } + } + + public static class MoveRowsForm extends ApiSaveRowsForm + { + private String _dataRegionSelectionKey; + private boolean _useSnapshotSelection; + + public String getDataRegionSelectionKey() + { + return _dataRegionSelectionKey; + } + + public void setDataRegionSelectionKey(String dataRegionSelectionKey) + { + _dataRegionSelectionKey = dataRegionSelectionKey; + } + + public boolean isUseSnapshotSelection() + { + return _useSnapshotSelection; + } + + public void setUseSnapshotSelection(boolean useSnapshotSelection) + { + _useSnapshotSelection = useSnapshotSelection; + } + + @Override + public void bindJson(JSONObject json) + { + super.bindJson(json); + _dataRegionSelectionKey = json.optString("dataRegionSelectionKey", null); + _useSnapshotSelection = json.optBoolean("useSnapshotSelection", false); + } + + public Set getIds(ViewContext context, boolean clear) + { + if (_useSnapshotSelection) + return new HashSet<>(DataRegionSelection.getSnapshotSelectedIntegers(context, getDataRegionSelectionKey())); + else + return DataRegionSelection.getSelectedIntegers(context, getDataRegionSelectionKey(), clear); + } + } + + @RequiresNoPermission //will check below + public static class SaveRowsAction extends BaseSaveRowsAction + { + public static final String PROP_VALUES = "values"; + public static final String PROP_OLD_KEYS = "oldKeys"; + + @Override + protected boolean isFailure(BindException errors) + { + return !isSuccessOnValidationError() && super.isFailure(errors); + } + + @Override + protected boolean allowRowAttachments() + { + return true; + } + + @Override + public ApiResponse execute(ApiSaveRowsForm apiSaveRowsForm, BindException errors) throws Exception + { + // Issue 21850: Verify that the user has at least some sort of basic access to the container. We'll check for more + // specific permissions later once we've figured out exactly what they're trying to do. This helps us + // give a better HTTP response code when they're trying to access a resource that's not available to guests + if (!getContainer().hasPermission(getUser(), ReadPermission.class) && + !getContainer().hasPermission(getUser(), DeletePermission.class) && + !getContainer().hasPermission(getUser(), InsertPermission.class) && + !getContainer().hasPermission(getUser(), UpdatePermission.class)) + { + throw new UnauthorizedException(); + } + + JSONObject json = getJsonObject(); + if (json == null) + throw new IllegalArgumentException("Empty request"); + + JSONArray commands = json.optJSONArray("commands"); + if (commands == null || commands.isEmpty()) + { + throw new NotFoundException("Empty request"); + } + + boolean validateOnly = json.optBoolean("validateOnly", false); + // If we are going to validate and not commit, we need to be sure we're transacted as well. Otherwise, + // respect the client's request. + boolean transacted = validateOnly || json.optBoolean("transacted", true); + + // Keep track of whether we end up committing or not + boolean committed = false; + + DbScope scope = null; + if (transacted) + { + for (int i = 0; i < commands.length(); i++) + { + JSONObject commandJSON = commands.getJSONObject(i); + String schemaName = commandJSON.getString(PROP_SCHEMA_NAME); + String queryName = commandJSON.getString(PROP_QUERY_NAME); + Container container = getContainerForCommand(commandJSON); + TableInfo tableInfo = getTableInfo(container, getUser(), schemaName, queryName); + if (scope == null) + { + scope = tableInfo.getSchema().getScope(); + } + else if (scope != tableInfo.getSchema().getScope()) + { + throw new IllegalArgumentException("All queries must be from the same source database"); + } + } + assert scope != null; + } + + JSONArray resultArray = new JSONArray(); + JSONObject extraContext = json.optJSONObject("extraContext"); + + int startingErrorIndex = 0; + int errorCount = 0; + // 11741: A transaction may already be active if we're trying to + // insert/update/delete from within a transformation/validation script. + + try (DbScope.Transaction transaction = transacted ? scope.ensureTransaction() : NO_OP_TRANSACTION) + { + for (int i = 0; i < commands.length(); i++) + { + JSONObject commandObject = commands.getJSONObject(i); + String commandName = commandObject.getString(PROP_COMMAND); + if (commandName == null) + { + throw new ApiUsageException(PROP_COMMAND + " is required but was missing"); + } + CommandType command = CommandType.valueOf(commandName); + + // Copy the top-level 'extraContext' and merge in the command-level extraContext. + Map commandExtraContext = new HashMap<>(); + if (extraContext != null) + commandExtraContext.putAll(extraContext.toMap()); + if (commandObject.has("extraContext")) + { + commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); + } + commandObject.put("extraContext", commandExtraContext); + + JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); + // Bail out immediately if we're going to return a failure-type response message + if (commandResponse == null || (errors.hasErrors() && !isSuccessOnValidationError())) + return null; + + //this would be populated in executeJson when a BatchValidationException is thrown + if (commandResponse.has("errors")) + { + errorCount += commandResponse.getJSONObject("errors").getInt("errorCount"); + } + + // If we encountered errors with this particular command and the client requested that don't treat + // the whole request as a failure (non-200 HTTP status code), stash the errors for this particular + // command in its response section. + // NOTE: executeJson should handle and serialize BatchValidationException + // these errors upstream + if (errors.getErrorCount() > startingErrorIndex && isSuccessOnValidationError()) + { + commandResponse.put("errors", ApiResponseWriter.convertToJSON(errors, startingErrorIndex).getValue()); + startingErrorIndex = errors.getErrorCount(); + } + + resultArray.put(commandResponse); + } + + // Don't commit if we had errors or if the client requested that we only validate (and not commit) + if (!errors.hasErrors() && !validateOnly && errorCount == 0) + { + transaction.commit(); + committed = true; + } + } + + errorCount += errors.getErrorCount(); + JSONObject result = new JSONObject(); + result.put("result", resultArray); + result.put("committed", committed); + result.put("errorCount", errorCount); + + return new ApiSimpleResponse(result); + } + } + + @RequiresPermission(ReadPermission.class) + public static class ApiTestAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + return new JspView<>("/org/labkey/query/view/apitest.jsp"); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("API Test"); + } + } + + + @RequiresPermission(AdminPermission.class) + public static class AdminAction extends SimpleViewAction + { + @SuppressWarnings("UnusedDeclaration") + public AdminAction() + { + } + + public AdminAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/admin.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Schema Administration", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ResetRemoteConnectionsForm + { + private boolean _reset; + + public boolean isReset() + { + return _reset; + } + + public void setReset(boolean reset) + { + _reset = reset; + } + } + + + @RequiresPermission(AdminPermission.class) + public static class ManageRemoteConnectionsAction extends FormViewAction + { + @Override + public void validateCommand(ResetRemoteConnectionsForm target, Errors errors) {} + + @Override + public boolean handlePost(ResetRemoteConnectionsForm form, BindException errors) + { + if (form.isReset()) + { + PropertyManager.getEncryptedStore().deletePropertySet(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); + } + return true; + } + + @Override + public URLHelper getSuccessURL(ResetRemoteConnectionsForm queryForm) + { + return new ActionURL(ManageRemoteConnectionsAction.class, getContainer()); + } + + @Override + public ModelAndView getView(ResetRemoteConnectionsForm queryForm, boolean reshow, BindException errors) + { + Map connectionMap; + try + { + // if the encrypted property store is configured but no values have yet been set, and empty map is returned + connectionMap = PropertyManager.getEncryptedStore().getProperties(getContainer(), RemoteConnections.REMOTE_QUERY_CONNECTIONS_CATEGORY); + } + catch (Exception e) + { + connectionMap = null; // render the failure page + } + setHelpTopic("remoteConnection"); + return new JspView<>("/org/labkey/query/view/manageRemoteConnections.jsp", connectionMap, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Remote Connections", new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer())); + } + } + + private abstract static class BaseInsertExternalSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction + { + protected BaseInsertExternalSchemaAction(Class commandClass) + { + super(commandClass); + } + + @Override + public void validateCommand(F form, Errors errors) + { + form.validate(errors); + } + + @Override + public boolean handlePost(F form, BindException errors) throws Exception + { + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + form.doInsert(); + auditSchemaAdminActivity(form.getBean(), "created", getContainer(), getUser()); + QueryManager.get().updateExternalSchemas(getContainer()); + + t.commit(); + } + catch (RuntimeSQLException e) + { + if (e.isConstraintException()) + { + errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); + return false; + } + + throw e; + } + + return true; + } + + @Override + public ActionURL getSuccessURL(F form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext()).addNavTrail(root); + root.addChild("Define Schema", new ActionURL(getClass(), getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class InsertLinkedSchemaAction extends BaseInsertExternalSchemaAction + { + public InsertLinkedSchemaAction() + { + super(LinkedSchemaForm.class); + } + + @Override + public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) + { + setHelpTopic("filterSchema"); + return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), form.getBean(), true), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class InsertExternalSchemaAction extends BaseInsertExternalSchemaAction + { + public InsertExternalSchemaAction() + { + super(ExternalSchemaForm.class); + } + + @Override + public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) + { + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), form.getBean(), true), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class DeleteSchemaAction extends ConfirmAction + { + @Override + public String getConfirmText() + { + return "Delete"; + } + + @Override + public ModelAndView getConfirmView(SchemaForm form, BindException errors) + { + if (getPageConfig().getTitle() == null) + setTitle("Delete Schema"); + + AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + String schemaName = isBlank(def.getUserSchemaName()) ? "this schema" : "the schema '" + def.getUserSchemaName() + "'"; + return new HtmlView(HtmlString.of("Are you sure you want to delete " + schemaName + "? The tables and queries defined in this schema will no longer be accessible.")); + } + + @Override + public boolean handlePost(SchemaForm form, BindException errors) + { + AbstractExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), AbstractExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + auditSchemaAdminActivity(def, "deleted", getContainer(), getUser()); + QueryManager.get().delete(def); + t.commit(); + } + return true; + } + + @Override + public void validateCommand(SchemaForm form, Errors errors) + { + } + + @Override + @NotNull + public ActionURL getSuccessURL(SchemaForm form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + } + + private static void auditSchemaAdminActivity(AbstractExternalSchemaDef def, String action, Container container, User user) + { + String comment = StringUtils.capitalize(def.getSchemaType().toString()) + " schema '" + def.getUserSchemaName() + "' " + action; + AuditTypeEvent event = new AuditTypeEvent(ContainerAuditProvider.CONTAINER_AUDIT_EVENT, container, comment); + AuditLogService.get().addEvent(user, event); + } + + + private abstract static class BaseEditSchemaAction, T extends AbstractExternalSchemaDef> extends FormViewAction + { + protected BaseEditSchemaAction(Class commandClass) + { + super(commandClass); + } + + @Override + public void validateCommand(F form, Errors errors) + { + form.validate(errors); + } + + @Nullable + protected abstract T getCurrent(int externalSchemaId); + + @NotNull + protected T getDef(F form, boolean reshow) + { + T def; + Container defContainer; + + if (reshow) + { + def = form.getBean(); + T current = getCurrent(def.getExternalSchemaId()); + if (current == null) + throw new NotFoundException(); + + defContainer = current.lookupContainer(); + } + else + { + form.refreshFromDb(); + if (!form.isDataLoaded()) + throw new NotFoundException(); + + def = form.getBean(); + if (def == null) + throw new NotFoundException(); + + defContainer = def.lookupContainer(); + } + + if (!getContainer().equals(defContainer)) + throw new UnauthorizedException(); + + return def; + } + + @Override + public boolean handlePost(F form, BindException errors) throws Exception + { + T def = form.getBean(); + T fromDb = getCurrent(def.getExternalSchemaId()); + + // Unauthorized if def in the database reports a different container + if (!getContainer().equals(fromDb.lookupContainer())) + throw new UnauthorizedException(); + + try (DbScope.Transaction t = QueryManager.get().getDbSchema().getScope().ensureTransaction()) + { + form.doUpdate(); + auditSchemaAdminActivity(def, "updated", getContainer(), getUser()); + QueryManager.get().updateExternalSchemas(getContainer()); + t.commit(); + } + catch (RuntimeSQLException e) + { + if (e.isConstraintException()) + { + errors.reject(ERROR_MSG, "A schema by that name is already defined in this folder"); + return false; + } + + throw e; + } + return true; + } + + @Override + public ActionURL getSuccessURL(F externalSchemaForm) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new AdminAction(getViewContext()).addNavTrail(root); + root.addChild("Edit Schema", new ActionURL(getClass(), getContainer())); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditLinkedSchemaAction extends BaseEditSchemaAction + { + public EditLinkedSchemaAction() + { + super(LinkedSchemaForm.class); + } + + @Nullable + @Override + protected LinkedSchemaDef getCurrent(int externalId) + { + return QueryManager.get().getLinkedSchemaDef(getContainer(), externalId); + } + + @Override + public ModelAndView getView(LinkedSchemaForm form, boolean reshow, BindException errors) + { + LinkedSchemaDef def = getDef(form, reshow); + + setHelpTopic("filterSchema"); + return new JspView<>("/org/labkey/query/view/linkedSchema.jsp", new LinkedSchemaBean(getContainer(), def, false), errors); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class EditExternalSchemaAction extends BaseEditSchemaAction + { + public EditExternalSchemaAction() + { + super(ExternalSchemaForm.class); + } + + @Nullable + @Override + protected ExternalSchemaDef getCurrent(int externalId) + { + return QueryManager.get().getExternalSchemaDef(getContainer(), externalId); + } + + @Override + public ModelAndView getView(ExternalSchemaForm form, boolean reshow, BindException errors) + { + ExternalSchemaDef def = getDef(form, reshow); + + setHelpTopic("externalSchemas"); + return new JspView<>("/org/labkey/query/view/externalSchema.jsp", new ExternalSchemaBean(getContainer(), def, false), errors); + } + } + + + public static class DataSourceInfo + { + public final String sourceName; + public final String displayName; + public final boolean editable; + + public DataSourceInfo(DbScope scope) + { + this(scope.getDataSourceName(), scope.getDisplayName(), scope.getSqlDialect().isEditable()); + } + + public DataSourceInfo(Container c) + { + this(c.getId(), c.getName(), false); + } + + public DataSourceInfo(String sourceName, String displayName, boolean editable) + { + this.sourceName = sourceName; + this.displayName = displayName; + this.editable = editable; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DataSourceInfo that = (DataSourceInfo) o; + return sourceName != null ? sourceName.equals(that.sourceName) : that.sourceName == null; + } + + @Override + public int hashCode() + { + return sourceName != null ? sourceName.hashCode() : 0; + } + } + + public static abstract class BaseExternalSchemaBean + { + protected final Container _c; + protected final T _def; + protected final boolean _insert; + protected final Map _help = new HashMap<>(); + + public BaseExternalSchemaBean(Container c, T def, boolean insert) + { + _c = c; + _def = def; + _insert = insert; + + TableInfo ti = QueryManager.get().getTableInfoExternalSchema(); + + ti.getColumns() + .stream() + .filter(ci -> null != ci.getDescription()) + .forEach(ci -> _help.put(ci.getName(), ci.getDescription())); + } + + public abstract DataSourceInfo getInitialSource(); + + public T getSchemaDef() + { + return _def; + } + + public boolean isInsert() + { + return _insert; + } + + public ActionURL getReturnURL() + { + return new ActionURL(AdminAction.class, _c); + } + + public ActionURL getDeleteURL() + { + return new QueryUrlsImpl().urlDeleteSchema(_c, _def); + } + + public String getHelpHTML(String fieldName) + { + return _help.get(fieldName); + } + } + + public static class LinkedSchemaBean extends BaseExternalSchemaBean + { + public LinkedSchemaBean(Container c, LinkedSchemaDef def, boolean insert) + { + super(c, def, insert); + } + + @Override + public DataSourceInfo getInitialSource() + { + Container sourceContainer = getInitialContainer(); + return new DataSourceInfo(sourceContainer); + } + + private @NotNull Container getInitialContainer() + { + LinkedSchemaDef def = getSchemaDef(); + Container sourceContainer = def.lookupSourceContainer(); + if (sourceContainer == null) + sourceContainer = def.lookupContainer(); + if (sourceContainer == null) + sourceContainer = _c; + return sourceContainer; + } + } + + public static class ExternalSchemaBean extends BaseExternalSchemaBean + { + protected final Map> _sourcesAndSchemas = new LinkedHashMap<>(); + protected final Map> _sourcesAndSchemasIncludingSystem = new LinkedHashMap<>(); + + public ExternalSchemaBean(Container c, ExternalSchemaDef def, boolean insert) + { + super(c, def, insert); + initSources(); + } + + public Collection getSources() + { + return _sourcesAndSchemas.keySet(); + } + + public Collection getSchemaNames(DataSourceInfo source, boolean includeSystem) + { + if (includeSystem) + return _sourcesAndSchemasIncludingSystem.get(source); + else + return _sourcesAndSchemas.get(source); + } + + @Override + public DataSourceInfo getInitialSource() + { + ExternalSchemaDef def = getSchemaDef(); + DbScope scope = def.lookupDbScope(); + if (scope == null) + scope = DbScope.getLabKeyScope(); + return new DataSourceInfo(scope); + } + + protected void initSources() + { + ModuleLoader moduleLoader = ModuleLoader.getInstance(); + + for (DbScope scope : DbScope.getDbScopes()) + { + SqlDialect dialect = scope.getSqlDialect(); + + Collection schemaNames = new LinkedList<>(); + Collection schemaNamesIncludingSystem = new LinkedList<>(); + + for (String schemaName : scope.getSchemaNames()) + { + schemaNamesIncludingSystem.add(schemaName); + + if (dialect.isSystemSchema(schemaName)) + continue; + + if (null != moduleLoader.getModule(scope, schemaName)) + continue; + + schemaNames.add(schemaName); + } + + DataSourceInfo source = new DataSourceInfo(scope); + _sourcesAndSchemas.put(source, schemaNames); + _sourcesAndSchemasIncludingSystem.put(source, schemaNamesIncludingSystem); + } + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetTablesForm + { + private String _dataSource; + private String _schemaName; + private boolean _sorted; + + public String getDataSource() + { + return _dataSource; + } + + public void setDataSource(String dataSource) + { + _dataSource = dataSource; + } + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isSorted() + { + return _sorted; + } + + public void setSorted(boolean sorted) + { + _sorted = sorted; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class GetTablesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(GetTablesForm form, BindException errors) + { + List> rows = new LinkedList<>(); + List tableNames = new ArrayList<>(); + + if (null != form.getSchemaName()) + { + DbScope scope = DbScope.getDbScope(form.getDataSource()); + if (null != scope) + { + DbSchema schema = scope.getSchema(form.getSchemaName(), DbSchemaType.Bare); + tableNames.addAll(schema.getTableNames()); + } + else + { + Container c = ContainerManager.getForId(form.getDataSource()); + if (null != c) + { + UserSchema schema = QueryService.get().getUserSchema(getUser(), c, form.getSchemaName()); + if (null != schema) + { + if (form.isSorted()) + for (TableInfo table : schema.getSortedTables()) + tableNames.add(table.getName()); + else + tableNames.addAll(schema.getTableAndQueryNames(true)); + } + } + } + } + + Collections.sort(tableNames); + + for (String tableName : tableNames) + { + Map row = new LinkedHashMap<>(); + row.put("table", tableName); + rows.add(row); + } + + Map properties = new HashMap<>(); + properties.put("rows", rows); + + return new ApiSimpleResponse(properties); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SchemaTemplateForm + { + private String _name; + + public String getName() + { + return _name; + } + + public void setName(String name) + { + _name = name; + } + } + + + @RequiresPermission(AdminOperationsPermission.class) + public static class SchemaTemplateAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(SchemaTemplateForm form, BindException errors) + { + String name = form.getName(); + if (name == null) + throw new IllegalArgumentException("name required"); + + Container c = getContainer(); + TemplateSchemaType template = QueryServiceImpl.get().getSchemaTemplate(c, name); + if (template == null) + throw new NotFoundException("template not found"); + + JSONObject templateJson = QueryServiceImpl.get().schemaTemplateJson(name, template); + + return new ApiSimpleResponse("template", templateJson); + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class SchemaTemplatesAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object form, BindException errors) + { + Container c = getContainer(); + QueryServiceImpl svc = QueryServiceImpl.get(); + Map templates = svc.getSchemaTemplates(c); + + JSONArray ret = new JSONArray(); + for (String key : templates.keySet()) + { + TemplateSchemaType template = templates.get(key); + JSONObject templateJson = svc.schemaTemplateJson(key, template); + ret.put(templateJson); + } + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("templates", ret); + resp.put("success", true); + return resp; + } + } + + @RequiresPermission(AdminPermission.class) + public static class ReloadExternalSchemaAction extends FormHandlerAction + { + private String _userSchemaName; + + @Override + public void validateCommand(SchemaForm form, Errors errors) + { + } + + @Override + public boolean handlePost(SchemaForm form, BindException errors) + { + ExternalSchemaDef def = ExternalSchemaDefCache.getSchemaDef(getContainer(), form.getExternalSchemaId(), ExternalSchemaDef.class); + if (def == null) + throw new NotFoundException(); + + QueryManager.get().reloadExternalSchema(def); + _userSchemaName = def.getUserSchemaName(); + + return true; + } + + @Override + public ActionURL getSuccessURL(SchemaForm form) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Schema " + _userSchemaName + " was reloaded successfully."); + } + } + + + @RequiresPermission(AdminPermission.class) + public static class ReloadAllUserSchemas extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + QueryManager.get().reloadAllExternalSchemas(getContainer()); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "All schemas in this folder were reloaded successfully."); + } + } + + @RequiresPermission(AdminPermission.class) + public static class ReloadFailedConnectionsAction extends FormHandlerAction + { + @Override + public void validateCommand(Object target, Errors errors) + { + } + + @Override + public boolean handlePost(Object o, BindException errors) + { + DbScope.clearFailedDbScopes(); + return true; + } + + @Override + public URLHelper getSuccessURL(Object o) + { + return new QueryUrlsImpl().urlExternalSchemaAdmin(getContainer(), "Reconnection was attempted on all data sources that failed previous connection attempts."); + } + } + + @RequiresPermission(ReadPermission.class) + public static class TableInfoAction extends SimpleViewAction + { + @Override + public ModelAndView getView(TableInfoForm form, BindException errors) throws Exception + { + TablesDocument ret = TablesDocument.Factory.newInstance(); + TablesType tables = ret.addNewTables(); + + FieldKey[] fields = form.getFieldKeys(); + if (fields.length != 0) + { + TableInfo tinfo = QueryView.create(form, errors).getTable(); + Map columnMap = CustomViewImpl.getColumnInfos(tinfo, Arrays.asList(fields)); + TableXML.initTable(tables.addNewTable(), tinfo, null, columnMap.values()); + } + + for (FieldKey tableKey : form.getTableKeys()) + { + TableInfo tableInfo = form.getTableInfo(tableKey); + TableType xbTable = tables.addNewTable(); + TableXML.initTable(xbTable, tableInfo, tableKey); + } + getViewContext().getResponse().setContentType("text/xml"); + getViewContext().getResponse().getWriter().write(ret.toString()); + return null; + } + + @Override + public void addNavTrail(NavTree root) + { + } + } + + + // Issue 18870: Guest user can't revert unsaved custom view changes + // Permission will be checked inline (guests are allowed to delete their session custom views) + @RequiresNoPermission + @Action(ActionType.Configure.class) + public static class DeleteViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(DeleteViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + + if (getUser().isGuest()) + { + // Guests can only delete session custom views. + if (!view.isSession()) + throw new UnauthorizedException(); + } + else + { + // Logged in users must have read permission + if (!getContainer().hasPermission(getUser(), ReadPermission.class)) + throw new UnauthorizedException(); + } + + if (view.isShared()) + { + if (!getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) + throw new UnauthorizedException(); + } + + view.delete(getUser(), getViewContext().getRequest()); + + // Delete the first shadowed custom view, if available. + if (form.isComplete()) + { + form.reset(); + CustomView shadowed = form.getCustomView(); + if (shadowed != null && shadowed.isEditable() && !(shadowed instanceof ModuleCustomView)) + { + if (!shadowed.isShared() || getContainer().hasPermission(getUser(), EditSharedViewPermission.class)) + shadowed.delete(getUser(), getViewContext().getRequest()); + } + } + + // Try to get a custom view of the same name as the view we just deleted. + // The deleted view may have been a session view or a personal view masking shared view with the same name. + form.reset(); + view = form.getCustomView(); + String nextViewName = null; + if (view != null) + nextViewName = view.getName(); + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("viewName", nextViewName); + return response; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SaveSessionViewForm extends QueryForm + { + private String newName; + private boolean inherit; + private boolean shared; + private boolean hidden; + private boolean replace; + private String containerPath; + + public String getNewName() + { + return newName; + } + + public void setNewName(String newName) + { + this.newName = newName; + } + + public boolean isInherit() + { + return inherit; + } + + public void setInherit(boolean inherit) + { + this.inherit = inherit; + } + + public boolean isShared() + { + return shared; + } + + public void setShared(boolean shared) + { + this.shared = shared; + } + + public String getContainerPath() + { + return containerPath; + } + + public void setContainerPath(String containerPath) + { + this.containerPath = containerPath; + } + + public boolean isHidden() + { + return hidden; + } + + public void setHidden(boolean hidden) + { + this.hidden = hidden; + } + + public boolean isReplace() + { + return replace; + } + + public void setReplace(boolean replace) + { + this.replace = replace; + } + } + + // Moves a session view into the database. + @RequiresPermission(ReadPermission.class) + public static class SaveSessionViewAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SaveSessionViewForm form, BindException errors) + { + CustomView view = form.getCustomView(); + if (view == null) + { + throw new NotFoundException(); + } + if (!view.isSession()) + throw new IllegalArgumentException("This action only supports saving session views."); + + //if (!getContainer().getId().equals(view.getContainer().getId())) + // throw new IllegalArgumentException("View may only be saved from container it was created in."); + + assert !view.canInherit() && !view.isShared() && view.isEditable(): "Session view should never be inheritable or shared and always be editable"; + + // Users may save views to a location other than the current container + String containerPath = form.getContainerPath(); + Container container; + if (form.isInherit() && containerPath != null) + { + // Only respect this request if it's a view that is inheritable in subfolders + container = ContainerManager.getForPath(containerPath); + } + else + { + // Otherwise, save it in the current container + container = getContainer(); + } + + if (container == null) + throw new NotFoundException("No such container: " + containerPath); + + if (form.isShared() || form.isInherit()) + { + if (!container.hasPermission(getUser(), EditSharedViewPermission.class)) + throw new UnauthorizedException(); + } + + DbScope scope = QueryManager.get().getDbSchema().getScope(); + try (DbScope.Transaction tx = scope.ensureTransaction()) + { + // Delete the session view. The view will be restored if an exception is thrown. + view.delete(getUser(), getViewContext().getRequest()); + + // Get any previously existing non-session view. + // The session custom view and the view-to-be-saved may have different names. + // If they do have different names, we may need to delete an existing session view with that name. + // UNDONE: If the view has a different name, we will clobber it without asking. + CustomView existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); + if (existingView != null && existingView.isSession()) + { + // Delete any session view we are overwriting. + existingView.delete(getUser(), getViewContext().getRequest()); + existingView = form.getQueryDef().getCustomView(getUser(), null, form.getNewName()); + } + + // save a new private view if shared is false but existing view is shared + if (existingView != null && !form.isShared() && existingView.getOwner() == null) + { + existingView = null; + } + + if (existingView != null && !form.isReplace() && !StringUtils.isEmpty(form.getNewName())) + throw new IllegalArgumentException("A saved view by the name \"" + form.getNewName() + "\" already exists. "); + + if (existingView == null || (existingView instanceof ModuleCustomView && existingView.isEditable())) + { + User owner = form.isShared() ? null : getUser(); + + CustomViewImpl viewCopy = new CustomViewImpl(form.getQueryDef(), owner, form.getNewName()); + viewCopy.setColumns(view.getColumns()); + viewCopy.setCanInherit(form.isInherit()); + viewCopy.setFilterAndSort(view.getFilterAndSort()); + viewCopy.setColumnProperties(view.getColumnProperties()); + viewCopy.setIsHidden(form.isHidden()); + if (form.isInherit()) + viewCopy.setContainer(container); + + viewCopy.save(getUser(), getViewContext().getRequest()); + } + else if (!existingView.isEditable()) + { + throw new IllegalArgumentException("Existing view '" + form.getNewName() + "' is not editable. You may save this view with a different name."); + } + else + { + // UNDONE: changing shared property of an existing view is unimplemented. Not sure if it makes sense from a usability point of view. + existingView.setColumns(view.getColumns()); + existingView.setFilterAndSort(view.getFilterAndSort()); + existingView.setColumnProperties(view.getColumnProperties()); + existingView.setCanInherit(form.isInherit()); + if (form.isInherit()) + ((CustomViewImpl)existingView).setContainer(container); + existingView.setIsHidden(form.isHidden()); + + existingView.save(getUser(), getViewContext().getRequest()); + } + + tx.commit(); + return new ApiSimpleResponse("success", true); + } + catch (Exception e) + { + // dirty the view then save the deleted session view back in session state + view.setName(view.getName()); + view.save(getUser(), getViewContext().getRequest()); + + throw e; + } + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class ManageViewsAction extends SimpleViewAction + { + @SuppressWarnings("UnusedDeclaration") + public ManageViewsAction() + { + } + + public ManageViewsAction(ViewContext ctx) + { + setViewContext(ctx); + } + + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return new JspView<>("/org/labkey/query/view/manageViews.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + new BeginAction(getViewContext()).addNavTrail(root); + root.addChild("Manage Views", QueryController.this.getViewContext().getActionURL()); + } + } + + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalDeleteView extends ConfirmAction + { + @Override + public ModelAndView getConfirmView(InternalViewForm form, BindException errors) + { + return new JspView<>("/org/labkey/query/view/internalDeleteView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalViewForm form, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + QueryManager.get().delete(view); + return true; + } + + @Override + public void validateCommand(InternalViewForm internalViewForm, Errors errors) + { + } + + @Override + @NotNull + public ActionURL getSuccessURL(InternalViewForm internalViewForm) + { + return new ActionURL(ManageViewsAction.class, getContainer()); + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalSourceViewAction extends FormViewAction + { + @Override + public void validateCommand(InternalSourceViewForm target, Errors errors) + { + } + + @Override + public ModelAndView getView(InternalSourceViewForm form, boolean reshow, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + form.ff_inherit = QueryManager.get().canInherit(view.getFlags()); + form.ff_hidden = QueryManager.get().isHidden(view.getFlags()); + form.ff_columnList = view.getColumns(); + form.ff_filter = view.getFilter(); + return new JspView<>("/org/labkey/query/view/internalSourceView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalSourceViewForm form, BindException errors) + { + CstmView view = form.getViewAndCheckPermission(); + int flags = view.getFlags(); + flags = QueryManager.get().setCanInherit(flags, form.ff_inherit); + flags = QueryManager.get().setIsHidden(flags, form.ff_hidden); + view.setFlags(flags); + view.setColumns(form.ff_columnList); + view.setFilter(form.ff_filter); + QueryManager.get().update(getUser(), view); + return true; + } + + @Override + public ActionURL getSuccessURL(InternalSourceViewForm form) + { + return new ActionURL(ManageViewsAction.class, getContainer()); + } + + @Override + public void addNavTrail(NavTree root) + { + new ManageViewsAction(getViewContext()).addNavTrail(root); + root.addChild("Edit source of Grid View"); + } + } + + /** Minimalist, secret UI to help users recover if they've created a broken view somehow */ + @RequiresPermission(AdminPermission.class) + public class InternalNewViewAction extends FormViewAction + { + int _customViewId = 0; + + @Override + public void validateCommand(InternalNewViewForm form, Errors errors) + { + if (StringUtils.trimToNull(form.ff_schemaName) == null) + { + errors.reject(ERROR_MSG, "Schema name cannot be blank."); + } + if (StringUtils.trimToNull(form.ff_queryName) == null) + { + errors.reject(ERROR_MSG, "Query name cannot be blank"); + } + } + + @Override + public ModelAndView getView(InternalNewViewForm form, boolean reshow, BindException errors) + { + return new JspView<>("/org/labkey/query/view/internalNewView.jsp", form, errors); + } + + @Override + public boolean handlePost(InternalNewViewForm form, BindException errors) + { + if (form.ff_share) + { + if (!getContainer().hasPermission(getUser(), AdminPermission.class)) + throw new UnauthorizedException(); + } + List existing = QueryManager.get().getCstmViews(getContainer(), form.ff_schemaName, form.ff_queryName, form.ff_viewName, form.ff_share ? null : getUser(), false, false); + CstmView view; + if (!existing.isEmpty()) + { + } + else + { + view = new CstmView(); + view.setSchema(form.ff_schemaName); + view.setQueryName(form.ff_queryName); + view.setName(form.ff_viewName); + view.setContainerId(getContainer().getId()); + if (form.ff_share) + { + view.setCustomViewOwner(null); + } + else + { + view.setCustomViewOwner(getUser().getUserId()); + } + if (form.ff_inherit) + { + view.setFlags(QueryManager.get().setCanInherit(view.getFlags(), form.ff_inherit)); + } + InternalViewForm.checkEdit(getViewContext(), view); + try + { + view = QueryManager.get().insert(getUser(), view); + } + catch (Exception e) + { + LogManager.getLogger(QueryController.class).error("Error", e); + errors.reject(ERROR_MSG, "An exception occurred: " + e); + return false; + } + _customViewId = view.getCustomViewId(); + } + return true; + } + + @Override + public ActionURL getSuccessURL(InternalNewViewForm form) + { + ActionURL forward = new ActionURL(InternalSourceViewAction.class, getContainer()); + forward.addParameter("customViewId", Integer.toString(_customViewId)); + return forward; + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Create New Grid View"); + } + } + + + @ActionNames("clearSelected, selectNone") + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class SelectNoneAction extends MutatingApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) + { + errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + if (form.getQueryName() == null) + { + DataRegionSelection.clearAll(getViewContext(), form.getKey()); + return new DataRegionSelection.SelectionResponse(0); + } + + int count = DataRegionSelection.setSelectedFromForm(form); + return new DataRegionSelection.SelectionResponse(count); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SelectForm extends QueryForm + { + protected boolean clearSelected; + protected String key; + + public boolean isClearSelected() + { + return clearSelected; + } + + public void setClearSelected(boolean clearSelected) + { + this.clearSelected = clearSelected; + } + + public String getKey() + { + return key; + } + + public void setKey(String key) + { + this.key = key; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectData.class) + public static class SelectAllAction extends MutatingApiAction + { + @Override + public void validateForm(QueryForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() || form.getQueryName() == null) + { + errors.reject(ERROR_MSG, "schemaName and queryName required"); + } + } + + @Override + public ApiResponse execute(final QueryForm form, BindException errors) throws Exception + { + int count = DataRegionSelection.setSelectionForAll(form, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetSelectedAction extends ReadOnlyApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (form.getSchemaName().isEmpty() != (form.getQueryName() == null)) + { + errors.reject(ERROR_MSG, "Both schemaName and queryName are required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + getViewContext().getResponse().setHeader("Content-Type", CONTENT_TYPE_JSON); + Set selected; + + if (form.getQueryName() == null) + selected = DataRegionSelection.getSelected(getViewContext(), form.getKey(), form.isClearSelected()); + else + selected = DataRegionSelection.getSelected(form, form.isClearSelected()); + + return new ApiSimpleResponse("selected", selected); + } + } + + @ActionNames("setSelected, setCheck") + @RequiresPermission(ReadPermission.class) + public static class SetCheckAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) throws Exception + { + String[] ids = form.getId(getViewContext().getRequest()); + Set selection = new LinkedHashSet<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + int count; + if (form.getQueryName() != null && form.isValidateIds() && form.isChecked()) + { + selection = DataRegionSelection.getValidatedIds(selection, form); + } + + count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, form.isChecked()); + + return new DataRegionSelection.SelectionResponse(count); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class SetCheckForm extends SelectForm + { + protected String[] ids; + protected boolean checked; + protected boolean validateIds; + + public String[] getId(HttpServletRequest request) + { + // 5025 : DataRegion checkbox names may contain comma + // Beehive parses a single parameter value with commas into an array + // which is not what we want. + String[] paramIds = request.getParameterValues("id"); + return paramIds == null ? ids: paramIds; + } + + public void setId(String[] ids) + { + this.ids = ids; + } + + public boolean isChecked() + { + return checked; + } + + public void setChecked(boolean checked) + { + this.checked = checked; + } + + public boolean isValidateIds() + { + return validateIds; + } + + public void setValidateIds(boolean validateIds) + { + this.validateIds = validateIds; + } + } + + @RequiresPermission(ReadPermission.class) + public static class ReplaceSelectedAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) + { + String[] ids = form.getId(getViewContext().getRequest()); + List selection = new ArrayList<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + + DataRegionSelection.clearAll(getViewContext(), form.getKey()); + int count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class SetSnapshotSelectionAction extends MutatingApiAction + { + @Override + public ApiResponse execute(final SetCheckForm form, BindException errors) + { + String[] ids = form.getId(getViewContext().getRequest()); + List selection = new ArrayList<>(); + if (ids != null) + { + for (String id : ids) + { + if (StringUtils.isNotBlank(id)) + selection.add(id); + } + } + + DataRegionSelection.clearAll(getViewContext(), form.getKey(), true); + int count = DataRegionSelection.setSelected( + getViewContext(), form.getKey(), + selection, true, true); + return new DataRegionSelection.SelectionResponse(count); + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetSnapshotSelectionAction extends ReadOnlyApiAction + { + @Override + public void validateForm(SelectForm form, Errors errors) + { + if (StringUtils.isEmpty(form.getKey())) + { + errors.reject(ERROR_MSG, "Selection key is required"); + } + } + + @Override + public ApiResponse execute(final SelectForm form, BindException errors) throws Exception + { + List selected = DataRegionSelection.getSnapshotSelected(getViewContext(), form.getKey()); + return new ApiSimpleResponse("selected", selected); + } + } + + public static String getMessage(SqlDialect d, SQLException x) + { + return x.getMessage(); + } + + + public static class GetSchemasForm + { + private boolean _includeHidden = true; + private SchemaKey _schemaName; + + public SchemaKey getSchemaName() + { + return _schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(SchemaKey schemaName) + { + _schemaName = schemaName; + } + + public boolean isIncludeHidden() + { + return _includeHidden; + } + + @SuppressWarnings("unused") + public void setIncludeHidden(boolean includeHidden) + { + _includeHidden = includeHidden; + } + } + + + @RequiresPermission(ReadPermission.class) + @ApiVersion(12.3) + public static class GetSchemasAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetSchemasForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetSchemasForm form, BindException errors) + { + final Container container = getContainer(); + final User user = getUser(); + + final boolean includeHidden = form.isIncludeHidden(); + if (getRequestedApiVersion() >= 9.3) + { + SimpleSchemaTreeVisitor visitor = new SimpleSchemaTreeVisitor<>(includeHidden) + { + @Override + public Void visitUserSchema(UserSchema schema, Path path, JSONObject json) + { + JSONObject schemaProps = new JSONObject(); + + schemaProps.put("schemaName", schema.getName()); + schemaProps.put("fullyQualifiedName", schema.getSchemaName()); + schemaProps.put("description", schema.getDescription()); + schemaProps.put("hidden", schema.isHidden()); + NavTree tree = schema.getSchemaBrowserLinks(user); + if (tree != null && tree.hasChildren()) + schemaProps.put("menu", tree.toJSON()); + + // Collect children schemas + JSONObject children = new JSONObject(); + visit(schema.getSchemas(_includeHidden), path, children); + if (!children.isEmpty()) + schemaProps.put("schemas", children); + + // Add node's schemaProps to the parent's json. + json.put(schema.getName(), schemaProps); + return null; + } + }; + + // By default, start from the root. + QuerySchema schema; + if (form.getSchemaName() != null) + schema = DefaultSchema.get(user, container, form.getSchemaName()); + else + schema = DefaultSchema.get(user, container); + + // Ensure consistent exception as other query actions + QueryForm.ensureSchemaNotNull(schema); + + // Create the JSON response by visiting the schema children. The parent schema information isn't included. + JSONObject ret = new JSONObject(); + visitor.visitTop(schema.getSchemas(includeHidden), ret); + + return new ApiSimpleResponse(ret); + } + else + { + return new ApiSimpleResponse("schemas", DefaultSchema.get(user, container).getUserSchemaPaths(includeHidden)); + } + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetQueriesForm + { + private String _schemaName; + private boolean _includeUserQueries = true; + private boolean _includeSystemQueries = true; + private boolean _includeColumns = true; + private boolean _includeViewDataUrl = true; + private boolean _includeTitle = true; + private boolean _queryDetailColumns = false; + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isIncludeUserQueries() + { + return _includeUserQueries; + } + + public void setIncludeUserQueries(boolean includeUserQueries) + { + _includeUserQueries = includeUserQueries; + } + + public boolean isIncludeSystemQueries() + { + return _includeSystemQueries; + } + + public void setIncludeSystemQueries(boolean includeSystemQueries) + { + _includeSystemQueries = includeSystemQueries; + } + + public boolean isIncludeColumns() + { + return _includeColumns; + } + + public void setIncludeColumns(boolean includeColumns) + { + _includeColumns = includeColumns; + } + + public boolean isQueryDetailColumns() + { + return _queryDetailColumns; + } + + public void setQueryDetailColumns(boolean queryDetailColumns) + { + _queryDetailColumns = queryDetailColumns; + } + + public boolean isIncludeViewDataUrl() + { + return _includeViewDataUrl; + } + + public void setIncludeViewDataUrl(boolean includeViewDataUrl) + { + _includeViewDataUrl = includeViewDataUrl; + } + + public boolean isIncludeTitle() + { + return _includeTitle; + } + + public void setIncludeTitle(boolean includeTitle) + { + _includeTitle = includeTitle; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class GetQueriesAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetQueriesForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetQueriesForm form, BindException errors) + { + if (null == StringUtils.trimToNull(form.getSchemaName())) + throw new IllegalArgumentException("You must supply a value for the 'schemaName' parameter!"); + + ApiSimpleResponse response = new ApiSimpleResponse(); + UserSchema uschema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); + if (null == uschema) + throw new NotFoundException("The schema name '" + form.getSchemaName() + + "' was not found within the folder '" + getContainer().getPath() + "'"); + + response.put("schemaName", form.getSchemaName()); + + List> qinfos = new ArrayList<>(); + + //user-defined queries + if (form.isIncludeUserQueries()) + { + for (QueryDefinition qdef : uschema.getQueryDefs().values()) + { + if (!qdef.isTemporary()) + { + ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; + qinfos.add(getQueryProps(qdef, viewDataUrl, true, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); + } + } + } + + //built-in tables + if (form.isIncludeSystemQueries()) + { + for (String qname : uschema.getVisibleTableNames()) + { + // Go direct against the UserSchema instead of calling into QueryService, which takes a schema and + // query name as strings and therefore has to create new instances + QueryDefinition qdef = uschema.getQueryDefForTable(qname); + if (qdef != null) + { + ActionURL viewDataUrl = form.isIncludeViewDataUrl() ? uschema.urlFor(QueryAction.executeQuery, qdef) : null; + qinfos.add(getQueryProps(qdef, viewDataUrl, false, uschema, form.isIncludeColumns(), form.isQueryDetailColumns(), form.isIncludeTitle())); + } + } + } + response.put("queries", qinfos); + + return response; + } + + protected Map getQueryProps(QueryDefinition qdef, ActionURL viewDataUrl, boolean isUserDefined, UserSchema schema, boolean includeColumns, boolean useQueryDetailColumns, boolean includeTitle) + { + Map qinfo = new HashMap<>(); + qinfo.put("hidden", qdef.isHidden()); + qinfo.put("snapshot", qdef.isSnapshot()); + qinfo.put("inherit", qdef.canInherit()); + qinfo.put("isUserDefined", isUserDefined); + boolean canEdit = qdef.canEdit(getUser()); + qinfo.put("canEdit", canEdit); + qinfo.put("canEditSharedViews", getContainer().hasPermission(getUser(), EditSharedViewPermission.class)); + // CONSIDER: do we want to separate the 'canEditMetadata' property and 'isMetadataOverridable' properties to differentiate between capability and the permission check? + qinfo.put("isMetadataOverrideable", qdef.isMetadataEditable() && qdef.canEditMetadata(getUser())); + + if (isUserDefined) + qinfo.put("moduleName", qdef.getModuleName()); + boolean isInherited = qdef.canInherit() && !getContainer().equals(qdef.getDefinitionContainer()); + qinfo.put("isInherited", isInherited); + if (isInherited) + qinfo.put("containerPath", qdef.getDefinitionContainer().getPath()); + qinfo.put("isIncludedForLookups", qdef.isIncludedForLookups()); + + if (null != qdef.getDescription()) + qinfo.put("description", qdef.getDescription()); + if (viewDataUrl != null) + qinfo.put("viewDataUrl", viewDataUrl); + + String title = qdef.getName(); + String name = qdef.getName(); + try + { + // get the TableInfo if the user requested column info or title, otherwise skip (it can be expensive) + if (includeColumns || includeTitle) + { + TableInfo table = qdef.getTable(schema, null, true); + + if (null != table) + { + if (includeColumns) + { + Collection> columns; + + if (useQueryDetailColumns) + { + columns = JsonWriter + .getNativeColProps(table, Collections.emptyList(), null, false, false) + .values(); + } + else + { + columns = new ArrayList<>(); + for (ColumnInfo col : table.getColumns()) + { + Map cinfo = new HashMap<>(); + cinfo.put("name", col.getName()); + if (null != col.getLabel()) + cinfo.put("caption", col.getLabel()); + if (null != col.getShortLabel()) + cinfo.put("shortCaption", col.getShortLabel()); + if (null != col.getDescription()) + cinfo.put("description", col.getDescription()); + + columns.add(cinfo); + } + } + + if (!columns.isEmpty()) + qinfo.put("columns", columns); + } + + if (includeTitle) + { + name = table.getPublicName(); + title = table.getTitle(); + } + } + } + } + catch(Exception e) + { + //may happen due to query failing parse + } + + qinfo.put("title", title); + qinfo.put("name", name); + return qinfo; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class GetQueryViewsForm + { + private String _schemaName; + private String _queryName; + private String _viewName; + private boolean _metadata; + private boolean _excludeSessionView; + + public String getSchemaName() + { + return _schemaName; + } + + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public String getQueryName() + { + return _queryName; + } + + public void setQueryName(String queryName) + { + _queryName = queryName; + } + + public String getViewName() + { + return _viewName; + } + + public void setViewName(String viewName) + { + _viewName = viewName; + } + + public boolean isMetadata() + { + return _metadata; + } + + public void setMetadata(boolean metadata) + { + _metadata = metadata; + } + + public boolean isExcludeSessionView() + { + return _excludeSessionView; + } + + public void setExcludeSessionView(boolean excludeSessionView) + { + _excludeSessionView = excludeSessionView; + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class GetQueryViewsAction extends ReadOnlyApiAction + { + @Override + protected long getLastModified(GetQueryViewsForm form) + { + return QueryService.get().metadataLastModified(); + } + + @Override + public ApiResponse execute(GetQueryViewsForm form, BindException errors) + { + if (null == StringUtils.trimToNull(form.getSchemaName())) + throw new IllegalArgumentException("You must pass a value for the 'schemaName' parameter!"); + if (null == StringUtils.trimToNull(form.getQueryName())) + throw new IllegalArgumentException("You must pass a value for the 'queryName' parameter!"); + + UserSchema schema = QueryService.get().getUserSchema(getUser(), getContainer(), form.getSchemaName()); + if (null == schema) + throw new NotFoundException("The schema name '" + form.getSchemaName() + + "' was not found within the folder '" + getContainer().getPath() + "'"); + + QueryDefinition querydef = QueryService.get().createQueryDefForTable(schema, form.getQueryName()); + if (null == querydef || querydef.getTable(null, true) == null) + throw new NotFoundException("The query '" + form.getQueryName() + "' was not found within the '" + + form.getSchemaName() + "' schema in the container '" + + getContainer().getPath() + "'!"); + + Map views = querydef.getCustomViews(getUser(), getViewContext().getRequest(), true, false, form.isExcludeSessionView()); + if (null == views) + views = Collections.emptyMap(); + + Map> columnMetadata = new HashMap<>(); + + List> viewInfos = Collections.emptyList(); + if (getViewContext().getBindPropertyValues().contains("viewName")) + { + // Get info for a named view or the default view (null) + String viewName = StringUtils.trimToNull(form.getViewName()); + CustomView view = views.get(viewName); + if (view != null) + { + viewInfos = Collections.singletonList(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); + } + else if (viewName == null) + { + // The default view was requested but it hasn't been customized yet. Create the 'default default' view. + viewInfos = Collections.singletonList(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); + } + } + else + { + boolean foundDefault = false; + viewInfos = new ArrayList<>(views.size()); + for (CustomView view : views.values()) + { + if (view.getName() == null) + foundDefault = true; + viewInfos.add(CustomViewUtil.toMap(view, getUser(), form.isMetadata())); + } + + if (!foundDefault) + { + // The default view hasn't been customized yet. Create the 'default default' view. + viewInfos.add(CustomViewUtil.toMap(getViewContext(), schema, form.getQueryName(), null, form.isMetadata(), true, columnMetadata)); + } + } + + ApiSimpleResponse response = new ApiSimpleResponse(); + response.put("schemaName", form.getSchemaName()); + response.put("queryName", form.getQueryName()); + response.put("views", viewInfos); + + return response; + } + } + + @RequiresNoPermission + public static class GetServerDateAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(Object o, BindException errors) + { + return new ApiSimpleResponse("date", new Date()); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + private static class SaveApiTestForm + { + private String _getUrl; + private String _postUrl; + private String _postData; + private String _response; + + public String getGetUrl() + { + return _getUrl; + } + + public void setGetUrl(String getUrl) + { + _getUrl = getUrl; + } + + public String getPostUrl() + { + return _postUrl; + } + + public void setPostUrl(String postUrl) + { + _postUrl = postUrl; + } + + public String getResponse() + { + return _response; + } + + public void setResponse(String response) + { + _response = response; + } + + public String getPostData() + { + return _postData; + } + + public void setPostData(String postData) + { + _postData = postData; + } + } + + + @RequiresPermission(ReadPermission.class) + public static class SaveApiTestAction extends MutatingApiAction + { + @Override + public ApiResponse execute(SaveApiTestForm form, BindException errors) + { + ApiSimpleResponse response = new ApiSimpleResponse(); + + ApiTestsDocument doc = ApiTestsDocument.Factory.newInstance(); + + TestCaseType test = doc.addNewApiTests().addNewTest(); + test.setName("recorded test case"); + ActionURL url = null; + + if (!StringUtils.isEmpty(form.getGetUrl())) + { + test.setType("get"); + url = new ActionURL(form.getGetUrl()); + } + else if (!StringUtils.isEmpty(form.getPostUrl())) + { + test.setType("post"); + test.setFormData(form.getPostData()); + url = new ActionURL(form.getPostUrl()); + } + + if (url != null) + { + String uri = url.getLocalURIString(); + if (uri.startsWith(url.getContextPath())) + uri = uri.substring(url.getContextPath().length() + 1); + + test.setUrl(uri); + } + test.setResponse(form.getResponse()); + + XmlOptions opts = new XmlOptions(); + opts.setSaveCDataEntityCountThreshold(0); + opts.setSaveCDataLengthThreshold(0); + opts.setSavePrettyPrint(); + opts.setUseDefaultNamespace(); + + response.put("xml", doc.xmlText(opts)); + + return response; + } + } + + + private abstract static class ParseAction extends SimpleViewAction + { + @Override + public ModelAndView getView(Object o, BindException errors) + { + List qpe = new ArrayList<>(); + String expr = getViewContext().getRequest().getParameter("q"); + ArrayList html = new ArrayList<>(); + PageConfig config = getPageConfig(); + var inputId = config.makeId("submit_"); + config.addHandler(inputId, "click", "Ext.getBody().mask();"); + html.add("
\n" + + "" + ); + + QNode e = null; + if (null != expr) + { + try + { + e = _parse(expr,qpe); + } + catch (RuntimeException x) + { + qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); + } + } + + Tree tree = null; + if (null != expr) + { + try + { + tree = _tree(expr); + } catch (Exception x) + { + qpe.add(new QueryParseException(x.getMessage(),x, 0, 0)); + } + } + + for (Throwable x : qpe) + { + if (null != x.getCause() && x != x.getCause()) + x = x.getCause(); + html.add("
" + PageFlowUtil.filter(x.toString())); + LogManager.getLogger(QueryController.class).debug(expr,x); + } + if (null != e) + { + String prefix = SqlParser.toPrefixString(e); + html.add("
"); + html.add(PageFlowUtil.filter(prefix)); + } + if (null != tree) + { + String prefix = SqlParser.toPrefixString(tree); + html.add("
"); + html.add(PageFlowUtil.filter(prefix)); + } + html.add(""); + return HtmlView.unsafe(StringUtils.join(html,"")); + } + + @Override + public void addNavTrail(NavTree root) + { + } + + abstract QNode _parse(String e, List errors); + abstract Tree _tree(String e) throws Exception; + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ParseExpressionAction extends ParseAction + { + @Override + QNode _parse(String s, List errors) + { + return new SqlParser().parseExpr(s, true, errors); + } + + @Override + Tree _tree(String e) + { + return null; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public static class ParseQueryAction extends ParseAction + { + @Override + QNode _parse(String s, List errors) + { + return new SqlParser().parseQuery(s, errors, null); + } + + @Override + Tree _tree(String s) throws Exception + { + return new SqlParser().rawQuery(s); + } + } + + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.SelectMetaData.class) + public static class ValidateQueryMetadataAction extends ReadOnlyApiAction + { + @Override + public ApiResponse execute(QueryForm form, BindException errors) + { + UserSchema schema = form.getSchema(); + + if (null == schema) + { + errors.reject(ERROR_MSG, "could not resolve schema: " + form.getSchemaName()); + return null; + } + + List parseErrors = new ArrayList<>(); + List parseWarnings = new ArrayList<>(); + ApiSimpleResponse response = new ApiSimpleResponse(); + + try + { + TableInfo table = schema.getTable(form.getQueryName(), null); + + if (null == table) + { + errors.reject(ERROR_MSG, "could not resolve table: " + form.getQueryName()); + return null; + } + + if (!QueryManager.get().validateQuery(table, true, parseErrors, parseWarnings)) + { + for (QueryParseException e : parseErrors) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + return response; + } + + SchemaKey schemaKey = SchemaKey.fromString(form.getSchemaName()); + QueryManager.get().validateQueryMetadata(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); + QueryManager.get().validateQueryViews(schemaKey, form.getQueryName(), getUser(), getContainer(), parseErrors, parseWarnings); + } + catch (QueryParseException e) + { + parseErrors.add(e); + } + + for (QueryParseException e : parseErrors) + { + errors.reject(ERROR_MSG, e.getMessage()); + } + + for (QueryParseException e : parseWarnings) + { + errors.reject(ERROR_MSG, "WARNING: " + e.getMessage()); + } + + return response; + } + + @Override + protected ApiResponseWriter createResponseWriter() throws IOException + { + ApiResponseWriter result = super.createResponseWriter(); + // Issue 44875 - don't send a 400 or 500 response code when there's a bogus query or metadata + result.setErrorResponseStatus(HttpServletResponse.SC_OK); + return result; + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class QueryExportAuditForm + { + private int rowId; + + public int getRowId() + { + return rowId; + } + + public void setRowId(int rowId) + { + this.rowId = rowId; + } + } + + /** + * Action used to redirect QueryAuditProvider [details] column to the exported table's grid view. + */ + @RequiresPermission(AdminPermission.class) + public static class QueryExportAuditRedirectAction extends SimpleRedirectAction + { + @Override + public URLHelper getRedirectURL(QueryExportAuditForm form) + { + if (form.getRowId() == 0) + throw new NotFoundException("Query export audit rowid required"); + + UserSchema auditSchema = QueryService.get().getUserSchema(getUser(), getContainer(), AbstractAuditTypeProvider.QUERY_SCHEMA_NAME); + TableInfo queryExportAuditTable = auditSchema.getTable(QueryExportAuditProvider.QUERY_AUDIT_EVENT, null); + if (null == queryExportAuditTable) + throw new NotFoundException(); + + TableSelector selector = new TableSelector(queryExportAuditTable, + PageFlowUtil.set( + QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME, + QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME, + QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL), + new SimpleFilter(FieldKey.fromParts(AbstractAuditTypeProvider.COLUMN_NAME_ROW_ID), form.getRowId()), null); + + Map result = selector.getMap(); + if (result == null) + throw new NotFoundException("Query export audit event not found for rowId"); + + String schemaName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_SCHEMA_NAME); + String queryName = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_QUERY_NAME); + String detailsURL = (String)result.get(QueryExportAuditProvider.COLUMN_NAME_DETAILS_URL); + + if (schemaName == null || queryName == null) + throw new NotFoundException("Query export audit event has not schemaName or queryName"); + + ActionURL url = new ActionURL(ExecuteQueryAction.class, getContainer()); + + // Apply the sorts and filters + if (detailsURL != null) + { + ActionURL sortFilterURL = new ActionURL(detailsURL); + url.setPropertyValues(sortFilterURL.getPropertyValues()); + } + + if (url.getParameter(QueryParam.schemaName) == null) + url.addParameter(QueryParam.schemaName, schemaName); + if (url.getParameter(QueryParam.queryName) == null && url.getParameter(QueryView.DATAREGIONNAME_DEFAULT + "." + QueryParam.queryName) == null) + url.addParameter(QueryParam.queryName, queryName); + + return url; + } + } + + @RequiresPermission(ReadPermission.class) + public static class AuditHistoryAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryForm form, BindException errors) + { + return QueryUpdateAuditProvider.createHistoryQueryView(getViewContext(), form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Audit History"); + } + } + + @RequiresPermission(ReadPermission.class) + public static class AuditDetailsAction extends SimpleViewAction + { + @Override + public ModelAndView getView(QueryDetailsForm form, BindException errors) + { + return QueryUpdateAuditProvider.createDetailsQueryView(getViewContext(), form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Audit History"); + } + } + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class QueryDetailsForm extends QueryForm + { + String _keyValue; + + public String getKeyValue() + { + return _keyValue; + } + + public void setKeyValue(String keyValue) + { + _keyValue = keyValue; + } + } + + @RequiresPermission(ReadPermission.class) + @Action(ActionType.Export.class) + public static class ExportTablesAction extends FormViewAction + { + private ActionURL _successUrl; + + @Override + public void validateCommand(ExportTablesForm form, Errors errors) + { + } + + @Override + public boolean handlePost(ExportTablesForm form, BindException errors) + { + HttpServletResponse httpResponse = getViewContext().getResponse(); + Container container = getContainer(); + QueryServiceImpl svc = (QueryServiceImpl)QueryService.get(); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream outputStream = new BufferedOutputStream(baos)) + { + try (ZipFile zip = new ZipFile(outputStream, true)) + { + svc.writeTables(container, getUser(), zip, form.getSchemas(), form.getHeaderType()); + } + + PageFlowUtil.streamFileBytes(httpResponse, FileUtil.makeFileNameWithTimestamp(container.getName(), "tables.zip"), baos.toByteArray(), false); + } + catch (Exception e) + { + errors.reject(ERROR_MSG, e.getMessage() != null ? e.getMessage() : e.getClass().getName()); + LOG.error("Errror exporting tables", e); + } + + if (errors.hasErrors()) + { + _successUrl = new ActionURL(ExportTablesAction.class, getContainer()); + } + + return !errors.hasErrors(); + } + + @Override + public ModelAndView getView(ExportTablesForm form, boolean reshow, BindException errors) + { + // When exporting the zip to the browser, the base action will attempt to reshow the view since we returned + // null as the success URL; returning null here causes the base action to stop pestering the action. + if (reshow && !errors.hasErrors()) + return null; + + return new JspView<>("/org/labkey/query/view/exportTables.jsp", form, errors); + } + + @Override + public void addNavTrail(NavTree root) + { + root.addChild("Export Tables"); + } + + @Override + public ActionURL getSuccessURL(ExportTablesForm form) + { + return _successUrl; + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class ExportTablesForm implements HasBindParameters + { + ColumnHeaderType _headerType = ColumnHeaderType.DisplayFieldKey; + Map>> _schemas = new HashMap<>(); + + public ColumnHeaderType getHeaderType() + { + return _headerType; + } + + public void setHeaderType(ColumnHeaderType headerType) + { + _headerType = headerType; + } + + public Map>> getSchemas() + { + return _schemas; + } + + public void setSchemas(Map>> schemas) + { + _schemas = schemas; + } + + @Override + public @NotNull BindException bindParameters(PropertyValues values) + { + BindException errors = new NullSafeBindException(this, "form"); + + PropertyValue schemasProperty = values.getPropertyValue("schemas"); + if (schemasProperty != null && schemasProperty.getValue() != null) + { + try + { + _schemas = JsonUtil.DEFAULT_MAPPER.readValue((String)schemasProperty.getValue(), _schemas.getClass()); + } + catch (IOException e) + { + errors.rejectValue("schemas", ERROR_MSG, e.getMessage()); + } + } + + PropertyValue headerTypeProperty = values.getPropertyValue("headerType"); + if (headerTypeProperty != null && headerTypeProperty.getValue() != null) + { + try + { + _headerType = ColumnHeaderType.valueOf(String.valueOf(headerTypeProperty.getValue())); + } + catch (IllegalArgumentException ex) + { + // ignore + } + } + + return errors; + } + } + + + @RequiresPermission(ReadPermission.class) + public static class SaveNamedSetAction extends MutatingApiAction + { + @Override + public Object execute(NamedSetForm namedSetForm, BindException errors) + { + QueryService.get().saveNamedSet(namedSetForm.getSetName(), namedSetForm.parseSetList()); + return new ApiSimpleResponse("success", true); + } + } + + + @SuppressWarnings({"unused", "WeakerAccess"}) + public static class NamedSetForm + { + String setName; + String[] setList; + + public String getSetName() + { + return setName; + } + + public void setSetName(String setName) + { + this.setName = setName; + } + + public String[] getSetList() + { + return setList; + } + + public void setSetList(String[] setList) + { + this.setList = setList; + } + + public List parseSetList() + { + return Arrays.asList(setList); + } + } + + + @RequiresPermission(ReadPermission.class) + public static class DeleteNamedSetAction extends MutatingApiAction + { + + @Override + public Object execute(NamedSetForm namedSetForm, BindException errors) + { + QueryService.get().deleteNamedSet(namedSetForm.getSetName()); + return new ApiSimpleResponse("success", true); + } + } + + @RequiresPermission(ReadPermission.class) + public static class AnalyzeQueriesAction extends ReadOnlyApiAction + { + @Override + public Object execute(Object o, BindException errors) throws Exception + { + JSONObject ret = new JSONObject(); + + try + { + QueryService.QueryAnalysisService analysisService = QueryService.get().getQueryAnalysisService(); + if (analysisService != null) + { + DefaultSchema start = DefaultSchema.get(getUser(), getContainer()); + var deps = new HashSetValuedHashMap(); + + analysisService.analyzeFolder(start, deps); + ret.put("success", true); + + JSONObject objects = new JSONObject(); + for (var from : deps.keySet()) + { + objects.put(from.getKey(), from.toJSON()); + for (var to : deps.get(from)) + objects.put(to.getKey(), to.toJSON()); + } + ret.put("objects", objects); + + JSONArray dependants = new JSONArray(); + for (var from : deps.keySet()) + { + for (var to : deps.get(from)) + dependants.put(new String[] {from.getKey(), to.getKey()}); + } + ret.put("graph", dependants); + } + else + { + ret.put("success", false); + } + return ret; + } + catch (Throwable e) + { + LOG.error(e); + throw UnexpectedException.wrap(e); + } + } + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) + public static class GetQueryEditorMetadataAction extends ReadOnlyApiAction + { + @Override + protected ObjectMapper createRequestObjectMapper() + { + PropertyService propertyService = PropertyService.get(); + if (null != propertyService) + { + ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); + mapper.addMixIn(GWTPropertyDescriptor.class, MetadataTableJSONMixin.class); + return mapper; + } + else + { + throw new RuntimeException("Could not serialize request object"); + } + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return createRequestObjectMapper(); + } + + @Override + public Object execute(QueryForm queryForm, BindException errors) throws Exception + { + QueryDefinition queryDef = queryForm.getQueryDef(); + return MetadataTableJSON.getMetadata(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); + } + } + + @Marshal(Marshaller.Jackson) + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + public static class SaveQueryMetadataAction extends MutatingApiAction + { + @Override + protected ObjectMapper createRequestObjectMapper() + { + PropertyService propertyService = PropertyService.get(); + if (null != propertyService) + { + ObjectMapper mapper = JsonUtil.DEFAULT_MAPPER.copy(); + propertyService.configureObjectMapper(mapper, null); + return mapper; + } + else + { + throw new RuntimeException("Could not serialize request object"); + } + } + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return createRequestObjectMapper(); + } + + @Override + public Object execute(QueryMetadataApiForm queryMetadataApiForm, BindException errors) throws Exception + { + String schemaName = queryMetadataApiForm.getSchemaName(); + MetadataTableJSON domain = queryMetadataApiForm.getDomain(); + MetadataTableJSON.saveMetadata(schemaName, domain.getName(), null, domain.getFields(true), queryMetadataApiForm.isUserDefinedQuery(), false, getUser(), getContainer()); + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + resp.put("domain", MetadataTableJSON.getMetadata(schemaName, domain.getName(), getUser(), getContainer())); + return resp; + } + } + + @Marshal(Marshaller.Jackson) + @RequiresAllOf({EditQueriesPermission.class, UpdatePermission.class}) + public static class ResetQueryMetadataAction extends MutatingApiAction + { + @Override + public Object execute(QueryForm queryForm, BindException errors) throws Exception + { + QueryDefinition queryDef = queryForm.getQueryDef(); + return MetadataTableJSON.resetToDefault(queryDef.getSchema().getSchemaName(), queryDef.getName(), getUser(), getContainer()); + } + } + + private static class QueryMetadataApiForm + { + private MetadataTableJSON _domain; + private String _schemaName; + private boolean _userDefinedQuery; + + public MetadataTableJSON getDomain() + { + return _domain; + } + + @SuppressWarnings("unused") + public void setDomain(MetadataTableJSON domain) + { + _domain = domain; + } + + public String getSchemaName() + { + return _schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(String schemaName) + { + _schemaName = schemaName; + } + + public boolean isUserDefinedQuery() + { + return _userDefinedQuery; + } + + @SuppressWarnings("unused") + public void setUserDefinedQuery(boolean userDefinedQuery) + { + _userDefinedQuery = userDefinedQuery; + } + } + + @RequiresPermission(ReadPermission.class) + public static class GetDefaultVisibleColumnsAction extends ReadOnlyApiAction + { + @Override + public Object execute(GetQueryDetailsAction.Form form, BindException errors) throws Exception + { + ApiSimpleResponse resp = new ApiSimpleResponse(); + + Container container = getContainer(); + User user = getUser(); + + if (StringUtils.isEmpty(form.getSchemaName())) + throw new NotFoundException("SchemaName not specified"); + + QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); + if (!(querySchema instanceof UserSchema schema)) + throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'"); + + QuerySettings settings = schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); + QueryDefinition queryDef = settings.getQueryDef(schema); + if (null == queryDef) + // Don't echo the provided query name, but schema name is legit since it was found. See #44528. + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'"); + + TableInfo tinfo = queryDef.getTable(null, true); + if (null == tinfo) + throw new NotFoundException("Could not find the specified query '" + form.getQueryName() + "' in the schema '" + form.getSchemaName() + "'"); + + List fields = tinfo.getDefaultVisibleColumns(); + + List displayColumns = QueryService.get().getColumns(tinfo, fields) + .values() + .stream() + .filter(cinfo -> fields.contains(cinfo.getFieldKey())) + .map(cinfo -> cinfo.getDisplayColumnFactory().createRenderer(cinfo)) + .collect(Collectors.toList()); + + resp.put("columns", JsonWriter.getNativeColProps(displayColumns, null, false)); + + return resp; + } + } + + public static class ParseForm implements ApiJsonForm + { + String expression = ""; + Map columnMap = new HashMap<>(); + List phiColumns = new ArrayList<>(); + + Map getColumnMap() + { + return columnMap; + } + + public String getExpression() + { + return expression; + } + + public void setExpression(String expression) + { + this.expression = expression; + } + + public List getPhiColumns() + { + return phiColumns; + } + + public void setPhiColumns(List phiColumns) + { + this.phiColumns = phiColumns; + } + + @Override + public void bindJson(JSONObject json) + { + if (json.has("expression")) + setExpression(json.getString("expression")); + if (json.has("phiColumns")) + setPhiColumns(json.getJSONArray("phiColumns").toList().stream().map(s -> FieldKey.fromParts(s.toString())).collect(Collectors.toList())); + if (json.has("columnMap")) + { + JSONObject columnMap = json.getJSONObject("columnMap"); + for (String key : columnMap.keySet()) + { + try + { + getColumnMap().put(FieldKey.fromParts(key), JdbcType.valueOf(String.valueOf(columnMap.get(key)))); + } + catch (IllegalArgumentException iae) + { + getColumnMap().put(FieldKey.fromParts(key), JdbcType.OTHER); + } + } + } + } + } + + + /** + * Since this api purpose is to return parse errors, it does not generally return success:false. + *
+ * The API expects JSON like this, note that column names should be in FieldKey.toString() encoded to match the response JSON format. + * 
+     *     { "expression": "A$ + B", "columnMap":{"A$D":"VARCHAR", "X":"VARCHAR"}}
+     *
+ * and returns a response like this + * 
+     *     {
+     *       "jdbcType" : "OTHER",
+     *       "success" : true,
+     *       "columnMap" : {"A$D":"VARCHAR", "B":"OTHER"}
+     *       "errors" : [ { "msg" : "\"B\" not found.", "type" : "sql" } ]
+     *     }
+     *
+ * The columnMap object keys are the names of columns found in the expression. Names are returned + * in FieldKey.toString() formatting e.g. dollar-sign encoded. The object structure + * is compatible with the columnMap input parameter, so it can be used as a template to make a second request + * with types filled in. If provided, the type will be copied from the input columnMap, otherwise it will be "OTHER". + *
+ * Parse exceptions may contain a line (usually 1) and col location e.g. + * 
+     * {
+     *     "msg" : "Error on line 1: Syntax error near 'error', expected 'EOF'
+     *     "col" : 2,
+     *     "line" : 1,
+     *     "type" : "sql",
+     *     "errorStr" : "A error B"
+     *   }
+     *
+ */ + @RequiresNoPermission + @CSRF(CSRF.Method.NONE) + public static class ParseCalculatedColumnAction extends ReadOnlyApiAction + { + @Override + public Object execute(ParseForm form, BindException errors) throws Exception + { + if (errors.hasErrors()) + return errors; + JSONObject result = new JSONObject(Map.of("success",true)); + var requiredColumns = new HashSet(); + JdbcType jdbcType = JdbcType.OTHER; + try + { + var schema = DefaultSchema.get(getViewContext().getUser(), getViewContext().getContainer()).getUserSchema("core"); + var table = new VirtualTable<>(schema.getDbSchema(), "EXPR", schema){}; + ColumnInfo calculatedCol = QueryServiceImpl.get().createQueryExpressionColumn(table, new FieldKey(null, "expr"), form.getExpression(), null); + Map columns = new HashMap<>(); + for (var entry : form.getColumnMap().entrySet()) + { + BaseColumnInfo entryCol = new BaseColumnInfo(entry.getKey(), entry.getValue()); + // bindQueryExpressionColumn has a check that restricts PHI columns from being used in expressions + // so we need to set the PHI level to something other than NotPHI on these fake BaseColumnInfo objects + if (form.getPhiColumns().contains(entry.getKey())) + entryCol.setPHI(PHI.PHI); + columns.put(entry.getKey(), entryCol); + table.addColumn(entryCol); + } + // TODO: calculating jdbcType still uses calculatedCol.getParentTable().getColumns() + QueryServiceImpl.get().bindQueryExpressionColumn(calculatedCol, columns, false, requiredColumns); + jdbcType = calculatedCol.getJdbcType(); + } + catch (QueryException x) + { + JSONArray parseErrors = new JSONArray(); + parseErrors.put(x.toJSON(form.getExpression())); + result.put("errors", parseErrors); + } + finally + { + if (!requiredColumns.isEmpty()) + { + JSONObject columnMap = new JSONObject(); + for (FieldKey fk : requiredColumns) + { + JdbcType type = Objects.requireNonNullElse(form.getColumnMap().get(fk), JdbcType.OTHER); + columnMap.put(fk.toString(), type); + } + result.put("columnMap", columnMap); + } + } + result.put("jdbcType", jdbcType.name()); + return result; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class QueryImportTemplateForm + { + private String schemaName; + private String queryName; + private String auditUserComment; + private List templateLabels; + private List templateUrls; + private Long _lastKnownModified; + + public void setQueryName(String queryName) + { + this.queryName = queryName; + } + + public List getTemplateLabels() + { + return templateLabels == null ? Collections.emptyList() : templateLabels; + } + + public void setTemplateLabels(List templateLabels) + { + this.templateLabels = templateLabels; + } + + public List getTemplateUrls() + { + return templateUrls == null ? Collections.emptyList() : templateUrls; + } + + public void setTemplateUrls(List templateUrls) + { + this.templateUrls = templateUrls; + } + + public String getSchemaName() + { + return schemaName; + } + + @SuppressWarnings("unused") + public void setSchemaName(String schemaName) + { + this.schemaName = schemaName; + } + + public String getQueryName() + { + return queryName; + } + + public Long getLastKnownModified() + { + return _lastKnownModified; + } + + public void setLastKnownModified(Long lastKnownModified) + { + _lastKnownModified = lastKnownModified; + } + + public String getAuditUserComment() + { + return auditUserComment; + } + + public void setAuditUserComment(String auditUserComment) + { + this.auditUserComment = auditUserComment; + } + + } + + @Marshal(Marshaller.Jackson) + @RequiresPermission(ReadPermission.class) //Real permissions will be enforced later on by the DomainKind + public static class UpdateQueryImportTemplateAction extends MutatingApiAction + { + private DomainKind _kind; + private UserSchema _schema; + private TableInfo _tInfo; + private QueryDefinition _queryDef; + private Domain _domain; + + @Override + protected ObjectMapper createResponseObjectMapper() + { + return this.createRequestObjectMapper(); + } + + @Override + public void validateForm(QueryImportTemplateForm form, Errors errors) + { + User user = getUser(); + Container container = getContainer(); + String domainURI = PropertyService.get().getDomainURI(form.getSchemaName(), form.getQueryName(), container, user); + _kind = PropertyService.get().getDomainKind(domainURI); + _domain = PropertyService.get().getDomain(container, domainURI); + if (_domain == null) + throw new IllegalArgumentException("Domain '" + domainURI + "' not found."); + + if (!_kind.canEditDefinition(user, _domain)) + throw new UnauthorizedException("You don't have permission to update import templates for this domain."); + + QuerySchema querySchema = DefaultSchema.get(user, container, form.getSchemaName()); + if (!(querySchema instanceof UserSchema _schema)) + throw new NotFoundException("Could not find the specified schema in the folder '" + container.getPath() + "'."); + QuerySettings settings = _schema.getSettings(getViewContext(), QueryView.DATAREGIONNAME_DEFAULT, form.getQueryName()); + _queryDef = settings.getQueryDef(_schema); + if (null == _queryDef) + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); + if (!_queryDef.isMetadataEditable()) + throw new UnsupportedOperationException("Query metadata is not editable."); + _tInfo = _queryDef.getTable(_schema, new ArrayList<>(), true, true); + if (_tInfo == null) + throw new NotFoundException("Could not find the specified query in the schema '" + form.getSchemaName() + "'."); + + } + + private Map getRowFiles() + { + Map rowFiles = new IntHashMap<>(); + if (getFileMap() != null) + { + for (Map.Entry fileEntry : getFileMap().entrySet()) + { + // allow for the fileMap key to include the row index for defining which row to attach this file to + // ex: "templateFile::0", "templateFile::1" + String fieldKey = fileEntry.getKey(); + int delimIndex = fieldKey.lastIndexOf("::"); + if (delimIndex > -1) + { + Integer fieldRowIndex = Integer.parseInt(fieldKey.substring(delimIndex + 2)); + SpringAttachmentFile file = new SpringAttachmentFile(fileEntry.getValue()); + rowFiles.put(fieldRowIndex, file.isEmpty() ? null : file); + } + } + } + return rowFiles; + } + + private List> getUploadedTemplates(QueryImportTemplateForm form, DomainKind kind) throws ValidationException, QueryUpdateServiceException, ExperimentException + { + FileContentService fcs = FileContentService.get(); + if (fcs == null) + throw new IllegalStateException("Unable to load file service."); + + User user = getUser(); + Container container = getContainer(); + + Map rowFiles = getRowFiles(); + List templateLabels = form.getTemplateLabels(); + Set labels = new HashSet<>(templateLabels); + if (labels.size() < templateLabels.size()) + throw new IllegalArgumentException("Duplicate template name is not allowed."); + + List templateUrls = form.getTemplateUrls(); + List> uploadedTemplates = new ArrayList<>(); + for (int rowIndex = 0; rowIndex < form.getTemplateLabels().size(); rowIndex++) + { + String templateLabel = templateLabels.get(rowIndex); + if (StringUtils.isBlank(templateLabel.trim())) + throw new IllegalArgumentException("Template name cannot be blank."); + String templateUrl = templateUrls.get(rowIndex); + Object file = rowFiles.get(rowIndex); + if (StringUtils.isEmpty(templateUrl) && file == null) + throw new IllegalArgumentException("Template file is not provided."); + + if (file instanceof MultipartFile || file instanceof SpringAttachmentFile) + { + String fileName; + if (file instanceof MultipartFile f) + fileName = f.getName(); + else + { + SpringAttachmentFile f = (SpringAttachmentFile) file; + fileName = f.getFilename(); + } + String fileNameValidation = FileUtil.validateFileName(fileName); + if (!StringUtils.isEmpty(fileNameValidation)) + throw new IllegalArgumentException(fileNameValidation); + + FileLike uploadDir = ensureUploadDirectory(container, kind.getDomainFileDirectory()); + uploadDir = uploadDir.resolveChild("_templates"); + Object savedFile = saveFile(user, container, "template file", file, uploadDir); + Path savedFilePath; + + if (savedFile instanceof File ioFile) + savedFilePath = ioFile.toPath(); + else if (savedFile instanceof FileLike fl) + savedFilePath = fl.toNioPathForRead(); + else + throw UnexpectedException.wrap(null,"Unable to upload template file."); + + templateUrl = fcs.getWebDavUrl(savedFilePath, container, FileContentService.PathType.serverRelative).toString(); + } + + uploadedTemplates.add(Pair.of(templateLabel, templateUrl)); + } + return uploadedTemplates; + } + + @Override + public Object execute(QueryImportTemplateForm form, BindException errors) throws ValidationException, QueryUpdateServiceException, SQLException, ExperimentException, MetadataUnavailableException + { + User user = getUser(); + Container container = getContainer(); + String schemaName = form.getSchemaName(); + String queryName = form.getQueryName(); + QueryDef queryDef = QueryManager.get().getQueryDef(container, schemaName, queryName, false); + if (queryDef != null && queryDef.getQueryDefId() != 0) + { + Long lastKnownModified = form.getLastKnownModified(); + if (lastKnownModified == null || lastKnownModified != queryDef.getModified().getTime()) + throw new ApiUsageException("Unable to save import templates. The templates appear out of date, reload the page and try again."); + } + + List> updatedTemplates = getUploadedTemplates(form, _kind); + + List> existingTemplates = _tInfo.getImportTemplates(getViewContext()); + List> existingCustomTemplates = new ArrayList<>(); + for (Pair template_ : existingTemplates) + { + if (!template_.second.toLowerCase().contains("exportexceltemplate")) + existingCustomTemplates.add(template_); + } + if (!updatedTemplates.equals(existingCustomTemplates)) + { + TablesDocument doc = null; + TableType xmlTable = null; + TableType.ImportTemplates xmlImportTemplates; + + if (queryDef != null) + { + try + { + doc = parseDocument(queryDef.getMetaData()); + } + catch (XmlException e) + { + throw new MetadataUnavailableException(e.getMessage()); + } + xmlTable = getTableType(form.getQueryName(), doc); + // when there is a queryDef but xmlTable is null it means the xmlMetaData contains tableName which does not + // match with actual queryName then reconstruct the xml table metadata : See Issue 43523 + if (xmlTable == null) + { + doc = null; + } + } + else + { + queryDef = new QueryDef(); + queryDef.setSchema(schemaName); + queryDef.setContainer(container.getId()); + queryDef.setName(queryName); + } + + if (doc == null) + { + doc = TablesDocument.Factory.newInstance(); + } + + if (xmlTable == null) + { + TablesType tables = doc.addNewTables(); + xmlTable = tables.addNewTable(); + xmlTable.setTableName(queryName); + } + + if (xmlTable.getTableDbType() == null) + { + xmlTable.setTableDbType("NOT_IN_DB"); + } + + // remove existing templates + if (xmlTable.isSetImportTemplates()) + xmlTable.unsetImportTemplates(); + xmlImportTemplates = xmlTable.addNewImportTemplates(); + + // set new templates + if (!updatedTemplates.isEmpty()) + { + for (Pair template_ : updatedTemplates) + { + ImportTemplateType importTemplateType = xmlImportTemplates.addNewTemplate(); + importTemplateType.setLabel(template_.first); + importTemplateType.setUrl(template_.second); + } + } + + XmlOptions xmlOptions = new XmlOptions(); + xmlOptions.setSavePrettyPrint(); + // Don't use an explicit namespace, making the XML much more readable + xmlOptions.setUseDefaultNamespace(); + queryDef.setMetaData(doc.xmlText(xmlOptions)); + if (queryDef.getQueryDefId() == 0) + { + QueryManager.get().insert(user, queryDef); + } + else + { + QueryManager.get().update(user, queryDef); + } + + DomainAuditProvider.DomainAuditEvent event = new DomainAuditProvider.DomainAuditEvent(getContainer(), "Import templates updated."); + event.setUserComment(form.getAuditUserComment()); + event.setDomainUri(_domain.getTypeURI()); + event.setDomainName(_domain.getName()); + AuditLogService.get().addEvent(user, event); + } + + ApiSimpleResponse resp = new ApiSimpleResponse(); + resp.put("success", true); + return resp; + } + } + + + public static class TestCase extends AbstractActionPermissionTest + { + @Override + public void testActionPermissions() + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + QueryController controller = new QueryController(); + + // @RequiresPermission(ReadPermission.class) + assertForReadPermission(user, false, + new BrowseAction(), + new BeginAction(), + controller.new SchemaAction(), + controller.new SourceQueryAction(), + controller.new ExecuteQueryAction(), + controller.new PrintRowsAction(), + new ExportScriptAction(), + new ExportRowsExcelAction(), + new ExportRowsXLSXAction(), + new ExportQueriesXLSXAction(), + new ExportExcelTemplateAction(), + new ExportRowsTsvAction(), + new ExcelWebQueryDefinitionAction(), + controller.new SaveQueryViewsAction(), + controller.new PropertiesQueryAction(), + controller.new SelectRowsAction(), + new GetDataAction(), + controller.new ExecuteSqlAction(), + controller.new SelectDistinctAction(), + controller.new GetColumnSummaryStatsAction(), + controller.new ImportAction(), + new ExportSqlAction(), + new UpdateRowsAction(), + new ImportRowsAction(), + new DeleteRowsAction(), + new TableInfoAction(), + new SaveSessionViewAction(), + new GetSchemasAction(), + new GetQueriesAction(), + new GetQueryViewsAction(), + new SaveApiTestAction(), + new ValidateQueryMetadataAction(), + new AuditHistoryAction(), + new AuditDetailsAction(), + new ExportTablesAction(), + new SaveNamedSetAction(), + new DeleteNamedSetAction(), + new ApiTestAction(), + new GetDefaultVisibleColumnsAction() + ); + + + // submitter should be allowed for InsertRows + assertForReadPermission(user, true, new InsertRowsAction()); + + // @RequiresPermission(DeletePermission.class) + assertForUpdateOrDeletePermission(user, + new DeleteQueryRowsAction() + ); + + // @RequiresPermission(AdminPermission.class) + assertForAdminPermission(user, + new DeleteQueryAction(), + controller.new MetadataQueryAction(), + controller.new NewQueryAction(), + new SaveSourceQueryAction(), + + new TruncateTableAction(), + new AdminAction(), + new ManageRemoteConnectionsAction(), + new ReloadExternalSchemaAction(), + new ReloadAllUserSchemas(), + controller.new ManageViewsAction(), + controller.new InternalDeleteView(), + controller.new InternalSourceViewAction(), + controller.new InternalNewViewAction(), + new QueryExportAuditRedirectAction() + ); + + // @RequiresPermission(AdminOperationsPermission.class) + assertForAdminOperationsPermission(user, + new EditRemoteConnectionAction(), + new DeleteRemoteConnectionAction(), + new TestRemoteConnectionAction(), + controller.new RawTableMetaDataAction(), + controller.new RawSchemaMetaDataAction(), + new InsertLinkedSchemaAction(), + new InsertExternalSchemaAction(), + new DeleteSchemaAction(), + new EditLinkedSchemaAction(), + new EditExternalSchemaAction(), + new GetTablesAction(), + new SchemaTemplateAction(), + new SchemaTemplatesAction(), + new ParseExpressionAction(), + new ParseQueryAction() + ); + + // @AdminConsoleAction + assertForAdminPermission(ContainerManager.getRoot(), user, + new DataSourceAdminAction() + ); + + // In addition to administrators (tested above), trusted analysts who are editors can create and edit queries + assertTrustedEditorPermission( + new DeleteQueryAction(), + controller.new MetadataQueryAction(), + controller.new NewQueryAction(), + new SaveSourceQueryAction() + ); + } + } + + public static class SaveRowsTestCase extends Assert + { + private static final String PROJECT_NAME1 = "SaveRowsTestProject1"; + private static final String PROJECT_NAME2 = "SaveRowsTestProject2"; + + private static final String USER_EMAIL = "saveRows@action.test"; + + private static final String LIST1 = "List1"; + private static final String LIST2 = "List2"; + + @Before + public void doSetup() throws Exception + { + doCleanup(); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME1, TestContext.get().getUser()); + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT_NAME2, TestContext.get().getUser()); + + //disable search so we dont get conflicts when deleting folder quickly + ContainerManager.updateSearchable(project1, false, TestContext.get().getUser()); + ContainerManager.updateSearchable(project2, false, TestContext.get().getUser()); + + ListDefinition ld1 = ListService.get().createList(project1, LIST1, ListDefinition.KeyType.Varchar); + ld1.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); + ld1.setKeyName("TextField"); + ld1.save(TestContext.get().getUser()); + + ListDefinition ld2 = ListService.get().createList(project2, LIST2, ListDefinition.KeyType.Varchar); + ld2.getDomain().addProperty(new PropertyStorageSpec("TextField", JdbcType.VARCHAR)); + ld2.setKeyName("TextField"); + ld2.save(TestContext.get().getUser()); + } + + @After + public void doCleanup() throws Exception + { + Container project = ContainerManager.getForPath(PROJECT_NAME1); + if (project != null) + { + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + if (project2 != null) + { + ContainerManager.deleteAll(project2, TestContext.get().getUser()); + } + + User u = UserManager.getUser(new ValidEmail(USER_EMAIL)); + if (u != null) + { + UserManager.deleteUser(u.getUserId()); + } + } + + private JSONObject getCommand(String val1, String val2) + { + JSONObject command1 = new JSONObject(); + command1.put("containerPath", ContainerManager.getForPath(PROJECT_NAME1).getPath()); + command1.put("command", "insert"); + command1.put("schemaName", "lists"); + command1.put("queryName", LIST1); + command1.put("rows", getTestRows(val1)); + + JSONObject command2 = new JSONObject(); + command2.put("containerPath", ContainerManager.getForPath(PROJECT_NAME2).getPath()); + command2.put("command", "insert"); + command2.put("schemaName", "lists"); + command2.put("queryName", LIST2); + command2.put("rows", getTestRows(val2)); + + JSONObject json = new JSONObject(); + json.put("commands", Arrays.asList(command1, command2)); + + return json; + } + + private MockHttpServletResponse makeRequest(JSONObject json, User user) throws Exception + { + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/json"); + + HttpServletRequest request = ViewServlet.mockRequest(RequestMethod.POST.name(), DetailsURL.fromString("/query/saveRows.view").copy(ContainerManager.getForPath(PROJECT_NAME1)).getActionURL(), user, headers, json.toString()); + return ViewServlet.mockDispatch(request, null); + } + + @Test + public void testCrossFolderSaveRows() throws Exception + { + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + JSONObject json = getCommand(PROJECT_NAME1, PROJECT_NAME2); + MockHttpServletResponse response = makeRequest(json, TestContext.get().getUser()); + if (response.getStatus() != HttpServletResponse.SC_OK) + { + JSONObject responseJson = new JSONObject(response.getContentAsString()); + throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); + } + + Container project1 = ContainerManager.getForPath(PROJECT_NAME1); + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + + TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); + TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); + + assertEquals("Incorrect row count, list1", 1L, new TableSelector(list1).getRowCount()); + assertEquals("Incorrect row count, list2", 1L, new TableSelector(list2).getRowCount()); + + assertEquals("Incorrect value", PROJECT_NAME1, new TableSelector(list1, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME1, String.class)); + assertEquals("Incorrect value", PROJECT_NAME2, new TableSelector(list2, PageFlowUtil.set("TextField")).getObject(PROJECT_NAME2, String.class)); + + list1.getUpdateService().truncateRows(TestContext.get().getUser(), project1, null, null); + list2.getUpdateService().truncateRows(TestContext.get().getUser(), project2, null, null); + } + + @Test + public void testWithoutPermissions() throws Exception + { + // Now test failure without appropriate permissions: + User withoutPermissions = SecurityManager.addUser(new ValidEmail(USER_EMAIL), TestContext.get().getUser()).getUser(); + + User user = TestContext.get().getUser(); + assertTrue(user.hasSiteAdminPermission()); + + Container project1 = ContainerManager.getForPath(PROJECT_NAME1); + Container project2 = ContainerManager.getForPath(PROJECT_NAME2); + + MutableSecurityPolicy securityPolicy = new MutableSecurityPolicy(SecurityPolicyManager.getPolicy(project1)); + securityPolicy.addRoleAssignment(withoutPermissions, EditorRole.class); + SecurityPolicyManager.savePolicyForTests(securityPolicy, TestContext.get().getUser()); + + assertTrue("Should have insert permission", project1.hasPermission(withoutPermissions, InsertPermission.class)); + assertFalse("Should not have insert permission", project2.hasPermission(withoutPermissions, InsertPermission.class)); + + // repeat insert: + JSONObject json = getCommand("ShouldFail1", "ShouldFail2"); + MockHttpServletResponse response = makeRequest(json, withoutPermissions); + if (response.getStatus() != HttpServletResponse.SC_FORBIDDEN) + { + JSONObject responseJson = new JSONObject(response.getContentAsString()); + throw new RuntimeException("Problem saving rows across folders: " + responseJson.getString("exception")); + } + + TableInfo list1 = ListService.get().getList(project1, LIST1).getTable(TestContext.get().getUser()); + TableInfo list2 = ListService.get().getList(project2, LIST2).getTable(TestContext.get().getUser()); + + // The insert should have failed + assertEquals("Incorrect row count, list1", 0L, new TableSelector(list1).getRowCount()); + assertEquals("Incorrect row count, list2", 0L, new TableSelector(list2).getRowCount()); + } + + private JSONArray getTestRows(String val) + { + JSONArray rows = new JSONArray(); + rows.put(Map.of("TextField", val)); + + return rows; + } + } +} diff --git a/study/src/org/labkey/study/query/DatasetUpdateService.java b/study/src/org/labkey/study/query/DatasetUpdateService.java index d64019bd1a3..5c8857de5f8 100644 --- a/study/src/org/labkey/study/query/DatasetUpdateService.java +++ b/study/src/org/labkey/study/query/DatasetUpdateService.java @@ -1,1144 +1,1144 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.study.query; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.labkey.api.audit.AbstractAuditTypeProvider; -import org.labkey.api.assay.AssayFileWriter; -import org.labkey.api.audit.TransactionAuditProvider; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.collections.ResultSetRowMapFactory; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MvUtil; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.DataIteratorBuilder; -import org.labkey.api.dataiterator.DataIteratorContext; -import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; -import org.labkey.api.dataiterator.MapDataIterator; -import org.labkey.api.dataiterator.SimpleTranslator; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.MvFieldWrapper; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.gwt.client.AuditBehaviorType; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.DefaultQueryUpdateService; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.InvalidKeyException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.query.QueryUpdateServiceException; -import org.labkey.api.query.SimpleValidationError; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.TimepointType; -import org.labkey.api.study.security.StudySecurityEscalator; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JunitUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.TestContext; -import org.labkey.study.model.DatasetDataIteratorBuilder; -import org.labkey.study.dataset.DatasetAuditProvider; -import org.labkey.study.model.DatasetDefinition; -import org.labkey.study.model.DatasetDomainKind; -import org.labkey.study.model.DatasetLsidImportHelper; -import org.labkey.study.model.ParticipantIdImportHelper; -import org.labkey.study.model.ParticipantSeqNumImportHelper; -import org.labkey.study.model.QCStateImportHelper; -import org.labkey.study.model.SecurityType; -import org.labkey.study.model.SequenceNumImportHelper; -import org.labkey.study.model.StudyImpl; -import org.labkey.study.model.StudyManager; -import org.labkey.study.visitmanager.PurgeParticipantsJob.ParticipantPurger; - -import java.nio.file.Path; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -import static org.labkey.api.util.IntegerUtils.asInteger; -import static org.labkey.api.gwt.client.AuditBehaviorType.DETAILED; -import static org.labkey.api.gwt.client.AuditBehaviorType.NONE; - -/* -* User: Dave -* Date: Jun 13, 2008 -* Time: 4:15:51 PM -*/ - -/** - * QueryUpdateService implementation for Study datasets. - *

- * Since datasets are of an unpredictable shape, this class just implements - * the QueryUpdateService directly, working with Map<String,Object> - * collections for the row data. - */ -public class DatasetUpdateService extends DefaultQueryUpdateService -{ - // These are that can be passed into DatasetUpdateService via DataIteratorContext.configParameters. - // These used to be passed to DatasetDataIterator via - // DatasetDefinition.importDatasetData()->DatasetDefinition.insertData(). - // Moving these options into DataInteratorContext allows for even more consistency and code sharing - // also see QueryUpdateService.ConfigParameters.Logger - public enum Config - { - CheckForDuplicates, // expected: enum CheckForDuplicates - DefaultQCState, // expected: class QCState - SkipResyncStudy, // expected: Boolean - - // NOTE: There really has to be better way to handle the functionality of StudyImportContext.getTableIdMap() - // NOTE: Could this be handled by a method on StudySchema or something??? - // see StudyImportContext.getTableIdMapMap() - StudyImportMaps, // expected: Map> - - KeyList, // expected: List - AllowImportManagedKey // expected: Boolean - } - - private static final Logger LOG = LogManager.getLogger(DatasetUpdateService.class); - - private final DatasetDefinition _dataset; - private final Set _potentiallyNewParticipants = new HashSet<>(); - private final Set _potentiallyDeletedParticipants = new HashSet<>(); - private boolean _participantVisitResyncRequired = false; - - private final boolean _skipAuditLogging = false; - - public DatasetUpdateService(DatasetTableImpl table) - { - super(table, table.getDatasetDefinition().getStorageTableInfo(false), createMVMapping(table.getDatasetDefinition().getDomain())); - _dataset = table.getDatasetDefinition(); - } - - @Override - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) - { - if (StudySecurityEscalator.isEscalated()) { - return true; - } - else { - return super.hasPermission(user, acl); - } - } - - - @Override - protected Map getRow(User user, Container container, Map keys) - throws InvalidKeyException - { - String lsid = keyFromMap(keys); - SimpleFilter filter = new SimpleFilter() - .addCondition(new FieldKey(null,"container"), container.getId()) - .addCondition(new FieldKey(null,"lsid"),lsid); - - // NOTE getQueryTable().getColumns() returns a bunch of columns that getDatasetRow() did not such as: - // Container, Dataset, DatasetId, Datasets, Folder, Modified, ModifiedBy, MouseVisit, ParticipantSequenceNum, VisitDay, VisitRowId - // Mostly this is harmless, but there is some noise. - HashSet nameset = new HashSet<>(getQueryTable().getColumnNameSet()); - List.of("Container","Datasets","DatasetId","Dataset","Folder").forEach(nameset::remove); - List columns = new ArrayList<>(getQueryTable().getColumns(nameset.toArray(new String[0]))); - - // filter out calculated columns which can be expensive to reselect - columns.removeIf(ColumnInfo::isCalculated); - - // This is a general version of DatasetDefinition.canonicalizeDatasetRow() - // The caller needs to make sure names are unique. Not suitable for use w/ lookups etc where there can be name collisions. - // CONSIDER: might be nice to make this a TableSelector method. - var map = new CaseInsensitiveHashMap<>(); - try (var str = new TableSelector(getQueryTable(), columns, filter, null).uncachedResultSetStream()) - { - str.forEach(rs -> { - try - { - for (int i = 0; i < columns.size(); i++) - { - Object o = rs.getObject(i + 1); - o = ResultSetRowMapFactory.translateResultSetObject(o, false); - map.put(columns.get(i).getName(), o); - } - } - catch (SQLException x) - { - throw new RuntimeSQLException(x); - } - }); - } - return map.isEmpty() ? null : map; - } - - - /* TODO for performance, NOTE need to return rows in order of input list - @Override - public List> getRows(User user, Container container, List> keys) throws InvalidKeyException - { - if (!hasPermission(user, ReadPermission.class)) - throw new UnauthorizedException("You do not have permission to read data from this table."); - ArrayList lsids = new ArrayList<>(keys.size()); - for (var m : keys) - lsids.add(keyFromMap(m)); - var result = (List)(new TableSelector(getQueryTable(), - TableSelector.ALL_COLUMNS, - new SimpleFilter(new FieldKey(null,"lsid"), lsids, CompareType.IN), - null)) - .getArrayList(Map.class); - return (List>)result; - } - */ - - - @Override - public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); - if (count > 0) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container, null, null, true); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - } - return count; - } - - @Override - public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) - { - int count = _importRowsUsingDIB(user, container, rows, null, context, extraScriptContext); - if (count > 0 && !Boolean.TRUE.equals(context.getConfigParameterBoolean(Config.SkipResyncStudy))) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container, null, null, true); - } - catch (ValidationException e) - { - context.getErrors().addRowError(e); - } - } - return count; - } - - @Override - public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) - { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); - - return loadRows(user, container, rows, context, extraScriptContext); - } - - @Override - public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws QueryUpdateServiceException - { - for (Map row : rows) - { - aliasColumns(_columnMapping, row); - } - - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); - - DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - if (_skipAuditLogging) - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE); - else if (!isBulkLoad()) - { - // default to DETAILED unless there is a metadata XML override - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, - getQueryTable().getXmlAuditBehaviorType() != null ? getQueryTable().getXmlAuditBehaviorType() : DETAILED); - } - - List> result = super._insertRowsUsingDIB(user, container, rows, context, extraScriptContext); - - if (null != result && result.size() > 0) - { - for (Map row : result) - { - String participantID = getParticipant(row, user, container); - _potentiallyNewParticipants.add(participantID); - } - - _participantVisitResyncRequired = true; // 13717 : Study failing to resync() on dataset insert - if (configParameters == null || !Boolean.TRUE.equals(configParameters.get(DatasetUpdateService.Config.SkipResyncStudy))) - { - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - } - } - return result; - } - - @Override - protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) - { - // If we're using a managed GUID as a key, wire it up here so that it's available to trigger scripts - if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER && - _dataset.getKeyManagementType() == Dataset.KeyManagementType.GUID && - _dataset.getKeyPropertyName() != null) - { - return new DataIteratorBuilder() - { - @Override - public DataIterator getDataIterator(DataIteratorContext context) - { - DataIterator input = in.getDataIterator(context); - if (null == input) - return null; // Can happen if context has errors - - final SimpleTranslator result = new SimpleTranslator(input, context); - - boolean foundKeyCol = false; - for (int c = 1; c <= input.getColumnCount(); c++) - { - ColumnInfo col = input.getColumnInfo(c); - - // Incoming data has a matching field - if (col.getName().equalsIgnoreCase(_dataset.getKeyPropertyName())) - { - // make sure guid is not null (12884) - result.addCoalesceColumn(col.getName(), c, new SimpleTranslator.GuidColumn()); - foundKeyCol = true; - } - else - { - // Pass it through as-is - result.addColumn(c); - } - } - - if (!foundKeyCol) - { - // Inject a column with a new GUID - ColumnInfo key = getQueryTable().getColumn(_dataset.getKeyPropertyName()); - result.addColumn(new BaseColumnInfo(key), new SimpleTranslator.GuidColumn()); - } - - return result; - } - }; - } - return in; - } - - @Override - public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) - { - if (null == context.getConfigParameter(Config.DefaultQCState)) - { - context.putConfigParameter(Config.DefaultQCState, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); - } - - if (null == context.getConfigParameter(Config.CheckForDuplicates)) - { - DatasetDefinition.CheckForDuplicates dupePolicy; - if (isBulkLoad()) - dupePolicy = DatasetDefinition.CheckForDuplicates.never; - else if (context.getInsertOption().mergeRows) - dupePolicy = DatasetDefinition.CheckForDuplicates.sourceOnly; - else - dupePolicy = DatasetDefinition.CheckForDuplicates.sourceAndDestination; - context.putConfigParameter(Config.CheckForDuplicates, dupePolicy); - } - - // NOTE: This was done to help coalesce some old code paths. However, this is a little weird, because - // the DI is over the DatasetSchemaTableInfo not the DatasetTableImpl you'd expect. This all still works - // because of property URI matching in StatementDataIterator. - return _dataset.getInsertDataIterator(user, container, data, context); - } - - - @Override - protected int _pump(DataIteratorBuilder etl, final ArrayList> rows, DataIteratorContext context) - { - try - { - boolean hasRowId = _dataset.getKeyManagementType() == Dataset.KeyManagementType.RowId; - - if (null != rows) - { - // TODO: consider creating DataIterator metadata to mark "internal" cols (not to be returned via API) - DataIterator it = etl.getDataIterator(context); - DataIteratorBuilder cleanMap = new MapDataIterator.MapDataIteratorImpl(it, true, CaseInsensitiveHashSet.of( - it.getColumnInfo(0).getName() - )); - etl = cleanMap; - } - - if (!hasRowId) - { - return super._pump(etl, rows, context); - } - - synchronized (_dataset.getManagedKeyLock()) - { - return super._pump(etl, rows, context); - } - } - catch (RuntimeSQLException e) - { - String translated = _dataset.translateSQLException(e); - if (translated != null) - { - context.getErrors().addRowError(new ValidationException(translated)); - return 0; - } - throw e; - } - } - - - @NotNull String getParticipant(Map row, User user, Container container) throws QueryUpdateServiceException - { - String columnName = _dataset.getStudy().getSubjectColumnName(); - Object participant = row.get(columnName); - if (participant == null) - { - participant = row.get("ParticipantId"); - } - if (participant == null) - { - try - { - // This may be an update or delete where the user specified the LSID as the key, but didn't bother - // sending the participant, so look it up - Map originalRow = getRow(user, container, row); - participant = originalRow == null ? null : originalRow.get(columnName); - if (participant == null) - { - participant = originalRow.get("ParticipantId"); - } - } - catch (InvalidKeyException e) - { - throw new QueryUpdateServiceException(e); - } - } - if (participant == null) - { - throw new QueryUpdateServiceException("All dataset rows must include a value for " + columnName); - } - return participant.toString(); - } - - static class PurgeParticipantCommitTask implements Runnable - { - private final Container _container; - private final Set _potentiallyDeletedParticipants; - - PurgeParticipantCommitTask(Container container, Set potentiallyDeletedParticipants) - { - _container = container; - _potentiallyDeletedParticipants = new HashSet<>(potentiallyDeletedParticipants); - } - - @Override - public void run() - { - new ParticipantPurger(_container, _potentiallyDeletedParticipants, LOG::info, LOG::error).purgeParticipants(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - PurgeParticipantCommitTask that = (PurgeParticipantCommitTask) o; - - if (!Objects.equals(_container, that._container)) return false; - - return true; - } - - @Override - public int hashCode() - { - return _container != null ? _container.hashCode() : 0; - } - } - - - @Override - public List> updateRows(User user, final Container container, List> rows, List> oldKeys, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - List> result = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); - if (null != extraScriptContext && Boolean.TRUE.equals(extraScriptContext.get("synchronousParticipantPurge"))) - { - PurgeParticipantCommitTask addObj = new PurgeParticipantCommitTask(container, _potentiallyDeletedParticipants); - PurgeParticipantCommitTask setObj = getQueryTable().getSchema().getScope().addCommitTask(addObj, DbScope.CommitTaskOption.POSTCOMMIT); - setObj._potentiallyDeletedParticipants.addAll(addObj._potentiallyDeletedParticipants); - } - - try - { - StudyManager.datasetModified(_dataset, true); - resyncStudy(user, container); - } - catch (ValidationException e) - { - errors.addRowError(e); - } - return result; - } - - private void resyncStudy(User user, Container container) throws ValidationException - { - resyncStudy(user, container, _potentiallyNewParticipants, _potentiallyDeletedParticipants, _participantVisitResyncRequired); - - _participantVisitResyncRequired = false; - _potentiallyNewParticipants.clear(); - _potentiallyDeletedParticipants.clear(); - } - - /** - * Resyncs the study : updates the participant, visit, and (optionally) participant visit tables. Also updates automatic cohort assignments. - * - * @param potentiallyAddedParticipants optionally, the specific participants that may have been added to the study. - * If null, all the changedDatasets and specimens will be checked to see if they contain new participants - * @param potentiallyDeletedParticipants optionally, the specific participants that may have been removed from the - * study. If null, all participants will be checked to see if they are still in the study. - * @param participantVisitResyncRequired If true, will force an update of the ParticipantVisit mapping for this study - */ - private void resyncStudy(User user, Container container, @Nullable Set potentiallyAddedParticipants, - @Nullable Set potentiallyDeletedParticipants, - boolean participantVisitResyncRequired) throws ValidationException - { - StudyImpl study = StudyManager.getInstance().getStudy(container); - Study sharedStudy = StudyManager.getInstance().getSharedStudy(study); - - ValidationException errors = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singletonList(_dataset), - potentiallyAddedParticipants, potentiallyDeletedParticipants, participantVisitResyncRequired, - sharedStudy != null ? sharedStudy.isFailForUndefinedTimepoints() : study.isFailForUndefinedTimepoints(), null); - - if (errors.hasErrors()) - throw errors; - } - - @Override - protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException - { - // Issue 53320 : ensure a valid file link path - if (fileLinkDirPath == null) - fileLinkDirPath = AssayFileWriter.getUploadDirectoryPath(c, "datasetdata").toNioPathForWrite(); - - super.convertTypes(user, c, row, t, fileLinkDirPath); - } - - @Override - protected Map _update(User user, Container container, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException - { - try (DbScope.Transaction transaction = StudyService.get().getDatasetSchema().getScope().ensureTransaction()) - { - String lsid = keyFromMap(oldRow); - checkDuplicateUpdate(lsid); - // Make sure we've found the original participant before doing the update - String oldParticipant = getParticipant(oldRow, user, container); - String newLsid; - - Long rowId = (Long)oldRow.get(DatasetDomainKind.DSROWID); - Map oldData = _dataset.getDatasetRow(user, lsid); - - if (oldData == null) - { - // No old record found, so we can't update - ValidationException error = new ValidationException(); - error.addError(new SimpleValidationError("Record not found with lsid: " + lsid)); - throw error; - } - - // values that are always recalculated - getComputedValues(user, row, oldRow); - - newLsid = (String)row.get(DatasetDomainKind.LSID); - Table.update(user, getDbTable(), row, rowId); - - if (!isBulkLoad()) - { - DatasetTableImpl target = (DatasetTableImpl)_dataset.getTableInfo(user); - new DatasetDefinition.DatasetAuditHandler(_dataset).addAuditEvent(user, container, target, AuditBehaviorType.DETAILED, null, QueryService.AuditAction.UPDATE, - List.of(row), List.of(oldData)); - } - - // Successfully updated - transaction.commit(); - - // return updated row - var returnRow = getRow(user, container, Map.of(DatasetDomainKind.LSID, newLsid)); - - String newParticipant = getParticipant(returnRow, user, container); - if (!oldParticipant.equals(newParticipant)) - { - // Participant has changed - might be a reference to a new participant, or removal of the last reference to - // the old participant - _potentiallyNewParticipants.add(newParticipant); - _potentiallyDeletedParticipants.add(oldParticipant); - - // Need to resync the ParticipantVisit table too - _participantVisitResyncRequired = true; - } - // Check if the timepoint may have changed, but only if we don't already know we need to resync - else if (!_participantVisitResyncRequired) - { - String columnName = StudyManager.getInstance().getStudy(container).getTimepointType().isVisitBased() ? - "SequenceNum" : "Date"; - Object oldTimepoint = oldRow.get(columnName); - Object newTimepoint = returnRow.get(columnName); - if (!Objects.equals(oldTimepoint, newTimepoint)) - { - _participantVisitResyncRequired = true; - } - } - - return returnRow; - } - catch (QueryUpdateServiceException | InvalidKeyException e) - { - throw new ValidationException(e.getMessage()); - } - } - - private void getComputedValues(User user, Map row, Map oldRow) throws ValidationException - { - String subjectColumnName = _dataset.getStudy().getSubjectColumnName(); - TableInfo table = _dataset.getTableInfo(user); - ColumnInfo subjectColumn = table.getColumn(subjectColumnName); - ColumnInfo sequenceNumColumn = table.getColumn(DatasetDomainKind.SEQUENCENUM); - ColumnInfo dateColumn = table.getColumn(DatasetDomainKind.DATE); - String managedKey = null; - if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER) - managedKey = _dataset.getKeyPropertyName(); - ColumnInfo managedKeyColumn = managedKey != null ? table.getColumn(managedKey) : null; - - Object inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(row, subjectColumn); - if (inputSubjectId == null) - inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(oldRow, subjectColumn); - Object inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(row, sequenceNumColumn); - if (inputSeqNum == null) - inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(oldRow, sequenceNumColumn); - Date inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(row, dateColumn); - if (inputDate == null) - inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(oldRow, dateColumn); - Object inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(row, managedKeyColumn); - if (inputManagedKey == null) - inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(oldRow, managedKeyColumn); - Integer inputQCState = asInteger(DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_ID_COLNAME))); - - SequenceNumImportHelper snih = new SequenceNumImportHelper(_dataset.getStudy(), _dataset); - Double sequenceNum = snih.translateSequenceNum(inputSeqNum, inputDate); - - ParticipantIdImportHelper helper = new ParticipantIdImportHelper(_dataset.getStudy(), user, _dataset); - String subjectId = helper.translateParticipantId(inputSubjectId); - - // generate participant sequence number - String participantSeqNum = ParticipantSeqNumImportHelper.translateParticipantSeqNum(subjectId, sequenceNum); - - // re-generate a new lsid - DatasetLsidImportHelper dlih = new DatasetLsidImportHelper(_dataset); - String lsid = dlih.translateLsid(subjectId, sequenceNum, inputDate, inputManagedKey, null); - - // handle default QC states - if (inputQCState == null) - { - String inputQCText = (String)DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_LABEL_COLNAME)); - QCStateImportHelper qcih = new QCStateImportHelper(user, _dataset, true, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); - Long qcState = qcih.translateQCState(inputQCText); - if (qcState != null) - row.put(DatasetTableImpl.QCSTATE_ID_COLNAME, qcState); - } - row.put(DatasetDomainKind.LSID, lsid); - row.put(DatasetDomainKind.PARTICIPANTSEQUENCENUM, participantSeqNum); - row.put(DatasetDomainKind.PARTICIPANTID, subjectId); - } - - @Override - public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) - throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException - { - List> result = super.deleteRows(user, container, keys, configParameters, extraScriptContext); - try - { - resyncStudy(user, container); - } - catch (ValidationException e) - { - throw new BatchValidationException(e); - } - return result; - } - - @Override - protected Map deleteRow(User user, Container container, Map oldRow) - throws InvalidKeyException, QueryUpdateServiceException - { - // Make sure we've found the original participant before doing the delete - String participant = getParticipant(oldRow, user, container); - _dataset.deleteDatasetRows(user, Collections.singleton(keyFromMap(oldRow)), isBulkLoad()); - _potentiallyDeletedParticipants.add(participant); - _participantVisitResyncRequired = true; - return oldRow; - } - - @Override - protected int truncateRows(User user, Container container) - { - return _dataset.deleteRows((Date) null); - } - - @Override - public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws BatchValidationException, QueryUpdateServiceException, SQLException - { - Map updatedParams = configParameters; - if (updatedParams == null) - updatedParams = new HashMap<>(); - updatedParams.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.SUMMARY); - return super.truncateRows(user, container, updatedParams, extraScriptContext); - } - - public String keyFromMap(Map map) throws InvalidKeyException - { - Object lsid = map.get("lsid"); - if (lsid != null) - return lsid.toString(); - lsid = map.get("LSID"); - if (lsid != null) - return lsid.toString(); - - boolean isDemographic = _dataset.isDemographicData(); - - // if there was no explicit lsid and KeyManagementType == None, there is no non-lsid key that is unique by itself. - // Unless of course it is a demographic table. - if (!isDemographic && _dataset.getKeyManagementType() == DatasetDefinition.KeyManagementType.None) - { - throw new InvalidKeyException("No lsid, and no KeyManagement"); - } - - String keyPropertyName = isDemographic ? _dataset.getStudy().getSubjectColumnName() : _dataset.getKeyPropertyName(); - Object id = map.get(keyPropertyName); - - if (null == id) - { - id = map.get("Key"); - } - - // if there was no other type of key, this query is invalid - if (null == id) - { - throw new InvalidKeyException(String.format("key needs one of 'lsid', '%s' or 'Key', none of which were found in %s", keyPropertyName, map)); - } - - // now look up lsid - // if one is found, return that - // if 0, it's legal to return null - // if > 1, there is an integrity problem that should raise alarm - String[] lsids = new TableSelector(getQueryTable().getColumn("LSID"), new SimpleFilter(keyPropertyName, id), null).getArray(String.class); - - if (lsids.length == 0) - { - return null; - } - else if (lsids.length > 1) - { - throw new IllegalStateException("More than one row matched for key '" + id + "' in column " + - _dataset.getKeyPropertyName() + " in dataset " + - _dataset.getName() + " in folder " + - _dataset.getContainer().getPath()); - } - else return lsids[0]; - } - - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends Assert - { - private static final String SUBJECT_COLUMN_NAME = "SubjectID"; - private static final String DATASET_NAME = "DS1"; - TestContext _context = null; - User _user = null; - Container _container = null; - StudyImpl _junitStudy = null; - StudyManager _manager = StudyManager.getInstance(); - String longName = "this is a very long name (with punctuation) that raises many questions \"?\" about your database design choices"; - - private void createDataset() throws Exception - { - if (DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME) != null) - { - return; - } - - var dsd = new DatasetDefinition(_junitStudy, 1001, DATASET_NAME, DATASET_NAME, null, null, null); - _manager.createDatasetDefinition(_user, dsd); - dsd = _manager.getDatasetDefinition(_junitStudy, 1001); - dsd.refreshDomain(); - { - var domain = dsd.getDomain(true); - DomainProperty p; - - p = domain.addProperty(); - p.setName("Field1"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName("SELECT"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName(longName); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); - - p = domain.addProperty(); - p.setName("Value1"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - p = domain.addProperty(); - p.setName("Value2"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - p = domain.addProperty(); - p.setName("Value3"); - p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); - p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); - p.setMvEnabled(true); - - domain.save(_user); - } - } - - private long getDatasetAuditRowCount() - { - return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT)).getRowCount(); - } - - private String getLatestAuditMessage() - { - return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT), PageFlowUtil.set("Comment"), null, new Sort("-rowId")).setMaxRows(1).getObject(String.class); - } - - @Test - public void testAuditing() throws Exception - { - createDataset(); - TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); - t.getUpdateService().truncateRows(_user, _container, null, null); - - final QueryUpdateService qus = t.getUpdateService(); - BatchValidationException errors = new BatchValidationException(); - - long actualAuditRows = getDatasetAuditRowCount(); - long expectedAuditRows; - - List> insertedRows = qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S1", - "SequenceNum", "1.2345", - longName, "NA"), - Map.of( - SUBJECT_COLUMN_NAME, "S2", - "SequenceNum", "1.2345", - longName, "WithoutBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A new dataset record was inserted", getLatestAuditMessage()); - - qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S3", - "SequenceNum", "1.2345", - longName, "WithoutBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 1; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - // Now update: - insertedRows.get(0).put(longName, "NewValue"); - insertedRows.get(1).put(longName, "NewValue"); - List> oldKeys = Arrays.asList( - Map.of("lsid", insertedRows.get(0).get("lsid")), - Map.of("lsid", insertedRows.get(1).get("lsid")) - ); - qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A dataset record was modified", getLatestAuditMessage()); - - qus.deleteRows(_user, _container, oldKeys, null, null); - expectedAuditRows = actualAuditRows + 2; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - Assert.assertEquals("Incorrect comment", "A dataset record was deleted", getLatestAuditMessage()); - - // Repeat using bulkLoad=true: - qus.setBulkLoad(true); - - insertedRows = qus.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, "S4", - "SequenceNum", "1.2345", - longName, "WithBulkLoad"), - Map.of( - SUBJECT_COLUMN_NAME, "S5", - "SequenceNum", "1.2345", - longName, "WithBulkLoad")), - errors, null, null); - - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - // Now update: - insertedRows.get(0).put(longName, "NewValue"); - insertedRows.get(1).put(longName, "NewValue"); - oldKeys = Arrays.asList( - Map.of("lsid", insertedRows.get(0).get("lsid")), - Map.of("lsid", insertedRows.get(1).get("lsid")) - ); - qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); - if (errors.hasErrors()) - { - fail(errors.getMessage()); - } - - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - - qus.deleteRows(_user, _container, oldKeys, null, null); - expectedAuditRows = actualAuditRows; - actualAuditRows = getDatasetAuditRowCount(); - Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); - } - - @Test - public void updateRowTest() throws Exception - { - createDataset(); - - TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); - assertNotNull(t); - assertTrue("Field1".equalsIgnoreCase(t.getColumn("Field1").getAlias().getId())); - assertFalse("SELECT".equalsIgnoreCase(t.getColumn("SELECT").getAlias().getId())); - assertFalse(longName.equalsIgnoreCase(t.getColumn(longName).getAlias().getId())); - var up = t.getUpdateService(); - assertNotNull(up); - var errors = new BatchValidationException(); - - var result = up.insertRows(_user, _container, - List.of(Map.of( - SUBJECT_COLUMN_NAME, " S1 \t", - "SequenceNum", "1.2345", - "Field1", "f", - "SELECT", "s", - longName, "l", - "value1", "1.0", - "value2", "NA", - "VALUE3", "NA")), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertFalse(errors.hasErrors()); - assertNotNull(result); - assertEquals(1, result.size()); - var map = result.get(0); - assertEquals("S1", map.get(SUBJECT_COLUMN_NAME)); - assertEquals("f", map.get("Field1")); - assertEquals("s", map.get("SELECT")); - assertEquals("l", map.get(longName)); - assertEquals( 1.0d, map.get("value1")); // 1.0 - var v2 = map.get("value2"); // NA - assertTrue(v2 instanceof MvFieldWrapper); - assertEquals("NA", ((MvFieldWrapper)v2).getMvIndicator()); - var v3 = map.get("value3"); // NA - assertTrue(v3 instanceof MvFieldWrapper); - assertEquals("NA", ((MvFieldWrapper)v3).getMvIndicator()); - assertNotNull(map.get("lsid")); - assertTrue(((String)map.get("lsid")).endsWith(":1001.S1.1.2345")); - String lsid = (String)map.get("lsid"); - - // update subjectid column - result = up.updateRows(_user, _container, - List.of(Map.of(SUBJECT_COLUMN_NAME, "\tS2 ")), - List.of(Map.of("lsid", lsid)), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertNotNull(result); - assertEquals(1, result.size()); - map = result.get(0); - assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); - // All other columns are preserved - assertEquals("f", map.get("Field1")); - assertEquals("s", map.get("SELECT")); - assertEquals("l", map.get(longName)); - assertEquals( 1.0d, map.get("value1")); // 1.0 - // DIFFERENCE - updateRows() does not return MvFieldWrapper - assertNull(map.get("value2")); // NA - assertEquals("NA", map.get("value2mvindicator")); - assertNull(map.get("VALUE3")); // NA - assertEquals("NA", map.get("value3MvIndicator")); - // LSID is updated - assertNotNull(map.get("lsid")); - assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); - lsid = (String)map.get("lsid"); - - // update other columns - result = up.updateRows(_user, _container, - List.of(Map.of( - "Field1", "fUpdated", - "SELECT", "sUpdated", - longName, "lUpdated", - "value1", "NA", // 1.0 -> NA - "value2", "2.0", // NA -> 2.0 - "value3", "QA") // NA -> QA - ), - List.of(Map.of("lsid", lsid)), - errors, null, null); - if (errors.hasErrors()) - fail(errors.getMessage()); - assertNotNull(result); - assertEquals(1, result.size()); - map = result.get(0); - assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); - assertEquals("fUpdated", map.get("Field1")); - assertEquals("sUpdated", map.get("SELECT")); - assertEquals("lUpdated", map.get(longName)); - assertNull(map.get("value1")); // NA - assertEquals("NA", map.get("Value1MVIndicator")); - assertEquals(2.0d, map.get("value2")); // 2.0 - assertNull(map.get("Value2MVIndicator")); - assertNull(map.get("value3")); // QA - assertEquals("QA", map.get("value3mVindicator")); - assertNotNull(map.get("lsid")); - // unchanged - assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); - map.get("lsid"); - } - - @Before - public void createStudy() - { - _context = TestContext.get(); - Container junit = JunitUtil.getTestContainer(); - String name = GUID.makeHash(); - Container c = ContainerManager.createContainer(junit, name, _context.getUser()); - MvUtil.assignMvIndicators(c, new String[] {"NA","QA"}, new String[] {"NA","QA"}); - StudyImpl s = new StudyImpl(c, "Junit Study"); - s.setTimepointType(TimepointType.VISIT); - s.setStartDate(new Date(DateUtil.parseDateTime(c, "2014-01-01"))); - s.setSubjectColumnName(SUBJECT_COLUMN_NAME); - s.setSubjectNounPlural("Subjects"); - s.setSubjectNounSingular("Subject"); - s.setSecurityType(SecurityType.BASIC_WRITE); - _junitStudy = StudyManager.getInstance().createStudy(_context.getUser(), s); - _user = _context.getUser(); - _container = _junitStudy.getContainer(); - } - - @After - public void tearDown() - { - if (null != _junitStudy) - { - assertTrue(ContainerManager.delete(_junitStudy.getContainer(), _context.getUser())); - } - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.study.query; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.labkey.api.audit.AbstractAuditTypeProvider; +import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.audit.TransactionAuditProvider; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.collections.ResultSetRowMapFactory; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MvUtil; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.DataIteratorBuilder; +import org.labkey.api.dataiterator.DataIteratorContext; +import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; +import org.labkey.api.dataiterator.MapDataIterator; +import org.labkey.api.dataiterator.SimpleTranslator; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.MvFieldWrapper; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.gwt.client.AuditBehaviorType; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.DefaultQueryUpdateService; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.InvalidKeyException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.query.QueryUpdateServiceException; +import org.labkey.api.query.SimpleValidationError; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.TimepointType; +import org.labkey.api.study.security.StudySecurityEscalator; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JunitUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.TestContext; +import org.labkey.study.model.DatasetDataIteratorBuilder; +import org.labkey.study.dataset.DatasetAuditProvider; +import org.labkey.study.model.DatasetDefinition; +import org.labkey.study.model.DatasetDomainKind; +import org.labkey.study.model.DatasetLsidImportHelper; +import org.labkey.study.model.ParticipantIdImportHelper; +import org.labkey.study.model.ParticipantSeqNumImportHelper; +import org.labkey.study.model.QCStateImportHelper; +import org.labkey.study.model.SecurityType; +import org.labkey.study.model.SequenceNumImportHelper; +import org.labkey.study.model.StudyImpl; +import org.labkey.study.model.StudyManager; +import org.labkey.study.visitmanager.PurgeParticipantsJob.ParticipantPurger; + +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static org.labkey.api.util.IntegerUtils.asInteger; +import static org.labkey.api.gwt.client.AuditBehaviorType.DETAILED; +import static org.labkey.api.gwt.client.AuditBehaviorType.NONE; + +/* +* User: Dave +* Date: Jun 13, 2008 +* Time: 4:15:51 PM +*/ + +/** + * QueryUpdateService implementation for Study datasets. + *

+ * Since datasets are of an unpredictable shape, this class just implements + * the QueryUpdateService directly, working with Map<String,Object> + * collections for the row data. + */ +public class DatasetUpdateService extends DefaultQueryUpdateService +{ + // These are that can be passed into DatasetUpdateService via DataIteratorContext.configParameters. + // These used to be passed to DatasetDataIterator via + // DatasetDefinition.importDatasetData()->DatasetDefinition.insertData(). + // Moving these options into DataInteratorContext allows for even more consistency and code sharing + // also see QueryUpdateService.ConfigParameters.Logger + public enum Config + { + CheckForDuplicates, // expected: enum CheckForDuplicates + DefaultQCState, // expected: class QCState + SkipResyncStudy, // expected: Boolean + + // NOTE: There really has to be better way to handle the functionality of StudyImportContext.getTableIdMap() + // NOTE: Could this be handled by a method on StudySchema or something??? + // see StudyImportContext.getTableIdMapMap() + StudyImportMaps, // expected: Map> + + KeyList, // expected: List + AllowImportManagedKey // expected: Boolean + } + + private static final Logger LOG = LogManager.getLogger(DatasetUpdateService.class); + + private final DatasetDefinition _dataset; + private final Set _potentiallyNewParticipants = new HashSet<>(); + private final Set _potentiallyDeletedParticipants = new HashSet<>(); + private boolean _participantVisitResyncRequired = false; + + private final boolean _skipAuditLogging = false; + + public DatasetUpdateService(DatasetTableImpl table) + { + super(table, table.getDatasetDefinition().getStorageTableInfo(false), createMVMapping(table.getDatasetDefinition().getDomain())); + _dataset = table.getDatasetDefinition(); + } + + @Override + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class acl) + { + if (StudySecurityEscalator.isEscalated()) { + return true; + } + else { + return super.hasPermission(user, acl); + } + } + + + @Override + protected Map getRow(User user, Container container, Map keys) + throws InvalidKeyException + { + String lsid = keyFromMap(keys); + SimpleFilter filter = new SimpleFilter() + .addCondition(new FieldKey(null,"container"), container.getId()) + .addCondition(new FieldKey(null,"lsid"),lsid); + + // NOTE getQueryTable().getColumns() returns a bunch of columns that getDatasetRow() did not such as: + // Container, Dataset, DatasetId, Datasets, Folder, Modified, ModifiedBy, MouseVisit, ParticipantSequenceNum, VisitDay, VisitRowId + // Mostly this is harmless, but there is some noise. + HashSet nameset = new HashSet<>(getQueryTable().getColumnNameSet()); + List.of("Container","Datasets","DatasetId","Dataset","Folder").forEach(nameset::remove); + List columns = new ArrayList<>(getQueryTable().getColumns(nameset.toArray(new String[0]))); + + // filter out calculated columns which can be expensive to reselect + columns.removeIf(ColumnInfo::isCalculated); + + // This is a general version of DatasetDefinition.canonicalizeDatasetRow() + // The caller needs to make sure names are unique. Not suitable for use w/ lookups etc where there can be name collisions. + // CONSIDER: might be nice to make this a TableSelector method. + var map = new CaseInsensitiveHashMap<>(); + try (var str = new TableSelector(getQueryTable(), columns, filter, null).uncachedResultSetStream()) + { + str.forEach(rs -> { + try + { + for (int i = 0; i < columns.size(); i++) + { + Object o = rs.getObject(i + 1); + o = ResultSetRowMapFactory.translateResultSetObject(o, false); + map.put(columns.get(i).getName(), o); + } + } + catch (SQLException x) + { + throw new RuntimeSQLException(x); + } + }); + } + return map.isEmpty() ? null : map; + } + + + /* TODO for performance, NOTE need to return rows in order of input list + @Override + public List> getRows(User user, Container container, List> keys) throws InvalidKeyException + { + if (!hasPermission(user, ReadPermission.class)) + throw new UnauthorizedException("You do not have permission to read data from this table."); + ArrayList lsids = new ArrayList<>(keys.size()); + for (var m : keys) + lsids.add(keyFromMap(m)); + var result = (List)(new TableSelector(getQueryTable(), + TableSelector.ALL_COLUMNS, + new SimpleFilter(new FieldKey(null,"lsid"), lsids, CompareType.IN), + null)) + .getArrayList(Map.class); + return (List>)result; + } + */ + + + @Override + public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); + if (count > 0) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container, null, null, true); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + } + return count; + } + + @Override + public int loadRows(User user, Container container, DataIteratorBuilder rows, DataIteratorContext context, @Nullable Map extraScriptContext) + { + int count = _importRowsUsingDIB(user, container, rows, null, context, extraScriptContext); + if (count > 0 && !Boolean.TRUE.equals(context.getConfigParameterBoolean(Config.SkipResyncStudy))) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container, null, null, true); + } + catch (ValidationException e) + { + context.getErrors().addRowError(e); + } + } + return count; + } + + @Override + public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) + { + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); + + return loadRows(user, container, rows, context, extraScriptContext); + } + + @Override + public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws QueryUpdateServiceException + { + for (Map row : rows) + { + aliasColumns(_columnMapping, row); + } + + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + if (_skipAuditLogging) + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE); + else if (!isBulkLoad()) + { + // default to DETAILED unless there is a metadata XML override + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, + getQueryTable().getXmlAuditBehaviorType() != null ? getQueryTable().getXmlAuditBehaviorType() : DETAILED); + } + + List> result = super._insertRowsUsingDIB(user, container, rows, context, extraScriptContext); + + if (null != result && result.size() > 0) + { + for (Map row : result) + { + String participantID = getParticipant(row, user, container); + _potentiallyNewParticipants.add(participantID); + } + + _participantVisitResyncRequired = true; // 13717 : Study failing to resync() on dataset insert + if (configParameters == null || !Boolean.TRUE.equals(configParameters.get(DatasetUpdateService.Config.SkipResyncStudy))) + { + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + } + } + return result; + } + + @Override + protected DataIteratorBuilder preTriggerDataIterator(DataIteratorBuilder in, DataIteratorContext context) + { + // If we're using a managed GUID as a key, wire it up here so that it's available to trigger scripts + if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER && + _dataset.getKeyManagementType() == Dataset.KeyManagementType.GUID && + _dataset.getKeyPropertyName() != null) + { + return new DataIteratorBuilder() + { + @Override + public DataIterator getDataIterator(DataIteratorContext context) + { + DataIterator input = in.getDataIterator(context); + if (null == input) + return null; // Can happen if context has errors + + final SimpleTranslator result = new SimpleTranslator(input, context); + + boolean foundKeyCol = false; + for (int c = 1; c <= input.getColumnCount(); c++) + { + ColumnInfo col = input.getColumnInfo(c); + + // Incoming data has a matching field + if (col.getName().equalsIgnoreCase(_dataset.getKeyPropertyName())) + { + // make sure guid is not null (12884) + result.addCoalesceColumn(col.getName(), c, new SimpleTranslator.GuidColumn()); + foundKeyCol = true; + } + else + { + // Pass it through as-is + result.addColumn(c); + } + } + + if (!foundKeyCol) + { + // Inject a column with a new GUID + ColumnInfo key = getQueryTable().getColumn(_dataset.getKeyPropertyName()); + result.addColumn(new BaseColumnInfo(key), new SimpleTranslator.GuidColumn()); + } + + return result; + } + }; + } + return in; + } + + @Override + public DataIteratorBuilder createImportDIB(User user, Container container, DataIteratorBuilder data, DataIteratorContext context) + { + if (null == context.getConfigParameter(Config.DefaultQCState)) + { + context.putConfigParameter(Config.DefaultQCState, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); + } + + if (null == context.getConfigParameter(Config.CheckForDuplicates)) + { + DatasetDefinition.CheckForDuplicates dupePolicy; + if (isBulkLoad()) + dupePolicy = DatasetDefinition.CheckForDuplicates.never; + else if (context.getInsertOption().mergeRows) + dupePolicy = DatasetDefinition.CheckForDuplicates.sourceOnly; + else + dupePolicy = DatasetDefinition.CheckForDuplicates.sourceAndDestination; + context.putConfigParameter(Config.CheckForDuplicates, dupePolicy); + } + + // NOTE: This was done to help coalesce some old code paths. However, this is a little weird, because + // the DI is over the DatasetSchemaTableInfo not the DatasetTableImpl you'd expect. This all still works + // because of property URI matching in StatementDataIterator. + return _dataset.getInsertDataIterator(user, container, data, context); + } + + + @Override + protected int _pump(DataIteratorBuilder etl, final ArrayList> rows, DataIteratorContext context) + { + try + { + boolean hasRowId = _dataset.getKeyManagementType() == Dataset.KeyManagementType.RowId; + + if (null != rows) + { + // TODO: consider creating DataIterator metadata to mark "internal" cols (not to be returned via API) + DataIterator it = etl.getDataIterator(context); + DataIteratorBuilder cleanMap = new MapDataIterator.MapDataIteratorImpl(it, true, CaseInsensitiveHashSet.of( + it.getColumnInfo(0).getName() + )); + etl = cleanMap; + } + + if (!hasRowId) + { + return super._pump(etl, rows, context); + } + + synchronized (_dataset.getManagedKeyLock()) + { + return super._pump(etl, rows, context); + } + } + catch (RuntimeSQLException e) + { + String translated = _dataset.translateSQLException(e); + if (translated != null) + { + context.getErrors().addRowError(new ValidationException(translated)); + return 0; + } + throw e; + } + } + + + @NotNull String getParticipant(Map row, User user, Container container) throws QueryUpdateServiceException + { + String columnName = _dataset.getStudy().getSubjectColumnName(); + Object participant = row.get(columnName); + if (participant == null) + { + participant = row.get("ParticipantId"); + } + if (participant == null) + { + try + { + // This may be an update or delete where the user specified the LSID as the key, but didn't bother + // sending the participant, so look it up + Map originalRow = getRow(user, container, row); + participant = originalRow == null ? null : originalRow.get(columnName); + if (participant == null) + { + participant = originalRow.get("ParticipantId"); + } + } + catch (InvalidKeyException e) + { + throw new QueryUpdateServiceException(e); + } + } + if (participant == null) + { + throw new QueryUpdateServiceException("All dataset rows must include a value for " + columnName); + } + return participant.toString(); + } + + static class PurgeParticipantCommitTask implements Runnable + { + private final Container _container; + private final Set _potentiallyDeletedParticipants; + + PurgeParticipantCommitTask(Container container, Set potentiallyDeletedParticipants) + { + _container = container; + _potentiallyDeletedParticipants = new HashSet<>(potentiallyDeletedParticipants); + } + + @Override + public void run() + { + new ParticipantPurger(_container, _potentiallyDeletedParticipants, LOG::info, LOG::error).purgeParticipants(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PurgeParticipantCommitTask that = (PurgeParticipantCommitTask) o; + + if (!Objects.equals(_container, that._container)) return false; + + return true; + } + + @Override + public int hashCode() + { + return _container != null ? _container.hashCode() : 0; + } + } + + + @Override + public List> updateRows(User user, final Container container, List> rows, List> oldKeys, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + List> result = super.updateRows(user, container, rows, oldKeys, errors, configParameters, extraScriptContext); + if (null != extraScriptContext && Boolean.TRUE.equals(extraScriptContext.get("synchronousParticipantPurge"))) + { + PurgeParticipantCommitTask addObj = new PurgeParticipantCommitTask(container, _potentiallyDeletedParticipants); + PurgeParticipantCommitTask setObj = getQueryTable().getSchema().getScope().addCommitTask(addObj, DbScope.CommitTaskOption.POSTCOMMIT); + setObj._potentiallyDeletedParticipants.addAll(addObj._potentiallyDeletedParticipants); + } + + try + { + StudyManager.datasetModified(_dataset, true); + resyncStudy(user, container); + } + catch (ValidationException e) + { + errors.addRowError(e); + } + return result; + } + + private void resyncStudy(User user, Container container) throws ValidationException + { + resyncStudy(user, container, _potentiallyNewParticipants, _potentiallyDeletedParticipants, _participantVisitResyncRequired); + + _participantVisitResyncRequired = false; + _potentiallyNewParticipants.clear(); + _potentiallyDeletedParticipants.clear(); + } + + /** + * Resyncs the study : updates the participant, visit, and (optionally) participant visit tables. Also updates automatic cohort assignments. + * + * @param potentiallyAddedParticipants optionally, the specific participants that may have been added to the study. + * If null, all the changedDatasets and specimens will be checked to see if they contain new participants + * @param potentiallyDeletedParticipants optionally, the specific participants that may have been removed from the + * study. If null, all participants will be checked to see if they are still in the study. + * @param participantVisitResyncRequired If true, will force an update of the ParticipantVisit mapping for this study + */ + private void resyncStudy(User user, Container container, @Nullable Set potentiallyAddedParticipants, + @Nullable Set potentiallyDeletedParticipants, + boolean participantVisitResyncRequired) throws ValidationException + { + StudyImpl study = StudyManager.getInstance().getStudy(container); + Study sharedStudy = StudyManager.getInstance().getSharedStudy(study); + + ValidationException errors = StudyManager.getInstance().getVisitManager(study).updateParticipantVisits(user, Collections.singletonList(_dataset), + potentiallyAddedParticipants, potentiallyDeletedParticipants, participantVisitResyncRequired, + sharedStudy != null ? sharedStudy.isFailForUndefinedTimepoints() : study.isFailForUndefinedTimepoints(), null); + + if (errors.hasErrors()) + throw errors; + } + + @Override + protected void convertTypes(User user, Container c, Map row, TableInfo t, @Nullable Path fileLinkDirPath) throws ValidationException + { + // Issue 53320 : ensure a valid file link path + if (fileLinkDirPath == null) + fileLinkDirPath = AssayFileWriter.getUploadDirectoryPath(c, "datasetdata").toNioPathForWrite(); + + super.convertTypes(user, c, row, t, fileLinkDirPath); + } + + @Override + protected Map _update(User user, Container container, Map row, Map oldRow, Object[] keys) throws SQLException, ValidationException + { + try (DbScope.Transaction transaction = StudyService.get().getDatasetSchema().getScope().ensureTransaction()) + { + String lsid = keyFromMap(oldRow); + checkDuplicateUpdate(lsid); + // Make sure we've found the original participant before doing the update + String oldParticipant = getParticipant(oldRow, user, container); + String newLsid; + + Long rowId = (Long)oldRow.get(DatasetDomainKind.DSROWID); + Map oldData = _dataset.getDatasetRow(user, lsid); + + if (oldData == null) + { + // No old record found, so we can't update + ValidationException error = new ValidationException(); + error.addError(new SimpleValidationError("Record not found with lsid: " + lsid)); + throw error; + } + + // values that are always recalculated + getComputedValues(user, row, oldRow); + + newLsid = (String)row.get(DatasetDomainKind.LSID); + Table.update(user, getDbTable(), row, rowId); + + if (!isBulkLoad()) + { + DatasetTableImpl target = (DatasetTableImpl)_dataset.getTableInfo(user); + new DatasetDefinition.DatasetAuditHandler(_dataset).addAuditEvent(user, container, target, AuditBehaviorType.DETAILED, null, QueryService.AuditAction.UPDATE, + List.of(row), List.of(oldData)); + } + + // Successfully updated + transaction.commit(); + + // return updated row + var returnRow = getRow(user, container, Map.of(DatasetDomainKind.LSID, newLsid)); + + String newParticipant = getParticipant(returnRow, user, container); + if (!oldParticipant.equals(newParticipant)) + { + // Participant has changed - might be a reference to a new participant, or removal of the last reference to + // the old participant + _potentiallyNewParticipants.add(newParticipant); + _potentiallyDeletedParticipants.add(oldParticipant); + + // Need to resync the ParticipantVisit table too + _participantVisitResyncRequired = true; + } + // Check if the timepoint may have changed, but only if we don't already know we need to resync + else if (!_participantVisitResyncRequired) + { + String columnName = StudyManager.getInstance().getStudy(container).getTimepointType().isVisitBased() ? + "SequenceNum" : "Date"; + Object oldTimepoint = oldRow.get(columnName); + Object newTimepoint = returnRow.get(columnName); + if (!Objects.equals(oldTimepoint, newTimepoint)) + { + _participantVisitResyncRequired = true; + } + } + + return returnRow; + } + catch (QueryUpdateServiceException | InvalidKeyException e) + { + throw new ValidationException(e.getMessage()); + } + } + + private void getComputedValues(User user, Map row, Map oldRow) throws ValidationException + { + String subjectColumnName = _dataset.getStudy().getSubjectColumnName(); + TableInfo table = _dataset.getTableInfo(user); + ColumnInfo subjectColumn = table.getColumn(subjectColumnName); + ColumnInfo sequenceNumColumn = table.getColumn(DatasetDomainKind.SEQUENCENUM); + ColumnInfo dateColumn = table.getColumn(DatasetDomainKind.DATE); + String managedKey = null; + if (_dataset.getKeyType() == Dataset.KeyType.SUBJECT_VISIT_OTHER) + managedKey = _dataset.getKeyPropertyName(); + ColumnInfo managedKeyColumn = managedKey != null ? table.getColumn(managedKey) : null; + + Object inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(row, subjectColumn); + if (inputSubjectId == null) + inputSubjectId = DatasetDataIteratorBuilder.findColumnInMap(oldRow, subjectColumn); + Object inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(row, sequenceNumColumn); + if (inputSeqNum == null) + inputSeqNum = DatasetDataIteratorBuilder.findColumnInMap(oldRow, sequenceNumColumn); + Date inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(row, dateColumn); + if (inputDate == null) + inputDate = (Date)DatasetDataIteratorBuilder.findColumnInMap(oldRow, dateColumn); + Object inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(row, managedKeyColumn); + if (inputManagedKey == null) + inputManagedKey = DatasetDataIteratorBuilder.findColumnInMap(oldRow, managedKeyColumn); + Integer inputQCState = asInteger(DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_ID_COLNAME))); + + SequenceNumImportHelper snih = new SequenceNumImportHelper(_dataset.getStudy(), _dataset); + Double sequenceNum = snih.translateSequenceNum(inputSeqNum, inputDate); + + ParticipantIdImportHelper helper = new ParticipantIdImportHelper(_dataset.getStudy(), user, _dataset); + String subjectId = helper.translateParticipantId(inputSubjectId); + + // generate participant sequence number + String participantSeqNum = ParticipantSeqNumImportHelper.translateParticipantSeqNum(subjectId, sequenceNum); + + // re-generate a new lsid + DatasetLsidImportHelper dlih = new DatasetLsidImportHelper(_dataset); + String lsid = dlih.translateLsid(subjectId, sequenceNum, inputDate, inputManagedKey, null); + + // handle default QC states + if (inputQCState == null) + { + String inputQCText = (String)DatasetDataIteratorBuilder.findColumnInMap(row, table.getColumn(DatasetTableImpl.QCSTATE_LABEL_COLNAME)); + QCStateImportHelper qcih = new QCStateImportHelper(user, _dataset, true, StudyManager.getInstance().getDefaultQCState(_dataset.getStudy())); + Long qcState = qcih.translateQCState(inputQCText); + if (qcState != null) + row.put(DatasetTableImpl.QCSTATE_ID_COLNAME, qcState); + } + row.put(DatasetDomainKind.LSID, lsid); + row.put(DatasetDomainKind.PARTICIPANTSEQUENCENUM, participantSeqNum); + row.put(DatasetDomainKind.PARTICIPANTID, subjectId); + } + + @Override + public List> deleteRows(User user, Container container, List> keys, @Nullable Map configParameters, @Nullable Map extraScriptContext) + throws InvalidKeyException, BatchValidationException, QueryUpdateServiceException, SQLException + { + List> result = super.deleteRows(user, container, keys, configParameters, extraScriptContext); + try + { + resyncStudy(user, container); + } + catch (ValidationException e) + { + throw new BatchValidationException(e); + } + return result; + } + + @Override + protected Map deleteRow(User user, Container container, Map oldRow) + throws InvalidKeyException, QueryUpdateServiceException + { + // Make sure we've found the original participant before doing the delete + String participant = getParticipant(oldRow, user, container); + _dataset.deleteDatasetRows(user, Collections.singleton(keyFromMap(oldRow)), isBulkLoad()); + _potentiallyDeletedParticipants.add(participant); + _participantVisitResyncRequired = true; + return oldRow; + } + + @Override + protected int truncateRows(User user, Container container) + { + return _dataset.deleteRows((Date) null); + } + + @Override + public int truncateRows(User user, Container container, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws BatchValidationException, QueryUpdateServiceException, SQLException + { + Map updatedParams = configParameters; + if (updatedParams == null) + updatedParams = new HashMap<>(); + updatedParams.put(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, AuditBehaviorType.SUMMARY); + return super.truncateRows(user, container, updatedParams, extraScriptContext); + } + + public String keyFromMap(Map map) throws InvalidKeyException + { + Object lsid = map.get("lsid"); + if (lsid != null) + return lsid.toString(); + lsid = map.get("LSID"); + if (lsid != null) + return lsid.toString(); + + boolean isDemographic = _dataset.isDemographicData(); + + // if there was no explicit lsid and KeyManagementType == None, there is no non-lsid key that is unique by itself. + // Unless of course it is a demographic table. + if (!isDemographic && _dataset.getKeyManagementType() == DatasetDefinition.KeyManagementType.None) + { + throw new InvalidKeyException("No lsid, and no KeyManagement"); + } + + String keyPropertyName = isDemographic ? _dataset.getStudy().getSubjectColumnName() : _dataset.getKeyPropertyName(); + Object id = map.get(keyPropertyName); + + if (null == id) + { + id = map.get("Key"); + } + + // if there was no other type of key, this query is invalid + if (null == id) + { + throw new InvalidKeyException(String.format("key needs one of 'lsid', '%s' or 'Key', none of which were found in %s", keyPropertyName, map)); + } + + // now look up lsid + // if one is found, return that + // if 0, it's legal to return null + // if > 1, there is an integrity problem that should raise alarm + String[] lsids = new TableSelector(getQueryTable().getColumn("LSID"), new SimpleFilter(keyPropertyName, id), null).getArray(String.class); + + if (lsids.length == 0) + { + return null; + } + else if (lsids.length > 1) + { + throw new IllegalStateException("More than one row matched for key '" + id + "' in column " + + _dataset.getKeyPropertyName() + " in dataset " + + _dataset.getName() + " in folder " + + _dataset.getContainer().getPath()); + } + else return lsids[0]; + } + + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends Assert + { + private static final String SUBJECT_COLUMN_NAME = "SubjectID"; + private static final String DATASET_NAME = "DS1"; + TestContext _context = null; + User _user = null; + Container _container = null; + StudyImpl _junitStudy = null; + StudyManager _manager = StudyManager.getInstance(); + String longName = "this is a very long name (with punctuation) that raises many questions \"?\" about your database design choices"; + + private void createDataset() throws Exception + { + if (DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME) != null) + { + return; + } + + var dsd = new DatasetDefinition(_junitStudy, 1001, DATASET_NAME, DATASET_NAME, null, null, null); + _manager.createDatasetDefinition(_user, dsd); + dsd = _manager.getDatasetDefinition(_junitStudy, 1001); + dsd.refreshDomain(); + { + var domain = dsd.getDomain(true); + DomainProperty p; + + p = domain.addProperty(); + p.setName("Field1"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName("SELECT"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName(longName); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.VARCHAR).getTypeUri()); + + p = domain.addProperty(); + p.setName("Value1"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + p = domain.addProperty(); + p.setName("Value2"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + p = domain.addProperty(); + p.setName("Value3"); + p.setPropertyURI(domain.getTypeURI() + "." + Lsid.encodePart(p.getName())); + p.setRangeURI(PropertyType.getFromJdbcType(JdbcType.DOUBLE).getTypeUri()); + p.setMvEnabled(true); + + domain.save(_user); + } + } + + private long getDatasetAuditRowCount() + { + return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT)).getRowCount(); + } + + private String getLatestAuditMessage() + { + return new TableSelector(QueryService.get().getUserSchema(_user, _container, AbstractAuditTypeProvider.QUERY_SCHEMA_NAME).getTable(DatasetAuditProvider.DATASET_AUDIT_EVENT), PageFlowUtil.set("Comment"), null, new Sort("-rowId")).setMaxRows(1).getObject(String.class); + } + + @Test + public void testAuditing() throws Exception + { + createDataset(); + TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); + t.getUpdateService().truncateRows(_user, _container, null, null); + + final QueryUpdateService qus = t.getUpdateService(); + BatchValidationException errors = new BatchValidationException(); + + long actualAuditRows = getDatasetAuditRowCount(); + long expectedAuditRows; + + List> insertedRows = qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S1", + "SequenceNum", "1.2345", + longName, "NA"), + Map.of( + SUBJECT_COLUMN_NAME, "S2", + "SequenceNum", "1.2345", + longName, "WithoutBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A new dataset record was inserted", getLatestAuditMessage()); + + qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S3", + "SequenceNum", "1.2345", + longName, "WithoutBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 1; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + // Now update: + insertedRows.get(0).put(longName, "NewValue"); + insertedRows.get(1).put(longName, "NewValue"); + List> oldKeys = Arrays.asList( + Map.of("lsid", insertedRows.get(0).get("lsid")), + Map.of("lsid", insertedRows.get(1).get("lsid")) + ); + qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A dataset record was modified", getLatestAuditMessage()); + + qus.deleteRows(_user, _container, oldKeys, null, null); + expectedAuditRows = actualAuditRows + 2; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + Assert.assertEquals("Incorrect comment", "A dataset record was deleted", getLatestAuditMessage()); + + // Repeat using bulkLoad=true: + qus.setBulkLoad(true); + + insertedRows = qus.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, "S4", + "SequenceNum", "1.2345", + longName, "WithBulkLoad"), + Map.of( + SUBJECT_COLUMN_NAME, "S5", + "SequenceNum", "1.2345", + longName, "WithBulkLoad")), + errors, null, null); + + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + // Now update: + insertedRows.get(0).put(longName, "NewValue"); + insertedRows.get(1).put(longName, "NewValue"); + oldKeys = Arrays.asList( + Map.of("lsid", insertedRows.get(0).get("lsid")), + Map.of("lsid", insertedRows.get(1).get("lsid")) + ); + qus.updateRows(_user, _container, insertedRows, oldKeys, errors, null, null); + if (errors.hasErrors()) + { + fail(errors.getMessage()); + } + + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + + qus.deleteRows(_user, _container, oldKeys, null, null); + expectedAuditRows = actualAuditRows; + actualAuditRows = getDatasetAuditRowCount(); + Assert.assertEquals("Incorrect number of audit records", expectedAuditRows, actualAuditRows); + } + + @Test + public void updateRowTest() throws Exception + { + createDataset(); + + TableInfo t = DefaultSchema.get(_user, _container).getSchema(StudyQuerySchema.SCHEMA_NAME).getTable(DATASET_NAME); + assertNotNull(t); + assertTrue("Field1".equalsIgnoreCase(t.getColumn("Field1").getAlias().getId())); + assertFalse("SELECT".equalsIgnoreCase(t.getColumn("SELECT").getAlias().getId())); + assertFalse(longName.equalsIgnoreCase(t.getColumn(longName).getAlias().getId())); + var up = t.getUpdateService(); + assertNotNull(up); + var errors = new BatchValidationException(); + + var result = up.insertRows(_user, _container, + List.of(Map.of( + SUBJECT_COLUMN_NAME, " S1 \t", + "SequenceNum", "1.2345", + "Field1", "f", + "SELECT", "s", + longName, "l", + "value1", "1.0", + "value2", "NA", + "VALUE3", "NA")), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertFalse(errors.hasErrors()); + assertNotNull(result); + assertEquals(1, result.size()); + var map = result.get(0); + assertEquals("S1", map.get(SUBJECT_COLUMN_NAME)); + assertEquals("f", map.get("Field1")); + assertEquals("s", map.get("SELECT")); + assertEquals("l", map.get(longName)); + assertEquals( 1.0d, map.get("value1")); // 1.0 + var v2 = map.get("value2"); // NA + assertTrue(v2 instanceof MvFieldWrapper); + assertEquals("NA", ((MvFieldWrapper)v2).getMvIndicator()); + var v3 = map.get("value3"); // NA + assertTrue(v3 instanceof MvFieldWrapper); + assertEquals("NA", ((MvFieldWrapper)v3).getMvIndicator()); + assertNotNull(map.get("lsid")); + assertTrue(((String)map.get("lsid")).endsWith(":1001.S1.1.2345")); + String lsid = (String)map.get("lsid"); + + // update subjectid column + result = up.updateRows(_user, _container, + List.of(Map.of(SUBJECT_COLUMN_NAME, "\tS2 ")), + List.of(Map.of("lsid", lsid)), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertNotNull(result); + assertEquals(1, result.size()); + map = result.get(0); + assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); + // All other columns are preserved + assertEquals("f", map.get("Field1")); + assertEquals("s", map.get("SELECT")); + assertEquals("l", map.get(longName)); + assertEquals( 1.0d, map.get("value1")); // 1.0 + // DIFFERENCE - updateRows() does not return MvFieldWrapper + assertNull(map.get("value2")); // NA + assertEquals("NA", map.get("value2mvindicator")); + assertNull(map.get("VALUE3")); // NA + assertEquals("NA", map.get("value3MvIndicator")); + // LSID is updated + assertNotNull(map.get("lsid")); + assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); + lsid = (String)map.get("lsid"); + + // update other columns + result = up.updateRows(_user, _container, + List.of(Map.of( + "Field1", "fUpdated", + "SELECT", "sUpdated", + longName, "lUpdated", + "value1", "NA", // 1.0 -> NA + "value2", "2.0", // NA -> 2.0 + "value3", "QA") // NA -> QA + ), + List.of(Map.of("lsid", lsid)), + errors, null, null); + if (errors.hasErrors()) + fail(errors.getMessage()); + assertNotNull(result); + assertEquals(1, result.size()); + map = result.get(0); + assertEquals("S2", map.get(SUBJECT_COLUMN_NAME)); + assertEquals("fUpdated", map.get("Field1")); + assertEquals("sUpdated", map.get("SELECT")); + assertEquals("lUpdated", map.get(longName)); + assertNull(map.get("value1")); // NA + assertEquals("NA", map.get("Value1MVIndicator")); + assertEquals(2.0d, map.get("value2")); // 2.0 + assertNull(map.get("Value2MVIndicator")); + assertNull(map.get("value3")); // QA + assertEquals("QA", map.get("value3mVindicator")); + assertNotNull(map.get("lsid")); + // unchanged + assertTrue(((String)map.get("lsid")).endsWith(":1001.S2.1.2345")); + map.get("lsid"); + } + + @Before + public void createStudy() + { + _context = TestContext.get(); + Container junit = JunitUtil.getTestContainer(); + String name = GUID.makeHash(); + Container c = ContainerManager.createContainer(junit, name, _context.getUser()); + MvUtil.assignMvIndicators(c, new String[] {"NA","QA"}, new String[] {"NA","QA"}); + StudyImpl s = new StudyImpl(c, "Junit Study"); + s.setTimepointType(TimepointType.VISIT); + s.setStartDate(new Date(DateUtil.parseDateTime(c, "2014-01-01"))); + s.setSubjectColumnName(SUBJECT_COLUMN_NAME); + s.setSubjectNounPlural("Subjects"); + s.setSubjectNounSingular("Subject"); + s.setSecurityType(SecurityType.BASIC_WRITE); + _junitStudy = StudyManager.getInstance().createStudy(_context.getUser(), s); + _user = _context.getUser(); + _container = _junitStudy.getContainer(); + } + + @After + public void tearDown() + { + if (null != _junitStudy) + { + assertTrue(ContainerManager.delete(_junitStudy.getContainer(), _context.getUser())); + } + } + } +} From 191963762e2a2a1960ec37aca94d439ca7c89302 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 26 Oct 2025 15:03:30 -0700 Subject: [PATCH 04/13] ETL transaction audit and details --- api/src/org/labkey/api/action/BaseViewAction.java | 2 +- .../labkey/api/audit/TransactionAuditProvider.java | 6 ++++-- .../labkey/api/query/AbstractQueryImportAction.java | 2 +- .../labkey/api/query/SimpleQueryUpdateService.java | 8 ++++---- .../experiment/api/ExpDataClassDataTableImpl.java | 2 +- .../experiment/api/SampleTypeUpdateServiceDI.java | 2 +- .../labkey/list/model/ListQueryUpdateService.java | 8 +++----- .../labkey/query/controllers/QueryController.java | 13 +++++++++++-- .../labkey/study/query/DatasetUpdateService.java | 6 +++--- 9 files changed, 29 insertions(+), 20 deletions(-) diff --git a/api/src/org/labkey/api/action/BaseViewAction.java b/api/src/org/labkey/api/action/BaseViewAction.java index be9a16e452b..fcbef9eb7e2 100644 --- a/api/src/org/labkey/api/action/BaseViewAction.java +++ b/api/src/org/labkey/api/action/BaseViewAction.java @@ -559,7 +559,7 @@ public static Map getTransac else // LKS { String refererRelativeURL = HttpUtil.getRefererRelativeURL(viewContext.getRequest()); - map.put(TransactionAuditProvider.TransactionDetail.Product, refererRelativeURL); + map.put(TransactionAuditProvider.TransactionDetail.RequestSource, refererRelativeURL); } } return map; diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index d3585a8a6f1..2b01acf31ca 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -133,10 +133,12 @@ public enum TransactionDetail Product(false, "The product (Sample Manager, etc) this action originated from"), Action(false, "The controller-action for this request"), QueryCommand(true, "The query commands (insert.update) executed during the transaction"), - BatchAction(false, "If data iterator was used for insert/update"), + DataIteratorUsed(false, "If data iterator was used for insert/update"), ImportOptions(true, "Various import parameters (CrossType, CrossFolder, etc) used during the import action"), EditMethod(false, "The method used to insert/update data from the app (e.g., 'DetailEdit', 'GridEdit', etc)"), - RequestSource(false, "The URL where the request originated from"); + RequestSource(false, "The URL where the request originated from"), + ETL(true, "The ETL process name involved in the transaction"), + FileWatcher(true, "File watcher source(s) involved in the transaction"); private final boolean multiValue; TransactionDetail(boolean multiValue, String description) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 98cd0be2b6d..0b079dfb48c 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -885,7 +885,7 @@ public static int importData(DataLoader dl, TableInfo target, QueryUpdateService return 0; if (auditEvent != null) { - auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true /* qus.loadRows always use DIB*/); + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true /* qus.loadRows always use DIB*/); auditEvent.addComment(auditAction, count); } diff --git a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java index 456ba2298e1..02ef9c422db 100644 --- a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java +++ b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java @@ -63,7 +63,7 @@ public SimpleQueryUpdateService(final SimpleTable queryTable, TableInfo dbTable, public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.IMPORT, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -73,7 +73,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -83,7 +83,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws DuplicateKeyException, QueryUpdateServiceException, SQLException { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); List> result = super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); afterInsertUpdate(result == null ? 0 : result.size(), errors); return result; @@ -146,7 +146,7 @@ public List> updateRows(User user, Container container, List if (shouldUpdateUsingDIB(container, rows, oldKeys, configParameters)) { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, configParameters); context.putConfigParameter(PreferPKOverObjectUriAsKey, shouldPreferPKOverObjectUriAsUpdateKey(rows)); diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 6dd2c93ebc4..e6baca27bf4 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -1509,7 +1509,7 @@ public List> updateRows(User user, Container container, List finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); results = super._updateRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters), extraScriptContext); } diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 3fe4ac3893b..86b6ebf50dc 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -562,7 +562,7 @@ public List> updateRows(User user, Container container, List finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); try { diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index bb86fed1d37..782f781c156 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -88,10 +88,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import static org.labkey.api.util.IntegerUtils.isIntegral; @@ -175,7 +173,7 @@ public List> insertRows(User user, Container container, List DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); if (null != result) @@ -280,7 +278,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B throw new UnauthorizedException("You do not have permission to update data in this table."); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); } @@ -293,7 +291,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, throw new UnauthorizedException("You do not have permission to insert data into this table."); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); if (count > 0 && !errors.hasErrors()) diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 0b8f41b0dca..908a9dae6ea 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -4713,8 +4713,8 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole if (auditEvent != null) { auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); - if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.BatchAction))) - auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.BatchAction, true); + if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.DataIteratorUsed))) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); } if (commandType == CommandType.moveRows) @@ -5140,6 +5140,7 @@ else if (scope != tableInfo.getSchema().getScope()) JSONArray resultArray = new JSONArray(); JSONObject extraContext = json.optJSONObject("extraContext"); + JSONObject auditDetails = json.optJSONObject("auditDetails"); int startingErrorIndex = 0; int errorCount = 0; @@ -5167,6 +5168,14 @@ else if (scope != tableInfo.getSchema().getScope()) commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); } commandObject.put("extraContext", commandExtraContext); + Map commandAuditDetails = new HashMap<>(); + if (auditDetails != null) + commandAuditDetails.putAll(auditDetails.toMap()); + if (commandObject.has("auditDetails")) + { + commandAuditDetails.putAll(commandObject.getJSONObject("auditDetails").toMap()); + } + commandObject.put("auditDetails", commandAuditDetails); JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); // Bail out immediately if we're going to return a failure-type response message diff --git a/study/src/org/labkey/study/query/DatasetUpdateService.java b/study/src/org/labkey/study/query/DatasetUpdateService.java index 5c8857de5f8..8011380dd80 100644 --- a/study/src/org/labkey/study/query/DatasetUpdateService.java +++ b/study/src/org/labkey/study/query/DatasetUpdateService.java @@ -238,7 +238,7 @@ public List> getRows(User user, Container container, List configParameters, Map extraScriptContext) { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); if (count > 0) { @@ -278,7 +278,7 @@ public int loadRows(User user, Container container, DataIteratorBuilder rows, Da public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) { if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); return loadRows(user, container, rows, context, extraScriptContext); @@ -294,7 +294,7 @@ public List> insertRows(User user, Container container, List } if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.BatchAction, true); + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); if (_skipAuditLogging) From bddf23b43bbb54a480b0d6602e64f16a8bd30fa7 Mon Sep 17 00:00:00 2001 From: XingY Date: Sun, 26 Oct 2025 19:26:19 -0700 Subject: [PATCH 05/13] File watcher transaction audit --- .../org/labkey/api/query/QueryService.java | 5 +- .../assay/pipeline/AssayImportRunTask.java | 17 +++++- .../experiment/pipeline/SampleReloadTask.java | 56 ++++++++++++------- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index bbccad189f6..bb6d87b7614 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -408,7 +408,10 @@ enum AuditAction "deleted"), MERGE("%s row(s) were inserted or updated.", "%s was inserted or updated.", - "inserted or updated"); + "inserted or updated"), + RELOAD("%s row(s) were reloaded.", + "%s was reloaded.", + "reloaded"); final String _commentDetailed; final String _commentDetailedFormat; diff --git a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java index 6a48f825d5e..97a69e27694 100644 --- a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java +++ b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java @@ -51,7 +51,9 @@ import org.labkey.api.pipeline.XMLBeanTaskFactoryFactory; import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.DataLoaderService; @@ -799,13 +801,24 @@ public RecordedActionSet run() throws PipelineJobException if (matchedFile == null) throw new PipelineJobException("No output files matched assay file type: " + assayFileType); + User user = getJob().getUser(); + Container container = getJob().getContainer(); + + String fileWatcherDescription = getJob().getDescription(); + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent = tx.getAuditEvent(); + if (transactionAuditEvent != null) + transactionAuditEvent.addDetail(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription); + else + { + transactionAuditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, QueryService.AuditAction.RELOAD, Map.of(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription)); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, transactionAuditEvent); + } + // Issue 22587: Create ExpData for the output file from the RecordedActions if necessary so that we // ensure the generated bit is set on the ExpData. Otherwise, the DefaultAssayRunCreator will create // the ExpData but without the generated bit. createData(matchedFile, assayDataType); - User user = getJob().getUser(); - Container container = getJob().getContainer(); AssayRunUploadContext.Factory factory = provider.createRunUploadFactory(protocol, user, container); diff --git a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java index 5237faa9b48..aa6c3ee30e2 100644 --- a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java @@ -3,7 +3,9 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.Container; +import org.labkey.api.data.DbScope; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorContext; import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; @@ -19,6 +21,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.RecordedActionSet; import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.QueryService; import org.labkey.api.query.QueryUpdateService; @@ -198,29 +201,44 @@ private void importSamples(Container c, User user, @Nullable ExpSampleType sampl QueryUpdateService qus = tinfo.getUpdateService(); if (qus != null) { - try (DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, c, null)) + try (DbScope.Transaction transaction = tinfo.getSchema().getScope().ensureTransaction()) { - BatchValidationException errors = new BatchValidationException(); - DataIteratorContext context = new DataIteratorContext(errors); - - if (_insertOption != null) - context.setInsertOption(_insertOption); - - if (_auditBehavior != null) - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, DETAILED); - context.setAllowImportLookupByAlternateKey(_alternateKeyLookup); + String fileWatcherDescription = getJob().getDescription(); + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent = transaction.getAuditEvent(); + if (transactionAuditEvent != null) + transactionAuditEvent.addDetail(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription); + else + { + transactionAuditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(c, QueryService.AuditAction.RELOAD, Map.of(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription)); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, transactionAuditEvent); + } - int count = qus.loadRows(user, c, loader, context, null); - log.info("Imported a total of " + count + " rows into : " + sampleType.getName()); - if (context.getErrors().hasErrors()) + try (DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, c, null)) { - for (ValidationException error : context.getErrors().getRowErrors()) - log.error(error.getMessage()); + BatchValidationException errors = new BatchValidationException(); + DataIteratorContext context = new DataIteratorContext(errors); + + if (_insertOption != null) + context.setInsertOption(_insertOption); + + if (_auditBehavior != null) + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, DETAILED); + context.setAllowImportLookupByAlternateKey(_alternateKeyLookup); + + int count = qus.loadRows(user, c, loader, context, null); + log.info("Imported a total of " + count + " rows into : " + sampleType.getName()); + if (context.getErrors().hasErrors()) + { + for (ValidationException error : context.getErrors().getRowErrors()) + log.error(error.getMessage()); + } } - } - catch (Exception e) - { - log.error("import failed", e); + catch (Exception e) + { + log.error("import failed", e); + } + + transaction.commit(); } } else From 49aa0002dd7ae7744746633fca25246be3c15117 Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 27 Oct 2025 13:33:31 -0700 Subject: [PATCH 06/13] Storage view actions --- .../labkey/api/audit/TransactionAuditProvider.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 2b01acf31ca..e06994160a3 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -127,18 +127,18 @@ public static TransactionAuditEvent getCurrentTransactionAuditEvent() public enum TransactionDetail { + Action(false, "The controller-action for this request"), AuditEvents(true, "The types of audit events generated during the transaction"), - ImportFileName(true, "The input filenames used for the import action"), ClientLibrary(false, "The client library (R, Python, etc) used to perform the action"), - Product(false, "The product (Sample Manager, etc) this action originated from"), - Action(false, "The controller-action for this request"), - QueryCommand(true, "The query commands (insert.update) executed during the transaction"), DataIteratorUsed(false, "If data iterator was used for insert/update"), - ImportOptions(true, "Various import parameters (CrossType, CrossFolder, etc) used during the import action"), EditMethod(false, "The method used to insert/update data from the app (e.g., 'DetailEdit', 'GridEdit', etc)"), - RequestSource(false, "The URL where the request originated from"), ETL(true, "The ETL process name involved in the transaction"), - FileWatcher(true, "File watcher source(s) involved in the transaction"); + FileWatcher(true, "File watcher source(s) involved in the transaction"), + ImportFileName(true, "The input filenames used for the import action"), + ImportOptions(true, "Various import parameters (CrossType, CrossFolder, etc) used during the import action"), + Product(false, "The product (Sample Manager, etc) this action originated from"), + QueryCommand(true, "The query commands (insert.update) executed during the transaction"), + RequestSource(false, "The URL where the request originated from"); private final boolean multiValue; TransactionDetail(boolean multiValue, String description) From ef8abd39da9c02e861f131457b812b098282a561 Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 28 Oct 2025 21:36:50 -0700 Subject: [PATCH 07/13] code review --- .../labkey/api/assay/pipeline/AssayUploadPipelineJob.java | 2 +- api/src/org/labkey/api/audit/TransactionAuditProvider.java | 7 ++++--- .../src/org/labkey/query/controllers/QueryController.java | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java index 283c1f7025d..e096edf09c4 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java @@ -161,7 +161,7 @@ public void doWork() transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, _primaryFile.getName()); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayUploadPipelineJob"); - ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails/*TODO*/); + ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails); setStatus(TaskStatus.complete); getLogger().info("Finished assay upload"); diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index e06994160a3..8b2320676ed 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -179,7 +179,7 @@ public void add(Map detailMap, Object value) detailMap.put(this, value); return; } - Object existing = detailMap.get(this);; + Object existing = detailMap.get(this); Set values; if (existing == null) values = new HashSet<>(); @@ -189,8 +189,9 @@ else if (existing instanceof Set) values = new HashSet<>(List.of(existing.toString())); if (value instanceof String) values.add((String) value); - else if (value instanceof Collections) - values.addAll((Set) value); + else if (value instanceof Collection) + for (Object v : (Collection) value) + values.add(v.toString()); detailMap.put(this, values); } } diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 908a9dae6ea..35ba9a5d411 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -4695,7 +4695,6 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole if (auditTransaction.getAuditEvent() != null) { auditEvent = auditTransaction.getAuditEvent(); - // detect trigger event? } else { From af89e8282d29364ebc958f03769f8979b8d41460 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 29 Oct 2025 09:28:14 -0700 Subject: [PATCH 08/13] fix build --- api/src/org/labkey/api/audit/TransactionAuditProvider.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 8b2320676ed..66486a10a02 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -23,6 +23,7 @@ import org.labkey.api.util.UnexpectedException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; From f3a4d7bbf549c6e4d40c58dda3ad4454acf37a92 Mon Sep 17 00:00:00 2001 From: XingY Date: Wed, 29 Oct 2025 15:57:04 -0700 Subject: [PATCH 09/13] add test for ETL and file watcher --- api/src/org/labkey/api/query/AbstractQueryUpdateService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index edc190429e2..92751e0464f 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -269,6 +269,8 @@ protected final DataIteratorContext getDataIteratorContext(BatchValidationExcept context.setInsertOption(forImport); context.setConfigParameters(configParameters); configureDataIteratorContext(context); + if (configParameters != null) + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); return context; } From b68466a1a8f084803758f87088579ca8483832f5 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 30 Oct 2025 21:06:43 -0700 Subject: [PATCH 10/13] fix build --- .../labkey/api/query/AbstractQueryUpdateService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index 92751e0464f..2984e741704 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -270,7 +270,15 @@ protected final DataIteratorContext getDataIteratorContext(BatchValidationExcept context.setConfigParameters(configParameters); configureDataIteratorContext(context); if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + { + try + { + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } catch (UnsupportedOperationException ignore) + { + // configParameters is immutable, likely originated from a junit test + } + } return context; } From 03600ccef376f03d72c2da26ab9472437d4a66e5 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 31 Oct 2025 11:45:26 -0700 Subject: [PATCH 11/13] Bug fixes --- .../api/assay/DefaultAssayRunCreator.java | 10 +++++++-- .../api/audit/TransactionAuditProvider.java | 19 ++++++++++++++++ .../api/query/AbstractQueryImportAction.java | 4 ++++ .../assay/actions/ImportRunApiAction.java | 22 ++++++++++++++++++- 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index c5046892d29..f09beff06a0 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -150,9 +150,10 @@ public Pair saveExperimentRun( try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) { - if (transaction.getAuditId() == null) + TransactionAuditProvider.TransactionAuditEvent auditEvent = transaction.getAuditEvent(); + if (auditEvent == null) { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); } context.init(); @@ -166,6 +167,7 @@ public Pair saveExperimentRun( throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); } FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); run.setComments(context.getComments()); run.setWorkflowTaskId(context.getWorkflowTask()); @@ -181,6 +183,10 @@ public Pair saveExperimentRun( { context.uploadComplete(null); context.setTransactionAuditId(transaction.getAuditId()); + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + if (primaryFile != null) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); exp = saveExperimentRunAsync(context, exp); } transaction.commit(); diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 66486a10a02..7555a2dd87f 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -1,8 +1,10 @@ package org.labkey.api.audit; import com.fasterxml.jackson.core.JsonProcessingException; +import io.micrometer.common.util.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; @@ -170,6 +172,23 @@ public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull String auditDetailsJson) + { + if (StringUtils.isEmpty(auditDetailsJson)) + return; + + Map auditDetails = new HashMap<>(); + try + { + auditDetails = new JSONObject(auditDetailsJson).toMap(); + } + catch (Exception ignore) + { + } + + addAuditDetails(transactionDetails, auditDetails); + } + public void add(Map detailMap, Object value) { if (value == null) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 0b079dfb48c..624fd4b03a5 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -310,6 +310,7 @@ public enum Params crossFolderImport, useTransactionAuditCache, lookupResolutionType, + auditDetails, } @Nullable @@ -442,6 +443,7 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except String text = getParam(Params.text); String path = getParam(Params.path); + String auditDetailsJson = getParam(Params.auditDetails); String moduleName = getParam(Params.module); String moduleResource = getParam(Params.moduleResource); @@ -458,6 +460,8 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except Map transactionDetails = getTransactionAuditDetails(); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, getTransactionImportParams(_insertOption.name(), _useAsync)); + if (!StringUtils.isEmpty(auditDetailsJson)) + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetailsJson); try { diff --git a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java index 9eac8ca22c6..3f7ce48a844 100644 --- a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java +++ b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java @@ -130,6 +130,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E boolean allowLookupByAlternateKey; String auditUserComment; Map outputData = new HashMap<>(); + String auditDetailsJsonStr; // 'json' form field -- allows for multipart forms JSONObject json = form.getJson(); @@ -173,6 +174,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E runFilePath = json.optString("runFilePath", null); moduleName = json.optString("module", null); auditUserComment = json.optString("auditUserComment", null); + auditDetailsJsonStr = json.optString("auditUserComment", null); JSONArray dataRows = json.optJSONArray(AssayJSONConverter.DATA_ROWS); if (dataRows != null) rawData = JsonUtil.toMapList(dataRows); @@ -204,6 +206,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E allowCrossRunFileInputs = form.isAllowCrossRunFileInputs(); allowLookupByAlternateKey = form.isAllowLookupByAlternateKey(); auditUserComment = form.getAuditUserComment(); + auditDetailsJsonStr = form.getAuditDetails(); } if (reImportOption == null) @@ -319,7 +322,10 @@ else if (rawData != null && !rawData.isEmpty()) try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, getTransactionAuditDetails()); + Map transactionDetails = getTransactionAuditDetails(); + if (!StringUtils.isEmpty(auditDetailsJsonStr)) + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetailsJsonStr); + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); var auditTransactionEvent = transaction.getAuditEvent(); Long auditTransactionId = auditTransactionEvent == null ? null : auditTransactionEvent.getRowId(); @@ -356,7 +362,10 @@ else if (rawData != null && !rawData.isEmpty()) String asyncJobGUID = uploadContext.getPipelineJobGUID(); if (!StringUtils.isEmpty(asyncJobGUID)) + { + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportOptions, "backgroundImport"); resp.put("jobId", PipelineService.get().getJobId(getUser(), getContainer(), asyncJobGUID)); + } return resp; } @@ -459,6 +468,7 @@ protected static class ImportRunApiForm extends SimpleApiJsonForm implements Has private boolean _allowCrossRunFileInputs; private boolean _allowLookupByAlternateKey = true; private String _auditUserComment = null; + private String _auditDetails = null; public JSONObject getJson() { @@ -680,6 +690,16 @@ public void setAuditUserComment(String auditUserComment) _auditUserComment = auditUserComment; } + public String getAuditDetails() + { + return _auditDetails; + } + + public void setAuditDetails(String auditDetails) + { + _auditDetails = auditDetails; + } + @Override public @NotNull BindException bindParameters(PropertyValues m) { From eb4c0811570df04b8cb951074592facf3baf03a5 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 31 Oct 2025 12:14:15 -0700 Subject: [PATCH 12/13] fix junit tests --- .../assay/pipeline/AssayUploadPipelineJob.java | 3 ++- .../api/query/DefaultQueryUpdateService.java | 16 ++++++++++++++++ .../api/query/SimpleQueryUpdateService.java | 13 ++++--------- .../api/ExpDataClassDataTableImpl.java | 4 +--- .../api/SampleTypeUpdateServiceDI.java | 4 +--- .../list/model/ListQueryUpdateService.java | 9 +++------ .../labkey/study/query/DatasetUpdateService.java | 9 +++------ 7 files changed, 30 insertions(+), 28 deletions(-) diff --git a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java index e096edf09c4..dbc66f79de4 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java @@ -156,11 +156,12 @@ public void doWork() _forceSaveBatchProps = true; } - // Do all the real work of the import Map transactionDetails = new HashMap<>(); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, _primaryFile.getName()); transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayUploadPipelineJob"); + + // Do all the real work of the import ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails); setStatus(TaskStatus.complete); getLogger().info("Finished assay upload"); diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index 1afb13580d0..46d9fa2d61e 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.ArrayListMap; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -934,4 +935,19 @@ protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIterator context.setCrossFolderImport(false); } } + + protected void recordDataIteratorUsed(@Nullable Map configParameters) + { + if (configParameters != null) + { + try + { + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } catch (UnsupportedOperationException ignore) + { + // configParameters is immutable, likely originated from a junit test + } + } + } + } diff --git a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java index 02ef9c422db..5d008f531ee 100644 --- a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java +++ b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java @@ -62,8 +62,7 @@ public SimpleQueryUpdateService(final SimpleTable queryTable, TableInfo dbTable, @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.IMPORT, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -72,8 +71,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -82,8 +80,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B @Override public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws DuplicateKeyException, QueryUpdateServiceException, SQLException { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); List> result = super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); afterInsertUpdate(result == null ? 0 : result.size(), errors); return result; @@ -145,9 +142,7 @@ public List> updateRows(User user, Container container, List { if (shouldUpdateUsingDIB(container, rows, oldKeys, configParameters)) { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); - + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, configParameters); context.putConfigParameter(PreferPKOverObjectUriAsKey, shouldPreferPKOverObjectUriAsUpdateKey(rows)); List> result = super._updateRowsUsingDIB(user, container, rows, context, extraScriptContext); diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index e6baca27bf4..858833a24b9 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -1508,9 +1508,7 @@ public List> updateRows(User user, Container container, List Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); - + recordDataIteratorUsed(configParameters); results = super._updateRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters), extraScriptContext); } else diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 86b6ebf50dc..021036ffdfb 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -560,9 +560,7 @@ public List> updateRows(User user, Container container, List { Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); - - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); try { diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 782f781c156..9e6fae5c12f 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -172,8 +172,7 @@ public List> insertRows(User user, Container container, List } DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); if (null != result) @@ -277,8 +276,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B if (!_list.isVisible(user)) throw new UnauthorizedException("You do not have permission to update data in this table."); - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); } @@ -290,8 +288,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, if (!_list.isVisible(user)) throw new UnauthorizedException("You do not have permission to insert data into this table."); - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); if (count > 0 && !errors.hasErrors()) diff --git a/study/src/org/labkey/study/query/DatasetUpdateService.java b/study/src/org/labkey/study/query/DatasetUpdateService.java index 8011380dd80..729c110246c 100644 --- a/study/src/org/labkey/study/query/DatasetUpdateService.java +++ b/study/src/org/labkey/study/query/DatasetUpdateService.java @@ -237,8 +237,7 @@ public List> getRows(User user, Container container, List configParameters, Map extraScriptContext) { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); if (count > 0) { @@ -277,8 +276,7 @@ public int loadRows(User user, Container container, DataIteratorBuilder rows, Da @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) { - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); return loadRows(user, container, rows, context, extraScriptContext); @@ -293,8 +291,7 @@ public List> insertRows(User user, Container container, List aliasColumns(_columnMapping, row); } - if (configParameters != null) - configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); if (_skipAuditLogging) From 043d9dcd54f63918bf93e04cd7b8371dd8ff23ce Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 31 Oct 2025 13:44:15 -0700 Subject: [PATCH 13/13] fix NPE --- api/src/org/labkey/api/assay/DefaultAssayRunCreator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index f09beff06a0..4d22109f358 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -167,7 +167,8 @@ public Pair saveExperimentRun( throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); } FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); - auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); + if (primaryFile != null) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); run.setComments(context.getComments()); run.setWorkflowTaskId(context.getWorkflowTask());