diff --git a/api/src/org/labkey/api/action/BaseViewAction.java b/api/src/org/labkey/api/action/BaseViewAction.java index d31dda91433..fcbef9eb7e2 100644 --- a/api/src/org/labkey/api/action/BaseViewAction.java +++ b/api/src/org/labkey/api/action/BaseViewAction.java @@ -28,13 +28,16 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.attachments.AttachmentFile; import org.labkey.api.attachments.SpringAttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ConvertHelper; import org.labkey.api.security.User; import org.labkey.api.util.HelpTopic; +import org.labkey.api.util.HttpUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewContext; import org.labkey.api.view.template.PageConfig; import org.labkey.api.writer.ContainerUser; import org.springframework.beans.AbstractPropertyAccessor; @@ -71,6 +74,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -535,6 +539,31 @@ public boolean supports(Class clazz) return getCommandClass().isAssignableFrom(clazz); } + public Map getTransactionAuditDetails() + { + return getTransactionAuditDetails(getViewContext()); + } + + public static Map getTransactionAuditDetails(ViewContext viewContext) + { + Map map = new HashMap<>(); + map.put(TransactionAuditProvider.TransactionDetail.Action, viewContext.getActionURL().getController() + "-" + viewContext.getActionURL().getAction()); + String clientLibrary = HttpUtil.getClientLibrary(viewContext.getRequest()); + if (null != clientLibrary) + map.put(TransactionAuditProvider.TransactionDetail.ClientLibrary, clientLibrary); + else + { + String productName = HttpUtil.getProductNameFromReferer(viewContext.getRequest()); // app + if (null != productName) + map.put(TransactionAuditProvider.TransactionDetail.Product, productName); + else // LKS + { + String refererRelativeURL = HttpUtil.getRefererRelativeURL(viewContext.getRequest()); + map.put(TransactionAuditProvider.TransactionDetail.RequestSource, refererRelativeURL); + } + } + return map; + } /* for TableViewForm, uses BeanUtils to work with DynaBeans */ static public class BeanUtilsPropertyBindingResult extends BeanPropertyBindingResult diff --git a/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java b/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java index 4d7c60f0458..3c489ba02a1 100644 --- a/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java +++ b/api/src/org/labkey/api/assay/AssayFilePropertyWriter.java @@ -2,6 +2,7 @@ import org.apache.logging.log4j.Logger; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.provider.FileSystemAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; @@ -59,7 +60,7 @@ public static void cleanupPostedFiles(Container container, Set files, } } - public CaseInsensitiveHashMap savePostedFiles(Container container, User user, Map filePropertyMap, Long auditTransactionId, String auditComment) throws ExperimentException + public CaseInsensitiveHashMap savePostedFiles(Container container, User user, Map filePropertyMap, TransactionAuditProvider.TransactionAuditEvent auditTransactionEvent, String auditComment) throws ExperimentException { FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(container); CaseInsensitiveHashMap properties = new CaseInsensitiveHashMap<>(); @@ -87,7 +88,7 @@ public CaseInsensitiveHashMap savePostedFiles(Container container, Use event.setProvidedFileName(originalName); event.setFile(file.getName()); event.setDirectory(file.getParent()); - event.setTransactionId(auditTransactionId); + event.setTransactionEvent(auditTransactionEvent, FileSystemAuditProvider.EVENT_TYPE); event.setFieldName(entry.getKey()); AuditLogService.get().addEvent(user, event); diff --git a/api/src/org/labkey/api/assay/AssayRunCreator.java b/api/src/org/labkey/api/assay/AssayRunCreator.java index 86766eff41f..15bbbd5c12f 100644 --- a/api/src/org/labkey/api/assay/AssayRunCreator.java +++ b/api/src/org/labkey/api/assay/AssayRunCreator.java @@ -16,12 +16,15 @@ package org.labkey.api.assay; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpRun; import org.labkey.api.query.ValidationException; import org.labkey.api.util.Pair; +import java.util.Map; + /** * An AssayRunCreator does the actual work of constructing an assay run and saving it to the database. It gets * data about the run to be created from a AssayRunUploadContext and translates it into objects in the Experiment @@ -40,14 +43,17 @@ public interface AssayRunCreator * @param batchId if not null, the run group that's already created for this batch. If null, a new one will be created. * @return Pair of batch and run that were inserted. ExpBatch will not be null, but ExpRun may be null when inserting the run async. */ - Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) - throws ExperimentException, ValidationException; + default Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) + throws ExperimentException, ValidationException + { + return saveExperimentRun(context, batchId, false, null); + } - Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId, boolean forceAsync) + Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId, boolean forceAsync, Map transactionDetails) throws ExperimentException, ValidationException; /** * @return the batch to which the run has been assigned */ - ExpExperiment saveExperimentRun(AssayRunUploadContext context, @Nullable ExpExperiment batch, ExpRun run, boolean forceSaveBatchProps) + ExpExperiment saveExperimentRun(AssayRunUploadContext context, @Nullable ExpExperiment batch, ExpRun run, boolean forceSaveBatchProps, @Nullable Map transactionDetails) throws ExperimentException, ValidationException; } diff --git a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java index f48283bf482..4d22109f358 100644 --- a/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java +++ b/api/src/org/labkey/api/assay/DefaultAssayRunCreator.java @@ -123,13 +123,6 @@ public TransformResult transform(AssayRunUploadContext context, Ex { return DataTransformService.get().transformAndValidate(context, run, DataTransformService.TransformOperation.INSERT); } - - @Override - public Pair saveExperimentRun(AssayRunUploadContext context, @Nullable Long batchId) throws ExperimentException, ValidationException - { - return saveExperimentRun(context, batchId, false); - } - /** * Create and save an experiment run synchronously or asynchronously in a background job depending upon the assay design. * @@ -141,7 +134,8 @@ public Pair saveExperimentRun(AssayRunUploadContext saveExperimentRun( AssayRunUploadContext context, @Nullable Long batchId, - boolean forceAsync + boolean forceAsync, + Map transactionDetails ) throws ExperimentException, ValidationException { ExpExperiment exp = null; @@ -156,9 +150,10 @@ public Pair saveExperimentRun( try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) { - if (transaction.getAuditId() == null) + TransactionAuditProvider.TransactionAuditEvent auditEvent = transaction.getAuditEvent(); + if (auditEvent == null) { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(context.getContainer(), context.getReRunId() == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); } context.init(); @@ -172,11 +167,13 @@ public Pair saveExperimentRun( throw new ClassCastException("FileLike expected: " + errFile + " context: " + context.getClass() + " " + context); } FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + if (primaryFile != null) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); run = AssayService.get().createExperimentRun(context.getName(), context.getContainer(), protocol, null == primaryFile ? null : primaryFile.toNioPathForRead().toFile()); run.setComments(context.getComments()); run.setWorkflowTaskId(context.getWorkflowTask()); - exp = saveExperimentRun(context, exp, run, false); + exp = saveExperimentRun(context, exp, run, false, transactionDetails); // re-fetch the run after it has been fully constructed run = ExperimentService.get().getExpRun(run.getRowId()); @@ -187,6 +184,10 @@ public Pair saveExperimentRun( { context.uploadComplete(null); context.setTransactionAuditId(transaction.getAuditId()); + FileLike primaryFile = context.getUploadedData().get(AssayDataCollector.PRIMARY_FILE); + if (primaryFile != null) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportFileName, primaryFile.getName()); + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); exp = saveExperimentRunAsync(context, exp); } transaction.commit(); @@ -267,7 +268,8 @@ public ExpExperiment saveExperimentRun( final AssayRunUploadContext context, @Nullable ExpExperiment batch, @NotNull ExpRun run, - boolean forceSaveBatchProps + boolean forceSaveBatchProps, + @Nullable Map transactionDetails ) throws ExperimentException, ValidationException { context.setAutoFillDefaultResultColumns(run.getRowId() > 0); // need to setAutoFillDefaultResultColumns before run is saved @@ -327,7 +329,7 @@ public ExpExperiment saveExperimentRun( } else { - var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction); + var auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, auditAction, transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent); } } diff --git a/api/src/org/labkey/api/assay/actions/UploadWizardAction.java b/api/src/org/labkey/api/assay/actions/UploadWizardAction.java index e5520b9731c..b4564122006 100644 --- a/api/src/org/labkey/api/assay/actions/UploadWizardAction.java +++ b/api/src/org/labkey/api/assay/actions/UploadWizardAction.java @@ -1027,7 +1027,7 @@ public class RunStepHandler extends StepHandler public final ExpRun saveExperimentRun(FormType form) throws ExperimentException, ValidationException { - Pair pair = form.getProvider().getRunCreator().saveExperimentRun(form, form.getBatchId()); + Pair pair = form.getProvider().getRunCreator().saveExperimentRun(form, form.getBatchId(), false, getTransactionAuditDetails()); assert pair != null && pair.first != null; ExpExperiment exp = pair.first; ExpRun run = pair.second; diff --git a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java index a364fdbf70e..dbc66f79de4 100644 --- a/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java +++ b/api/src/org/labkey/api/assay/pipeline/AssayUploadPipelineJob.java @@ -22,6 +22,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; import org.labkey.api.assay.AssayUrls; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.exp.ExperimentException; import org.labkey.api.exp.api.ExpExperiment; import org.labkey.api.exp.api.ExpRun; @@ -155,8 +156,13 @@ public void doWork() _forceSaveBatchProps = true; } + Map transactionDetails = new HashMap<>(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, _primaryFile.getName()); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayUploadPipelineJob"); + // Do all the real work of the import - ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps); + ExpExperiment result = _context.getProvider().getRunCreator().saveExperimentRun(_context, batch, _run, _forceSaveBatchProps, transactionDetails); setStatus(TaskStatus.complete); getLogger().info("Finished assay upload"); diff --git a/api/src/org/labkey/api/audit/AuditTypeEvent.java b/api/src/org/labkey/api/audit/AuditTypeEvent.java index af57a884112..15778411cab 100644 --- a/api/src/org/labkey/api/audit/AuditTypeEvent.java +++ b/api/src/org/labkey/api/audit/AuditTypeEvent.java @@ -191,6 +191,15 @@ public void setTransactionId(Long transactionId) _transactionId = transactionId; } + public void setTransactionEvent(@Nullable TransactionAuditProvider.TransactionAuditEvent transactionEvent, String auditEventType) + { + if (transactionEvent == null) + return; + + _transactionId = transactionEvent.getRowId(); + transactionEvent.addDetail(TransactionAuditProvider.TransactionDetail.AuditEvents, auditEventType); + } + protected String getContainerMessageElement(@NotNull Container container) { String value = " (" + container.getId() + ")"; diff --git a/api/src/org/labkey/api/audit/ExperimentAuditEvent.java b/api/src/org/labkey/api/audit/ExperimentAuditEvent.java index 73ad958c9c7..1427e9d3531 100644 --- a/api/src/org/labkey/api/audit/ExperimentAuditEvent.java +++ b/api/src/org/labkey/api/audit/ExperimentAuditEvent.java @@ -21,13 +21,12 @@ public class ExperimentAuditEvent extends AuditTypeEvent public ExperimentAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public ExperimentAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getProtocolLsid() diff --git a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java index eb60d7424b1..3ed3637595f 100644 --- a/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java +++ b/api/src/org/labkey/api/audit/SampleTimelineAuditEvent.java @@ -104,13 +104,12 @@ public static SampleTimelineEventType getTypeFromName(@Nullable String name) public SampleTimelineAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public SampleTimelineAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getSampleLsid() diff --git a/api/src/org/labkey/api/audit/TransactionAuditProvider.java b/api/src/org/labkey/api/audit/TransactionAuditProvider.java index 0dc6af6e889..7555a2dd87f 100644 --- a/api/src/org/labkey/api/audit/TransactionAuditProvider.java +++ b/api/src/org/labkey/api/audit/TransactionAuditProvider.java @@ -1,12 +1,17 @@ package org.labkey.api.audit; +import com.fasterxml.jackson.core.JsonProcessingException; +import io.micrometer.common.util.StringUtils; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.DbScope; import org.labkey.api.data.JdbcType; +import org.labkey.api.data.JsonPrettyPrintDisplayColumnFactory; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.PropertyStorageSpec; import org.labkey.api.data.TableInfo; @@ -16,12 +21,18 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; +import org.labkey.api.util.JsonUtil; +import org.labkey.api.util.UnexpectedException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -32,6 +43,7 @@ public class TransactionAuditProvider extends AbstractAuditTypeProvider implemen public static final String COLUMN_NAME_START_TIME = "StartTime"; public static final String COLUMN_NAME_TRANSACTION_TYPE = "TransactionType"; + public static final String COLUMN_NAME_TRANSACTION_DETAILS = "TransactionDetails"; static final List defaultVisibleColumns = new ArrayList<>(); @@ -81,6 +93,11 @@ protected void initColumn(MutableColumnInfo col) col.setLabel("Start Time"); else if (COLUMN_NAME_TRANSACTION_TYPE.equalsIgnoreCase(col.getName())) col.setLabel("Transaction Type"); + else if (COLUMN_NAME_TRANSACTION_DETAILS.equalsIgnoreCase(col.getName())) + { + col.setLabel("Transaction Details"); + col.setDisplayColumnFactory(new JsonPrettyPrintDisplayColumnFactory()); + } } }; } @@ -104,6 +121,101 @@ public static Long getCurrentTransactionAuditId() return DbScope.getLabKeyScope().getCurrentTransaction().getAuditId(); } + public static TransactionAuditEvent getCurrentTransactionAuditEvent() + { + if (null == DbScope.getLabKeyScope().getCurrentTransaction()) + return null; + return DbScope.getLabKeyScope().getCurrentTransaction().getAuditEvent(); + } + + public enum TransactionDetail + { + Action(false, "The controller-action for this request"), + AuditEvents(true, "The types of audit events generated during the transaction"), + ClientLibrary(false, "The client library (R, Python, etc) used to perform the action"), + DataIteratorUsed(false, "If data iterator was used for insert/update"), + EditMethod(false, "The method used to insert/update data from the app (e.g., 'DetailEdit', 'GridEdit', etc)"), + ETL(true, "The ETL process name involved in the transaction"), + FileWatcher(true, "File watcher source(s) involved in the transaction"), + ImportFileName(true, "The input filenames used for the import action"), + ImportOptions(true, "Various import parameters (CrossType, CrossFolder, etc) used during the import action"), + Product(false, "The product (Sample Manager, etc) this action originated from"), + QueryCommand(true, "The query commands (insert.update) executed during the transaction"), + RequestSource(false, "The URL where the request originated from"); + + private final boolean multiValue; + TransactionDetail(boolean multiValue, String description) + { + this.multiValue = multiValue; + } + + public static TransactionDetail fromString(String key) + { + for (TransactionDetail detail : values()) + { + if (detail.name().equalsIgnoreCase(key)) + return detail; + } + return null; + } + + public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull Map auditDetails) + { + if (!auditDetails.isEmpty()) + { + for (Map.Entry entry : auditDetails.entrySet()) + { + TransactionAuditProvider.TransactionDetail detail = TransactionAuditProvider.TransactionDetail.fromString(entry.getKey()); + if (detail != null) + detail.add(transactionDetails, entry.getValue()); + } + } + } + + public static void addAuditDetails(@NotNull Map transactionDetails, @NotNull String auditDetailsJson) + { + if (StringUtils.isEmpty(auditDetailsJson)) + return; + + Map auditDetails = new HashMap<>(); + try + { + auditDetails = new JSONObject(auditDetailsJson).toMap(); + } + catch (Exception ignore) + { + } + + addAuditDetails(transactionDetails, auditDetails); + } + + public void add(Map detailMap, Object value) + { + if (value == null) + return; + + if (!this.multiValue) + { + detailMap.put(this, value); + return; + } + Object existing = detailMap.get(this); + Set values; + if (existing == null) + values = new HashSet<>(); + else if (existing instanceof Set) + values = (Set) existing; + else + values = new HashSet<>(List.of(existing.toString())); + if (value instanceof String) + values.add((String) value); + else if (value instanceof Collection) + for (Object v : (Collection) value) + values.add(v.toString()); + detailMap.put(this, values); + } + } + public static class TransactionAuditEvent extends AuditTypeEvent { private Date _startTime; @@ -112,6 +224,8 @@ public static class TransactionAuditEvent extends AuditTypeEvent private int _commentCount = 0; // the audit event comment might have been updated/appended multiple times, for example, the original insert triggers additional insert/update via trigger scripts + private final Map _detailMap = new HashMap<>(); + public TransactionAuditEvent() { super(); @@ -146,6 +260,28 @@ public void setTransactionType(String transactionType) _transactionType = transactionType; } + public String getTransactionDetails() + { + try + { + return JsonUtil.DEFAULT_MAPPER.writeValueAsString(_detailMap); + } + catch (JsonProcessingException e) + { + throw UnexpectedException.wrap(e); + } + } + + public void addDetail(TransactionDetail key, Object value) + { + key.add(_detailMap, value); + } + + public void addDetails(Map details) + { + _detailMap.putAll(details); + } + @Override public void setComment(String comment) { @@ -197,6 +333,7 @@ public TransactionAuditDomainKind() Set fields = new LinkedHashSet<>(); fields.add(createPropertyDescriptor(COLUMN_NAME_START_TIME, PropertyType.DATE_TIME)); fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_TYPE, PropertyType.STRING)); + fields.add(createPropertyDescriptor(COLUMN_NAME_TRANSACTION_DETAILS, PropertyType.STRING, -1)); _fields = Collections.unmodifiableSet(fields); // We override the base fields so we can use a DbSequence as the RowId and make it available diff --git a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java index b103f6ac0b8..441f5862e21 100644 --- a/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java +++ b/api/src/org/labkey/api/audit/provider/FileSystemAuditProvider.java @@ -119,13 +119,12 @@ public FileSystemAuditEvent() { super(); setEventType(EVENT_TYPE); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public FileSystemAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getDirectory() diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 136e9d9186a..624fd4b03a5 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -51,13 +51,11 @@ import org.labkey.api.resource.Resource; import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CPUTimer; import org.labkey.api.util.FileStream; import org.labkey.api.util.FileUtil; -import org.labkey.api.util.NetworkDrive; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.Path; @@ -75,7 +73,6 @@ import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.servlet.ModelAndView; -import java.io.File; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -83,6 +80,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import static org.labkey.api.action.SpringActionController.ERROR_GENERIC; import static org.labkey.api.query.AbstractQueryUpdateService.addTransactionAuditEvent; @@ -312,6 +310,7 @@ public enum Params crossFolderImport, useTransactionAuditCache, lookupResolutionType, + auditDetails, } @Nullable @@ -334,6 +333,23 @@ protected Map getOptionParamsMap() return _optionParamsMap; } + protected Set getTransactionImportParams(String insertOption, boolean useAsync) + { + Set importParams = new TreeSet<>(); + importParams.add(insertOption); + if (useAsync) + importParams.add("backgroundImport"); + if (Boolean.valueOf(getParam(Params.crossTypeImport))) + importParams.add(Params.crossTypeImport.name()); + if (Boolean.valueOf(getParam(Params.crossFolderImport))) + importParams.add(Params.crossFolderImport.name()); + if (Boolean.valueOf(getParam(Params.useTransactionAuditCache))) + importParams.add(Params.useTransactionAuditCache.name()); + if (Boolean.valueOf(getParam(Params.allowCreateStorage))) + importParams.add(Params.allowCreateStorage.name()); + return importParams; + } + protected boolean getOptionParamValue(Params p) { return getOptionParamsMap().getOrDefault(p, false); @@ -427,6 +443,7 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except String text = getParam(Params.text); String path = getParam(Params.path); + String auditDetailsJson = getParam(Params.auditDetails); String moduleName = getParam(Params.module); String moduleResource = getParam(Params.moduleResource); @@ -441,6 +458,11 @@ public final ApiResponse _execute(FORM form, BindException errors) throws Except // If not defined there, check for the audit behavior defined in the action form (getAuditBehaviorType()). AuditBehaviorType behaviorType = (_target != null) ? _target.getEffectiveAuditBehavior(getAuditBehaviorType()) : getAuditBehaviorType(); + Map transactionDetails = getTransactionAuditDetails(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, getTransactionImportParams(_insertOption.name(), _useAsync)); + if (!StringUtils.isEmpty(auditDetailsJson)) + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetailsJson); + try { if (null != StringUtils.trimToNull(text)) @@ -471,6 +493,7 @@ else if (StringUtils.isNotBlank(path)) loader = DataLoader.get().createLoader(resource, _hasColumnHeaders, null, null); file = resource.getFileStream(user); originalName = resource.getName(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); } if (!hasPostData) @@ -514,6 +537,7 @@ else if (_target != null) hasPostData = true; loader = DataLoader.get().createLoader(r, _hasColumnHeaders, null, TabLoader.TSV_FILE_TYPE); originalName = p.getName(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, originalName); // Set file to null so assay import doesn't copy the file file = null; } @@ -525,6 +549,7 @@ else if (getViewContext().getRequest() instanceof MultipartHttpServletRequest) MultipartFile multipartfile = null==files ? null : files.get("file"); if (null != multipartfile && multipartfile.getSize() > 0) { + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, multipartfile.getOriginalFilename()); hasPostData = true; originalName = multipartfile.getOriginalFilename(); // can't read the multipart file twice so create temp file (12800) @@ -582,6 +607,7 @@ else if (!dataFileDir.exists()) .setJobDescription(getQueryImportDescription()) .setJobNotificationProvider(getQueryImportJobNotificationProviderName()); + importContextBuilder.setTransactionDetails(transactionDetails); QueryImportPipelineJob job = new QueryImportPipelineJob(getQueryImportProviderName(), info, root, importContextBuilder); PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName()); @@ -613,7 +639,7 @@ else if (!dataFileDir.exists()) TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (isCrossTypeImport || (behaviorType != null && behaviorType != AuditBehaviorType.NONE)) - auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction); + auditEvent = createTransactionAuditEvent(getContainer(), _insertOption.auditAction, transactionDetails); int rowCount = importData(loader, file, originalName, ve, behaviorType, auditEvent, _auditUserComment); @@ -862,7 +888,10 @@ public static int importData(DataLoader dl, TableInfo target, QueryUpdateService if (context.getErrors().hasErrors()) return 0; if (auditEvent != null) + { + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true /* qus.loadRows always use DIB*/); auditEvent.addComment(auditAction, count); + } incrementRowCountMetric(count, context.getInsertOption(), getMetricPrefix(target)); transaction.commit(); diff --git a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java index b6d9256fba2..2984e741704 100644 --- a/api/src/org/labkey/api/query/AbstractQueryUpdateService.java +++ b/api/src/org/labkey/api/query/AbstractQueryUpdateService.java @@ -230,9 +230,17 @@ public boolean hasExistingRowsInOtherContainers(Container container, Map details) { long auditId = DbSequenceManager.get(ContainerManager.getRoot(), DB_SEQUENCE_NAME).next(); - return new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); + TransactionAuditProvider.TransactionAuditEvent event = new TransactionAuditProvider.TransactionAuditEvent(container, auditAction, auditId); + if (details != null) + event.addDetails(details); + return event; } public static void addTransactionAuditEvent(DbScope.Transaction transaction, User user, TransactionAuditProvider.TransactionAuditEvent auditEvent) @@ -261,6 +269,16 @@ protected final DataIteratorContext getDataIteratorContext(BatchValidationExcept context.setInsertOption(forImport); context.setConfigParameters(configParameters); configureDataIteratorContext(context); + if (configParameters != null) + { + try + { + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } catch (UnsupportedOperationException ignore) + { + // configParameters is immutable, likely originated from a junit test + } + } return context; } diff --git a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java index 1afb13580d0..46d9fa2d61e 100644 --- a/api/src/org/labkey/api/query/DefaultQueryUpdateService.java +++ b/api/src/org/labkey/api/query/DefaultQueryUpdateService.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.ArrayListMap; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -934,4 +935,19 @@ protected void configureCrossFolderImport(DataIteratorBuilder rows, DataIterator context.setCrossFolderImport(false); } } + + protected void recordDataIteratorUsed(@Nullable Map configParameters) + { + if (configParameters != null) + { + try + { + configParameters.put(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } catch (UnsupportedOperationException ignore) + { + // configParameters is immutable, likely originated from a junit test + } + } + } + } diff --git a/api/src/org/labkey/api/query/QueryImportPipelineJob.java b/api/src/org/labkey/api/query/QueryImportPipelineJob.java index 3534b4a8c70..a4820826fab 100644 --- a/api/src/org/labkey/api/query/QueryImportPipelineJob.java +++ b/api/src/org/labkey/api/query/QueryImportPipelineJob.java @@ -67,6 +67,8 @@ public static class QueryImportAsyncContextBuilder String _jobNotificationProvider; + Map _transactionDetails; + public QueryImportAsyncContextBuilder() { @@ -233,6 +235,17 @@ public QueryImportAsyncContextBuilder setJobDescription(String jobDescription) return this; } + public QueryImportAsyncContextBuilder setTransactionDetails(Map transactionDetails) + { + _transactionDetails = transactionDetails; + return this; + } + + public Map getTransactionDetails() + { + return _transactionDetails; + } + } @Override @@ -300,7 +313,7 @@ public void run() TransactionAuditProvider.TransactionAuditEvent auditEvent = null; if (diContext.isCrossTypeImport() || (_importContextBuilder.getAuditBehaviorType() != null && _importContextBuilder.getAuditBehaviorType() != AuditBehaviorType.NONE)) - auditEvent = createTransactionAuditEvent(getContainer(), diContext.getInsertOption().auditAction); + auditEvent = createTransactionAuditEvent(getContainer(), diContext.getInsertOption().auditAction, _importContextBuilder.getTransactionDetails()); int importedCount = AbstractQueryImportAction.importData(loader, target, updateService, diContext, auditEvent, getInfo().getUser(), getInfo().getContainer()); diff --git a/api/src/org/labkey/api/query/QueryService.java b/api/src/org/labkey/api/query/QueryService.java index bbccad189f6..bb6d87b7614 100644 --- a/api/src/org/labkey/api/query/QueryService.java +++ b/api/src/org/labkey/api/query/QueryService.java @@ -408,7 +408,10 @@ enum AuditAction "deleted"), MERGE("%s row(s) were inserted or updated.", "%s was inserted or updated.", - "inserted or updated"); + "inserted or updated"), + RELOAD("%s row(s) were reloaded.", + "%s was reloaded.", + "reloaded"); final String _commentDetailed; final String _commentDetailedFormat; diff --git a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java index 30240d4c353..5d008f531ee 100644 --- a/api/src/org/labkey/api/query/SimpleQueryUpdateService.java +++ b/api/src/org/labkey/api/query/SimpleQueryUpdateService.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; import org.labkey.api.data.DatabaseTableType; @@ -61,6 +62,7 @@ public SimpleQueryUpdateService(final SimpleTable queryTable, TableInfo dbTable, @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { + recordDataIteratorUsed(configParameters); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.IMPORT, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -69,6 +71,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, @Override public int mergeRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, @Nullable Map configParameters, Map extraScriptContext) { + recordDataIteratorUsed(configParameters); var count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); afterInsertUpdate(count, errors); return count; @@ -77,6 +80,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B @Override public List> insertRows(User user, Container container, List> rows, BatchValidationException errors, @Nullable Map configParameters, @Nullable Map extraScriptContext) throws DuplicateKeyException, QueryUpdateServiceException, SQLException { + recordDataIteratorUsed(configParameters); List> result = super._insertRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.INSERT, configParameters), extraScriptContext); afterInsertUpdate(result == null ? 0 : result.size(), errors); return result; @@ -138,6 +142,7 @@ public List> updateRows(User user, Container container, List { if (shouldUpdateUsingDIB(container, rows, oldKeys, configParameters)) { + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.UPDATE, configParameters); context.putConfigParameter(PreferPKOverObjectUriAsKey, shouldPreferPKOverObjectUriAsUpdateKey(rows)); List> result = super._updateRowsUsingDIB(user, container, rows, context, extraScriptContext); diff --git a/api/src/org/labkey/api/query/UserSchemaAction.java b/api/src/org/labkey/api/query/UserSchemaAction.java index 12f94df02c1..4f267a9c678 100644 --- a/api/src/org/labkey/api/query/UserSchemaAction.java +++ b/api/src/org/labkey/api/query/UserSchemaAction.java @@ -264,7 +264,7 @@ else if (table.getPkColumnNames().size() > 1) auditEvent = transaction.getAuditEvent(); else { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), auditAction, getTransactionAuditDetails()); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); } } diff --git a/api/src/org/labkey/api/util/HttpUtil.java b/api/src/org/labkey/api/util/HttpUtil.java index 2ee8ef53aa5..7657e2ba5f2 100644 --- a/api/src/org/labkey/api/util/HttpUtil.java +++ b/api/src/org/labkey/api/util/HttpUtil.java @@ -36,6 +36,7 @@ import org.labkey.api.module.DefaultModule; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; import org.labkey.api.view.BadRequestException; import org.springframework.web.servlet.mvc.Controller; import org.w3c.dom.Document; @@ -257,7 +258,7 @@ public static void trackClientApiRequests(HttpServletRequest request) SimpleMetricsService.get().increment(DefaultModule.CORE_MODULE_NAME, "clientApiRequests", clientLibrary); } - private static @Nullable String getClientLibrary(HttpServletRequest request) + public static @Nullable String getClientLibrary(HttpServletRequest request) { String userAgent = getUserAgent(request); if (null != userAgent) @@ -299,4 +300,50 @@ else if (userAgent.startsWith("LabKey SAS API")) } return request.getRemoteAddr(); } + + public static @Nullable String getProductNameFromReferer(HttpServletRequest request) + { + if (!isBrowser(request)) + return null; + + String referer = request.getHeader("Referer"); + if (referer != null) + { + try + { + ActionURL url = new ActionURL(referer); + String actionName = url.getAction(); + if ("app".equalsIgnoreCase(actionName) || "appdev".equalsIgnoreCase(actionName)) + return url.getController(); + } + catch (Exception e) + { + return null; + } + } + + return null; + } + + public static @Nullable String getRefererRelativeURL(HttpServletRequest request) + { + if (!isBrowser(request)) + return null; + + String referer = request.getHeader("Referer"); + if (referer != null) + { + try + { + ActionURL url = new ActionURL(referer); + return url.toContainerRelativeURL(); + } + catch (Exception e) + { + return null; + } + } + + return null; + } } diff --git a/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java b/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java index 2edcdca159f..7505d817d26 100644 --- a/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java +++ b/assay/api-src/org/labkey/api/assay/DefaultAssaySaveHandler.java @@ -22,6 +22,8 @@ import org.jetbrains.annotations.Nullable; import org.json.JSONArray; import org.json.JSONObject; +import org.labkey.api.action.BaseViewAction; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ExpDataFileConverter; import org.labkey.api.data.MvUtil; @@ -219,7 +221,7 @@ protected ExpExperiment saveExperimentRun( AssayRunUploadContext uploadContext = createRunUploadContext(context, protocol, runJson, dataRows, inputData, outputData, inputMaterial, outputMaterial); - return saveAssayRun(uploadContext, batch, run); + return saveAssayRun(uploadContext, batch, run, BaseViewAction.getTransactionAuditDetails(context)); } return null; } @@ -316,10 +318,10 @@ protected AssayRunUploadContext createRunUploadContext( return provider.createRunUploadFactory(protocol, context); } - protected ExpExperiment saveAssayRun(AssayRunUploadContext context, ExpExperiment batch, ExpRun run) throws ExperimentException, ValidationException + protected ExpExperiment saveAssayRun(AssayRunUploadContext context, ExpExperiment batch, ExpRun run, Map transactionDetail) throws ExperimentException, ValidationException { AssayRunCreator runCreator = getProvider().getRunCreator(); - return runCreator.saveExperimentRun(context, batch, run, false); + return runCreator.saveExperimentRun(context, batch, run, false, transactionDetail); } @Override diff --git a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java index a4bc9a84ac8..3f7ce48a844 100644 --- a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java +++ b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java @@ -130,6 +130,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E boolean allowLookupByAlternateKey; String auditUserComment; Map outputData = new HashMap<>(); + String auditDetailsJsonStr; // 'json' form field -- allows for multipart forms JSONObject json = form.getJson(); @@ -173,6 +174,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E runFilePath = json.optString("runFilePath", null); moduleName = json.optString("module", null); auditUserComment = json.optString("auditUserComment", null); + auditDetailsJsonStr = json.optString("auditUserComment", null); JSONArray dataRows = json.optJSONArray(AssayJSONConverter.DATA_ROWS); if (dataRows != null) rawData = JsonUtil.toMapList(dataRows); @@ -204,6 +206,7 @@ public ApiResponse execute(ImportRunApiForm form, BindException errors) throws E allowCrossRunFileInputs = form.isAllowCrossRunFileInputs(); allowLookupByAlternateKey = form.isAllowLookupByAlternateKey(); auditUserComment = form.getAuditUserComment(); + auditDetailsJsonStr = form.getAuditDetails(); } if (reImportOption == null) @@ -319,15 +322,19 @@ else if (rawData != null && !rawData.isEmpty()) try (DbScope.Transaction transaction = ExperimentService.get().getSchema().getScope().ensureTransaction(ExperimentService.get().getProtocolImportLock())) { - TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT); + Map transactionDetails = getTransactionAuditDetails(); + if (!StringUtils.isEmpty(auditDetailsJsonStr)) + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetailsJsonStr); + TransactionAuditProvider.TransactionAuditEvent auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(getContainer(), reRunId == null ? QueryService.AuditAction.UPDATE : QueryService.AuditAction.INSERT, transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(transaction, getUser(), auditEvent); - Long auditTransactionId = transaction.getAuditId(); + var auditTransactionEvent = transaction.getAuditEvent(); + Long auditTransactionId = auditTransactionEvent == null ? null : auditTransactionEvent.getRowId(); // Bind file property values and persist files to the file system. { Map fileMap = getFileMap(); - bindAndPersistFilePropertyValues(AssayJSONConverter.BATCH_PROPERTIES, batchProperties, fileMap, filePropertyWriter, auditTransactionId, "Assay batch property file uploaded."); - bindAndPersistFilePropertyValues(AssayJSONConverter.RUN_PROPERTIES, runProperties, fileMap, filePropertyWriter, auditTransactionId, "Assay run property file uploaded."); + bindAndPersistFilePropertyValues(AssayJSONConverter.BATCH_PROPERTIES, batchProperties, fileMap, filePropertyWriter, auditTransactionEvent, "Assay batch property file uploaded."); + bindAndPersistFilePropertyValues(AssayJSONConverter.RUN_PROPERTIES, runProperties, fileMap, filePropertyWriter, auditTransactionEvent, "Assay run property file uploaded."); } AssayRunUploadContext uploadContext = factory.setOutputDatas(outputData) @@ -337,7 +344,7 @@ else if (rawData != null && !rawData.isEmpty()) .setUploadedFiles(filePropertyWriter.getUploadedFiles()) .create(); - Pair result = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, forceAsync); + Pair result = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, forceAsync, getTransactionAuditDetails()); ExpRun run = result.second; transaction.commit(); @@ -355,7 +362,10 @@ else if (rawData != null && !rawData.isEmpty()) String asyncJobGUID = uploadContext.getPipelineJobGUID(); if (!StringUtils.isEmpty(asyncJobGUID)) + { + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.ImportOptions, "backgroundImport"); resp.put("jobId", PipelineService.get().getJobId(getUser(), getContainer(), asyncJobGUID)); + } return resp; } @@ -377,7 +387,7 @@ private void bindAndPersistFilePropertyValues( CaseInsensitiveHashMap properties, Map fileMap, AssayFilePropertyWriter fileWriter, - Long auditTransactionId, + TransactionAuditProvider.TransactionAuditEvent auditTransactionEvent, String auditComment ) throws ExperimentException, ValidationException { @@ -388,7 +398,7 @@ private void bindAndPersistFilePropertyValues( if (filePropertyMap.isEmpty()) return; - var fileProperties = fileWriter.savePostedFiles(getContainer(), getUser(), filePropertyMap, auditTransactionId, auditComment); + var fileProperties = fileWriter.savePostedFiles(getContainer(), getUser(), filePropertyMap, auditTransactionEvent, auditComment); for (var entry : fileProperties.entrySet()) properties.put(entry.getKey(), entry.getValue().toNioPathForRead().toString()); } @@ -458,6 +468,7 @@ protected static class ImportRunApiForm extends SimpleApiJsonForm implements Has private boolean _allowCrossRunFileInputs; private boolean _allowLookupByAlternateKey = true; private String _auditUserComment = null; + private String _auditDetails = null; public JSONObject getJson() { @@ -679,6 +690,16 @@ public void setAuditUserComment(String auditUserComment) _auditUserComment = auditUserComment; } + public String getAuditDetails() + { + return _auditDetails; + } + + public void setAuditDetails(String auditDetails) + { + _auditDetails = auditDetails; + } + @Override public @NotNull BindException bindParameters(PropertyValues m) { diff --git a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java index a13e34b7293..97a69e27694 100644 --- a/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java +++ b/assay/src/org/labkey/assay/pipeline/AssayImportRunTask.java @@ -24,6 +24,7 @@ import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayRunUploadContext; import org.labkey.api.assay.AssayService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.data.Container; import org.labkey.api.data.DbScope; @@ -50,7 +51,9 @@ import org.labkey.api.pipeline.XMLBeanTaskFactoryFactory; import org.labkey.api.pipeline.file.AbstractFileAnalysisJob; import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.QueryService; import org.labkey.api.query.ValidationException; import org.labkey.api.reader.DataLoader; import org.labkey.api.reader.DataLoaderService; @@ -798,13 +801,24 @@ public RecordedActionSet run() throws PipelineJobException if (matchedFile == null) throw new PipelineJobException("No output files matched assay file type: " + assayFileType); + User user = getJob().getUser(); + Container container = getJob().getContainer(); + + String fileWatcherDescription = getJob().getDescription(); + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent = tx.getAuditEvent(); + if (transactionAuditEvent != null) + transactionAuditEvent.addDetail(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription); + else + { + transactionAuditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, QueryService.AuditAction.RELOAD, Map.of(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription)); + AbstractQueryUpdateService.addTransactionAuditEvent(tx, user, transactionAuditEvent); + } + // Issue 22587: Create ExpData for the output file from the RecordedActions if necessary so that we // ensure the generated bit is set on the ExpData. Otherwise, the DefaultAssayRunCreator will create // the ExpData but without the generated bit. createData(matchedFile, assayDataType); - User user = getJob().getUser(); - Container container = getJob().getContainer(); AssayRunUploadContext.Factory factory = provider.createRunUploadFactory(protocol, user, container); @@ -837,7 +851,11 @@ public RecordedActionSet run() throws PipelineJobException Long batchId = null; // Import the assay run - Pair pair = provider.getRunCreator().saveExperimentRun(uploadContext, batchId); + Map transactionDetails = new HashMap<>(); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportFileName, uploadedData.getName()); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.ImportOptions, "BackgroundImport"); + transactionDetails.put(TransactionAuditProvider.TransactionDetail.Action, "AssayImportRunTask"); + Pair pair = provider.getRunCreator().saveExperimentRun(uploadContext, batchId, false, transactionDetails); ExpRun run = pair.second; if (getJob() instanceof FileAnalysisJobSupport) diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 81f3b709ea4..9c055d795e7 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -2918,7 +2918,7 @@ public PlateSetImpl createPlateSet( // Example comment: "Plate set was created. Created via reformat. Initially contains 5 plates." int plateCount = plates == null ? 0 : plates.size(); String comment = StringUtilsLabKey.joinNonBlank(" ", StringUtils.trimToEmpty(additionalAuditComment), String.format("Initially contains %s.", StringUtilsLabKey.pluralize(plateCount, "plate"))); - PlateSetAuditEvent auditEvent = PlateSetAuditProvider.EventFactory.plateSetCreated(container, tx.getAuditId(), newPlateSet, comment); + PlateSetAuditEvent auditEvent = PlateSetAuditProvider.EventFactory.plateSetCreated(container, tx.getAuditEvent(), newPlateSet, comment); AuditLogService.get().addEvent(user, auditEvent); } @@ -3060,7 +3060,7 @@ public void archive(Container container, User user, @Nullable List plateSe archive(container, user, AssayDbSchema.getInstance().getTableInfoPlateSet(), "plate sets", plateSetIds, archive); tx.addCommitTask(() -> clearPlateSetCache(container, plateSetIds), DbScope.CommitTaskOption.POSTCOMMIT); - List auditEvents = PlateSetAuditProvider.EventFactory.plateSetsArchived(container, tx.getAuditId(), plateSetIds, archive); + List auditEvents = PlateSetAuditProvider.EventFactory.plateSetsArchived(container, tx.getAuditEvent(), plateSetIds, archive); AuditLogService.get().addEvents(user, auditEvents, true); } @@ -5162,17 +5162,17 @@ private void addPlateAuditEvents(User user, Collection plates, Function

plates, @Nullable String additionalComment) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateCreated(container, tx.getAuditId(), plate, additionalComment)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateCreated(container, tx.getAuditEvent(), plate, additionalComment)); } public void addPlateDeletedAuditEvents(Container container, User user, DbScope.Transaction tx, Collection plates) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateDeleted(container, tx.getAuditId(), plate)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateDeleted(container, tx.getAuditEvent(), plate)); } public void addPlateImportAuditEvents(Container container, User user, DbScope.Transaction tx, Collection plates, ExpRun run, boolean isReimport) { - addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateImported(container, tx.getAuditId(), plate, run, isReimport)); + addPlateAuditEvents(user, plates, plate -> PlateAuditProvider.EventFactory.plateImported(container, tx.getAuditEvent(), plate, run, isReimport)); } public void ensureTransactionAuditId(DbScope.Transaction tx, Container container, User user, QueryService.AuditAction auditAction) diff --git a/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java b/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java index 5bfafa4e892..f6c8fbd1be3 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateAuditEvent.java @@ -2,6 +2,7 @@ import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.Container; @@ -36,7 +37,7 @@ protected PlateAuditEvent( PlateAuditProvider.PlateEventType eventType, Container container, PlateImpl plate, - Long transactionAuditId + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent ) { super(EVENT_NAME, container, eventType.getComment(plate.isTemplate())); @@ -47,7 +48,7 @@ protected PlateAuditEvent( setPlateTypeRowId(plate.getPlateType().getRowId()); setSourcePlateRowId(plate.getSourcePlateRowId()); setTemplate(plate.isTemplate()); - setTransactionId(transactionAuditId); + setTransactionEvent(transactionAuditEvent, EVENT_NAME); } public String getPlateEventType() diff --git a/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java b/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java index 9c9697f226b..ff5bad3d5bc 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateAuditProvider.java @@ -3,6 +3,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.data.Container; @@ -185,12 +186,12 @@ public static class EventFactory { public static PlateAuditEvent plateCreated( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate, @Nullable String additionalComment ) { - var event = new PlateAuditEvent(PlateEventType.CREATE_PLATE, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.CREATE_PLATE, container, plate, transactionAuditEvent); event.setNewRecordMap(container, plate); if (additionalComment != null) @@ -199,17 +200,17 @@ public static PlateAuditEvent plateCreated( return event; } - public static PlateAuditEvent plateDeleted(Container container, Long transactionAuditId, PlateImpl plate) + public static PlateAuditEvent plateDeleted(Container container, TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate) { - var event = new PlateAuditEvent(PlateEventType.DELETE_PLATE, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.DELETE_PLATE, container, plate, transactionAuditEvent); event.setOldRecordMap(container, plate); return event; } - public static PlateAuditEvent plateImported(Container container, Long transactionAuditId, PlateImpl plate, ExpRun run, boolean isReimport) + public static PlateAuditEvent plateImported(Container container, TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateImpl plate, ExpRun run, boolean isReimport) { - var event = new PlateAuditEvent(PlateEventType.PLATE_IMPORT, container, plate, transactionAuditId); + var event = new PlateAuditEvent(PlateEventType.PLATE_IMPORT, container, plate, transactionAuditEvent); event.setImportRunId(run.getRowId()); event.setReimport(isReimport); diff --git a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java index 4398a8bef3b..9ddfb49f2f9 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditEvent.java @@ -2,6 +2,7 @@ import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.DetailedAuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.Container; @@ -33,7 +34,7 @@ public PlateSetAuditEvent( PlateSetAuditProvider.PlateSetEventType eventType, Container container, PlateSetImpl plateSet, - Long transactionAuditId + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent ) { super(EVENT_NAME, container, eventType.getComment()); @@ -45,7 +46,7 @@ public PlateSetAuditEvent( setPrimaryPlateSetRowId(plateSet.getPrimaryPlateSetId()); setParentPlateSetRowId(plateSet.getParentPlateSetId()); setRootPlateSetRowId(plateSet.getRootPlateSetId()); - setTransactionId(transactionAuditId); + setTransactionEvent(transactionAuditEvent, EVENT_NAME); } public Boolean getArchived() diff --git a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java index b9e1ccc175a..651757c9ee1 100644 --- a/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java +++ b/assay/src/org/labkey/assay/plate/audit/PlateSetAuditProvider.java @@ -3,6 +3,7 @@ import org.jetbrains.annotations.Nullable; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditTypeEvent; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.audit.query.AbstractAuditDomainKind; import org.labkey.api.audit.query.DefaultAuditTypeTable; import org.labkey.api.collections.CaseInsensitiveHashMap; @@ -185,12 +186,12 @@ public static class EventFactory { public static PlateSetAuditEvent plateSetCreated( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, PlateSetImpl plateSet, @Nullable String additionalComment ) { - var event = new PlateSetAuditEvent(PlateSetEventType.CREATE_PLATE_SET, container, plateSet, transactionAuditId); + var event = new PlateSetAuditEvent(PlateSetEventType.CREATE_PLATE_SET, container, plateSet, transactionAuditEvent); event.setNewRecordMap(container, plateSet); if (additionalComment != null) @@ -201,7 +202,7 @@ public static PlateSetAuditEvent plateSetCreated( public static List plateSetsArchived( Container container, - Long transactionAuditId, + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent, List plateSetIds, boolean archive ) throws ValidationException @@ -220,7 +221,7 @@ public static List plateSetsArchived( plateSet.setArchived(archive); - var event = new PlateSetAuditEvent(eventType, container, plateSet, transactionAuditId); + var event = new PlateSetAuditEvent(eventType, container, plateSet, transactionAuditEvent); event.setOldRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of(PlateTable.Column.Archived.name(), String.valueOf(!archive)))); event.setNewRecordMap(AbstractAuditTypeProvider.encodeForDataMap(CaseInsensitiveHashMap.of(PlateTable.Column.Archived.name(), String.valueOf(archive)))); events.add(event); diff --git a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java index 0ad0a24832a..3a5f1ea4fa7 100644 --- a/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java +++ b/experiment/src/org/labkey/experiment/SampleTypeAuditProvider.java @@ -126,13 +126,12 @@ public static class SampleTypeAuditEvent extends AuditTypeEvent public SampleTypeAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public SampleTypeAuditEvent(Container container, String comment) { super(EVENT_TYPE, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), EVENT_TYPE); } public String getSourceLsid() diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 9b3bd485f40..858833a24b9 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -28,6 +28,7 @@ import org.labkey.api.attachments.AttachmentParent; import org.labkey.api.attachments.AttachmentParentFactory; import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.Sets; @@ -1506,6 +1507,8 @@ public List> updateRows(User user, Container container, List { Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); + + recordDataIteratorUsed(configParameters); results = super._updateRowsUsingDIB(user, container, rows, getDataIteratorContext(errors, InsertOption.UPDATE, finalConfigParameters), extraScriptContext); } else diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 23ba91d58dc..f3d51bf42b7 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -9817,7 +9817,7 @@ public Map moveAssayRuns(@NotNull List assayR for (List runFileRenameData : assayMoveData.fileMovesByRunId().values()) { for (AbstractAssayProvider.AssayFileMoveData renameData : runFileRenameData) - moveFile(renameData, container, user, transaction.getAuditId()); + moveFile(renameData, container, user, transaction.getAuditEvent()); } }, POSTCOMMIT); @@ -9827,7 +9827,7 @@ public Map moveAssayRuns(@NotNull List assayR return assayMoveData.counts(); } - private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Container sourceContainer, User user, Long txAuditId) + private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) { String fieldName = renameData.fieldName() == null ? "datafileurl" : renameData.fieldName(); File targetFile = renameData.targetFile(); @@ -9850,7 +9850,7 @@ private boolean moveFile(AbstractAssayProvider.AssayFileMoveData renameData, Con String changeDetail = String.format("assay '%s' run '%s'", assayName, runName); - return moveFileLinkFile(sourceFile, targetFile, sourceContainer, user, changeDetail, txAuditId, fieldName); + return moveFileLinkFile(sourceFile, targetFile, sourceContainer, user, changeDetail, txAuditEvent, fieldName); } @Override @@ -10030,7 +10030,7 @@ public boolean isLookupToMaterials(DomainProperty dp) * * Move file (post-commit) after moving sample/assay data */ - public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sourceFileContainer, User user, String actionComment, Long txAuditId, String fieldName) + public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sourceFileContainer, User user, String actionComment, TransactionAuditProvider.TransactionAuditEvent txAuditEvent, String fieldName) { if (!sourceFile.exists()) return false; @@ -10062,8 +10062,8 @@ public boolean moveFileLinkFile(File sourceFile, File targetFile, Container sour return false; } } - if (txAuditId != null && event.getTransactionId() == null) - event.setTransactionId(txAuditId); + if (txAuditEvent != null && event.getTransactionId() == null) + event.setTransactionEvent(txAuditEvent, FileSystemAuditProvider.EVENT_TYPE); event.setDirectory(sourceFile.getParentFile().getAbsolutePath()); event.setFile(targetFile.getName()); event.setProvidedFileName(sourceFile.getName()); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java index 9109ecd0e27..013bf5d01ac 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeServiceImpl.java @@ -1975,7 +1975,7 @@ public Map moveSamples(Collection sample for (List sampleFileRenameData : fileMovesBySampleId.values()) { for (FileFieldRenameData renameData : sampleFileRenameData) - moveFile(renameData, sourceContainer, user, transaction.getAuditId()); + moveFile(renameData, sourceContainer, user, transaction.getAuditEvent()); } }, POSTCOMMIT); @@ -2165,7 +2165,7 @@ private Map> updateSampleFilePaths(ExpSampleType return sampleFileRenames; } - private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, Long txAuditId) + private boolean moveFile(FileFieldRenameData renameData, Container sourceContainer, User user, TransactionAuditProvider.TransactionAuditEvent txAuditEvent) { if (!renameData.targetFile.getParentFile().exists()) { @@ -2190,7 +2190,7 @@ private boolean moveFile(FileFieldRenameData renameData, Container sourceContain } String changeDetail = String.format("sample type '%s' sample '%s'", renameData.sampleType.getName(), renameData.sampleName); - return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditId, renameData.fieldName); + return ExperimentServiceImpl.get().moveFileLinkFile(renameData.sourceFile, renameData.targetFile, sourceContainer, user, changeDetail, txAuditEvent, renameData.fieldName); } @Override diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index 8417fd01071..021036ffdfb 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -28,6 +28,7 @@ import org.labkey.api.assay.AssayFileWriter; import org.labkey.api.attachments.AttachmentFile; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.CaseInsensitiveMapWrapper; @@ -559,6 +560,7 @@ public List> updateRows(User user, Container container, List { Map finalConfigParameters = configParameters == null ? new HashMap<>() : configParameters; finalConfigParameters.put(ExperimentService.QueryOptions.UseLsidForUpdate, true); + recordDataIteratorUsed(configParameters); try { diff --git a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java index 5237faa9b48..aa6c3ee30e2 100644 --- a/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java +++ b/experiment/src/org/labkey/experiment/pipeline/SampleReloadTask.java @@ -3,7 +3,9 @@ import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.data.Container; +import org.labkey.api.data.DbScope; import org.labkey.api.data.TableInfo; import org.labkey.api.dataiterator.DataIteratorContext; import org.labkey.api.dataiterator.DetailedAuditLogDataIterator; @@ -19,6 +21,7 @@ import org.labkey.api.pipeline.PipelineJob; import org.labkey.api.pipeline.RecordedActionSet; import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.query.AbstractQueryUpdateService; import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.QueryService; import org.labkey.api.query.QueryUpdateService; @@ -198,29 +201,44 @@ private void importSamples(Container c, User user, @Nullable ExpSampleType sampl QueryUpdateService qus = tinfo.getUpdateService(); if (qus != null) { - try (DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, c, null)) + try (DbScope.Transaction transaction = tinfo.getSchema().getScope().ensureTransaction()) { - BatchValidationException errors = new BatchValidationException(); - DataIteratorContext context = new DataIteratorContext(errors); - - if (_insertOption != null) - context.setInsertOption(_insertOption); - - if (_auditBehavior != null) - context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, DETAILED); - context.setAllowImportLookupByAlternateKey(_alternateKeyLookup); + String fileWatcherDescription = getJob().getDescription(); + TransactionAuditProvider.TransactionAuditEvent transactionAuditEvent = transaction.getAuditEvent(); + if (transactionAuditEvent != null) + transactionAuditEvent.addDetail(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription); + else + { + transactionAuditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(c, QueryService.AuditAction.RELOAD, Map.of(TransactionAuditProvider.TransactionDetail.FileWatcher, fileWatcherDescription)); + AbstractQueryUpdateService.addTransactionAuditEvent(transaction, user, transactionAuditEvent); + } - int count = qus.loadRows(user, c, loader, context, null); - log.info("Imported a total of " + count + " rows into : " + sampleType.getName()); - if (context.getErrors().hasErrors()) + try (DataLoader loader = DataLoader.get().createLoader(dataFile, null, true, c, null)) { - for (ValidationException error : context.getErrors().getRowErrors()) - log.error(error.getMessage()); + BatchValidationException errors = new BatchValidationException(); + DataIteratorContext context = new DataIteratorContext(errors); + + if (_insertOption != null) + context.setInsertOption(_insertOption); + + if (_auditBehavior != null) + context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, DETAILED); + context.setAllowImportLookupByAlternateKey(_alternateKeyLookup); + + int count = qus.loadRows(user, c, loader, context, null); + log.info("Imported a total of " + count + " rows into : " + sampleType.getName()); + if (context.getErrors().hasErrors()) + { + for (ValidationException error : context.getErrors().getRowErrors()) + log.error(error.getMessage()); + } } - } - catch (Exception e) - { - log.error("import failed", e); + catch (Exception e) + { + log.error("import failed", e); + } + + transaction.commit(); } } else diff --git a/list/src/org/labkey/list/model/ListAuditProvider.java b/list/src/org/labkey/list/model/ListAuditProvider.java index e91414869f8..ac40e3dab9b 100644 --- a/list/src/org/labkey/list/model/ListAuditProvider.java +++ b/list/src/org/labkey/list/model/ListAuditProvider.java @@ -156,7 +156,6 @@ public static class ListAuditEvent extends DetailedAuditTypeEvent public ListAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public ListAuditEvent(Container container, String comment, ListDefinitionImpl list) @@ -165,7 +164,7 @@ public ListAuditEvent(Container container, String comment, ListDefinitionImpl li setListDomainUri(list.getDomain().getTypeURI()); setListId(list.getListId()); setListName(list.getName()); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), ListManager.LIST_AUDIT_EVENT); } public int getListId() diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 75b679af3ae..9e6fae5c12f 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -88,10 +88,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import static org.labkey.api.util.IntegerUtils.isIntegral; @@ -174,6 +172,7 @@ public List> insertRows(User user, Container container, List } DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); + recordDataIteratorUsed(configParameters); List> result = this._insertRowsUsingDIB(getListUser(user, container), container, rows, context, extraScriptContext); if (null != result) @@ -277,6 +276,7 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B if (!_list.isVisible(user)) throw new UnauthorizedException("You do not have permission to update data in this table."); + recordDataIteratorUsed(configParameters); return _importRowsUsingDIB(getListUser(user, container), container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); } @@ -288,6 +288,7 @@ public int importRows(User user, Container container, DataIteratorBuilder rows, if (!_list.isVisible(user)) throw new UnauthorizedException("You do not have permission to insert data into this table."); + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); int count = _importRowsUsingDIB(getListUser(user, container), container, rows, null, context, extraScriptContext); if (count > 0 && !errors.hasErrors()) diff --git a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java index 76f970a6e05..ce48da2a02c 100644 --- a/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java +++ b/query/src/org/labkey/query/audit/QueryUpdateAuditProvider.java @@ -208,13 +208,12 @@ public static class QueryUpdateAuditEvent extends DetailedAuditTypeEvent public QueryUpdateAuditEvent() { super(); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); } public QueryUpdateAuditEvent(Container container, String comment) { super(QUERY_UPDATE_AUDIT_EVENT, container, comment); - setTransactionId(TransactionAuditProvider.getCurrentTransactionAuditId()); + setTransactionEvent(TransactionAuditProvider.getCurrentTransactionAuditEvent(), QUERY_UPDATE_AUDIT_EVENT); } public String getRowPk() diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 4b37da7c7dd..35ba9a5d411 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -4645,6 +4645,8 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole Map extraContext = json.has("extraContext") ? json.getJSONObject("extraContext").toMap() : new CaseInsensitiveHashMap<>(); + Map auditDetails = json.has("auditDetails") ? json.getJSONObject("auditDetails").toMap() : new CaseInsensitiveHashMap<>(); + Map configParameters = new HashMap<>(); // Check first if the audit behavior has been defined for the table either in code or through XML. @@ -4691,19 +4693,28 @@ protected JSONObject executeJson(JSONObject json, CommandType commandType, boole auditTransaction = NO_OP_TRANSACTION; if (auditTransaction.getAuditEvent() != null) + { auditEvent = auditTransaction.getAuditEvent(); + } else { - auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction()); + Map transactionDetails = getTransactionAuditDetails(); + TransactionAuditProvider.TransactionDetail.addAuditDetails(transactionDetails, auditDetails); + auditEvent = AbstractQueryUpdateService.createTransactionAuditEvent(container, commandType.getAuditAction(), transactionDetails); AbstractQueryUpdateService.addTransactionAuditEvent(auditTransaction, getUser(), auditEvent); } + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.QueryCommand, commandType.name()); } QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, container); List> responseRows = commandType.saveRows(qus, rowsToProcess, getUser(), container, configParameters, extraContext); if (auditEvent != null) + { auditEvent.addComment(commandType.getAuditAction(), responseRows.size()); + if (Boolean.TRUE.equals(configParameters.get(TransactionAuditProvider.TransactionDetail.DataIteratorUsed))) + auditEvent.addDetail(TransactionAuditProvider.TransactionDetail.DataIteratorUsed, true); + } if (commandType == CommandType.moveRows) { @@ -5128,6 +5139,7 @@ else if (scope != tableInfo.getSchema().getScope()) JSONArray resultArray = new JSONArray(); JSONObject extraContext = json.optJSONObject("extraContext"); + JSONObject auditDetails = json.optJSONObject("auditDetails"); int startingErrorIndex = 0; int errorCount = 0; @@ -5155,6 +5167,14 @@ else if (scope != tableInfo.getSchema().getScope()) commandExtraContext.putAll(commandObject.getJSONObject("extraContext").toMap()); } commandObject.put("extraContext", commandExtraContext); + Map commandAuditDetails = new HashMap<>(); + if (auditDetails != null) + commandAuditDetails.putAll(auditDetails.toMap()); + if (commandObject.has("auditDetails")) + { + commandAuditDetails.putAll(commandObject.getJSONObject("auditDetails").toMap()); + } + commandObject.put("auditDetails", commandAuditDetails); JSONObject commandResponse = executeJson(commandObject, command, !transacted, errors, transacted, i); // Bail out immediately if we're going to return a failure-type response message diff --git a/study/src/org/labkey/study/query/DatasetUpdateService.java b/study/src/org/labkey/study/query/DatasetUpdateService.java index 9a28eb8b9e1..729c110246c 100644 --- a/study/src/org/labkey/study/query/DatasetUpdateService.java +++ b/study/src/org/labkey/study/query/DatasetUpdateService.java @@ -25,6 +25,7 @@ import org.junit.Test; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.assay.AssayFileWriter; +import org.labkey.api.audit.TransactionAuditProvider; import org.labkey.api.collections.CaseInsensitiveHashMap; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.ResultSetRowMapFactory; @@ -236,6 +237,7 @@ public List> getRows(User user, Container container, List configParameters, Map extraScriptContext) { + recordDataIteratorUsed(configParameters); int count = _importRowsUsingDIB(user, container, rows, null, getDataIteratorContext(errors, InsertOption.MERGE, configParameters), extraScriptContext); if (count > 0) { @@ -274,6 +276,7 @@ public int loadRows(User user, Container container, DataIteratorBuilder rows, Da @Override public int importRows(User user, Container container, DataIteratorBuilder rows, BatchValidationException errors, Map configParameters, Map extraScriptContext) { + recordDataIteratorUsed(configParameters); DataIteratorContext context = getDataIteratorContext(errors, InsertOption.IMPORT, configParameters); return loadRows(user, container, rows, context, extraScriptContext); @@ -288,6 +291,8 @@ public List> insertRows(User user, Container container, List aliasColumns(_columnMapping, row); } + recordDataIteratorUsed(configParameters); + DataIteratorContext context = getDataIteratorContext(errors, InsertOption.INSERT, configParameters); if (_skipAuditLogging) context.putConfigParameter(DetailedAuditLogDataIterator.AuditConfigs.AuditBehavior, NONE);