diff --git a/src/main/java/org/commcare/formplayer/application/MenuController.java b/src/main/java/org/commcare/formplayer/application/MenuController.java index 2176f24e0..e046ce96a 100644 --- a/src/main/java/org/commcare/formplayer/application/MenuController.java +++ b/src/main/java/org/commcare/formplayer/application/MenuController.java @@ -11,10 +11,7 @@ import org.commcare.formplayer.beans.ResponseMetaData; import org.commcare.formplayer.beans.SessionNavigationBean; import org.commcare.formplayer.beans.SubmitResponseBean; -import org.commcare.formplayer.beans.menus.BaseResponseBean; -import org.commcare.formplayer.beans.menus.EntityDetailListResponse; -import org.commcare.formplayer.beans.menus.EntityDetailResponse; -import org.commcare.formplayer.beans.menus.LocationRelevantResponseBean; +import org.commcare.formplayer.beans.menus.*; import org.commcare.formplayer.services.FormplayerStorageFactory; import org.commcare.formplayer.services.MenuSessionFactory; import org.commcare.formplayer.services.ResponseMetaDataTracker; @@ -35,6 +32,8 @@ import org.springframework.web.server.ResponseStatusException; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import jakarta.servlet.http.HttpServletRequest; @@ -185,6 +184,8 @@ public BaseResponseBean navigateSessionWithAuth(@RequestBody SessionNavigationBe sessionNavigationBean.getFormSessionId() ); + logCaseTypeColumnIfPresent(response, "controller", log); + setResponseMetaData(response); SubmitResponseBean formSubmissionResponse = handleAutoFormSubmission(request, sessionNavigationBean, @@ -197,6 +198,38 @@ public BaseResponseBean navigateSessionWithAuth(@RequestBody SessionNavigationBe } } + public static void logCaseTypeColumnIfPresent(BaseResponseBean response, String label, Log log) { + + if (response instanceof EntityListResponse entityListResponse && + entityListResponse.getEntities().length > 0 && + entityListResponse.getHeaders().length > 0 && + entityListResponse.getHeaders()[0].equals("Case Type") + ) { + StringBuilder sb = new StringBuilder("USH-6370 Checking at '" + label + "' "); + + Set caseTypes = new HashSet<>(); + for (EntityBean entity : entityListResponse.getEntities()) { + caseTypes.add(entity.getData()[0].toString()); + } + if (caseTypes.size() > 1) { + sb.append("mismatch"); + sb.append("\nExpected all 'Case Type's to be the same at '") + .append(label) + .append("'. Got: ") + .append(caseTypes); + } else { + sb.append("ok"); + } + for (EntityBean entity : entityListResponse.getEntities()) { + sb.append("\n") + .append(entity.getId()) + .append(": ") + .append(entity.getData()[0].toString()); + } + log.error(sb.toString()); + } + } + private void setResponseMetaData(BaseResponseBean response) { ResponseMetaData responseMetaData = new ResponseMetaData(responseMetaDataTracker.isAttemptRestore(), responseMetaDataTracker.isNewInstall()); diff --git a/src/main/java/org/commcare/formplayer/beans/menus/EntityListResponse.java b/src/main/java/org/commcare/formplayer/beans/menus/EntityListResponse.java index 36478af50..8f5acec43 100644 --- a/src/main/java/org/commcare/formplayer/beans/menus/EntityListResponse.java +++ b/src/main/java/org/commcare/formplayer/beans/menus/EntityListResponse.java @@ -2,6 +2,8 @@ import static org.commcare.formplayer.util.Constants.TOGGLE_SPLIT_SCREEN_CASE_SEARCH; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.commcare.cases.entity.Entity; import org.commcare.core.graph.model.GraphData; import org.commcare.core.graph.util.GraphException; @@ -29,11 +31,7 @@ import org.springframework.web.util.UriBuilder; import org.springframework.web.util.UriTemplate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Vector; +import java.util.*; import java.util.function.Predicate; import datadog.trace.api.Trace; @@ -42,6 +40,8 @@ * Created by willpride on 4/13/16. */ public class EntityListResponse extends MenuBean { + private static final Log log = LogFactory.getLog(EntityListResponse.class); + private EntityBean[] entities; private DisplayElement[] actions; private String redoLast; @@ -83,12 +83,24 @@ public EntityListResponse(EntityScreen nextScreen) { // subscreen should be of type EntityListSubscreen in order to init this response class Subscreen subScreen = nextScreen.getCurrentScreen(); if (subScreen instanceof EntityListSubscreen) { - EntityListSubscreen entityListScreen = ((EntityListSubscreen)nextScreen.getCurrentScreen()); + EntityListSubscreen entityListScreen = ((EntityListSubscreen) nextScreen.getCurrentScreen()); Vector entityListActions = entityListScreen.getActions(); this.actions = processActions(nextScreen.getSession(), entityListActions); this.redoLast = processRedoLast(entityListActions); List> entityList = entityListScreen.getEntities(); + StringBuilder sb1 = new StringBuilder("USH-6370 Checking at 'entityListScreen.getEntities' "); + Set caseTypes = new HashSet<>(); + for (Entity e: entityList) { + caseTypes.add(e.getData()[0].toString()); + } + if (caseTypes.size() > 1) { + sb1.append("mismatch"); + sb1.append("\nExpected all 'Case Type's to be the same at. Got: ") + .append(caseTypes); + } else { + sb1.append("ok"); + } EntityScreenContext entityScreenContext = nextScreen.getEntityScreenContext(); int casesPerPage = entityScreenContext.getCasesPerPage(); casesPerPage = Math.min(casesPerPage, MAX_CASES_PER_PAGE); @@ -96,12 +108,44 @@ public EntityListResponse(EntityScreen nextScreen) { Detail detail = nextScreen.getShortDetail(); List> entitesForPage = paginateEntities(entityList, detail, casesPerPage, offset); +// StringBuilder sb2 = new StringBuilder("USH-6370 Checking at 'paginateEntities' "); +// caseTypes = new HashSet<>(); +// for (Entity e: entityList) { +// caseTypes.add(e.getData()[0].toString()); +// } +// if (caseTypes.size() > 1) { +// sb2.append("mismatch"); +// sb2.append("\nExpected all 'Case Type's to be the same. Got: ") +// .append(caseTypes); +// } else { +// sb2.append("ok"); +// } EvaluationContext ec = nextScreen.getEvalContext(); SessionWrapper session = nextScreen.getSession(); - EntityDatum neededDatum = (EntityDatum)session.getNeededDatum(); + EntityDatum neededDatum = (EntityDatum) session.getNeededDatum(); List entityBeans = processEntitiesForCaseList(entitesForPage, ec, neededDatum); entities = new EntityBean[entityBeans.size()]; entityBeans.toArray(entities); + +// caseTypes = new HashSet<>(); +// for (EntityBean entity : entities) { +// caseTypes.add(entity.getData()[0].toString()); +// } +// StringBuilder sb = new StringBuilder("USH-6370 Checking at 'processEntitiesForCaseList' "); +// if (caseTypes.size() > 1) { +// sb.append("mismatch"); +// sb.append("\nExpected all 'Case Type's to be the same. Got: ") +// .append(caseTypes); +// for (EntityBean entity : entities) { +// sb.append("\n") +// .append(entity.getId()) +// .append(": ") +// .append(entity.getData()[0].toString()); +// } +// } else { +// sb.append("ok"); +// } + setNoItemsText(getNoItemsTextLocaleString(detail)); setSelectText(getSelectTextLocaleString(detail)); hasDetails = nextScreen.getLongDetail() != null; @@ -117,7 +161,7 @@ public EntityListResponse(EntityScreen nextScreen) { this.sortIndices = detail.getOrderedFieldIndicesForSorting(); isMultiSelect = nextScreen instanceof MultiSelectEntityScreen; if (isMultiSelect) { - maxSelectValue = ((MultiSelectEntityScreen)nextScreen).getMaxSelectValue(); + maxSelectValue = ((MultiSelectEntityScreen) nextScreen).getMaxSelectValue(); } if (detail.getGroup() != null) { groupHeaderRows = detail.getGroup().getHeaderRows(); @@ -125,11 +169,17 @@ public EntityListResponse(EntityScreen nextScreen) { QueryScreen queryScreen = nextScreen.getQueryScreen(); if (queryScreen != null) { +// sb.append("\nqueryScreen"); setQueryKey(queryScreen.getQueryKey()); if (FeatureFlagChecker.isToggleEnabled(TOGGLE_SPLIT_SCREEN_CASE_SEARCH)) { queryResponse = new QueryResponseBean(queryScreen); } } + if (this.headers.length > 0 && this.headers[0].equals("Case Type")) { + log.error(sb1.toString()); +// log.error(sb2.toString()); +// log.error(sb.toString()); + } } } @@ -168,10 +218,22 @@ private static EntityBean[] processEntitiesForCaseDetail(Detail detail, TreeRefe public static List processEntitiesForCaseList(List> entityList, EvaluationContext ec, EntityDatum neededDatum) { + StringBuilder sb = new StringBuilder("USH-6370 Checking in 'processEntitiesForCaseList' "); + List entities = new ArrayList<>(); + StringBuilder innerSb = new StringBuilder(); for (Entity entity : entityList) { - entities.add(toEntityBean(entity, ec, neededDatum)); + EntityBean bean = toEntityBean(entity, ec, neededDatum, innerSb); + entities.add(bean); } + if (innerSb.isEmpty()) { + sb.append("ok"); + } else { + sb.append("missmatch") + .append(innerSb); + } + + log.error(sb.toString()); return entities; } @@ -235,14 +297,29 @@ public boolean isHasDetails() { // Converts the Given Entity to EntityBean @Trace private static EntityBean toEntityBean(Entity entity, - EvaluationContext ec, EntityDatum neededDatum) { + EvaluationContext ec, EntityDatum neededDatum, StringBuilder sb) { Object[] entityData = entity.getData(); Object[] data = new Object[entityData.length]; String id = getEntityId(entity.getElement(), neededDatum, ec); + + // Log entity conversion details +// log.error(String.format("toEntityBean: ref=%s, id=%s, rawData[0]=%s", +// entity.getElement().toString(), +// id, +// entityData.length > 0 ? entityData[0] : "none")); + + EntityBean entityBean = new EntityBean(id); for (int i = 0; i < entityData.length; i++) { data[i] = processData(entityData[i]); + if (i == 0 && !String.valueOf(entityData[i]).equals(String.valueOf(data[i]))) { + sb.append("\n").append(id) + .append(" Data[").append(i).append("] changed during processing: ") + .append(entityData[i]).append(" -> ").append(data[i]); + } } + + entityBean.setData(data); entityBean.setGroupKey(entity.getGroupKey()); entityBean.setAltText(entity.getAltText()); diff --git a/src/main/java/org/commcare/formplayer/services/CaseSearchHelper.java b/src/main/java/org/commcare/formplayer/services/CaseSearchHelper.java index 43b0c3401..15d57f266 100644 --- a/src/main/java/org/commcare/formplayer/services/CaseSearchHelper.java +++ b/src/main/java/org/commcare/formplayer/services/CaseSearchHelper.java @@ -3,8 +3,10 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; +import io.sentry.SentryLevel; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.commcare.cases.entity.Entity; import org.commcare.cases.instance.CaseInstanceTreeElement; import org.commcare.cases.model.Case; import org.commcare.core.parse.CaseInstanceXmlTransactionParserFactory; @@ -16,15 +18,13 @@ import org.commcare.formplayer.session.MenuSession; import org.commcare.formplayer.sqlitedb.CaseSearchDB; import org.commcare.formplayer.sqlitedb.SQLiteDB; +import org.commcare.formplayer.util.FormplayerSentry; import org.commcare.formplayer.util.SerializationUtil; import org.commcare.formplayer.web.client.WebClient; import org.commcare.util.screen.ScreenUtils; -import org.javarosa.core.model.instance.AbstractTreeElement; -import org.javarosa.core.model.instance.ExternalDataInstance; -import org.javarosa.core.model.instance.ExternalDataInstanceSource; -import org.javarosa.core.model.instance.InstanceBase; -import org.javarosa.core.model.instance.TreeElement; +import org.javarosa.core.model.instance.*; import org.javarosa.core.model.instance.utils.TreeUtilities; +import org.javarosa.core.services.storage.IStorageIterator; import org.javarosa.core.services.storage.IStorageUtilityIndexed; import org.javarosa.core.util.MD5; import org.javarosa.core.util.externalizable.ExtUtil; @@ -43,10 +43,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; @CacheConfig(cacheNames = "case_search") @Component @@ -82,6 +79,9 @@ public AbstractTreeElement getExternalRoot(String instanceId, ExternalDataInstan throws UnfullfilledRequirementsException, XmlPullParserException, InvalidStructureException, IOException { + skipCache = true; +// log.error("Error getExternalRoot skipCache: " + skipCache); + Multimap requestData = source.getRequestData(); String url = source.getSourceUri(); @@ -98,18 +98,22 @@ public AbstractTreeElement getExternalRoot(String instanceId, ExternalDataInstan IStorageUtilityIndexed caseSearchStorage = caseSearchSandbox.getCaseStorage(); FormplayerCaseIndexTable caseSearchIndexTable = getCaseIndexTable(caseSearchSandbox, caseSearchTableName); if (skipCache || !caseSearchStorage.isStorageExists()) { + Collection requestCaseTypes = requestData.get("case_type"); String responseString = webClient.postFormData(url, requestData); if (responseString != null) { byte[] responseBytes = responseString.getBytes(StandardCharsets.UTF_8); + validateCaseTypesInResponse(responseBytes, instanceId, requestCaseTypes); ByteArrayInputStream responeStream = new ByteArrayInputStream(responseBytes); if (shouldParseIntoCaseSearchStorage(source.useCaseTemplate())) { parseIntoCaseSearchStorage(caseSearchDb, caseSearchSandbox, caseSearchStorage, responeStream, caseSearchIndexTable); + validateCaseTypesInStorage(caseSearchStorage, requestCaseTypes, url, requestData); } else { TreeElement root = TreeUtilities.xmlStreamToTreeElement(responeStream, instanceId); if (root != null) { cache.put(cacheKey, root); } + return root; } } @@ -168,6 +172,80 @@ private boolean shouldParseIntoCaseSearchStorage(boolean useCaseTemplate) { return useCaseTemplate && storageFactory.getPropertyManager().isIndexCaseSearchResults(); } + private void validateCaseTypesInResponse(byte[] responseBytes, String instanceId, Collection requestCaseTypes) throws UnfullfilledRequirementsException, XmlPullParserException, InvalidStructureException,IOException { + + StringBuilder sb = new StringBuilder("USH-6370 Checking in 'validateCaseTypesInResponse' "); + + if (requestCaseTypes == null || requestCaseTypes.isEmpty()) { + sb.append("No case_type in request data - skipping validation"); + } else { + ByteArrayInputStream responseStream = new ByteArrayInputStream(responseBytes); + TreeElement root = TreeUtilities.xmlStreamToTreeElement(responseStream, instanceId); + + if (root == null) { + sb.append("Root element is null - cannot validate"); + return; + } else { + Set caseTypes = new HashSet<>(); + for (int i = 0; i < root.getNumChildren(); i++) { + TreeElement child = root.getChildAt(i); + caseTypes.add(child.getAttributeValue(null, "case_type")); + } + if (caseTypes.stream().anyMatch(type -> !requestCaseTypes.contains(type))) { + sb.append("mismatch requested: ") + .append(requestCaseTypes) + .append(" got: ") + .append(caseTypes); + + } else { + sb.append("ok"); + } + for (int i = 0; i < root.getNumChildren(); i++) { + TreeElement child = root.getChildAt(i); + String caseType = child.getAttributeValue(null, "case_type"); + String caseId = child.getAttributeValue(null, "case_id"); + sb.append("\n") + .append(caseId) + .append(": ") + .append(caseType); + } + } + } + + log.error(sb.toString()); + + } + + private void validateCaseTypesInStorage(IStorageUtilityIndexed caseSearchStorage, + Collection requestCaseTypes, String url, Multimap requestData) { + + StringBuilder sb = new StringBuilder("USH-6370 Checking in 'validateCaseTypesInStorage' "); + if (requestCaseTypes == null || requestCaseTypes.isEmpty()) { + sb.append("No case_type in request data - skipping validation"); + } else { + try { + Set caseTypes = new HashSet<>(); + IStorageIterator iterator = caseSearchStorage.iterate(); + while (iterator.hasMore()) { + Case caseObj = iterator.nextRecord(); + String caseType = caseObj.getTypeId(); + caseTypes.add(caseType); + } + if (caseTypes.stream().anyMatch(type -> !requestCaseTypes.contains(type))) { + sb.append("mismatch requested: ") + .append(requestCaseTypes) + .append(" got: ") + .append(caseTypes); + } else { + sb.append("ok"); + } + } catch (Exception e) { + sb.append("Error validating case types from storage"); + } + } + log.error(sb.toString()); + } + private TreeElement getCachedRoot(Cache cache, String cacheKey, String url, boolean skipCache) { if (skipCache) { log.info("Skipping cache check for case search results"); diff --git a/src/main/java/org/commcare/formplayer/services/MenuSessionFactory.java b/src/main/java/org/commcare/formplayer/services/MenuSessionFactory.java index b1d8d8ce6..e0339656e 100644 --- a/src/main/java/org/commcare/formplayer/services/MenuSessionFactory.java +++ b/src/main/java/org/commcare/formplayer/services/MenuSessionFactory.java @@ -119,7 +119,9 @@ public Screen rebuildSessionFromFrame(MenuSession menuSession, CaseSearchHelper } } else if (screen instanceof EntityScreen) { EntityScreen entityScreen = (EntityScreen)screen; - entityScreen.initReferences(menuSession.getSessionWrapper()); + StringBuilder sb = new StringBuilder(); + entityScreen.initReferences(menuSession.getSessionWrapper(), sb); + log.error(sb.toString()); SessionDatum neededDatum = entityScreen.getSession().getNeededDatum(); for (StackFrameStep step : steps) { if (step.getId().equals(neededDatum.getDataId())) { diff --git a/src/main/java/org/commcare/formplayer/services/MenuSessionRunnerService.java b/src/main/java/org/commcare/formplayer/services/MenuSessionRunnerService.java index d935bb2b3..74d8cebff 100644 --- a/src/main/java/org/commcare/formplayer/services/MenuSessionRunnerService.java +++ b/src/main/java/org/commcare/formplayer/services/MenuSessionRunnerService.java @@ -14,6 +14,7 @@ import org.apache.commons.logging.LogFactory; import org.commcare.core.interfaces.RemoteInstanceFetcher; import org.commcare.data.xml.VirtualInstances; +import org.commcare.formplayer.application.MenuController; import org.commcare.formplayer.beans.NewFormResponse; import org.commcare.formplayer.beans.NotificationMessage; import org.commcare.formplayer.beans.auth.FeatureFlagChecker; @@ -191,6 +192,7 @@ public BaseResponseBean getNextMenu(@Nullable Screen nextScreen, MenuSession men } addHereFuncHandler((EntityScreen)nextScreen, menuSession); menuResponseBean = new EntityListResponse((EntityScreen)nextScreen); +// MenuController.logCaseTypeColumnIfPresent(menuResponseBean, "getNextMenu", log); datadog.addRequestScopedTag(Constants.MODULE_TAG, "case_list"); Sentry.setTag(Constants.MODULE_TAG, "case_list"); // using getBestTitle to eliminate risk of showing private information @@ -301,6 +303,7 @@ public BaseResponseBean advanceSessionWithSelections(MenuSession menuSession, NotificationMessage.Tag.sync), true); } +// log.error("USH-6370_2: advanceSessionWithSelections: postSyncResponse"); return postSyncResponse; } } else { @@ -325,6 +328,7 @@ public BaseResponseBean advanceSessionWithSelections(MenuSession menuSession, queryData, entityScreenContext ); +// MenuController.logCaseTypeColumnIfPresent(nextResponse, "advanceSessionWithSelections", log); restoreFactory.cacheSessionSelections(menuSession.getSelections()); if (nextResponse != null) { @@ -333,11 +337,13 @@ public BaseResponseBean advanceSessionWithSelections(MenuSession menuSession, } log.info("Returning menu: " + nextResponse); nextResponse.setSelections(menuSession.getSelections()); +// log.error("USH-6370_2: advanceSessionWithSelections: nextResponse"); return nextResponse; } else { if (notificationMessage == null) { notificationMessage = new NotificationMessage(null, false, NotificationMessage.Tag.selection); } +// log.error("USH-6370_2: advanceSessionWithSelections: BaseResponseBean"); return new BaseResponseBean(null, notificationMessage, true); } } @@ -390,7 +396,9 @@ private Screen autoAdvanceSession( // Auto select if we have not advanced as part of auto launch // avoiding unnecessary screen init by skipping the original screen if (!sessionAdvanced && iterationCount != 0) { - nextScreen.init(menuSession.getSessionWrapper()); + StringBuilder sb = new StringBuilder(); + nextScreen.init(menuSession.getSessionWrapper(), sb); + log.error(sb.toString()); if (nextScreen.shouldBeSkipped()) { sessionAdvanced = ((EntityScreen)nextScreen).autoSelectEntities( menuSession.getSessionWrapper());