From ad74b23f549b1860761d5ed6edf3afb33853fe2a Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 17 Apr 2026 09:49:13 -0700 Subject: [PATCH 1/3] GitHub Issue 1073: Updating a List MVTC field shows array in audit for values with quotes --- .../api/audit/AbstractAuditTypeProvider.java | 787 ++--- .../org/labkey/list/model/ListManager.java | 2968 ++++++++--------- 2 files changed, 1881 insertions(+), 1874 deletions(-) diff --git a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java index 488691bd02c..5e811a43d4d 100644 --- a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java +++ b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java @@ -1,390 +1,397 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.audit.data.DataMapColumn; -import org.labkey.api.audit.data.DataMapDiffColumn; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.AbstractTableInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.ExistingRecordDataIterator; -import org.labkey.api.exp.ChangePropertyDescriptorException; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.ActionURL; - -import java.sql.Time; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.labkey.api.audit.query.AbstractAuditDomainKind.AUDIT_RECORD_DATA_MAP_CONCEPT_URI; -import static org.labkey.api.audit.query.AbstractAuditDomainKind.NEW_RECORD_PROP_NAME; -import static org.labkey.api.audit.query.AbstractAuditDomainKind.OLD_RECORD_PROP_NAME; - -public abstract class AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_SCHEMA_NAME = "auditLog"; - public static final String SCHEMA_NAME = "audit"; - - public static final String COLUMN_NAME_ROW_ID = "RowId"; - public static final String COLUMN_NAME_CONTAINER = "Container"; - public static final String COLUMN_NAME_COMMENT = "Comment"; - public static final String COLUMN_NAME_USER_COMMENT = "UserComment"; - public static final String COLUMN_NAME_EVENT_TYPE = "EventType"; - public static final String COLUMN_NAME_CREATED = "Created"; - public static final String COLUMN_NAME_CREATED_BY = "CreatedBy"; - public static final String COLUMN_NAME_IMPERSONATED_BY = "ImpersonatedBy"; - public static final String COLUMN_NAME_PROJECT_ID = "ProjectId"; - public static final String COLUMN_NAME_TRANSACTION_ID = "TransactionID"; - public static final String COLUMN_NAME_DATA_CHANGES = "DataChanges"; - - private final AbstractAuditDomainKind _domainKind; - - public AbstractAuditTypeProvider(@NotNull AbstractAuditDomainKind domainKind) - { - // TODO: consolidate domain kind initialization to this constructor and stop overriding getDomainKind() - _domainKind = domainKind; - // Register the DomainKind - PropertyService.get().registerDomainKind(getDomainKind()); - } - - protected final AbstractAuditDomainKind getDomainKind() - { - if (_domainKind == null) - throw new IllegalStateException(String.format("The audit type : \"%s\" has a null domain kind", getLabel())); - - return _domainKind; - } - - // Expose the domain kind to AbstractAuditDomainKind$TestCase without touching every subclass - public AbstractAuditDomainKind getAuditDomainKind() - { - return getDomainKind(); - } - - @Override - public final void initializeProvider(User user) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - domainKind.validate(); - - Domain domain = getDomain(true); - - // if the domain doesn't exist, create it - if (domain == null) - { - try - { - String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); - domain = PropertyService.get().createDomain(getDomainContainer(), domainURI, domainKind.getKindName()); - for (PropertyDescriptor pd : domainKind.getProperties()) - { - domain.addPropertyOfPropertyDescriptor(pd); - } - domain.save(user); - domain = getDomain(true); - } - catch (ChangePropertyDescriptorException e) - { - throw new RuntimeException(e); - } - } - - // adjust potential domain kind changes - ensureProperties(user, domain); - } - - // NOTE: Changing the name of an existing PropertyDescriptor will lose data! - private void ensureProperties(User user, Domain domain) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - if (domain != null && domainKind != null) - { - // Create a map of desired properties - Map props = new CaseInsensitiveHashMap<>(); - for (PropertyDescriptor pd : domainKind.getProperties()) - props.put(pd.getName(), pd); - - // Create a map of existing properties - Map current = new CaseInsensitiveHashMap<>(); - for (DomainProperty dp : domain.getProperties()) - { - current.put(dp.getName(), dp); - } - - Set toAdd = new LinkedHashSet<>(); - for (PropertyDescriptor pd : props.values()) - if (!current.containsKey(pd.getName())) - toAdd.add(pd); - - Set toUpdate = new LinkedHashSet<>(); - boolean changed = false; - - for (DomainProperty dp : current.values()) - { - if (props.containsKey(dp.getName())) - toUpdate.add(dp); - else - { - dp.delete(); - changed = true; - } - } - - for (PropertyDescriptor pd : toAdd) - { - domain.addPropertyOfPropertyDescriptor(pd); - } - - try (DbScope.Transaction transaction = domainKind.getScope().ensureTransaction()) - { - // CONSIDER: Avoid always updating the existing properties -- only update changed props. - for (DomainProperty dp : toUpdate) - { - PropertyDescriptor desired = props.get(dp.getName()); - assert desired != null; - - if (differ(desired, dp, domain.getContainer())) - { - changed = true; - copyTo(dp, desired, domain.getContainer()); - } - } - - changed = changed || !toAdd.isEmpty(); - if (changed) - { - domain.save(user); - } - - assert domain.getStorageTableName() != null; - assert domain.getDomainKind() != null; - assert domain.getDomainKind().getClass().equals(domainKind.getClass()); - - StorageProvisioner.get().ensureTableIndices(domain); - transaction.commit(); - } - catch (ChangePropertyDescriptorException e) - { - throw new RuntimeException(e); - } - } - } - - // Issue 26311: We want to trigger a save if the scale has changed - // CONSIDER: check for other differences here as well. - private boolean differ(PropertyDescriptor pd, DomainProperty dp, Container c) - { - return dp.getScale() != pd.getScale() || !dp.getRangeURI().equals(pd.getRangeURI()); - } - - private void copyTo(DomainProperty dp, PropertyDescriptor pd, Container c) - { - dp.setRangeURI(pd.getRangeURI()); - dp.setLabel(pd.getLabel()); - dp.setRequired(pd.isRequired()); - dp.setHidden(pd.isHidden()); - dp.setMvEnabled(pd.isMvEnabled()); - dp.setScale(pd.getScale()); - if (pd.getDefaultValueType() != null) - dp.setDefaultValueTypeEnum(DefaultValueType.valueOf(pd.getDefaultValueType())); - } - - @Override - public final Domain getDomain() - { - return getDomain(false); - } - - @Override - public final Domain getDomain(boolean forUpdate) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - - String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); - - return PropertyService.get().getDomain(getDomainContainer(), domainURI, forUpdate); - } - - - protected DbSchema getSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Provisioned); - } - - - public TableInfo createStorageTableInfo() - { - Domain domain = getDomain(); - if (null == domain) - throw new NullPointerException("Could not find domain for " + getEventName()); - return StorageProvisioner.createTableInfo(domain); - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()); - } - - public List getDefaultVisibleColumns() - { - return null; - } - - public static Container getDomainContainer() - { - return ContainerManager.getSharedContainer(); - } - - protected void appendValueMapColumns(AbstractTableInfo table) - { - appendValueMapColumns(table, null); - } - - protected void appendValueMapColumns(AbstractTableInfo table, String eventName) - { - appendValueMapColumns(table, eventName, false); - } - - protected void appendValueMapColumns(AbstractTableInfo table, String eventName, boolean noUrl) - { - MutableColumnInfo oldCol = table.getMutableColumn(FieldKey.fromString(OLD_RECORD_PROP_NAME)); - MutableColumnInfo newCol = table.getMutableColumn(FieldKey.fromString(NEW_RECORD_PROP_NAME)); - - if (oldCol != null) - { - var added = table.addColumn(new AliasedColumn(table, "OldValues", oldCol)); - added.setDisplayColumnFactory(DataMapColumn::new); - added.setLabel(AbstractAuditDomainKind.OLD_RECORD_PROP_CAPTION); - added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); - oldCol.setHidden(true); - } - - if (newCol != null) - { - var added = table.addColumn(new AliasedColumn(table, "NewValues", newCol)); - added.setDisplayColumnFactory(DataMapColumn::new); - added.setLabel(AbstractAuditDomainKind.NEW_RECORD_PROP_CAPTION); - added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); - newCol.setHidden(true); - } - - // add a column to show the differences between old and new values - if (oldCol != null && newCol != null) - table.addColumn(new DataMapDiffColumn(table, COLUMN_NAME_DATA_CHANGES, oldCol, newCol)); - - if (!noUrl) - { - String urlStr = "audit-detailedAuditChanges.view?auditRowId=${rowId}"; - if (!StringUtils.isEmpty(eventName)) - urlStr = urlStr + "&auditEventType=" + eventName; - DetailsURL url = DetailsURL.fromString(urlStr); - url.setStrictContainerContextEval(true); - table.setDetailsURL(url); - } - - } - - @Override - public ActionURL getAuditUrl() - { - return AuditLogService.get().getAuditUrl(); - } - - public static Map decodeFromDataMap(String properties) - { - try - { - if (properties != null) - { - return PageFlowUtil.mapFromQueryString(properties); - } - return Collections.emptyMap(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - public static String encodeForDataMap(Map properties) - { - if (properties == null) return null; - - Map stringMap = new LinkedHashMap<>(); - for (Map.Entry entry : properties.entrySet()) - { - // see AuditHandler.getRecordForInsert(), rather than create a new map just skip values here - if (entry.getKey().equals(DataIterator.ROWNUMBER_COLUMNNAME) || - entry.getKey().equals(ExistingRecordDataIterator.EXISTING_RECORD_COLUMN_NAME) || - entry.getKey().equals(ExperimentService.ALIASCOLUMNALIAS)) - continue; - Object value = entry.getValue(); - if (value instanceof Time time) - { - String formatted = DateUtil.formatIsoLongTime(time); - stringMap.put(entry.getKey(), formatted); - } - else if (value instanceof Date date) - { - // Issue 35002 - normalize Date values to avoid Timestamp/Date toString differences - // Issue 36472 - use iso format to show date-time values - String formatted = DateUtil.toISO(date); - stringMap.put(entry.getKey(), formatted); - } - else - stringMap.put(entry.getKey(), value == null ? null : value.toString()); - } - return PageFlowUtil.toQueryString(stringMap.entrySet()); - } - - public int moveEvents(Container targetContainer, String idColumnName, Collection ids) - { - return Table.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.audit.data.DataMapColumn; +import org.labkey.api.audit.data.DataMapDiffColumn; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.MultiChoice; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.ExistingRecordDataIterator; +import org.labkey.api.exp.ChangePropertyDescriptorException; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; + +import java.sql.Time; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.labkey.api.audit.query.AbstractAuditDomainKind.AUDIT_RECORD_DATA_MAP_CONCEPT_URI; +import static org.labkey.api.audit.query.AbstractAuditDomainKind.NEW_RECORD_PROP_NAME; +import static org.labkey.api.audit.query.AbstractAuditDomainKind.OLD_RECORD_PROP_NAME; + +public abstract class AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_SCHEMA_NAME = "auditLog"; + public static final String SCHEMA_NAME = "audit"; + + public static final String COLUMN_NAME_ROW_ID = "RowId"; + public static final String COLUMN_NAME_CONTAINER = "Container"; + public static final String COLUMN_NAME_COMMENT = "Comment"; + public static final String COLUMN_NAME_USER_COMMENT = "UserComment"; + public static final String COLUMN_NAME_EVENT_TYPE = "EventType"; + public static final String COLUMN_NAME_CREATED = "Created"; + public static final String COLUMN_NAME_CREATED_BY = "CreatedBy"; + public static final String COLUMN_NAME_IMPERSONATED_BY = "ImpersonatedBy"; + public static final String COLUMN_NAME_PROJECT_ID = "ProjectId"; + public static final String COLUMN_NAME_TRANSACTION_ID = "TransactionID"; + public static final String COLUMN_NAME_DATA_CHANGES = "DataChanges"; + + private final AbstractAuditDomainKind _domainKind; + + public AbstractAuditTypeProvider(@NotNull AbstractAuditDomainKind domainKind) + { + // TODO: consolidate domain kind initialization to this constructor and stop overriding getDomainKind() + _domainKind = domainKind; + // Register the DomainKind + PropertyService.get().registerDomainKind(getDomainKind()); + } + + protected final AbstractAuditDomainKind getDomainKind() + { + if (_domainKind == null) + throw new IllegalStateException(String.format("The audit type : \"%s\" has a null domain kind", getLabel())); + + return _domainKind; + } + + // Expose the domain kind to AbstractAuditDomainKind$TestCase without touching every subclass + public AbstractAuditDomainKind getAuditDomainKind() + { + return getDomainKind(); + } + + @Override + public final void initializeProvider(User user) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + domainKind.validate(); + + Domain domain = getDomain(true); + + // if the domain doesn't exist, create it + if (domain == null) + { + try + { + String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); + domain = PropertyService.get().createDomain(getDomainContainer(), domainURI, domainKind.getKindName()); + for (PropertyDescriptor pd : domainKind.getProperties()) + { + domain.addPropertyOfPropertyDescriptor(pd); + } + domain.save(user); + domain = getDomain(true); + } + catch (ChangePropertyDescriptorException e) + { + throw new RuntimeException(e); + } + } + + // adjust potential domain kind changes + ensureProperties(user, domain); + } + + // NOTE: Changing the name of an existing PropertyDescriptor will lose data! + private void ensureProperties(User user, Domain domain) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + if (domain != null && domainKind != null) + { + // Create a map of desired properties + Map props = new CaseInsensitiveHashMap<>(); + for (PropertyDescriptor pd : domainKind.getProperties()) + props.put(pd.getName(), pd); + + // Create a map of existing properties + Map current = new CaseInsensitiveHashMap<>(); + for (DomainProperty dp : domain.getProperties()) + { + current.put(dp.getName(), dp); + } + + Set toAdd = new LinkedHashSet<>(); + for (PropertyDescriptor pd : props.values()) + if (!current.containsKey(pd.getName())) + toAdd.add(pd); + + Set toUpdate = new LinkedHashSet<>(); + boolean changed = false; + + for (DomainProperty dp : current.values()) + { + if (props.containsKey(dp.getName())) + toUpdate.add(dp); + else + { + dp.delete(); + changed = true; + } + } + + for (PropertyDescriptor pd : toAdd) + { + domain.addPropertyOfPropertyDescriptor(pd); + } + + try (DbScope.Transaction transaction = domainKind.getScope().ensureTransaction()) + { + // CONSIDER: Avoid always updating the existing properties -- only update changed props. + for (DomainProperty dp : toUpdate) + { + PropertyDescriptor desired = props.get(dp.getName()); + assert desired != null; + + if (differ(desired, dp, domain.getContainer())) + { + changed = true; + copyTo(dp, desired, domain.getContainer()); + } + } + + changed = changed || !toAdd.isEmpty(); + if (changed) + { + domain.save(user); + } + + assert domain.getStorageTableName() != null; + assert domain.getDomainKind() != null; + assert domain.getDomainKind().getClass().equals(domainKind.getClass()); + + StorageProvisioner.get().ensureTableIndices(domain); + transaction.commit(); + } + catch (ChangePropertyDescriptorException e) + { + throw new RuntimeException(e); + } + } + } + + // Issue 26311: We want to trigger a save if the scale has changed + // CONSIDER: check for other differences here as well. + private boolean differ(PropertyDescriptor pd, DomainProperty dp, Container c) + { + return dp.getScale() != pd.getScale() || !dp.getRangeURI().equals(pd.getRangeURI()); + } + + private void copyTo(DomainProperty dp, PropertyDescriptor pd, Container c) + { + dp.setRangeURI(pd.getRangeURI()); + dp.setLabel(pd.getLabel()); + dp.setRequired(pd.isRequired()); + dp.setHidden(pd.isHidden()); + dp.setMvEnabled(pd.isMvEnabled()); + dp.setScale(pd.getScale()); + if (pd.getDefaultValueType() != null) + dp.setDefaultValueTypeEnum(DefaultValueType.valueOf(pd.getDefaultValueType())); + } + + @Override + public final Domain getDomain() + { + return getDomain(false); + } + + @Override + public final Domain getDomain(boolean forUpdate) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + + String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); + + return PropertyService.get().getDomain(getDomainContainer(), domainURI, forUpdate); + } + + + protected DbSchema getSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Provisioned); + } + + + public TableInfo createStorageTableInfo() + { + Domain domain = getDomain(); + if (null == domain) + throw new NullPointerException("Could not find domain for " + getEventName()); + return StorageProvisioner.createTableInfo(domain); + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()); + } + + public List getDefaultVisibleColumns() + { + return null; + } + + public static Container getDomainContainer() + { + return ContainerManager.getSharedContainer(); + } + + protected void appendValueMapColumns(AbstractTableInfo table) + { + appendValueMapColumns(table, null); + } + + protected void appendValueMapColumns(AbstractTableInfo table, String eventName) + { + appendValueMapColumns(table, eventName, false); + } + + protected void appendValueMapColumns(AbstractTableInfo table, String eventName, boolean noUrl) + { + MutableColumnInfo oldCol = table.getMutableColumn(FieldKey.fromString(OLD_RECORD_PROP_NAME)); + MutableColumnInfo newCol = table.getMutableColumn(FieldKey.fromString(NEW_RECORD_PROP_NAME)); + + if (oldCol != null) + { + var added = table.addColumn(new AliasedColumn(table, "OldValues", oldCol)); + added.setDisplayColumnFactory(DataMapColumn::new); + added.setLabel(AbstractAuditDomainKind.OLD_RECORD_PROP_CAPTION); + added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); + oldCol.setHidden(true); + } + + if (newCol != null) + { + var added = table.addColumn(new AliasedColumn(table, "NewValues", newCol)); + added.setDisplayColumnFactory(DataMapColumn::new); + added.setLabel(AbstractAuditDomainKind.NEW_RECORD_PROP_CAPTION); + added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); + newCol.setHidden(true); + } + + // add a column to show the differences between old and new values + if (oldCol != null && newCol != null) + table.addColumn(new DataMapDiffColumn(table, COLUMN_NAME_DATA_CHANGES, oldCol, newCol)); + + if (!noUrl) + { + String urlStr = "audit-detailedAuditChanges.view?auditRowId=${rowId}"; + if (!StringUtils.isEmpty(eventName)) + urlStr = urlStr + "&auditEventType=" + eventName; + DetailsURL url = DetailsURL.fromString(urlStr); + url.setStrictContainerContextEval(true); + table.setDetailsURL(url); + } + + } + + @Override + public ActionURL getAuditUrl() + { + return AuditLogService.get().getAuditUrl(); + } + + public static Map decodeFromDataMap(String properties) + { + try + { + if (properties != null) + { + return PageFlowUtil.mapFromQueryString(properties); + } + return Collections.emptyMap(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + public static String encodeForDataMap(Map properties) + { + if (properties == null) return null; + + Map stringMap = new LinkedHashMap<>(); + for (Map.Entry entry : properties.entrySet()) + { + // see AuditHandler.getRecordForInsert(), rather than create a new map just skip values here + if (entry.getKey().equals(DataIterator.ROWNUMBER_COLUMNNAME) || + entry.getKey().equals(ExistingRecordDataIterator.EXISTING_RECORD_COLUMN_NAME) || + entry.getKey().equals(ExperimentService.ALIASCOLUMNALIAS)) + continue; + Object value = entry.getValue(); + if (value instanceof Time time) + { + String formatted = DateUtil.formatIsoLongTime(time); + stringMap.put(entry.getKey(), formatted); + } + else if (value instanceof Date date) + { + // Issue 35002 - normalize Date values to avoid Timestamp/Date toString differences + // Issue 36472 - use iso format to show date-time values + String formatted = DateUtil.toISO(date); + stringMap.put(entry.getKey(), formatted); + } + else if (value instanceof java.sql.Array arr) + { + // GitHub Issue 1073: Updating a List MVTC field shows array in audit for values with quotes + var arrayVal = MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, arr); + stringMap.put(entry.getKey(), PageFlowUtil.joinValuesToStringForExport(arrayVal)); + } + else + stringMap.put(entry.getKey(), value == null ? null : value.toString()); + } + return PageFlowUtil.toQueryString(stringMap.entrySet()); + } + + public int moveEvents(Container targetContainer, String idColumnName, Collection ids) + { + return Table.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); + } +} diff --git a/list/src/org/labkey/list/model/ListManager.java b/list/src/org/labkey/list/model/ListManager.java index 9637309e819..ef5f7143174 100644 --- a/list/src/org/labkey/list/model/ListManager.java +++ b/list/src/org/labkey/list/model/ListManager.java @@ -1,1484 +1,1484 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.list.model; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.FileAttachmentFile; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.data.*; -import org.labkey.api.data.Selector.ForEachBlock; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.DomainNotFoundException; -import org.labkey.api.exp.DomainURIFactory; -import org.labkey.api.exp.ImportTypesHelper; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListDefinition.BodySetting; -import org.labkey.api.exp.list.ListDefinition.IndexSetting; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; -import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.list.controllers.ListController; -import org.labkey.list.model.ListImporter.ValidatorImporter; -import org.labkey.list.view.ListItemAttachmentParent; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.io.File; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.util.IntegerUtils.asInteger; - -public class ListManager implements SearchService.DocumentProvider -{ - private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); - private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; - private static final ListManager INSTANCE = new ListManager(); - - public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; - public static final String LISTID_FIELD_NAME = "listId"; - - private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; - - private class ListDefCacheLoader implements CacheLoader> - { - @Override - public List load(@NotNull String entityId, @Nullable Object argument) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); - ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); - return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); - } - } - - public static ListManager get() - { - return INSTANCE; - } - - DbSchema getListMetadataSchema() - { - return ExperimentService.get().getSchema(); - } - - TableInfo getListMetadataTable() - { - return getListMetadataSchema().getTable("list"); - } - - public Collection getPicklists(Container container) - { - return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getPicklists(Container container, boolean includeProjectAndShared) - { - return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getLists(Container container) - { - return getLists(container, false); - } - - public Collection getLists(Container container, boolean includeProjectAndShared) - { - return getLists(container, null, false, true, includeProjectAndShared); - } - - public Collection getLists( - @NotNull Container container, - @Nullable User user, - boolean checkVisibility, - boolean includePicklists, - boolean includeProjectAndShared - ) - { - Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); - if (!includePicklists) - scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); - if (checkVisibility) - return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); - else - return scopedLists; - } - - /** - * Returns all list definitions defined within the scope of the container. This can optionally include list - * definitions from the container's project as well as the Shared folder. In the event of a name collision the - * closest container's list definition will be returned (i.e. container > project > Shared). - */ - private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) - { - List ownLists = _listDefCache.get(container.getId()); - Map listDefMap = new CaseInsensitiveHashMap<>(); - - if (includeProjectAndShared) - { - for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) - listDefMap.put(sharedList.getName(), sharedList); - - Container project = container.getProject(); - if (project != null) - { - for (ListDef projectList : _listDefCache.get(project.getId())) - listDefMap.put(projectList.getName(), projectList); - } - } - - // Workbooks can see parent lists. - if (container.isWorkbook()) - { - Container parent = container.getParent(); - if (parent != null) - { - for (ListDef parentList : _listDefCache.get(parent.getId())) - listDefMap.put(parentList.getName(), parentList); - } - } - - for (ListDef ownList : ownLists) - listDefMap.put(ownList.getName(), ownList); - - return listDefMap.values(); - } - - /** - * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null - */ - String getListTableName(TableInfo ti) - { - if (ti instanceof ListTable lti) - return lti.getRealTable().getSelectName(); - return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances - } - - @Nullable - public ListDef getList(Container container, int listId) - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - } - return list; - } - - public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) - { - if (null == listId) - { - return new ListDomainKindProperties(); - } - else - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - } - return list; - } - } - - // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) - public ListDef insert(User user, final ListDef def, Collection preferredListIds) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - TableInfo tinfo = getListMetadataTable(); - DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); - ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); - - builder.setListId(-1); - - for (Integer preferredListId : preferredListIds) - { - SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); - - // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction - if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) - { - builder.setListId(preferredListId); - sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used - break; - } - } - - // If none of the preferred IDs is available then use the next sequence value - if (builder.getListId() == -1) - builder.setListId((int)sequence.next()); - - ListDef ret = Table.insert(user, tinfo, builder.build()); - _listDefCache.remove(c.getId()); - return ret; - } - - - // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) - ListDef update(User user, final ListDef def) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDef ret; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDef old = getList(c, def.getListId()); - ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); - handleIndexSettingChanges(scope, def, old, ret); - - String oldName = old.getName(); - String updatedName = ret.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - transaction.commit(); - } - - return ret; - } - - //Note: this is sort of a dupe of above update() which returns ListDef - ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) - { - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDomainKindProperties updated; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); - updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); - ListDef listDef = getList(c, listProps.getListId()); - handleIndexSettingChanges(scope, listDef, old, listProps); - String oldName = old.getName(); - String updatedName = updated.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - - transaction.commit(); - } - - return updated; - } - - // Queue up one-time operations related to turning indexing on or off - private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) - { - boolean oldEachItemIndex = old.isEachItemIndex(); - boolean newEachItemIndex = updated.isEachItemIndex(); - - String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); - String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); - - int oldEachItemBodySetting = old.getEachItemBodySetting(); - int newEachItemBodySetting = updated.getEachItemBodySetting(); - - String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); - String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); - - boolean oldEntireListIndex = old.isEntireListIndex(); - boolean newEntireListIndex = updated.isEntireListIndex(); - - boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); - boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); - - String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); - String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); - - int oldEntireListIndexSetting = old.getEntireListIndexSetting(); - int newEntireListIndexSetting = updated.getEntireListIndexSetting(); - - int oldEntireListBodySetting = old.getEntireListBodySetting(); - int newEntireListBodySetting = updated.getEntireListBodySetting(); - - String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); - String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); - - scope.addCommitTask(() -> { - ListDefinition list = ListDefinitionImpl.of(listDef); - - // Is each-item indexing turned on? - if (newEachItemIndex) - { - // Turning on each-item indexing, or changing document title template, body template, - // or body setting -> clear this list's LastIndexed column - if - ( - !oldEachItemIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || - !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || - newEachItemBodySetting != oldEachItemBodySetting - ) - { - clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); - } - } - else - { - // Turning off each-item indexing -> clear item docs from the index - if (oldEachItemIndex) - deleteIndexedItems(list); - } - - // Is attachment indexing turned on? - if (newFileAttachmentIndex) - { - // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column - if - ( - !oldFileAttachmentIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template - ) - { - clearAttachmentLastIndexed(list); - } - } - else - { - // Turning off attachment indexing -> clear attachment docs from the index - if (oldFileAttachmentIndex) - deleteIndexedAttachments(list); - } - - // Is entire-list indexing turned on? - if (newEntireListIndex) - { - // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or - // body settings -> clear this list's last indexed column - if - ( - !oldEntireListIndex || - !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || - !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || - newEntireListIndexSetting != oldEntireListIndexSetting || - newEntireListBodySetting != oldEntireListBodySetting - ) - { - SQLFragment sql = new SQLFragment("UPDATE ") - .append(getListMetadataTable().getSelectName()) - .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") - .add(list.getListId()); - - new SqlExecutor(scope).execute(sql); - } - } - else - { - // Turning off entire-list indexing -> clear entire-list doc from the index - if (oldEntireListIndex) - deleteIndexedEntireListDoc(list); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) - { - _listDefCache.remove(c.getId()); - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); - } - - void deleteList(User user, @NotNull ListDefinitionImpl list, @Nullable String auditUserComment) throws DomainNotFoundException - { - var table = list.getTable(user); - var container = list.getContainer(); - var listName = list.getName(); - - // In certain cases we may create a list that is not viable (i.e., one in which a table was never created because - // the metadata wasn't valid). Still allow deleting the list. - try (var tx = (table != null) ? table.getSchema().getScope().ensureTransaction() : ExperimentService.get().ensureTransaction()) - { - // remove related full-text search docs and attachments - if (table != null && table.getUpdateService() instanceof ListQueryUpdateService qus) - qus.deleteRelatedListData(null); - else - deleteIndexedList(list); - - try - { - Table.delete(getListMetadataTable(), new Object[]{container, list.getListId()}); - } - catch (OptimisticConflictException x) - { - // ok - } - _listDefCache.remove(container.getId()); - - list.getDomainOrThrow().delete(user, auditUserComment); - addAuditEvent(list, user, String.format("The list %s was deleted", listName)); - - tx.commit(); - } - - QueryService.get().fireQueryDeleted(user, container, null, SchemaKey.fromParts(ListQuerySchema.NAME), Collections.singleton(listName)); - } - - /** - * Deletes all lists and list data in the given container. - */ - public void deleteLists(Container c, User user, @Nullable String auditUserComment) - { - try - { - var containerId = c.getEntityId().toString(); - for (var list : ListManager.get().getLists(c, true)) - { - var listDef = ListDefinitionImpl.of(list); - - // Delete the entire list when the list's container is deleted - if (containerId.equals(list.getContainerId())) - { - deleteList(user, listDef, auditUserComment); - continue; - } - - var table = listDef.getTable(user); - if (table == null) - continue; - - var qus = table.getUpdateService(); - if (qus == null) - continue; - - // Otherwise, truncate the rows in this list in this container - qus.truncateRows(user, c, null, null); - } - } - catch (Exception e) - { - throw new RuntimeException(e.getMessage(), e); - } - } - - public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); - - // Index all lists in this container - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) - { - Consumer r = (q) -> { - Map lists = ListService.get().getLists(q.getContainer(), null, false); - - try - { - QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); - QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); - for (ListDefinition list : lists.values()) - { - try - { - boolean reindex = since == null; - indexList(q, list, reindex); - } - catch (Exception ex) - { - LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); - } - } - } - finally - { - QueryService.get().clearEnvironment(); - } - }; - - queue.addRunnable(r); - } - - public void indexList(final ListDefinition def) - { - indexList(((ListDefinitionImpl) def)._def); - } - - // Index a single list - public void indexList(final ListDef def) - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); - Consumer r = (q) -> - { - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string - ListDefinition list = ListService.get().getList(c, def.getListId()); - if (null != list) // Could have just been deleted - indexList(q, list, false); - } - }; - - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - queue.addRunnable(r); - } - } - - private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) - { - Domain domain = list.getDomain(); - - // List might have just been deleted - if (null != domain) - { - // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction - assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; - - indexEntireList(queue, list, reindex); - indexModifiedItems(queue, list, reindex); - indexAttachments(queue, list, reindex); - } - } - - // Delete a single list item from the index after item delete - public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) - { - // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() - // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). - getListMetadataSchema().getScope().addCommitTask(() -> - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); - if (list.getEachItemIndex()) - { - SearchService.get().deleteResource(getDocumentId(list, entityId)); - } - - // Reindex the entire list document iff data is being indexed - if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) - { - indexEntireList(queue, list, true); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private String getDocumentId(ListDefinition list) - { - return "list:" + ((ListDefinitionImpl)list).getEntityId(); - } - - // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but - // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. - private String getDocumentId(ListDefinition list, @Nullable String entityId) - { - return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); - } - - private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) - { - return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); - } - - // Index all modified items in this list - private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) - { - if (list.getEachItemIndex()) - { - String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing - - // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed - lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); - - indexItems(queue, list, filter); - } - } - - // Reindex items specified by filter - private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - - if (null != listTable) - { - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); - - FieldKey keyKey = new FieldKey(null, list.getKeyName()); - FieldKey entityIdKey = new FieldKey(null, "EntityId"); - - FieldKey createdKey = new FieldKey(null, "created"); - FieldKey createdByKey = new FieldKey(null, "createdBy"); - FieldKey modifiedKey = new FieldKey(null, "modified"); - FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); - - // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. - StringExpression se = listTable.getDetailsURL(null, list.getContainer()); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { - Map map = results.getFieldKeyRowMap(); - final Object pk = map.get(keyKey); - String entityId = (String) map.get(entityIdKey); - - String documentId = getDocumentId(list, entityId); - Map props = new HashMap<>(); - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - String displayTitle = titleTemplate.eval(map); - props.put(SearchService.PROPERTY.title.toString(), displayTitle); - - Date created = null; - if (map.get(createdKey) instanceof Date) - created = (Date) map.get(createdKey); - - Date modified = null; - if (map.get(modifiedKey) instanceof Date) - modified = (Date) map.get(modifiedKey); - - String body = bodyTemplate.eval(map); - - ActionURL itemURL; - - try - { - itemURL = new ActionURL(se.eval(map)); - } - catch (Exception e) - { - itemURL = list.urlDetails(pk); - } - - itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body, - itemURL, - UserManager.getUser(asInteger( map.get(createdByKey))), created, - UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - try - { - ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); - } - catch (BadSqlGrammarException e) - { - // This may occur due to a race condition between enumeration and list deletion. Issue #48878 - // expected P-sql expected MS-sql - if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) - LOG.debug("Attempt to set LastIndexed on list table failed", e); - else - throw e; - } - } - }; - - // Add navtrail that includes link to full list grid - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); - }); - } - } - - /** - * Add searchable resources to Indexing task for file attachments - * @param list containing file attachments - */ - private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) - { - //Get common objects & properties - AttachmentService as = AttachmentService.get(); - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - - // Breadcrumb link to list is the same for all attachments on all items - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - - // Enumerate all list rows in batches and re-index based on the value of reindex parameter - // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); - - // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned - // TODO: Pass modifiedSince into this method? - Date modifiedSince = reindex ? null : new Date(); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { - // RowEntityId -> List item RowMap - Map> lookupMap = batch.stream() - .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); - - // RowEntityId -> Document names that need to be indexed - MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() - .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); - - documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { - Map map = lookupMap.get(rowEntityId); - String title = titleTemplate.eval(map); - - documentNames.forEach(documentName -> { - ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); - - //Generate searchable resource - String displayTitle = title + " attachment file \"" + documentName + "\""; - WebdavResource attachmentRes = as.getDocumentResource( - new Path(rowEntityId, documentName), - downloadUrl, - displayTitle, - new ListItemAttachmentParent(rowEntityId, list.getContainer()), - documentName, - SearchService.fileCategory - ); - - attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - queue.addResource(attachmentRes); - LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); - }); - }); - }); - } - } - - private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) - { - if (list.getEntireListIndex()) - { - IndexSetting setting = list.getEntireListIndexSetting(); - String documentId = getDocumentId(list); - - // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed - boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); - - // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified - if (!needToIndex && setting.indexItemData()) - needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); - - needToIndex |= reindex; - - if (needToIndex) - { - StringBuilder body = new StringBuilder(); - Map props = new HashMap<>(); - - // Use standard title if template is null/whitespace - String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); - String title = null == templateString ? "List " + list.getName() : templateString; - - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), title); - - if (!StringUtils.isEmpty(list.getDescription())) - body.append(list.getDescription()).append("\n"); - - String sep = ""; - - if (setting.indexMetaData()) - { - String comma = ""; - for (DomainProperty property : list.getDomain().getProperties()) - { - String n = StringUtils.trimToEmpty(property.getName()); - String l = StringUtils.trimToEmpty(property.getLabel()); - if (n.equals(l)) - l = ""; - body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); - comma = ","; - sep = "\n"; - } - } - - if (setting.indexItemData()) - { - TableInfo ti = list.getTable(User.getSearchUser()); - int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); - - if (ti != null) - { - body.append(sep); - FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); - - // All columns, all rows, no filters, no sorts - new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() - { - @Override - public void exec(Results results) throws StopIteratingException - { - body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); - // Issue 25366: Short circuit for very large list - if (body.length() > fileSizeLimit) - { - body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit - stopIterating(); - } - } - }); - } - } - - ActionURL url = list.urlShowData(); - url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body.toString(), - url, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - ListManager.get().setLastIndexed(list, ms); - } - }; - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); - } - } - } - - void deleteIndexedList(ListDefinition list) - { - if (list.getEntireListIndex()) - deleteIndexedEntireListDoc(list); - - if (list.getEachItemIndex()) - deleteIndexedItems(list); - - if (list.getFileAttachmentIndex()) - deleteIndexedAttachments(list); - } - - private void deleteIndexedAttachments(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); - } - - private void clearAttachmentLastIndexed(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::clearLastIndexed); - } - - private interface AttachmentParentHandler - { - void handle(AttachmentService as, List parentIds); - } - - // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 - private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) - { - // make sure container still exists (race condition on container delete) - Container listContainer = list.getContainer(); - if (null == listContainer) - return; - TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); - if (null == listTable) - return; - - AttachmentService as = AttachmentService.get(); - - if (hasAttachmentColumns(listTable)) - { - new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); - } - } - - // Un-index the entire list doc, but leave the list items alone - private void deleteIndexedEntireListDoc(ListDefinition list) - { - SearchService.get().deleteResource(getDocumentId(list)); - } - - - // Un-index all list items, but leave the entire list doc alone - private void deleteIndexedItems(ListDefinition list) - { - SearchService.get().deleteResourcesForPrefix(getDocumentId(list, null)); - } - - - private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); - - if (null != templateString) - { - template = createValidStringExpression(templateString, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); - } - - // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out - String name = list.getName().replaceAll("\\$\\{", "_{"); - template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); - - return template; - } - - - private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - - if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) - { - template = createValidStringExpression(customTemplate, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, templateType, error)); - } - - StringBuilder sb = new StringBuilder(); - String sep = ""; - - for (ColumnInfo column : listTable.getColumns()) - { - if (setting.accept(column)) - { - sb.append(sep); - sb.append("${"); - sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode - sb.append("}"); - sep = " "; - } - } - - template = createValidStringExpression(sb.toString(), error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); - - return template; - } - - - // Issue 21726: Perform some simple validation of custom indexing template - private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) - { - // Don't URL encode and use lenient substitution (replace nulls with blank) - FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); - - try - { - // TODO: Is there a more official way to validate a StringExpression? - se.eval(Collections.emptyMap()); - } - catch (IllegalArgumentException e) - { - error.append(e.getMessage()); - se = null; - } - - return se; - } - - - private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) - { - return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; - } - - - private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) - { - return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; - } - - - // Checks for existence of list items that have been modified since the entire list was last indexed - private boolean hasModifiedItems(ListDefinition list) - { - TableInfo table = list.getTable(User.getSearchUser()); - - if (null != table && null != getListTableName(table)) - { - // Using EXISTS query should be reasonably efficient. - SQLFragment sql = new SQLFragment("SELECT 1 FROM "); - sql.append(getListTableName(table)); - sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); - sql.append(" WHERE ListId = ? AND Container = ?)"); - sql.add(list.getListId()); - sql.add(list.getContainer().getEntityId()); - - return new SqlSelector(getListMetadataSchema(), sql).exists(); - } - - return false; - } - - private void setLastIndexed(ListDefinition list, long ms) - { - // list table does not have an index on listid, so we should include container in the WHERE - SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) - .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); - new SqlExecutor(getListMetadataSchema()).execute(update); - _listDefCache.remove(list.getContainer().getId()); - list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); - long modified = list.getModified().getTime(); - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); - } - - - private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) - { - // The "search user" might not have access - if (null != ti) - { - // 'unwrap' ListTable to get schema table for update - TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); - ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); - if (null != keyColumn) - { - var keySelectName = keyColumn.getSelectIdentifier(); - SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) - .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) - .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); - new SqlExecutor(sti.getSchema()).execute(sqlf); - } - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); - } - } - - - @Override - public void indexDeleted() - { - TableInfo listTable = getListMetadataTable(); - DbScope scope = listTable.getSchema().getScope(); - - // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case - clearLastIndexed(scope, listTable.getSelectName()); - - String listSchemaName = ListSchema.getInstance().getSchemaName(); - - // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. - new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); - } - - private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) - { - // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 - if (listDef.isEachItemIndex()) - { - ListDefinition list = new ListDefinitionImpl(listDef); - Domain domain = list.getDomain(); - if (null != domain && null != domain.getStorageTableName()) - { - LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); - clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); - } - } - } - - private void clearLastIndexed(DbScope scope, String selectName) - { - try - { - // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases - new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); - } - catch (Exception e) - { - // Log the exception, but allow other tables to be cleared - ExceptionUtil.logExceptionToMothership(null, e); - } - } - - void addAuditEvent(ListDefinitionImpl list, User user, String comment) - { - if (null != user) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); - AuditLogService.get().addEvent(user, event); - } - } - - void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); - - event.setListItemEntityId(entityId); - if (oldRecord != null) event.setOldRecordMap(oldRecord); - if (newRecord != null) event.setNewRecordMap(newRecord); - - AuditLogService.get().addEvent(user, event); - } - - String formatAuditItem(ListDefinitionImpl list, User user, Map props) - { - String itemRecord = ""; - TableInfo ti = list.getTable(user); - - if (null != ti) - { - Map recordChangedMap = new CaseInsensitiveHashMap<>(); - Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); - - // Match props to columns - for (Map.Entry entry : props.entrySet()) - { - String baseKey = entry.getKey(); - - boolean isReserved = false; - for (String res : reserved) - { - if (res.equalsIgnoreCase(baseKey)) - { - isReserved = true; - break; - } - } - - if (isReserved) - continue; - - ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); - String value = Objects.toString(entry.getValue(), ""); - String key = null; - - if (null != col) - { - // Found the column - key = col.getName(); // best good - } - else - { - // See if there is a match in the domain properties - for (DomainProperty dp : list.getDomain().getProperties()) - { - if (dp.getName().equalsIgnoreCase(baseKey)) - { - key = dp.getName(); // middle good - } - } - - // Try by name - DomainProperty dp = list.getDomain().getPropertyByName(baseKey); - if (null != dp) - key = dp.getName(); - } - - if (null != key && null != value) - recordChangedMap.put(key, value); - } - - if (!recordChangedMap.isEmpty()) - itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); - } - - return itemRecord; - } - - boolean importListSchema( - ListDefinition unsavedList, - ImportTypesHelper importHelper, - User user, - Collection validatorImporters, - List errors - ) throws Exception - { - if (!errors.isEmpty()) - return false; - - final Container container = unsavedList.getContainer(); - final Domain domain = unsavedList.getDomain(); - final String typeURI = domain.getTypeURI(); - - DomainURIFactory factory = name -> new Pair<>(typeURI, container); - - ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - if (null == ipd.domainName || null == ipd.domainURI) - errors.add("List not specified for property: " + ipd.pd.getName()); - } - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); - domainProperty.setConditionalFormats(ipd.formats); - domainProperty.setDefaultValue(ipd.defaultValue); - } - - unsavedList.save(user); - - // Save validators later, after all the lists are imported, #40343 - validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); - - return true; - } - - public static class TestCase extends Assert - { - private static final String PROJECT_NAME = "BiologicsManagerTest Project"; - private static final String LIST_NAME = "Unit Test list"; - private static final String FIELD_NAME = "field"; - private static final String ATTACHMENT_FIELD_NAME = "attachment"; - private static final String PARENT_LIST_ITEM = "parentItem"; - private static final String CHILD1_LIST_ITEM = "child1Item"; - private static final String CHILD2_LIST_ITEM = "child2Item"; - private static final String KEY_FIELD_NAME = "Unit test list key"; - private static Integer PARENT_LI_KEY; - - private static ListDefinitionImpl list; - private static DomainProperty dp; - private static User u; - private static Container c; - - @BeforeClass - public static void setup() throws Exception - { - u = TestContext.get().getUser(); - - deleteTestContainer(); - - c = ContainerManager.ensureContainer(PROJECT_NAME, u); - list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); - list.setKeyName(KEY_FIELD_NAME); - - dp = list.getDomain().addProperty(); - dp.setName(FIELD_NAME); - dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); - dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); - - var attachmentDp = list.getDomain().addProperty(); - attachmentDp.setName(ATTACHMENT_FIELD_NAME); - attachmentDp.setType(PropertyService.get().getType(c, PropertyType.ATTACHMENT.getXmlName())); - attachmentDp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), ATTACHMENT_FIELD_NAME, c, list.getKeyType()).toString()); - list.save(u); - - PARENT_LI_KEY = addListItem(c, list, PARENT_LIST_ITEM, null); - } - - private static int addListItem(Container scopedContainer, ListDefinition scopedList, String value, @Nullable File attachment) throws Exception - { - var row = new CaseInsensitiveHashMap<>(); - row.put(FIELD_NAME, value); - if (attachment != null) - row.put(ATTACHMENT_FIELD_NAME, new FileAttachmentFile(attachment)); - - var errors = new BatchValidationException(); - var result = scopedList.getTable(u).getUpdateService().insertRows(u, scopedContainer, List.of(row), errors, null, null); - if (errors.hasErrors()) - throw errors.getLastRowError(); - - return MapUtils.getIntValue(result.getFirst(), KEY_FIELD_NAME); - } - - @AfterClass - public static void cleanup() - { - list = null; - dp = null; - c = null; - u = null; - - deleteTestContainer(); - } - - private static void deleteTestContainer() - { - var project = ContainerManager.getForPath(PROJECT_NAME); - - if (project != null) - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - @Test - public void testListServiceInOwnFolder() - { - Map lists = ListService.get().getLists(c); - assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); - ListItem li = lists.get(LIST_NAME).getListItem(PARENT_LI_KEY, u, c); - assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); - } - - @Test - public void testListServiceInWorkbook() throws Exception - { - Container workbook1 = createWorkbook("Unit Test Workbook 1"); - Container workbook2 = createWorkbook("Unit Test Workbook 2"); - Map lists = ListService.get().getLists(workbook1); - assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); - - ListDefinition childList1 = ListService.get().getLists(workbook1).get(LIST_NAME); - ListDefinition childList2 = ListService.get().getLists(workbook2).get(LIST_NAME); - - assertEquals("Lists available to each child container are not the same", childList1.toString(), childList2.toString()); - int childKey1 = addListItem(workbook1, childList1, CHILD1_LIST_ITEM, null); - int childKey2 = addListItem(workbook2, childList2, CHILD2_LIST_ITEM, null); - - assertNull("Parent item should not be visible in child container", childList1.getListItem(PARENT_LI_KEY, u, workbook1)); - assertNull("Sibling child item should not be visible in another child container", childList1.getListItem(childKey2, u, workbook1)); - assertEquals("Parent container should be able to see child container item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, c).getProperty(dp)); - assertEquals("Child container should be able to see its own list item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, workbook1).getProperty(dp)); - } - - /** - * GitHub Issue #1013: Rows added to a list in a subfolder must be cleaned up when that subfolder - * is deleted. Previously, those rows were left orphaned in the database. - */ - @Test - public void testSubfolderDeletionCleansUpListData() throws Exception - { - Container subfolder1 = createSubfolder("subfolder1"); - Container subfolder2 = createSubfolder("subfolder2"); - - File attachment1 = FileUtil.createTempFile("subfolder1", ".txt"); - attachment1.deleteOnExit(); - File attachment2 = FileUtil.createTempFile("subfolder2", ".txt"); - attachment2.deleteOnExit(); - - int key1 = addListItem(subfolder1, list, "subfolder1Item", attachment1); - int key2 = addListItem(subfolder2, list, "subfolder2Item", attachment2); - - // Precondition: parent can see both subfolder items (setUp inserts key 1; subfolders get keys 2 and 3) - assertNotNull("Subfolder1 item should be visible from subfolder1 before deletion", list.getListItem(key1, u, subfolder1)); - assertNotNull("Subfolder2 item should be visible from subfolder2 before deletion", list.getListItem(key2, u, subfolder2)); - - TableInfo table = list.getTable(u, c, ContainerFilter.getUnsafeEverythingFilter()); - assertNotNull("Expected list table to resolve", table); - - // Preverify rows in the database - long rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); - assertEquals("Expected only two rows (parent, subfolder1, subfolder2)", 3, rowCount); - - // Delete subfolder1 — should delete its list rows and attachments, - // not touch subfolder2 or the parent or the list definition - ListService.get().deleteLists(subfolder1, u, null); - - // Verify row is deleted from underlying table - rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); - assertEquals("Expected only two rows (parent, subfolder2)", 2, rowCount); - - // Items belonging to other containers should be unaffected - assertNotNull("Parent item should survive subfolder1 deletion", list.getListItem(PARENT_LI_KEY, u, c)); - assertNotNull("Subfolder2 item should survive subfolder1 deletion", list.getListItem(key2, u, subfolder2)); - } - - private Container createSubfolder(String folderName) - { - return ContainerManager.createContainer(c, folderName, u); - } - - private Container createWorkbook(String title) - { - return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.list.model; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.FileAttachmentFile; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.data.*; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.DomainNotFoundException; +import org.labkey.api.exp.DomainURIFactory; +import org.labkey.api.exp.ImportTypesHelper; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListDefinition.BodySetting; +import org.labkey.api.exp.list.ListDefinition.IndexSetting; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; +import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.list.controllers.ListController; +import org.labkey.list.model.ListImporter.ValidatorImporter; +import org.labkey.list.view.ListItemAttachmentParent; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.io.File; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.util.IntegerUtils.asInteger; + +public class ListManager implements SearchService.DocumentProvider +{ + private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); + private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; + private static final ListManager INSTANCE = new ListManager(); + + public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; + public static final String LISTID_FIELD_NAME = "listId"; + + private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; + + private class ListDefCacheLoader implements CacheLoader> + { + @Override + public List load(@NotNull String entityId, @Nullable Object argument) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); + ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); + return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); + } + } + + public static ListManager get() + { + return INSTANCE; + } + + DbSchema getListMetadataSchema() + { + return ExperimentService.get().getSchema(); + } + + TableInfo getListMetadataTable() + { + return getListMetadataSchema().getTable("list"); + } + + public Collection getPicklists(Container container) + { + return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getPicklists(Container container, boolean includeProjectAndShared) + { + return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getLists(Container container) + { + return getLists(container, false); + } + + public Collection getLists(Container container, boolean includeProjectAndShared) + { + return getLists(container, null, false, true, includeProjectAndShared); + } + + public Collection getLists( + @NotNull Container container, + @Nullable User user, + boolean checkVisibility, + boolean includePicklists, + boolean includeProjectAndShared + ) + { + Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); + if (!includePicklists) + scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); + if (checkVisibility) + return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); + else + return scopedLists; + } + + /** + * Returns all list definitions defined within the scope of the container. This can optionally include list + * definitions from the container's project as well as the Shared folder. In the event of a name collision the + * closest container's list definition will be returned (i.e. container > project > Shared). + */ + private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) + { + List ownLists = _listDefCache.get(container.getId()); + Map listDefMap = new CaseInsensitiveHashMap<>(); + + if (includeProjectAndShared) + { + for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) + listDefMap.put(sharedList.getName(), sharedList); + + Container project = container.getProject(); + if (project != null) + { + for (ListDef projectList : _listDefCache.get(project.getId())) + listDefMap.put(projectList.getName(), projectList); + } + } + + // Workbooks can see parent lists. + if (container.isWorkbook()) + { + Container parent = container.getParent(); + if (parent != null) + { + for (ListDef parentList : _listDefCache.get(parent.getId())) + listDefMap.put(parentList.getName(), parentList); + } + } + + for (ListDef ownList : ownLists) + listDefMap.put(ownList.getName(), ownList); + + return listDefMap.values(); + } + + /** + * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null + */ + String getListTableName(TableInfo ti) + { + if (ti instanceof ListTable lti) + return lti.getRealTable().getSelectName(); + return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances + } + + @Nullable + public ListDef getList(Container container, int listId) + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + } + return list; + } + + public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) + { + if (null == listId) + { + return new ListDomainKindProperties(); + } + else + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + } + return list; + } + } + + // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) + public ListDef insert(User user, final ListDef def, Collection preferredListIds) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + TableInfo tinfo = getListMetadataTable(); + DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); + ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); + + builder.setListId(-1); + + for (Integer preferredListId : preferredListIds) + { + SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); + + // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction + if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) + { + builder.setListId(preferredListId); + sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used + break; + } + } + + // If none of the preferred IDs is available then use the next sequence value + if (builder.getListId() == -1) + builder.setListId((int)sequence.next()); + + ListDef ret = Table.insert(user, tinfo, builder.build()); + _listDefCache.remove(c.getId()); + return ret; + } + + + // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) + ListDef update(User user, final ListDef def) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDef ret; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDef old = getList(c, def.getListId()); + ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); + handleIndexSettingChanges(scope, def, old, ret); + + String oldName = old.getName(); + String updatedName = ret.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + transaction.commit(); + } + + return ret; + } + + //Note: this is sort of a dupe of above update() which returns ListDef + ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) + { + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDomainKindProperties updated; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); + updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); + ListDef listDef = getList(c, listProps.getListId()); + handleIndexSettingChanges(scope, listDef, old, listProps); + String oldName = old.getName(); + String updatedName = updated.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + + transaction.commit(); + } + + return updated; + } + + // Queue up one-time operations related to turning indexing on or off + private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) + { + boolean oldEachItemIndex = old.isEachItemIndex(); + boolean newEachItemIndex = updated.isEachItemIndex(); + + String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); + String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); + + int oldEachItemBodySetting = old.getEachItemBodySetting(); + int newEachItemBodySetting = updated.getEachItemBodySetting(); + + String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); + String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); + + boolean oldEntireListIndex = old.isEntireListIndex(); + boolean newEntireListIndex = updated.isEntireListIndex(); + + boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); + boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); + + String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); + String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); + + int oldEntireListIndexSetting = old.getEntireListIndexSetting(); + int newEntireListIndexSetting = updated.getEntireListIndexSetting(); + + int oldEntireListBodySetting = old.getEntireListBodySetting(); + int newEntireListBodySetting = updated.getEntireListBodySetting(); + + String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); + String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); + + scope.addCommitTask(() -> { + ListDefinition list = ListDefinitionImpl.of(listDef); + + // Is each-item indexing turned on? + if (newEachItemIndex) + { + // Turning on each-item indexing, or changing document title template, body template, + // or body setting -> clear this list's LastIndexed column + if + ( + !oldEachItemIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || + !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || + newEachItemBodySetting != oldEachItemBodySetting + ) + { + clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); + } + } + else + { + // Turning off each-item indexing -> clear item docs from the index + if (oldEachItemIndex) + deleteIndexedItems(list); + } + + // Is attachment indexing turned on? + if (newFileAttachmentIndex) + { + // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column + if + ( + !oldFileAttachmentIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template + ) + { + clearAttachmentLastIndexed(list); + } + } + else + { + // Turning off attachment indexing -> clear attachment docs from the index + if (oldFileAttachmentIndex) + deleteIndexedAttachments(list); + } + + // Is entire-list indexing turned on? + if (newEntireListIndex) + { + // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or + // body settings -> clear this list's last indexed column + if + ( + !oldEntireListIndex || + !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || + !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || + newEntireListIndexSetting != oldEntireListIndexSetting || + newEntireListBodySetting != oldEntireListBodySetting + ) + { + SQLFragment sql = new SQLFragment("UPDATE ") + .append(getListMetadataTable().getSelectName()) + .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") + .add(list.getListId()); + + new SqlExecutor(scope).execute(sql); + } + } + else + { + // Turning off entire-list indexing -> clear entire-list doc from the index + if (oldEntireListIndex) + deleteIndexedEntireListDoc(list); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) + { + _listDefCache.remove(c.getId()); + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); + } + + void deleteList(User user, @NotNull ListDefinitionImpl list, @Nullable String auditUserComment) throws DomainNotFoundException + { + var table = list.getTable(user); + var container = list.getContainer(); + var listName = list.getName(); + + // In certain cases we may create a list that is not viable (i.e., one in which a table was never created because + // the metadata wasn't valid). Still allow deleting the list. + try (var tx = (table != null) ? table.getSchema().getScope().ensureTransaction() : ExperimentService.get().ensureTransaction()) + { + // remove related full-text search docs and attachments + if (table != null && table.getUpdateService() instanceof ListQueryUpdateService qus) + qus.deleteRelatedListData(null); + else + deleteIndexedList(list); + + try + { + Table.delete(getListMetadataTable(), new Object[]{container, list.getListId()}); + } + catch (OptimisticConflictException x) + { + // ok + } + _listDefCache.remove(container.getId()); + + list.getDomainOrThrow().delete(user, auditUserComment); + addAuditEvent(list, user, String.format("The list %s was deleted", listName)); + + tx.commit(); + } + + QueryService.get().fireQueryDeleted(user, container, null, SchemaKey.fromParts(ListQuerySchema.NAME), Collections.singleton(listName)); + } + + /** + * Deletes all lists and list data in the given container. + */ + public void deleteLists(Container c, User user, @Nullable String auditUserComment) + { + try + { + var containerId = c.getEntityId().toString(); + for (var list : ListManager.get().getLists(c, true)) + { + var listDef = ListDefinitionImpl.of(list); + + // Delete the entire list when the list's container is deleted + if (containerId.equals(list.getContainerId())) + { + deleteList(user, listDef, auditUserComment); + continue; + } + + var table = listDef.getTable(user); + if (table == null) + continue; + + var qus = table.getUpdateService(); + if (qus == null) + continue; + + // Otherwise, truncate the rows in this list in this container + qus.truncateRows(user, c, null, null); + } + } + catch (Exception e) + { + throw new RuntimeException(e.getMessage(), e); + } + } + + public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); + + // Index all lists in this container + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) + { + Consumer r = (q) -> { + Map lists = ListService.get().getLists(q.getContainer(), null, false); + + try + { + QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); + for (ListDefinition list : lists.values()) + { + try + { + boolean reindex = since == null; + indexList(q, list, reindex); + } + catch (Exception ex) + { + LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); + } + } + } + finally + { + QueryService.get().clearEnvironment(); + } + }; + + queue.addRunnable(r); + } + + public void indexList(final ListDefinition def) + { + indexList(((ListDefinitionImpl) def)._def); + } + + // Index a single list + public void indexList(final ListDef def) + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); + Consumer r = (q) -> + { + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string + ListDefinition list = ListService.get().getList(c, def.getListId()); + if (null != list) // Could have just been deleted + indexList(q, list, false); + } + }; + + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + queue.addRunnable(r); + } + } + + private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) + { + Domain domain = list.getDomain(); + + // List might have just been deleted + if (null != domain) + { + // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction + assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; + + indexEntireList(queue, list, reindex); + indexModifiedItems(queue, list, reindex); + indexAttachments(queue, list, reindex); + } + } + + // Delete a single list item from the index after item delete + public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) + { + // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() + // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). + getListMetadataSchema().getScope().addCommitTask(() -> + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); + if (list.getEachItemIndex()) + { + SearchService.get().deleteResource(getDocumentId(list, entityId)); + } + + // Reindex the entire list document iff data is being indexed + if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) + { + indexEntireList(queue, list, true); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private String getDocumentId(ListDefinition list) + { + return "list:" + ((ListDefinitionImpl)list).getEntityId(); + } + + // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but + // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. + private String getDocumentId(ListDefinition list, @Nullable String entityId) + { + return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); + } + + private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) + { + return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); + } + + // Index all modified items in this list + private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) + { + if (list.getEachItemIndex()) + { + String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing + + // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed + lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); + + indexItems(queue, list, filter); + } + } + + // Reindex items specified by filter + private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + + if (null != listTable) + { + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); + + FieldKey keyKey = new FieldKey(null, list.getKeyName()); + FieldKey entityIdKey = new FieldKey(null, "EntityId"); + + FieldKey createdKey = new FieldKey(null, "created"); + FieldKey createdByKey = new FieldKey(null, "createdBy"); + FieldKey modifiedKey = new FieldKey(null, "modified"); + FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); + + // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. + StringExpression se = listTable.getDetailsURL(null, list.getContainer()); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { + Map map = results.getFieldKeyRowMap(); + final Object pk = map.get(keyKey); + String entityId = (String) map.get(entityIdKey); + + String documentId = getDocumentId(list, entityId); + Map props = new HashMap<>(); + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + String displayTitle = titleTemplate.eval(map); + props.put(SearchService.PROPERTY.title.toString(), displayTitle); + + Date created = null; + if (map.get(createdKey) instanceof Date) + created = (Date) map.get(createdKey); + + Date modified = null; + if (map.get(modifiedKey) instanceof Date) + modified = (Date) map.get(modifiedKey); + + String body = bodyTemplate.eval(map); + + ActionURL itemURL; + + try + { + itemURL = new ActionURL(se.eval(map)); + } + catch (Exception e) + { + itemURL = list.urlDetails(pk); + } + + itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body, + itemURL, + UserManager.getUser(asInteger( map.get(createdByKey))), created, + UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + try + { + ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); + } + catch (BadSqlGrammarException e) + { + // This may occur due to a race condition between enumeration and list deletion. Issue #48878 + // expected P-sql expected MS-sql + if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) + LOG.debug("Attempt to set LastIndexed on list table failed", e); + else + throw e; + } + } + }; + + // Add navtrail that includes link to full list grid + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); + }); + } + } + + /** + * Add searchable resources to Indexing task for file attachments + * @param list containing file attachments + */ + private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) + { + //Get common objects & properties + AttachmentService as = AttachmentService.get(); + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + + // Breadcrumb link to list is the same for all attachments on all items + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + + // Enumerate all list rows in batches and re-index based on the value of reindex parameter + // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); + + // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned + // TODO: Pass modifiedSince into this method? + Date modifiedSince = reindex ? null : new Date(); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { + // RowEntityId -> List item RowMap + Map> lookupMap = batch.stream() + .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); + + // RowEntityId -> Document names that need to be indexed + MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() + .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); + + documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { + Map map = lookupMap.get(rowEntityId); + String title = titleTemplate.eval(map); + + documentNames.forEach(documentName -> { + ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); + + //Generate searchable resource + String displayTitle = title + " attachment file \"" + documentName + "\""; + WebdavResource attachmentRes = as.getDocumentResource( + new Path(rowEntityId, documentName), + downloadUrl, + displayTitle, + new ListItemAttachmentParent(rowEntityId, list.getContainer()), + documentName, + SearchService.fileCategory + ); + + attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + queue.addResource(attachmentRes); + LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); + }); + }); + }); + } + } + + private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) + { + if (list.getEntireListIndex()) + { + IndexSetting setting = list.getEntireListIndexSetting(); + String documentId = getDocumentId(list); + + // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed + boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); + + // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified + if (!needToIndex && setting.indexItemData()) + needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); + + needToIndex |= reindex; + + if (needToIndex) + { + StringBuilder body = new StringBuilder(); + Map props = new HashMap<>(); + + // Use standard title if template is null/whitespace + String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); + String title = null == templateString ? "List " + list.getName() : templateString; + + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), title); + + if (!StringUtils.isEmpty(list.getDescription())) + body.append(list.getDescription()).append("\n"); + + String sep = ""; + + if (setting.indexMetaData()) + { + String comma = ""; + for (DomainProperty property : list.getDomain().getProperties()) + { + String n = StringUtils.trimToEmpty(property.getName()); + String l = StringUtils.trimToEmpty(property.getLabel()); + if (n.equals(l)) + l = ""; + body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); + comma = ","; + sep = "\n"; + } + } + + if (setting.indexItemData()) + { + TableInfo ti = list.getTable(User.getSearchUser()); + int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); + + if (ti != null) + { + body.append(sep); + FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); + + // All columns, all rows, no filters, no sorts + new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() + { + @Override + public void exec(Results results) throws StopIteratingException + { + body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); + // Issue 25366: Short circuit for very large list + if (body.length() > fileSizeLimit) + { + body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit + stopIterating(); + } + } + }); + } + } + + ActionURL url = list.urlShowData(); + url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body.toString(), + url, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + ListManager.get().setLastIndexed(list, ms); + } + }; + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); + } + } + } + + void deleteIndexedList(ListDefinition list) + { + if (list.getEntireListIndex()) + deleteIndexedEntireListDoc(list); + + if (list.getEachItemIndex()) + deleteIndexedItems(list); + + if (list.getFileAttachmentIndex()) + deleteIndexedAttachments(list); + } + + private void deleteIndexedAttachments(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); + } + + private void clearAttachmentLastIndexed(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::clearLastIndexed); + } + + private interface AttachmentParentHandler + { + void handle(AttachmentService as, List parentIds); + } + + // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 + private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) + { + // make sure container still exists (race condition on container delete) + Container listContainer = list.getContainer(); + if (null == listContainer) + return; + TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); + if (null == listTable) + return; + + AttachmentService as = AttachmentService.get(); + + if (hasAttachmentColumns(listTable)) + { + new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); + } + } + + // Un-index the entire list doc, but leave the list items alone + private void deleteIndexedEntireListDoc(ListDefinition list) + { + SearchService.get().deleteResource(getDocumentId(list)); + } + + + // Un-index all list items, but leave the entire list doc alone + private void deleteIndexedItems(ListDefinition list) + { + SearchService.get().deleteResourcesForPrefix(getDocumentId(list, null)); + } + + + private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); + + if (null != templateString) + { + template = createValidStringExpression(templateString, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); + } + + // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out + String name = list.getName().replaceAll("\\$\\{", "_{"); + template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); + + return template; + } + + + private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + + if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) + { + template = createValidStringExpression(customTemplate, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, templateType, error)); + } + + StringBuilder sb = new StringBuilder(); + String sep = ""; + + for (ColumnInfo column : listTable.getColumns()) + { + if (setting.accept(column)) + { + sb.append(sep); + sb.append("${"); + sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode + sb.append("}"); + sep = " "; + } + } + + template = createValidStringExpression(sb.toString(), error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); + + return template; + } + + + // Issue 21726: Perform some simple validation of custom indexing template + private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) + { + // Don't URL encode and use lenient substitution (replace nulls with blank) + FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); + + try + { + // TODO: Is there a more official way to validate a StringExpression? + se.eval(Collections.emptyMap()); + } + catch (IllegalArgumentException e) + { + error.append(e.getMessage()); + se = null; + } + + return se; + } + + + private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) + { + return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; + } + + + private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) + { + return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; + } + + + // Checks for existence of list items that have been modified since the entire list was last indexed + private boolean hasModifiedItems(ListDefinition list) + { + TableInfo table = list.getTable(User.getSearchUser()); + + if (null != table && null != getListTableName(table)) + { + // Using EXISTS query should be reasonably efficient. + SQLFragment sql = new SQLFragment("SELECT 1 FROM "); + sql.append(getListTableName(table)); + sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); + sql.append(" WHERE ListId = ? AND Container = ?)"); + sql.add(list.getListId()); + sql.add(list.getContainer().getEntityId()); + + return new SqlSelector(getListMetadataSchema(), sql).exists(); + } + + return false; + } + + private void setLastIndexed(ListDefinition list, long ms) + { + // list table does not have an index on listid, so we should include container in the WHERE + SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) + .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); + new SqlExecutor(getListMetadataSchema()).execute(update); + _listDefCache.remove(list.getContainer().getId()); + list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); + long modified = list.getModified().getTime(); + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); + } + + + private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) + { + // The "search user" might not have access + if (null != ti) + { + // 'unwrap' ListTable to get schema table for update + TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); + ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); + if (null != keyColumn) + { + var keySelectName = keyColumn.getSelectIdentifier(); + SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) + .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) + .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); + new SqlExecutor(sti.getSchema()).execute(sqlf); + } + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); + } + } + + + @Override + public void indexDeleted() + { + TableInfo listTable = getListMetadataTable(); + DbScope scope = listTable.getSchema().getScope(); + + // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case + clearLastIndexed(scope, listTable.getSelectName()); + + String listSchemaName = ListSchema.getInstance().getSchemaName(); + + // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. + new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); + } + + private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) + { + // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 + if (listDef.isEachItemIndex()) + { + ListDefinition list = new ListDefinitionImpl(listDef); + Domain domain = list.getDomain(); + if (null != domain && null != domain.getStorageTableName()) + { + LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); + clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); + } + } + } + + private void clearLastIndexed(DbScope scope, String selectName) + { + try + { + // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases + new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); + } + catch (Exception e) + { + // Log the exception, but allow other tables to be cleared + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + void addAuditEvent(ListDefinitionImpl list, User user, String comment) + { + if (null != user) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); + AuditLogService.get().addEvent(user, event); + } + } + + void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); + + event.setListItemEntityId(entityId); + if (oldRecord != null) event.setOldRecordMap(oldRecord); + if (newRecord != null) event.setNewRecordMap(newRecord); + + AuditLogService.get().addEvent(user, event); + } + + String formatAuditItem(ListDefinitionImpl list, User user, Map props) + { + String itemRecord = ""; + TableInfo ti = list.getTable(user); + + if (null != ti) + { + Map recordChangedMap = new CaseInsensitiveHashMap<>(); + Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); + + // Match props to columns + for (Map.Entry entry : props.entrySet()) + { + String baseKey = entry.getKey(); + + boolean isReserved = false; + for (String res : reserved) + { + if (res.equalsIgnoreCase(baseKey)) + { + isReserved = true; + break; + } + } + + if (isReserved) + continue; + + ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); + Object value = entry.getValue(); + String key = null; + + if (null != col) + { + // Found the column + key = col.getName(); // best good + } + else + { + // See if there is a match in the domain properties + for (DomainProperty dp : list.getDomain().getProperties()) + { + if (dp.getName().equalsIgnoreCase(baseKey)) + { + key = dp.getName(); // middle good + } + } + + // Try by name + DomainProperty dp = list.getDomain().getPropertyByName(baseKey); + if (null != dp) + key = dp.getName(); + } + + if (null != key && null != value) + recordChangedMap.put(key, value); + } + + if (!recordChangedMap.isEmpty()) + itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); + } + + return itemRecord; + } + + boolean importListSchema( + ListDefinition unsavedList, + ImportTypesHelper importHelper, + User user, + Collection validatorImporters, + List errors + ) throws Exception + { + if (!errors.isEmpty()) + return false; + + final Container container = unsavedList.getContainer(); + final Domain domain = unsavedList.getDomain(); + final String typeURI = domain.getTypeURI(); + + DomainURIFactory factory = name -> new Pair<>(typeURI, container); + + ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + if (null == ipd.domainName || null == ipd.domainURI) + errors.add("List not specified for property: " + ipd.pd.getName()); + } + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); + domainProperty.setConditionalFormats(ipd.formats); + domainProperty.setDefaultValue(ipd.defaultValue); + } + + unsavedList.save(user); + + // Save validators later, after all the lists are imported, #40343 + validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); + + return true; + } + + public static class TestCase extends Assert + { + private static final String PROJECT_NAME = "BiologicsManagerTest Project"; + private static final String LIST_NAME = "Unit Test list"; + private static final String FIELD_NAME = "field"; + private static final String ATTACHMENT_FIELD_NAME = "attachment"; + private static final String PARENT_LIST_ITEM = "parentItem"; + private static final String CHILD1_LIST_ITEM = "child1Item"; + private static final String CHILD2_LIST_ITEM = "child2Item"; + private static final String KEY_FIELD_NAME = "Unit test list key"; + private static Integer PARENT_LI_KEY; + + private static ListDefinitionImpl list; + private static DomainProperty dp; + private static User u; + private static Container c; + + @BeforeClass + public static void setup() throws Exception + { + u = TestContext.get().getUser(); + + deleteTestContainer(); + + c = ContainerManager.ensureContainer(PROJECT_NAME, u); + list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); + list.setKeyName(KEY_FIELD_NAME); + + dp = list.getDomain().addProperty(); + dp.setName(FIELD_NAME); + dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); + dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); + + var attachmentDp = list.getDomain().addProperty(); + attachmentDp.setName(ATTACHMENT_FIELD_NAME); + attachmentDp.setType(PropertyService.get().getType(c, PropertyType.ATTACHMENT.getXmlName())); + attachmentDp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), ATTACHMENT_FIELD_NAME, c, list.getKeyType()).toString()); + list.save(u); + + PARENT_LI_KEY = addListItem(c, list, PARENT_LIST_ITEM, null); + } + + private static int addListItem(Container scopedContainer, ListDefinition scopedList, String value, @Nullable File attachment) throws Exception + { + var row = new CaseInsensitiveHashMap<>(); + row.put(FIELD_NAME, value); + if (attachment != null) + row.put(ATTACHMENT_FIELD_NAME, new FileAttachmentFile(attachment)); + + var errors = new BatchValidationException(); + var result = scopedList.getTable(u).getUpdateService().insertRows(u, scopedContainer, List.of(row), errors, null, null); + if (errors.hasErrors()) + throw errors.getLastRowError(); + + return MapUtils.getIntValue(result.getFirst(), KEY_FIELD_NAME); + } + + @AfterClass + public static void cleanup() + { + list = null; + dp = null; + c = null; + u = null; + + deleteTestContainer(); + } + + private static void deleteTestContainer() + { + var project = ContainerManager.getForPath(PROJECT_NAME); + + if (project != null) + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + @Test + public void testListServiceInOwnFolder() + { + Map lists = ListService.get().getLists(c); + assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); + ListItem li = lists.get(LIST_NAME).getListItem(PARENT_LI_KEY, u, c); + assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); + } + + @Test + public void testListServiceInWorkbook() throws Exception + { + Container workbook1 = createWorkbook("Unit Test Workbook 1"); + Container workbook2 = createWorkbook("Unit Test Workbook 2"); + Map lists = ListService.get().getLists(workbook1); + assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); + + ListDefinition childList1 = ListService.get().getLists(workbook1).get(LIST_NAME); + ListDefinition childList2 = ListService.get().getLists(workbook2).get(LIST_NAME); + + assertEquals("Lists available to each child container are not the same", childList1.toString(), childList2.toString()); + int childKey1 = addListItem(workbook1, childList1, CHILD1_LIST_ITEM, null); + int childKey2 = addListItem(workbook2, childList2, CHILD2_LIST_ITEM, null); + + assertNull("Parent item should not be visible in child container", childList1.getListItem(PARENT_LI_KEY, u, workbook1)); + assertNull("Sibling child item should not be visible in another child container", childList1.getListItem(childKey2, u, workbook1)); + assertEquals("Parent container should be able to see child container item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, c).getProperty(dp)); + assertEquals("Child container should be able to see its own list item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, workbook1).getProperty(dp)); + } + + /** + * GitHub Issue #1013: Rows added to a list in a subfolder must be cleaned up when that subfolder + * is deleted. Previously, those rows were left orphaned in the database. + */ + @Test + public void testSubfolderDeletionCleansUpListData() throws Exception + { + Container subfolder1 = createSubfolder("subfolder1"); + Container subfolder2 = createSubfolder("subfolder2"); + + File attachment1 = FileUtil.createTempFile("subfolder1", ".txt"); + attachment1.deleteOnExit(); + File attachment2 = FileUtil.createTempFile("subfolder2", ".txt"); + attachment2.deleteOnExit(); + + int key1 = addListItem(subfolder1, list, "subfolder1Item", attachment1); + int key2 = addListItem(subfolder2, list, "subfolder2Item", attachment2); + + // Precondition: parent can see both subfolder items (setUp inserts key 1; subfolders get keys 2 and 3) + assertNotNull("Subfolder1 item should be visible from subfolder1 before deletion", list.getListItem(key1, u, subfolder1)); + assertNotNull("Subfolder2 item should be visible from subfolder2 before deletion", list.getListItem(key2, u, subfolder2)); + + TableInfo table = list.getTable(u, c, ContainerFilter.getUnsafeEverythingFilter()); + assertNotNull("Expected list table to resolve", table); + + // Preverify rows in the database + long rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); + assertEquals("Expected only two rows (parent, subfolder1, subfolder2)", 3, rowCount); + + // Delete subfolder1 — should delete its list rows and attachments, + // not touch subfolder2 or the parent or the list definition + ListService.get().deleteLists(subfolder1, u, null); + + // Verify row is deleted from underlying table + rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); + assertEquals("Expected only two rows (parent, subfolder2)", 2, rowCount); + + // Items belonging to other containers should be unaffected + assertNotNull("Parent item should survive subfolder1 deletion", list.getListItem(PARENT_LI_KEY, u, c)); + assertNotNull("Subfolder2 item should survive subfolder1 deletion", list.getListItem(key2, u, subfolder2)); + } + + private Container createSubfolder(String folderName) + { + return ContainerManager.createContainer(c, folderName, u); + } + + private Container createWorkbook(String title) + { + return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); + } + } +} From 411f6ce17aa1a7ca0c0397c8614ac2eea9316319 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 17 Apr 2026 09:50:02 -0700 Subject: [PATCH 2/3] crlf --- .../org/labkey/list/model/ListManager.java | 2968 ++++++++--------- 1 file changed, 1484 insertions(+), 1484 deletions(-) diff --git a/list/src/org/labkey/list/model/ListManager.java b/list/src/org/labkey/list/model/ListManager.java index ef5f7143174..f65e898963c 100644 --- a/list/src/org/labkey/list/model/ListManager.java +++ b/list/src/org/labkey/list/model/ListManager.java @@ -1,1484 +1,1484 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.list.model; - -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.collections4.MultiValuedMap; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.attachments.FileAttachmentFile; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheLoader; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.LabKeyCollectors; -import org.labkey.api.data.*; -import org.labkey.api.data.Selector.ForEachBlock; -import org.labkey.api.exceptions.OptimisticConflictException; -import org.labkey.api.exp.DomainNotFoundException; -import org.labkey.api.exp.DomainURIFactory; -import org.labkey.api.exp.ImportTypesHelper; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; -import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListDefinition.BodySetting; -import org.labkey.api.exp.list.ListDefinition.IndexSetting; -import org.labkey.api.exp.list.ListItem; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryChangeListener; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.UserManager; -import org.labkey.api.util.ExceptionUtil; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.util.Path; -import org.labkey.api.util.StringExpression; -import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; -import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.webdav.SimpleDocumentResource; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.list.controllers.ListController; -import org.labkey.list.model.ListImporter.ValidatorImporter; -import org.labkey.list.view.ListItemAttachmentParent; -import org.springframework.jdbc.BadSqlGrammarException; - -import java.io.File; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static org.labkey.api.util.IntegerUtils.asInteger; - -public class ListManager implements SearchService.DocumentProvider -{ - private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); - private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; - private static final ListManager INSTANCE = new ListManager(); - - public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; - public static final String LISTID_FIELD_NAME = "listId"; - - private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; - - private class ListDefCacheLoader implements CacheLoader> - { - @Override - public List load(@NotNull String entityId, @Nullable Object argument) - { - SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); - ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); - return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); - } - } - - public static ListManager get() - { - return INSTANCE; - } - - DbSchema getListMetadataSchema() - { - return ExperimentService.get().getSchema(); - } - - TableInfo getListMetadataTable() - { - return getListMetadataSchema().getTable("list"); - } - - public Collection getPicklists(Container container) - { - return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getPicklists(Container container, boolean includeProjectAndShared) - { - return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); - } - - public Collection getLists(Container container) - { - return getLists(container, false); - } - - public Collection getLists(Container container, boolean includeProjectAndShared) - { - return getLists(container, null, false, true, includeProjectAndShared); - } - - public Collection getLists( - @NotNull Container container, - @Nullable User user, - boolean checkVisibility, - boolean includePicklists, - boolean includeProjectAndShared - ) - { - Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); - if (!includePicklists) - scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); - if (checkVisibility) - return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); - else - return scopedLists; - } - - /** - * Returns all list definitions defined within the scope of the container. This can optionally include list - * definitions from the container's project as well as the Shared folder. In the event of a name collision the - * closest container's list definition will be returned (i.e. container > project > Shared). - */ - private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) - { - List ownLists = _listDefCache.get(container.getId()); - Map listDefMap = new CaseInsensitiveHashMap<>(); - - if (includeProjectAndShared) - { - for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) - listDefMap.put(sharedList.getName(), sharedList); - - Container project = container.getProject(); - if (project != null) - { - for (ListDef projectList : _listDefCache.get(project.getId())) - listDefMap.put(projectList.getName(), projectList); - } - } - - // Workbooks can see parent lists. - if (container.isWorkbook()) - { - Container parent = container.getParent(); - if (parent != null) - { - for (ListDef parentList : _listDefCache.get(parent.getId())) - listDefMap.put(parentList.getName(), parentList); - } - } - - for (ListDef ownList : ownLists) - listDefMap.put(ownList.getName(), ownList); - - return listDefMap.values(); - } - - /** - * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null - */ - String getListTableName(TableInfo ti) - { - if (ti instanceof ListTable lti) - return lti.getRealTable().getSelectName(); - return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances - } - - @Nullable - public ListDef getList(Container container, int listId) - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); - } - return list; - } - - public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) - { - if (null == listId) - { - return new ListDomainKindProperties(); - } - else - { - SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); - ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - - // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time - if (list == null && container.isWorkbook()) - { - filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); - list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - } - return list; - } - } - - // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) - public ListDef insert(User user, final ListDef def, Collection preferredListIds) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - TableInfo tinfo = getListMetadataTable(); - DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); - ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); - - builder.setListId(-1); - - for (Integer preferredListId : preferredListIds) - { - SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); - - // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction - if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) - { - builder.setListId(preferredListId); - sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used - break; - } - } - - // If none of the preferred IDs is available then use the next sequence value - if (builder.getListId() == -1) - builder.setListId((int)sequence.next()); - - ListDef ret = Table.insert(user, tinfo, builder.build()); - _listDefCache.remove(c.getId()); - return ret; - } - - - // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) - ListDef update(User user, final ListDef def) - { - Container c = def.lookupContainer(); - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDef ret; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDef old = getList(c, def.getListId()); - ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); - handleIndexSettingChanges(scope, def, old, ret); - - String oldName = old.getName(); - String updatedName = ret.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - transaction.commit(); - } - - return ret; - } - - //Note: this is sort of a dupe of above update() which returns ListDef - ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) - { - if (null == c) - throw OptimisticConflictException.create(Table.ERROR_DELETED); - - DbScope scope = getListMetadataSchema().getScope(); - ListDomainKindProperties updated; - - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); - updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); - ListDef listDef = getList(c, listProps.getListId()); - handleIndexSettingChanges(scope, listDef, old, listProps); - String oldName = old.getName(); - String updatedName = updated.getName(); - queryChangeUpdate(user, c, oldName, updatedName); - - transaction.commit(); - } - - return updated; - } - - // Queue up one-time operations related to turning indexing on or off - private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) - { - boolean oldEachItemIndex = old.isEachItemIndex(); - boolean newEachItemIndex = updated.isEachItemIndex(); - - String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); - String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); - - int oldEachItemBodySetting = old.getEachItemBodySetting(); - int newEachItemBodySetting = updated.getEachItemBodySetting(); - - String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); - String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); - - boolean oldEntireListIndex = old.isEntireListIndex(); - boolean newEntireListIndex = updated.isEntireListIndex(); - - boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); - boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); - - String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); - String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); - - int oldEntireListIndexSetting = old.getEntireListIndexSetting(); - int newEntireListIndexSetting = updated.getEntireListIndexSetting(); - - int oldEntireListBodySetting = old.getEntireListBodySetting(); - int newEntireListBodySetting = updated.getEntireListBodySetting(); - - String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); - String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); - - scope.addCommitTask(() -> { - ListDefinition list = ListDefinitionImpl.of(listDef); - - // Is each-item indexing turned on? - if (newEachItemIndex) - { - // Turning on each-item indexing, or changing document title template, body template, - // or body setting -> clear this list's LastIndexed column - if - ( - !oldEachItemIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || - !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || - newEachItemBodySetting != oldEachItemBodySetting - ) - { - clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); - } - } - else - { - // Turning off each-item indexing -> clear item docs from the index - if (oldEachItemIndex) - deleteIndexedItems(list); - } - - // Is attachment indexing turned on? - if (newFileAttachmentIndex) - { - // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column - if - ( - !oldFileAttachmentIndex || - !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template - ) - { - clearAttachmentLastIndexed(list); - } - } - else - { - // Turning off attachment indexing -> clear attachment docs from the index - if (oldFileAttachmentIndex) - deleteIndexedAttachments(list); - } - - // Is entire-list indexing turned on? - if (newEntireListIndex) - { - // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or - // body settings -> clear this list's last indexed column - if - ( - !oldEntireListIndex || - !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || - !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || - newEntireListIndexSetting != oldEntireListIndexSetting || - newEntireListBodySetting != oldEntireListBodySetting - ) - { - SQLFragment sql = new SQLFragment("UPDATE ") - .append(getListMetadataTable().getSelectName()) - .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") - .add(list.getListId()); - - new SqlExecutor(scope).execute(sql); - } - } - else - { - // Turning off entire-list indexing -> clear entire-list doc from the index - if (oldEntireListIndex) - deleteIndexedEntireListDoc(list); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) - { - _listDefCache.remove(c.getId()); - QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); - } - - void deleteList(User user, @NotNull ListDefinitionImpl list, @Nullable String auditUserComment) throws DomainNotFoundException - { - var table = list.getTable(user); - var container = list.getContainer(); - var listName = list.getName(); - - // In certain cases we may create a list that is not viable (i.e., one in which a table was never created because - // the metadata wasn't valid). Still allow deleting the list. - try (var tx = (table != null) ? table.getSchema().getScope().ensureTransaction() : ExperimentService.get().ensureTransaction()) - { - // remove related full-text search docs and attachments - if (table != null && table.getUpdateService() instanceof ListQueryUpdateService qus) - qus.deleteRelatedListData(null); - else - deleteIndexedList(list); - - try - { - Table.delete(getListMetadataTable(), new Object[]{container, list.getListId()}); - } - catch (OptimisticConflictException x) - { - // ok - } - _listDefCache.remove(container.getId()); - - list.getDomainOrThrow().delete(user, auditUserComment); - addAuditEvent(list, user, String.format("The list %s was deleted", listName)); - - tx.commit(); - } - - QueryService.get().fireQueryDeleted(user, container, null, SchemaKey.fromParts(ListQuerySchema.NAME), Collections.singleton(listName)); - } - - /** - * Deletes all lists and list data in the given container. - */ - public void deleteLists(Container c, User user, @Nullable String auditUserComment) - { - try - { - var containerId = c.getEntityId().toString(); - for (var list : ListManager.get().getLists(c, true)) - { - var listDef = ListDefinitionImpl.of(list); - - // Delete the entire list when the list's container is deleted - if (containerId.equals(list.getContainerId())) - { - deleteList(user, listDef, auditUserComment); - continue; - } - - var table = listDef.getTable(user); - if (table == null) - continue; - - var qus = table.getUpdateService(); - if (qus == null) - continue; - - // Otherwise, truncate the rows in this list in this container - qus.truncateRows(user, c, null, null); - } - } - catch (Exception e) - { - throw new RuntimeException(e.getMessage(), e); - } - } - - public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); - - // Index all lists in this container - @Override - public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) - { - Consumer r = (q) -> { - Map lists = ListService.get().getLists(q.getContainer(), null, false); - - try - { - QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); - QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); - for (ListDefinition list : lists.values()) - { - try - { - boolean reindex = since == null; - indexList(q, list, reindex); - } - catch (Exception ex) - { - LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); - } - } - } - finally - { - QueryService.get().clearEnvironment(); - } - }; - - queue.addRunnable(r); - } - - public void indexList(final ListDefinition def) - { - indexList(((ListDefinitionImpl) def)._def); - } - - // Index a single list - public void indexList(final ListDef def) - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); - Consumer r = (q) -> - { - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string - ListDefinition list = ListService.get().getList(c, def.getListId()); - if (null != list) // Could have just been deleted - indexList(q, list, false); - } - }; - - Container c = def.lookupContainer(); - if (!ContainerManager.exists(c)) - { - LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); - } - else - { - queue.addRunnable(r); - } - } - - private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) - { - Domain domain = list.getDomain(); - - // List might have just been deleted - if (null != domain) - { - // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction - assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; - - indexEntireList(queue, list, reindex); - indexModifiedItems(queue, list, reindex); - indexAttachments(queue, list, reindex); - } - } - - // Delete a single list item from the index after item delete - public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) - { - // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() - // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). - getListMetadataSchema().getScope().addCommitTask(() -> - { - SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); - if (list.getEachItemIndex()) - { - SearchService.get().deleteResource(getDocumentId(list, entityId)); - } - - // Reindex the entire list document iff data is being indexed - if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) - { - indexEntireList(queue, list, true); - } - }, DbScope.CommitTaskOption.POSTCOMMIT); - } - - private String getDocumentId(ListDefinition list) - { - return "list:" + ((ListDefinitionImpl)list).getEntityId(); - } - - // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but - // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. - private String getDocumentId(ListDefinition list, @Nullable String entityId) - { - return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); - } - - private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) - { - return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); - } - - // Index all modified items in this list - private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) - { - if (list.getEachItemIndex()) - { - String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing - - // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed - lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); - - indexItems(queue, list, filter); - } - } - - // Reindex items specified by filter - private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - - if (null != listTable) - { - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); - - FieldKey keyKey = new FieldKey(null, list.getKeyName()); - FieldKey entityIdKey = new FieldKey(null, "EntityId"); - - FieldKey createdKey = new FieldKey(null, "created"); - FieldKey createdByKey = new FieldKey(null, "createdBy"); - FieldKey modifiedKey = new FieldKey(null, "modified"); - FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); - - // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. - StringExpression se = listTable.getDetailsURL(null, list.getContainer()); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { - Map map = results.getFieldKeyRowMap(); - final Object pk = map.get(keyKey); - String entityId = (String) map.get(entityIdKey); - - String documentId = getDocumentId(list, entityId); - Map props = new HashMap<>(); - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - String displayTitle = titleTemplate.eval(map); - props.put(SearchService.PROPERTY.title.toString(), displayTitle); - - Date created = null; - if (map.get(createdKey) instanceof Date) - created = (Date) map.get(createdKey); - - Date modified = null; - if (map.get(modifiedKey) instanceof Date) - modified = (Date) map.get(modifiedKey); - - String body = bodyTemplate.eval(map); - - ActionURL itemURL; - - try - { - itemURL = new ActionURL(se.eval(map)); - } - catch (Exception e) - { - itemURL = list.urlDetails(pk); - } - - itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body, - itemURL, - UserManager.getUser(asInteger( map.get(createdByKey))), created, - UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - try - { - ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); - } - catch (BadSqlGrammarException e) - { - // This may occur due to a race condition between enumeration and list deletion. Issue #48878 - // expected P-sql expected MS-sql - if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) - LOG.debug("Attempt to set LastIndexed on list table failed", e); - else - throw e; - } - } - }; - - // Add navtrail that includes link to full list grid - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); - }); - } - } - - /** - * Add searchable resources to Indexing task for file attachments - * @param list containing file attachments - */ - private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) - { - TableInfo listTable = list.getTable(User.getSearchUser()); - if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) - { - //Get common objects & properties - AttachmentService as = AttachmentService.get(); - FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); - - // Breadcrumb link to list is the same for all attachments on all items - ActionURL gridURL = list.urlShowData(); - gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - NavTree t = new NavTree("list", gridURL); - String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); - - // Enumerate all list rows in batches and re-index based on the value of reindex parameter - // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? - SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); - - // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned - // TODO: Pass modifiedSince into this method? - Date modifiedSince = reindex ? null : new Date(); - - new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { - // RowEntityId -> List item RowMap - Map> lookupMap = batch.stream() - .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); - - // RowEntityId -> Document names that need to be indexed - MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() - .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); - - documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { - Map map = lookupMap.get(rowEntityId); - String title = titleTemplate.eval(map); - - documentNames.forEach(documentName -> { - ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); - - //Generate searchable resource - String displayTitle = title + " attachment file \"" + documentName + "\""; - WebdavResource attachmentRes = as.getDocumentResource( - new Path(rowEntityId, documentName), - downloadUrl, - displayTitle, - new ListItemAttachmentParent(rowEntityId, list.getContainer()), - documentName, - SearchService.fileCategory - ); - - attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); - queue.addResource(attachmentRes); - LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); - }); - }); - }); - } - } - - private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) - { - if (list.getEntireListIndex()) - { - IndexSetting setting = list.getEntireListIndexSetting(); - String documentId = getDocumentId(list); - - // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed - boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); - - // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified - if (!needToIndex && setting.indexItemData()) - needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); - - needToIndex |= reindex; - - if (needToIndex) - { - StringBuilder body = new StringBuilder(); - Map props = new HashMap<>(); - - // Use standard title if template is null/whitespace - String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); - String title = null == templateString ? "List " + list.getName() : templateString; - - props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); - props.put(SearchService.PROPERTY.title.toString(), title); - - if (!StringUtils.isEmpty(list.getDescription())) - body.append(list.getDescription()).append("\n"); - - String sep = ""; - - if (setting.indexMetaData()) - { - String comma = ""; - for (DomainProperty property : list.getDomain().getProperties()) - { - String n = StringUtils.trimToEmpty(property.getName()); - String l = StringUtils.trimToEmpty(property.getLabel()); - if (n.equals(l)) - l = ""; - body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); - comma = ","; - sep = "\n"; - } - } - - if (setting.indexItemData()) - { - TableInfo ti = list.getTable(User.getSearchUser()); - int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); - - if (ti != null) - { - body.append(sep); - FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); - - // All columns, all rows, no filters, no sorts - new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() - { - @Override - public void exec(Results results) throws StopIteratingException - { - body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); - // Issue 25366: Short circuit for very large list - if (body.length() > fileSizeLimit) - { - body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit - stopIterating(); - } - } - }); - } - } - - ActionURL url = list.urlShowData(); - url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames - - SimpleDocumentResource r = new SimpleDocumentResource( - new Path(documentId), - documentId, - list.getContainer().getEntityId(), - "text/plain", - body.toString(), - url, - props) - { - @Override - public void setLastIndexed(long ms, long modified) - { - ListManager.get().setLastIndexed(list, ms); - } - }; - - queue.addResource(r); - LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); - } - } - } - - void deleteIndexedList(ListDefinition list) - { - if (list.getEntireListIndex()) - deleteIndexedEntireListDoc(list); - - if (list.getEachItemIndex()) - deleteIndexedItems(list); - - if (list.getFileAttachmentIndex()) - deleteIndexedAttachments(list); - } - - private void deleteIndexedAttachments(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); - } - - private void clearAttachmentLastIndexed(@NotNull ListDefinition list) - { - handleAttachmentParents(list, AttachmentService::clearLastIndexed); - } - - private interface AttachmentParentHandler - { - void handle(AttachmentService as, List parentIds); - } - - // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 - private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) - { - // make sure container still exists (race condition on container delete) - Container listContainer = list.getContainer(); - if (null == listContainer) - return; - TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); - if (null == listTable) - return; - - AttachmentService as = AttachmentService.get(); - - if (hasAttachmentColumns(listTable)) - { - new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); - } - } - - // Un-index the entire list doc, but leave the list items alone - private void deleteIndexedEntireListDoc(ListDefinition list) - { - SearchService.get().deleteResource(getDocumentId(list)); - } - - - // Un-index all list items, but leave the entire list doc alone - private void deleteIndexedItems(ListDefinition list) - { - SearchService.get().deleteResourcesForPrefix(getDocumentId(list, null)); - } - - - private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); - - if (null != templateString) - { - template = createValidStringExpression(templateString, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); - } - - // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out - String name = list.getName().replaceAll("\\$\\{", "_{"); - template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); - - return template; - } - - - private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) - { - FieldKeyStringExpression template; - StringBuilder error = new StringBuilder(); - - if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) - { - template = createValidStringExpression(customTemplate, error); - - if (null != template) - return template; - else - LOG.warn(getTemplateErrorMessage(list, templateType, error)); - } - - StringBuilder sb = new StringBuilder(); - String sep = ""; - - for (ColumnInfo column : listTable.getColumns()) - { - if (setting.accept(column)) - { - sb.append(sep); - sb.append("${"); - sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode - sb.append("}"); - sep = " "; - } - } - - template = createValidStringExpression(sb.toString(), error); - - if (null == template) - throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); - - return template; - } - - - // Issue 21726: Perform some simple validation of custom indexing template - private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) - { - // Don't URL encode and use lenient substitution (replace nulls with blank) - FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); - - try - { - // TODO: Is there a more official way to validate a StringExpression? - se.eval(Collections.emptyMap()); - } - catch (IllegalArgumentException e) - { - error.append(e.getMessage()); - se = null; - } - - return se; - } - - - private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) - { - return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; - } - - - private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) - { - return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; - } - - - // Checks for existence of list items that have been modified since the entire list was last indexed - private boolean hasModifiedItems(ListDefinition list) - { - TableInfo table = list.getTable(User.getSearchUser()); - - if (null != table && null != getListTableName(table)) - { - // Using EXISTS query should be reasonably efficient. - SQLFragment sql = new SQLFragment("SELECT 1 FROM "); - sql.append(getListTableName(table)); - sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); - sql.append(" WHERE ListId = ? AND Container = ?)"); - sql.add(list.getListId()); - sql.add(list.getContainer().getEntityId()); - - return new SqlSelector(getListMetadataSchema(), sql).exists(); - } - - return false; - } - - private void setLastIndexed(ListDefinition list, long ms) - { - // list table does not have an index on listid, so we should include container in the WHERE - SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) - .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); - new SqlExecutor(getListMetadataSchema()).execute(update); - _listDefCache.remove(list.getContainer().getId()); - list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); - long modified = list.getModified().getTime(); - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); - } - - - private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) - { - // The "search user" might not have access - if (null != ti) - { - // 'unwrap' ListTable to get schema table for update - TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); - ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); - if (null != keyColumn) - { - var keySelectName = keyColumn.getSelectIdentifier(); - SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) - .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) - .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); - new SqlExecutor(sti.getSchema()).execute(sqlf); - } - String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; - LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); - } - } - - - @Override - public void indexDeleted() - { - TableInfo listTable = getListMetadataTable(); - DbScope scope = listTable.getSchema().getScope(); - - // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case - clearLastIndexed(scope, listTable.getSelectName()); - - String listSchemaName = ListSchema.getInstance().getSchemaName(); - - // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. - new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); - } - - private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) - { - // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 - if (listDef.isEachItemIndex()) - { - ListDefinition list = new ListDefinitionImpl(listDef); - Domain domain = list.getDomain(); - if (null != domain && null != domain.getStorageTableName()) - { - LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); - clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); - } - } - } - - private void clearLastIndexed(DbScope scope, String selectName) - { - try - { - // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases - new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); - } - catch (Exception e) - { - // Log the exception, but allow other tables to be cleared - ExceptionUtil.logExceptionToMothership(null, e); - } - } - - void addAuditEvent(ListDefinitionImpl list, User user, String comment) - { - if (null != user) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); - AuditLogService.get().addEvent(user, event); - } - } - - void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) - { - ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); - - event.setListItemEntityId(entityId); - if (oldRecord != null) event.setOldRecordMap(oldRecord); - if (newRecord != null) event.setNewRecordMap(newRecord); - - AuditLogService.get().addEvent(user, event); - } - - String formatAuditItem(ListDefinitionImpl list, User user, Map props) - { - String itemRecord = ""; - TableInfo ti = list.getTable(user); - - if (null != ti) - { - Map recordChangedMap = new CaseInsensitiveHashMap<>(); - Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); - - // Match props to columns - for (Map.Entry entry : props.entrySet()) - { - String baseKey = entry.getKey(); - - boolean isReserved = false; - for (String res : reserved) - { - if (res.equalsIgnoreCase(baseKey)) - { - isReserved = true; - break; - } - } - - if (isReserved) - continue; - - ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); - Object value = entry.getValue(); - String key = null; - - if (null != col) - { - // Found the column - key = col.getName(); // best good - } - else - { - // See if there is a match in the domain properties - for (DomainProperty dp : list.getDomain().getProperties()) - { - if (dp.getName().equalsIgnoreCase(baseKey)) - { - key = dp.getName(); // middle good - } - } - - // Try by name - DomainProperty dp = list.getDomain().getPropertyByName(baseKey); - if (null != dp) - key = dp.getName(); - } - - if (null != key && null != value) - recordChangedMap.put(key, value); - } - - if (!recordChangedMap.isEmpty()) - itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); - } - - return itemRecord; - } - - boolean importListSchema( - ListDefinition unsavedList, - ImportTypesHelper importHelper, - User user, - Collection validatorImporters, - List errors - ) throws Exception - { - if (!errors.isEmpty()) - return false; - - final Container container = unsavedList.getContainer(); - final Domain domain = unsavedList.getDomain(); - final String typeURI = domain.getTypeURI(); - - DomainURIFactory factory = name -> new Pair<>(typeURI, container); - - ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - if (null == ipd.domainName || null == ipd.domainURI) - errors.add("List not specified for property: " + ipd.pd.getName()); - } - - if (!errors.isEmpty()) - return false; - - for (ImportPropertyDescriptor ipd : pds.properties) - { - DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); - domainProperty.setConditionalFormats(ipd.formats); - domainProperty.setDefaultValue(ipd.defaultValue); - } - - unsavedList.save(user); - - // Save validators later, after all the lists are imported, #40343 - validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); - - return true; - } - - public static class TestCase extends Assert - { - private static final String PROJECT_NAME = "BiologicsManagerTest Project"; - private static final String LIST_NAME = "Unit Test list"; - private static final String FIELD_NAME = "field"; - private static final String ATTACHMENT_FIELD_NAME = "attachment"; - private static final String PARENT_LIST_ITEM = "parentItem"; - private static final String CHILD1_LIST_ITEM = "child1Item"; - private static final String CHILD2_LIST_ITEM = "child2Item"; - private static final String KEY_FIELD_NAME = "Unit test list key"; - private static Integer PARENT_LI_KEY; - - private static ListDefinitionImpl list; - private static DomainProperty dp; - private static User u; - private static Container c; - - @BeforeClass - public static void setup() throws Exception - { - u = TestContext.get().getUser(); - - deleteTestContainer(); - - c = ContainerManager.ensureContainer(PROJECT_NAME, u); - list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); - list.setKeyName(KEY_FIELD_NAME); - - dp = list.getDomain().addProperty(); - dp.setName(FIELD_NAME); - dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); - dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); - - var attachmentDp = list.getDomain().addProperty(); - attachmentDp.setName(ATTACHMENT_FIELD_NAME); - attachmentDp.setType(PropertyService.get().getType(c, PropertyType.ATTACHMENT.getXmlName())); - attachmentDp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), ATTACHMENT_FIELD_NAME, c, list.getKeyType()).toString()); - list.save(u); - - PARENT_LI_KEY = addListItem(c, list, PARENT_LIST_ITEM, null); - } - - private static int addListItem(Container scopedContainer, ListDefinition scopedList, String value, @Nullable File attachment) throws Exception - { - var row = new CaseInsensitiveHashMap<>(); - row.put(FIELD_NAME, value); - if (attachment != null) - row.put(ATTACHMENT_FIELD_NAME, new FileAttachmentFile(attachment)); - - var errors = new BatchValidationException(); - var result = scopedList.getTable(u).getUpdateService().insertRows(u, scopedContainer, List.of(row), errors, null, null); - if (errors.hasErrors()) - throw errors.getLastRowError(); - - return MapUtils.getIntValue(result.getFirst(), KEY_FIELD_NAME); - } - - @AfterClass - public static void cleanup() - { - list = null; - dp = null; - c = null; - u = null; - - deleteTestContainer(); - } - - private static void deleteTestContainer() - { - var project = ContainerManager.getForPath(PROJECT_NAME); - - if (project != null) - ContainerManager.deleteAll(project, TestContext.get().getUser()); - } - - @Test - public void testListServiceInOwnFolder() - { - Map lists = ListService.get().getLists(c); - assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); - ListItem li = lists.get(LIST_NAME).getListItem(PARENT_LI_KEY, u, c); - assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); - } - - @Test - public void testListServiceInWorkbook() throws Exception - { - Container workbook1 = createWorkbook("Unit Test Workbook 1"); - Container workbook2 = createWorkbook("Unit Test Workbook 2"); - Map lists = ListService.get().getLists(workbook1); - assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); - - ListDefinition childList1 = ListService.get().getLists(workbook1).get(LIST_NAME); - ListDefinition childList2 = ListService.get().getLists(workbook2).get(LIST_NAME); - - assertEquals("Lists available to each child container are not the same", childList1.toString(), childList2.toString()); - int childKey1 = addListItem(workbook1, childList1, CHILD1_LIST_ITEM, null); - int childKey2 = addListItem(workbook2, childList2, CHILD2_LIST_ITEM, null); - - assertNull("Parent item should not be visible in child container", childList1.getListItem(PARENT_LI_KEY, u, workbook1)); - assertNull("Sibling child item should not be visible in another child container", childList1.getListItem(childKey2, u, workbook1)); - assertEquals("Parent container should be able to see child container item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, c).getProperty(dp)); - assertEquals("Child container should be able to see its own list item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, workbook1).getProperty(dp)); - } - - /** - * GitHub Issue #1013: Rows added to a list in a subfolder must be cleaned up when that subfolder - * is deleted. Previously, those rows were left orphaned in the database. - */ - @Test - public void testSubfolderDeletionCleansUpListData() throws Exception - { - Container subfolder1 = createSubfolder("subfolder1"); - Container subfolder2 = createSubfolder("subfolder2"); - - File attachment1 = FileUtil.createTempFile("subfolder1", ".txt"); - attachment1.deleteOnExit(); - File attachment2 = FileUtil.createTempFile("subfolder2", ".txt"); - attachment2.deleteOnExit(); - - int key1 = addListItem(subfolder1, list, "subfolder1Item", attachment1); - int key2 = addListItem(subfolder2, list, "subfolder2Item", attachment2); - - // Precondition: parent can see both subfolder items (setUp inserts key 1; subfolders get keys 2 and 3) - assertNotNull("Subfolder1 item should be visible from subfolder1 before deletion", list.getListItem(key1, u, subfolder1)); - assertNotNull("Subfolder2 item should be visible from subfolder2 before deletion", list.getListItem(key2, u, subfolder2)); - - TableInfo table = list.getTable(u, c, ContainerFilter.getUnsafeEverythingFilter()); - assertNotNull("Expected list table to resolve", table); - - // Preverify rows in the database - long rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); - assertEquals("Expected only two rows (parent, subfolder1, subfolder2)", 3, rowCount); - - // Delete subfolder1 — should delete its list rows and attachments, - // not touch subfolder2 or the parent or the list definition - ListService.get().deleteLists(subfolder1, u, null); - - // Verify row is deleted from underlying table - rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); - assertEquals("Expected only two rows (parent, subfolder2)", 2, rowCount); - - // Items belonging to other containers should be unaffected - assertNotNull("Parent item should survive subfolder1 deletion", list.getListItem(PARENT_LI_KEY, u, c)); - assertNotNull("Subfolder2 item should survive subfolder1 deletion", list.getListItem(key2, u, subfolder2)); - } - - private Container createSubfolder(String folderName) - { - return ContainerManager.createContainer(c, folderName, u); - } - - private Container createWorkbook(String title) - { - return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.list.model; + +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.collections4.MultiValuedMap; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.attachments.FileAttachmentFile; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheLoader; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.LabKeyCollectors; +import org.labkey.api.data.*; +import org.labkey.api.data.Selector.ForEachBlock; +import org.labkey.api.exceptions.OptimisticConflictException; +import org.labkey.api.exp.DomainNotFoundException; +import org.labkey.api.exp.DomainURIFactory; +import org.labkey.api.exp.ImportTypesHelper; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptor; +import org.labkey.api.exp.OntologyManager.ImportPropertyDescriptorsList; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListDefinition.BodySetting; +import org.labkey.api.exp.list.ListDefinition.IndexSetting; +import org.labkey.api.exp.list.ListItem; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryChangeListener; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.UserManager; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.StringExpression; +import org.labkey.api.util.StringExpressionFactory.AbstractStringExpression.NullValueBehavior; +import org.labkey.api.util.StringExpressionFactory.FieldKeyStringExpression; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.webdav.SimpleDocumentResource; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.list.controllers.ListController; +import org.labkey.list.model.ListImporter.ValidatorImporter; +import org.labkey.list.view.ListItemAttachmentParent; +import org.springframework.jdbc.BadSqlGrammarException; + +import java.io.File; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.labkey.api.util.IntegerUtils.asInteger; + +public class ListManager implements SearchService.DocumentProvider +{ + private static final Logger LOG = LogHelper.getLogger(ListManager.class, "List indexing events"); + private static final String LIST_SEQUENCE_NAME = "org.labkey.list.Lists"; + private static final ListManager INSTANCE = new ListManager(); + + public static final String LIST_AUDIT_EVENT = "ListAuditEvent"; + public static final String LISTID_FIELD_NAME = "listId"; + + private final Cache> _listDefCache = DatabaseCache.get(CoreSchema.getInstance().getScope(), CacheManager.UNLIMITED, CacheManager.DAY, "List definitions", new ListDefCacheLoader()) ; + + private class ListDefCacheLoader implements CacheLoader> + { + @Override + public List load(@NotNull String entityId, @Nullable Object argument) + { + SimpleFilter filter = new SimpleFilter(FieldKey.fromParts("Container"), entityId); + ArrayList ownLists = new TableSelector(getListMetadataTable(), filter, null).getArrayList(ListDef.class); + return ownLists.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(ownLists); + } + } + + public static ListManager get() + { + return INSTANCE; + } + + DbSchema getListMetadataSchema() + { + return ExperimentService.get().getSchema(); + } + + TableInfo getListMetadataTable() + { + return getListMetadataSchema().getTable("list"); + } + + public Collection getPicklists(Container container) + { + return getLists(container, true).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getPicklists(Container container, boolean includeProjectAndShared) + { + return getLists(container, includeProjectAndShared).stream().filter(ListDef::isPicklist).collect(Collectors.toList()); + } + + public Collection getLists(Container container) + { + return getLists(container, false); + } + + public Collection getLists(Container container, boolean includeProjectAndShared) + { + return getLists(container, null, false, true, includeProjectAndShared); + } + + public Collection getLists( + @NotNull Container container, + @Nullable User user, + boolean checkVisibility, + boolean includePicklists, + boolean includeProjectAndShared + ) + { + Collection scopedLists = getAllScopedLists(container, includeProjectAndShared); + if (!includePicklists) + scopedLists = scopedLists.stream().filter(listDef -> !listDef.isPicklist()).collect(Collectors.toList()); + if (checkVisibility) + return scopedLists.stream().filter(listDef -> listDef.isVisible(user)).collect(Collectors.toList()); + else + return scopedLists; + } + + /** + * Returns all list definitions defined within the scope of the container. This can optionally include list + * definitions from the container's project as well as the Shared folder. In the event of a name collision the + * closest container's list definition will be returned (i.e. container > project > Shared). + */ + private Collection getAllScopedLists(@NotNull Container container, boolean includeProjectAndShared) + { + List ownLists = _listDefCache.get(container.getId()); + Map listDefMap = new CaseInsensitiveHashMap<>(); + + if (includeProjectAndShared) + { + for (ListDef sharedList : _listDefCache.get(ContainerManager.getSharedContainer().getId())) + listDefMap.put(sharedList.getName(), sharedList); + + Container project = container.getProject(); + if (project != null) + { + for (ListDef projectList : _listDefCache.get(project.getId())) + listDefMap.put(projectList.getName(), projectList); + } + } + + // Workbooks can see parent lists. + if (container.isWorkbook()) + { + Container parent = container.getParent(); + if (parent != null) + { + for (ListDef parentList : _listDefCache.get(parent.getId())) + listDefMap.put(parentList.getName(), parentList); + } + } + + for (ListDef ownList : ownLists) + listDefMap.put(ownList.getName(), ownList); + + return listDefMap.values(); + } + + /** + * Utility method now that ListTable is ContainerFilter aware; TableInfo.getSelectName() returns now returns null + */ + String getListTableName(TableInfo ti) + { + if (ti instanceof ListTable lti) + return lti.getRealTable().getSelectName(); + return ti.getSelectName(); // if db is being upgraded from <= 13.1, lists are still SchemaTableInfo instances + } + + @Nullable + public ListDef getList(Container container, int listId) + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDef list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDef.class); + } + return list; + } + + public ListDomainKindProperties getListDomainKindProperties(Container container, @Nullable Integer listId) + { + if (null == listId) + { + return new ListDomainKindProperties(); + } + else + { + SimpleFilter filter = new PkFilter(getListMetadataTable(), new Object[]{container, listId}); + ListDomainKindProperties list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + + // Workbooks can see their parent's lists, so check that container if we didn't find the list the first time + if (list == null && container.isWorkbook()) + { + filter = new PkFilter(getListMetadataTable(), new Object[]{container.getParent(), listId}); + list = new TableSelector(getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + } + return list; + } + } + + // Note: callers must invoke indexer (can't invoke here since we may be in a transaction) + public ListDef insert(User user, final ListDef def, Collection preferredListIds) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + TableInfo tinfo = getListMetadataTable(); + DbSequence sequence = DbSequenceManager.get(c, LIST_SEQUENCE_NAME); + ListDef.ListDefBuilder builder = new ListDef.ListDefBuilder(def); + + builder.setListId(-1); + + for (Integer preferredListId : preferredListIds) + { + SimpleFilter filter = new SimpleFilter(tinfo.getColumn("Container").getFieldKey(), c).addCondition(tinfo.getColumn("ListId"), preferredListId); + + // Need to check proactively... unfortunately, calling insert and handling the constraint violation will cancel the current transaction + if (!new TableSelector(getListMetadataTable().getColumn("ListId"), filter, null).exists()) + { + builder.setListId(preferredListId); + sequence.ensureMinimum(preferredListId); // Ensure sequence is at or above the preferred ID we just used + break; + } + } + + // If none of the preferred IDs is available then use the next sequence value + if (builder.getListId() == -1) + builder.setListId((int)sequence.next()); + + ListDef ret = Table.insert(user, tinfo, builder.build()); + _listDefCache.remove(c.getId()); + return ret; + } + + + // Note: callers must invoke indexer (can't invoke here since we may already be in a transaction) + ListDef update(User user, final ListDef def) + { + Container c = def.lookupContainer(); + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDef ret; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDef old = getList(c, def.getListId()); + ret = Table.update(user, getListMetadataTable(), def, new Object[]{c, def.getListId()}); + handleIndexSettingChanges(scope, def, old, ret); + + String oldName = old.getName(); + String updatedName = ret.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + transaction.commit(); + } + + return ret; + } + + //Note: this is sort of a dupe of above update() which returns ListDef + ListDomainKindProperties update(User user, Container c, final ListDomainKindProperties listProps) + { + if (null == c) + throw OptimisticConflictException.create(Table.ERROR_DELETED); + + DbScope scope = getListMetadataSchema().getScope(); + ListDomainKindProperties updated; + + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + ListDomainKindProperties old = getListDomainKindProperties(c, listProps.getListId()); + updated = Table.update(user, getListMetadataTable(), listProps, new Object[]{c, listProps.getListId()}); + ListDef listDef = getList(c, listProps.getListId()); + handleIndexSettingChanges(scope, listDef, old, listProps); + String oldName = old.getName(); + String updatedName = updated.getName(); + queryChangeUpdate(user, c, oldName, updatedName); + + transaction.commit(); + } + + return updated; + } + + // Queue up one-time operations related to turning indexing on or off + private void handleIndexSettingChanges(DbScope scope, ListDef listDef, ListIndexingSettings old, ListIndexingSettings updated) + { + boolean oldEachItemIndex = old.isEachItemIndex(); + boolean newEachItemIndex = updated.isEachItemIndex(); + + String oldEachItemTitleTemplate = old.getEachItemTitleTemplate(); + String newEachItemTitleTemplate = updated.getEachItemTitleTemplate(); + + int oldEachItemBodySetting = old.getEachItemBodySetting(); + int newEachItemBodySetting = updated.getEachItemBodySetting(); + + String oldEachItemBodyTemplate = old.getEachItemBodyTemplate(); + String newEachItemBodyTemplate = updated.getEachItemBodyTemplate(); + + boolean oldEntireListIndex = old.isEntireListIndex(); + boolean newEntireListIndex = updated.isEntireListIndex(); + + boolean oldFileAttachmentIndex = old.isFileAttachmentIndex(); + boolean newFileAttachmentIndex = updated.isFileAttachmentIndex(); + + String oldEntireListTitleTemplate = old.getEntireListTitleTemplate(); + String newEntireListTitleTemplate = updated.getEntireListTitleTemplate(); + + int oldEntireListIndexSetting = old.getEntireListIndexSetting(); + int newEntireListIndexSetting = updated.getEntireListIndexSetting(); + + int oldEntireListBodySetting = old.getEntireListBodySetting(); + int newEntireListBodySetting = updated.getEntireListBodySetting(); + + String oldEntireListBodyTemplate = old.getEntireListBodyTemplate(); + String newEntireListBodyTemplate = updated.getEntireListBodyTemplate(); + + scope.addCommitTask(() -> { + ListDefinition list = ListDefinitionImpl.of(listDef); + + // Is each-item indexing turned on? + if (newEachItemIndex) + { + // Turning on each-item indexing, or changing document title template, body template, + // or body setting -> clear this list's LastIndexed column + if + ( + !oldEachItemIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) || + !Objects.equals(newEachItemBodyTemplate, oldEachItemBodyTemplate) || + newEachItemBodySetting != oldEachItemBodySetting + ) + { + clearLastIndexed(scope, ListSchema.getInstance().getSchemaName(), listDef); + } + } + else + { + // Turning off each-item indexing -> clear item docs from the index + if (oldEachItemIndex) + deleteIndexedItems(list); + } + + // Is attachment indexing turned on? + if (newFileAttachmentIndex) + { + // Turning on attachment indexing or changing title template -> clear attachment LastIndexed column + if + ( + !oldFileAttachmentIndex || + !Objects.equals(newEachItemTitleTemplate, oldEachItemTitleTemplate) // Attachment indexing uses the each-item title template + ) + { + clearAttachmentLastIndexed(list); + } + } + else + { + // Turning off attachment indexing -> clear attachment docs from the index + if (oldFileAttachmentIndex) + deleteIndexedAttachments(list); + } + + // Is entire-list indexing turned on? + if (newEntireListIndex) + { + // Turning on entire-list indexing, or changing the title template, body template, indexing settings, or + // body settings -> clear this list's last indexed column + if + ( + !oldEntireListIndex || + !Objects.equals(newEntireListTitleTemplate, oldEntireListTitleTemplate) || + !Objects.equals(newEntireListBodyTemplate, oldEntireListBodyTemplate) || + newEntireListIndexSetting != oldEntireListIndexSetting || + newEntireListBodySetting != oldEntireListBodySetting + ) + { + SQLFragment sql = new SQLFragment("UPDATE ") + .append(getListMetadataTable().getSelectName()) + .append(" SET LastIndexed = NULL WHERE ListId = ? AND LastIndexed IS NOT NULL") + .add(list.getListId()); + + new SqlExecutor(scope).execute(sql); + } + } + else + { + // Turning off entire-list indexing -> clear entire-list doc from the index + if (oldEntireListIndex) + deleteIndexedEntireListDoc(list); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private void queryChangeUpdate(User user, Container c, String oldName, String updatedName) + { + _listDefCache.remove(c.getId()); + QueryChangeListener.QueryPropertyChange.handleQueryNameChange(oldName, updatedName, new SchemaKey(null, ListQuerySchema.NAME), user, c); + } + + void deleteList(User user, @NotNull ListDefinitionImpl list, @Nullable String auditUserComment) throws DomainNotFoundException + { + var table = list.getTable(user); + var container = list.getContainer(); + var listName = list.getName(); + + // In certain cases we may create a list that is not viable (i.e., one in which a table was never created because + // the metadata wasn't valid). Still allow deleting the list. + try (var tx = (table != null) ? table.getSchema().getScope().ensureTransaction() : ExperimentService.get().ensureTransaction()) + { + // remove related full-text search docs and attachments + if (table != null && table.getUpdateService() instanceof ListQueryUpdateService qus) + qus.deleteRelatedListData(null); + else + deleteIndexedList(list); + + try + { + Table.delete(getListMetadataTable(), new Object[]{container, list.getListId()}); + } + catch (OptimisticConflictException x) + { + // ok + } + _listDefCache.remove(container.getId()); + + list.getDomainOrThrow().delete(user, auditUserComment); + addAuditEvent(list, user, String.format("The list %s was deleted", listName)); + + tx.commit(); + } + + QueryService.get().fireQueryDeleted(user, container, null, SchemaKey.fromParts(ListQuerySchema.NAME), Collections.singleton(listName)); + } + + /** + * Deletes all lists and list data in the given container. + */ + public void deleteLists(Container c, User user, @Nullable String auditUserComment) + { + try + { + var containerId = c.getEntityId().toString(); + for (var list : ListManager.get().getLists(c, true)) + { + var listDef = ListDefinitionImpl.of(list); + + // Delete the entire list when the list's container is deleted + if (containerId.equals(list.getContainerId())) + { + deleteList(user, listDef, auditUserComment); + continue; + } + + var table = listDef.getTable(user); + if (table == null) + continue; + + var qus = table.getUpdateService(); + if (qus == null) + continue; + + // Otherwise, truncate the rows in this list in this container + qus.truncateRows(user, c, null, null); + } + } + catch (Exception e) + { + throw new RuntimeException(e.getMessage(), e); + } + } + + public static final SearchService.SearchCategory listCategory = new SearchService.SearchCategory("list", "Lists"); + + // Index all lists in this container + @Override + public void enumerateDocuments(SearchService.TaskIndexingQueue queue, @Nullable Date since) + { + Consumer r = (q) -> { + Map lists = ListService.get().getLists(q.getContainer(), null, false); + + try + { + QueryService.get().setEnvironment(QueryService.Environment.USER, User.getSearchUser()); + QueryService.get().setEnvironment(QueryService.Environment.CONTAINER, q.getContainer()); + for (ListDefinition list : lists.values()) + { + try + { + boolean reindex = since == null; + indexList(q, list, reindex); + } + catch (Exception ex) + { + LOG.error("Error indexing list '" + list.getName() + "' in container '" + q.getContainer().getPath() + "'.", ex); + } + } + } + finally + { + QueryService.get().clearEnvironment(); + } + }; + + queue.addRunnable(r); + } + + public void indexList(final ListDefinition def) + { + indexList(((ListDefinitionImpl) def)._def); + } + + // Index a single list + public void indexList(final ListDef def) + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(def.lookupContainer(), SearchService.PRIORITY.modified); + Consumer r = (q) -> + { + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + //Refresh list definition -- Issue #42207 - MSSQL server returns entityId as uppercase string + ListDefinition list = ListService.get().getList(c, def.getListId()); + if (null != list) // Could have just been deleted + indexList(q, list, false); + } + }; + + Container c = def.lookupContainer(); + if (!ContainerManager.exists(c)) + { + LOG.info("List container has been deleted or is being deleted; not indexing list \"" + def.getName() + "\""); + } + else + { + queue.addRunnable(r); + } + } + + private void indexList(SearchService.TaskIndexingQueue queue, ListDefinition list, final boolean reindex) + { + Domain domain = list.getDomain(); + + // List might have just been deleted + if (null != domain) + { + // indexing methods turn off JDBC driver caching and use a side connection, so we must not be in a transaction + assert !DbScope.getLabKeyScope().isTransactionActive() : "Should not be in a transaction since this code path disables JDBC driver caching"; + + indexEntireList(queue, list, reindex); + indexModifiedItems(queue, list, reindex); + indexAttachments(queue, list, reindex); + } + } + + // Delete a single list item from the index after item delete + public void deleteItemIndex(final ListDefinition list, @NotNull final String entityId) + { + // Transaction-aware is good practice. But it happens to be critical in the case of calling indexEntireList() + // because it turns off JDBC caching, using a non-transacted connection (bad news if we call it mid-transaction). + getListMetadataSchema().getScope().addCommitTask(() -> + { + SearchService.TaskIndexingQueue queue = SearchService.get().defaultTask().getQueue(list.getContainer(), SearchService.PRIORITY.modified); + if (list.getEachItemIndex()) + { + SearchService.get().deleteResource(getDocumentId(list, entityId)); + } + + // Reindex the entire list document iff data is being indexed + if (list.getEntireListIndex() && list.getEntireListIndexSetting().indexItemData()) + { + indexEntireList(queue, list, true); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + } + + private String getDocumentId(ListDefinition list) + { + return "list:" + ((ListDefinitionImpl)list).getEntityId(); + } + + // Use each item's EntityId since PKs are mutable. ObjectIds maybe be the better choice (they're shorter) but + // that would require adding this column to the query definition. Consider: a private TableInfo just for indexing. + private String getDocumentId(ListDefinition list, @Nullable String entityId) + { + return getDocumentId(list) + ":" + (null != entityId ? entityId : ""); + } + + private static boolean hasAttachmentColumns(@NotNull TableInfo listTable) + { + return listTable.getColumns().stream().anyMatch(ci -> ci.getPropertyType() == PropertyType.ATTACHMENT); + } + + // Index all modified items in this list + private void indexModifiedItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, final boolean reindex) + { + if (list.getEachItemIndex()) + { + String lastIndexClause = reindex ? "(1=1) OR " : ""; //Prepend TRUE if we want to force a reindexing + + // Index all items that have never been indexed OR where either the list definition or list item itself has changed since last indexed + lastIndexClause += "LastIndexed IS NULL OR LastIndexed < ? OR (Modified IS NOT NULL AND LastIndexed < Modified)"; + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause(lastIndexClause, new Object[]{list.getModified()})); + + indexItems(queue, list, filter); + } + } + + // Reindex items specified by filter + private void indexItems(@NotNull SearchService.TaskIndexingQueue queue, final ListDefinition list, SimpleFilter filter) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + + if (null != listTable) + { + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + FieldKeyStringExpression bodyTemplate = createBodyTemplate(list, "\"each item as a separate document\" custom indexing template", list.getEachItemBodySetting(), list.getEachItemBodyTemplate(), listTable); + + FieldKey keyKey = new FieldKey(null, list.getKeyName()); + FieldKey entityIdKey = new FieldKey(null, "EntityId"); + + FieldKey createdKey = new FieldKey(null, "created"); + FieldKey createdByKey = new FieldKey(null, "createdBy"); + FieldKey modifiedKey = new FieldKey(null, "modified"); + FieldKey modifiedByKey = new FieldKey(null, "modifiedBy"); + + // TODO: Attempting to respect tableUrl for details link... but this doesn't actually work. See #28747. + StringExpression se = listTable.getDetailsURL(null, list.getContainer()); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachResults(results -> { + Map map = results.getFieldKeyRowMap(); + final Object pk = map.get(keyKey); + String entityId = (String) map.get(entityIdKey); + + String documentId = getDocumentId(list, entityId); + Map props = new HashMap<>(); + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + String displayTitle = titleTemplate.eval(map); + props.put(SearchService.PROPERTY.title.toString(), displayTitle); + + Date created = null; + if (map.get(createdKey) instanceof Date) + created = (Date) map.get(createdKey); + + Date modified = null; + if (map.get(modifiedKey) instanceof Date) + modified = (Date) map.get(modifiedKey); + + String body = bodyTemplate.eval(map); + + ActionURL itemURL; + + try + { + itemURL = new ActionURL(se.eval(map)); + } + catch (Exception e) + { + itemURL = list.urlDetails(pk); + } + + itemURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body, + itemURL, + UserManager.getUser(asInteger( map.get(createdByKey))), created, + UserManager.getUser(asInteger( map.get(modifiedByKey))), modified, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + try + { + ListManager.get().setItemLastIndexed(list, pk, listTable, ms, modified); + } + catch (BadSqlGrammarException e) + { + // This may occur due to a race condition between enumeration and list deletion. Issue #48878 + // expected P-sql expected MS-sql + if (e.getCause().getMessage().contains("does not exist") || e.getCause().getMessage().contains("Invalid object name")) + LOG.debug("Attempt to set LastIndexed on list table failed", e); + else + throw e; + } + } + }; + + // Add navtrail that includes link to full list grid + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + r.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of item with PK = " + pk); + }); + } + } + + /** + * Add searchable resources to Indexing task for file attachments + * @param list containing file attachments + */ + private void indexAttachments(@NotNull final SearchService.TaskIndexingQueue queue, ListDefinition list, boolean reindex) + { + TableInfo listTable = list.getTable(User.getSearchUser()); + if (listTable != null && list.getFileAttachmentIndex() && hasAttachmentColumns(listTable)) + { + //Get common objects & properties + AttachmentService as = AttachmentService.get(); + FieldKeyStringExpression titleTemplate = createEachItemTitleTemplate(list, listTable); + + // Breadcrumb link to list is the same for all attachments on all items + ActionURL gridURL = list.urlShowData(); + gridURL.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + NavTree t = new NavTree("list", gridURL); + String nav = NavTree.toJS(Collections.singleton(t), null, false, true).toString(); + + // Enumerate all list rows in batches and re-index based on the value of reindex parameter + // For now, enumerate all rows. In the future, pass in a PK filter for the single item change case? + SimpleFilter filter = new SimpleFilter(new SimpleFilter.SQLClause("(1=1)", null)); + + // Need to pass non-null modifiedSince for incremental indexing, otherwise all attachments will be returned + // TODO: Pass modifiedSince into this method? + Date modifiedSince = reindex ? null : new Date(); + + new TableSelector(listTable, filter, null).setJdbcCaching(false).setForDisplay(true).forEachMapBatch(10_000, batch -> { + // RowEntityId -> List item RowMap + Map> lookupMap = batch.stream() + .collect(Collectors.toMap(map -> (String) map.get("EntityId"), map -> map)); + + // RowEntityId -> Document names that need to be indexed + MultiValuedMap documentMultiMap = as.listAttachmentsForIndexing(lookupMap.keySet(), modifiedSince).stream() + .collect(LabKeyCollectors.toMultiValuedMap(stringStringPair -> stringStringPair.first, stringStringPair -> stringStringPair.second)); + + documentMultiMap.asMap().forEach((rowEntityId, documentNames) -> { + Map map = lookupMap.get(rowEntityId); + String title = titleTemplate.eval(map); + + documentNames.forEach(documentName -> { + ActionURL downloadUrl = ListController.getDownloadURL(list, rowEntityId, documentName); + + //Generate searchable resource + String displayTitle = title + " attachment file \"" + documentName + "\""; + WebdavResource attachmentRes = as.getDocumentResource( + new Path(rowEntityId, documentName), + downloadUrl, + displayTitle, + new ListItemAttachmentParent(rowEntityId, list.getContainer()), + documentName, + SearchService.fileCategory + ); + + attachmentRes.getMutableProperties().put(SearchService.PROPERTY.navtrail.toString(), nav); + queue.addResource(attachmentRes); + LOG.debug("List \"" + list + "\": Queued indexing of attachment \"" + documentName + "\" for item with PK = " + map.get(list.getKeyName())); + }); + }); + }); + } + } + + private void indexEntireList(SearchService.TaskIndexingQueue queue, final ListDefinition list, boolean reindex) + { + if (list.getEntireListIndex()) + { + IndexSetting setting = list.getEntireListIndexSetting(); + String documentId = getDocumentId(list); + + // First check if metadata needs to be indexed: if the setting is enabled and the definition has changed + boolean needToIndex = (setting.indexMetaData() && hasDefinitionChangedSinceLastIndex(list)); + + // If that didn't hold true then check for entire list data indexing: if the definition has changed or any item has been modified + if (!needToIndex && setting.indexItemData()) + needToIndex = hasDefinitionChangedSinceLastIndex(list) || hasModifiedItems(list); + + needToIndex |= reindex; + + if (needToIndex) + { + StringBuilder body = new StringBuilder(); + Map props = new HashMap<>(); + + // Use standard title if template is null/whitespace + String templateString = StringUtils.trimToNull(list.getEntireListTitleTemplate()); + String title = null == templateString ? "List " + list.getName() : templateString; + + props.put(SearchService.PROPERTY.categories.toString(), listCategory.toString()); + props.put(SearchService.PROPERTY.title.toString(), title); + + if (!StringUtils.isEmpty(list.getDescription())) + body.append(list.getDescription()).append("\n"); + + String sep = ""; + + if (setting.indexMetaData()) + { + String comma = ""; + for (DomainProperty property : list.getDomain().getProperties()) + { + String n = StringUtils.trimToEmpty(property.getName()); + String l = StringUtils.trimToEmpty(property.getLabel()); + if (n.equals(l)) + l = ""; + body.append(comma).append(sep).append(StringUtilsLabKey.joinNonBlank(" ", n, l)); + comma = ","; + sep = "\n"; + } + } + + if (setting.indexItemData()) + { + TableInfo ti = list.getTable(User.getSearchUser()); + int fileSizeLimit = (int) (SearchService.get().getFileSizeLimit() * .99); + + if (ti != null) + { + body.append(sep); + FieldKeyStringExpression template = createBodyTemplate(list, "\"entire list as a single document\" custom indexing template", list.getEntireListBodySetting(), list.getEntireListBodyTemplate(), ti); + + // All columns, all rows, no filters, no sorts + new TableSelector(ti).setJdbcCaching(false).setForDisplay(true).forEachResults(new ForEachBlock<>() + { + @Override + public void exec(Results results) throws StopIteratingException + { + body.append(template.eval(results.getFieldKeyRowMap())).append("\n"); + // Issue 25366: Short circuit for very large list + if (body.length() > fileSizeLimit) + { + body.setLength(fileSizeLimit); // indexer also checks size... make sure we're under the limit + stopIterating(); + } + } + }); + } + } + + ActionURL url = list.urlShowData(); + url.setExtraPath(list.getContainer().getId()); // Use ID to guard against folder moves/renames + + SimpleDocumentResource r = new SimpleDocumentResource( + new Path(documentId), + documentId, + list.getContainer().getEntityId(), + "text/plain", + body.toString(), + url, + props) + { + @Override + public void setLastIndexed(long ms, long modified) + { + ListManager.get().setLastIndexed(list, ms); + } + }; + + queue.addResource(r); + LOG.debug("List \"" + list + "\": Queued indexing of entire list document"); + } + } + } + + void deleteIndexedList(ListDefinition list) + { + if (list.getEntireListIndex()) + deleteIndexedEntireListDoc(list); + + if (list.getEachItemIndex()) + deleteIndexedItems(list); + + if (list.getFileAttachmentIndex()) + deleteIndexedAttachments(list); + } + + private void deleteIndexedAttachments(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::deleteIndexedAttachments); + } + + private void clearAttachmentLastIndexed(@NotNull ListDefinition list) + { + handleAttachmentParents(list, AttachmentService::clearLastIndexed); + } + + private interface AttachmentParentHandler + { + void handle(AttachmentService as, List parentIds); + } + + // If the list has any attachment columns, select all parent IDs and invoke the passed in handler in batches of 10,000 + private void handleAttachmentParents(@NotNull ListDefinition list, AttachmentParentHandler handler) + { + // make sure container still exists (race condition on container delete) + Container listContainer = list.getContainer(); + if (null == listContainer) + return; + TableInfo listTable = new ListQuerySchema(User.getSearchUser(), listContainer).getTable(list.getName()); + if (null == listTable) + return; + + AttachmentService as = AttachmentService.get(); + + if (hasAttachmentColumns(listTable)) + { + new TableSelector(listTable, Collections.singleton("EntityId")).setJdbcCaching(false).forEachBatch(String.class, 10_000, parentIds -> handler.handle(as, parentIds)); + } + } + + // Un-index the entire list doc, but leave the list items alone + private void deleteIndexedEntireListDoc(ListDefinition list) + { + SearchService.get().deleteResource(getDocumentId(list)); + } + + + // Un-index all list items, but leave the entire list doc alone + private void deleteIndexedItems(ListDefinition list) + { + SearchService.get().deleteResourcesForPrefix(getDocumentId(list, null)); + } + + + private FieldKeyStringExpression createEachItemTitleTemplate(ListDefinition list, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + String templateString = StringUtils.trimToNull(list.getEachItemTitleTemplate()); + + if (null != templateString) + { + template = createValidStringExpression(templateString, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, "\"each item as a separate document\" title template", error)); + } + + // Issue 21794: If you're devious enough to put ${ in your list name then we'll just strip it out + String name = list.getName().replaceAll("\\$\\{", "_{"); + template = createValidStringExpression("List " + name + " - ${" + PageFlowUtil.encode(listTable.getTitleColumn()) + "}", error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated title template", error)); + + return template; + } + + + private FieldKeyStringExpression createBodyTemplate(ListDefinition list, String templateType, BodySetting setting, @Nullable String customTemplate, TableInfo listTable) + { + FieldKeyStringExpression template; + StringBuilder error = new StringBuilder(); + + if (setting == BodySetting.Custom && !StringUtils.isBlank(customTemplate)) + { + template = createValidStringExpression(customTemplate, error); + + if (null != template) + return template; + else + LOG.warn(getTemplateErrorMessage(list, templateType, error)); + } + + StringBuilder sb = new StringBuilder(); + String sep = ""; + + for (ColumnInfo column : listTable.getColumns()) + { + if (setting.accept(column)) + { + sb.append(sep); + sb.append("${"); + sb.append(column.getFieldKey().encode()); // Issue 21794: Must encode + sb.append("}"); + sep = " "; + } + } + + template = createValidStringExpression(sb.toString(), error); + + if (null == template) + throw new IllegalStateException(getTemplateErrorMessage(list, "auto-generated indexing template", error)); + + return template; + } + + + // Issue 21726: Perform some simple validation of custom indexing template + private @Nullable FieldKeyStringExpression createValidStringExpression(String template, StringBuilder error) + { + // Don't URL encode and use lenient substitution (replace nulls with blank) + FieldKeyStringExpression se = FieldKeyStringExpression.create(template, false, NullValueBehavior.ReplaceNullWithBlank); + + try + { + // TODO: Is there a more official way to validate a StringExpression? + se.eval(Collections.emptyMap()); + } + catch (IllegalArgumentException e) + { + error.append(e.getMessage()); + se = null; + } + + return se; + } + + + private String getTemplateErrorMessage(ListDefinition list, String templateType, CharSequence message) + { + return "Invalid " + templateType + " for list \"" + list.getName() + "\" in " + list.getContainer().getPath() + ": " + message; + } + + + private boolean hasDefinitionChangedSinceLastIndex(ListDefinition list) + { + return list.getLastIndexed() == null || list.getModified().compareTo(list.getLastIndexed()) > 0; + } + + + // Checks for existence of list items that have been modified since the entire list was last indexed + private boolean hasModifiedItems(ListDefinition list) + { + TableInfo table = list.getTable(User.getSearchUser()); + + if (null != table && null != getListTableName(table)) + { + // Using EXISTS query should be reasonably efficient. + SQLFragment sql = new SQLFragment("SELECT 1 FROM "); + sql.append(getListTableName(table)); + sql.append(" WHERE Modified > (SELECT LastIndexed FROM ").append(getListMetadataTable()); + sql.append(" WHERE ListId = ? AND Container = ?)"); + sql.add(list.getListId()); + sql.add(list.getContainer().getEntityId()); + + return new SqlSelector(getListMetadataSchema(), sql).exists(); + } + + return false; + } + + private void setLastIndexed(ListDefinition list, long ms) + { + // list table does not have an index on listid, so we should include container in the WHERE + SQLFragment update = new SQLFragment("UPDATE ").append(getListMetadataTable()) + .append(" SET LastIndexed = ? WHERE Container = ? AND ListId = ?").addAll(new Timestamp(ms), list.getContainer(), list.getListId()); + new SqlExecutor(getListMetadataSchema()).execute(update); + _listDefCache.remove(list.getContainer().getId()); + list = ListDefinitionImpl.of(getList(list.getContainer(), list.getListId())); + long modified = list.getModified().getTime(); + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for entire list document" + warning); + } + + + private void setItemLastIndexed(ListDefinition list, Object pk, TableInfo ti, long ms, long modified) + { + // The "search user" might not have access + if (null != ti) + { + // 'unwrap' ListTable to get schema table for update + TableInfo sti = ((ListTable)ti).getSchemaTableInfo(); + ColumnInfo keyColumn = sti.getColumn(list.getKeyName()); + if (null != keyColumn) + { + var keySelectName = keyColumn.getSelectIdentifier(); + SQLFragment sqlf = new SQLFragment("UPDATE ").appendIdentifier(getListTableName(sti)) + .append(" SET LastIndexed = ").appendValue(new Timestamp(ms)) + .append(" WHERE ").appendIdentifier(keySelectName).append(" = ?").add(pk); + new SqlExecutor(sti.getSchema()).execute(sqlf); + } + String warning = ms < modified ? ". WARNING: LastIndexed is less than Modified! " + ms + " vs. " + modified : ""; + LOG.debug("List \"" + list + "\": Set LastIndexed for item with PK = " + pk + warning); + } + } + + + @Override + public void indexDeleted() + { + TableInfo listTable = getListMetadataTable(); + DbScope scope = listTable.getSchema().getScope(); + + // Clear LastIndexed column of the exp.List table, which addresses the "index the entire list as a single document" case + clearLastIndexed(scope, listTable.getSelectName()); + + String listSchemaName = ListSchema.getInstance().getSchemaName(); + + // Now clear LastIndexed column of every underlying list table, which addresses the "index each list item as a separate document" case. See #28748. + new TableSelector(getListMetadataTable()).forEach(ListDef.class, listDef -> clearLastIndexed(scope, listSchemaName, listDef)); + } + + private void clearLastIndexed(DbScope scope, String listSchemaName, ListDef listDef) + { + // Clear LastIndexed column only for lists that are set to index each item, Issue 47998 + if (listDef.isEachItemIndex()) + { + ListDefinition list = new ListDefinitionImpl(listDef); + Domain domain = list.getDomain(); + if (null != domain && null != domain.getStorageTableName()) + { + LOG.info("List " + listDef.getContainerPath() + " - " + listDef.getName() + ": Set to index each item, so clearing last indexed"); + clearLastIndexed(scope, listSchemaName + "." + domain.getStorageTableName()); + } + } + } + + private void clearLastIndexed(DbScope scope, String selectName) + { + try + { + // Yes, that WHERE clause is intentional and makes a big performance improvement in some cases + new SqlExecutor(scope).execute("UPDATE " + selectName + " SET LastIndexed = NULL WHERE LastIndexed IS NOT NULL"); + } + catch (Exception e) + { + // Log the exception, but allow other tables to be cleared + ExceptionUtil.logExceptionToMothership(null, e); + } + } + + void addAuditEvent(ListDefinitionImpl list, User user, String comment) + { + if (null != user) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(list.getContainer(), comment, list); + AuditLogService.get().addEvent(user, event); + } + } + + void addAuditEvent(ListDefinitionImpl list, User user, Container c, String comment, String entityId, @Nullable String oldRecord, @Nullable String newRecord) + { + ListAuditProvider.ListAuditEvent event = new ListAuditProvider.ListAuditEvent(c, comment, list); + + event.setListItemEntityId(entityId); + if (oldRecord != null) event.setOldRecordMap(oldRecord); + if (newRecord != null) event.setNewRecordMap(newRecord); + + AuditLogService.get().addEvent(user, event); + } + + String formatAuditItem(ListDefinitionImpl list, User user, Map props) + { + String itemRecord = ""; + TableInfo ti = list.getTable(user); + + if (null != ti) + { + Map recordChangedMap = new CaseInsensitiveHashMap<>(); + Set reserved = list.getDomain().getDomainKind().getReservedPropertyNames(list.getDomain(), user); + + // Match props to columns + for (Map.Entry entry : props.entrySet()) + { + String baseKey = entry.getKey(); + + boolean isReserved = false; + for (String res : reserved) + { + if (res.equalsIgnoreCase(baseKey)) + { + isReserved = true; + break; + } + } + + if (isReserved) + continue; + + ColumnInfo col = ti.getColumn(FieldKey.fromParts(baseKey)); + Object value = entry.getValue(); + String key = null; + + if (null != col) + { + // Found the column + key = col.getName(); // best good + } + else + { + // See if there is a match in the domain properties + for (DomainProperty dp : list.getDomain().getProperties()) + { + if (dp.getName().equalsIgnoreCase(baseKey)) + { + key = dp.getName(); // middle good + } + } + + // Try by name + DomainProperty dp = list.getDomain().getPropertyByName(baseKey); + if (null != dp) + key = dp.getName(); + } + + if (null != key && null != value) + recordChangedMap.put(key, value); + } + + if (!recordChangedMap.isEmpty()) + itemRecord = ListAuditProvider.encodeForDataMap(recordChangedMap); + } + + return itemRecord; + } + + boolean importListSchema( + ListDefinition unsavedList, + ImportTypesHelper importHelper, + User user, + Collection validatorImporters, + List errors + ) throws Exception + { + if (!errors.isEmpty()) + return false; + + final Container container = unsavedList.getContainer(); + final Domain domain = unsavedList.getDomain(); + final String typeURI = domain.getTypeURI(); + + DomainURIFactory factory = name -> new Pair<>(typeURI, container); + + ImportPropertyDescriptorsList pds = importHelper.getImportPropertyDescriptors(factory, errors, container); + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + if (null == ipd.domainName || null == ipd.domainURI) + errors.add("List not specified for property: " + ipd.pd.getName()); + } + + if (!errors.isEmpty()) + return false; + + for (ImportPropertyDescriptor ipd : pds.properties) + { + DomainProperty domainProperty = domain.addPropertyOfPropertyDescriptor(ipd.pd); + domainProperty.setConditionalFormats(ipd.formats); + domainProperty.setDefaultValue(ipd.defaultValue); + } + + unsavedList.save(user); + + // Save validators later, after all the lists are imported, #40343 + validatorImporters.add(new ValidatorImporter(domain.getTypeId(), pds.properties, user)); + + return true; + } + + public static class TestCase extends Assert + { + private static final String PROJECT_NAME = "BiologicsManagerTest Project"; + private static final String LIST_NAME = "Unit Test list"; + private static final String FIELD_NAME = "field"; + private static final String ATTACHMENT_FIELD_NAME = "attachment"; + private static final String PARENT_LIST_ITEM = "parentItem"; + private static final String CHILD1_LIST_ITEM = "child1Item"; + private static final String CHILD2_LIST_ITEM = "child2Item"; + private static final String KEY_FIELD_NAME = "Unit test list key"; + private static Integer PARENT_LI_KEY; + + private static ListDefinitionImpl list; + private static DomainProperty dp; + private static User u; + private static Container c; + + @BeforeClass + public static void setup() throws Exception + { + u = TestContext.get().getUser(); + + deleteTestContainer(); + + c = ContainerManager.ensureContainer(PROJECT_NAME, u); + list = (ListDefinitionImpl)ListService.get().createList(c, LIST_NAME, ListDefinition.KeyType.AutoIncrementInteger); + list.setKeyName(KEY_FIELD_NAME); + + dp = list.getDomain().addProperty(); + dp.setName(FIELD_NAME); + dp.setType(PropertyService.get().getType(c, PropertyType.STRING.getXmlName())); + dp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), FIELD_NAME, c, list.getKeyType()).toString()); + + var attachmentDp = list.getDomain().addProperty(); + attachmentDp.setName(ATTACHMENT_FIELD_NAME); + attachmentDp.setType(PropertyService.get().getType(c, PropertyType.ATTACHMENT.getXmlName())); + attachmentDp.setPropertyURI(ListDomainKind.createPropertyURI(list.getName(), ATTACHMENT_FIELD_NAME, c, list.getKeyType()).toString()); + list.save(u); + + PARENT_LI_KEY = addListItem(c, list, PARENT_LIST_ITEM, null); + } + + private static int addListItem(Container scopedContainer, ListDefinition scopedList, String value, @Nullable File attachment) throws Exception + { + var row = new CaseInsensitiveHashMap<>(); + row.put(FIELD_NAME, value); + if (attachment != null) + row.put(ATTACHMENT_FIELD_NAME, new FileAttachmentFile(attachment)); + + var errors = new BatchValidationException(); + var result = scopedList.getTable(u).getUpdateService().insertRows(u, scopedContainer, List.of(row), errors, null, null); + if (errors.hasErrors()) + throw errors.getLastRowError(); + + return MapUtils.getIntValue(result.getFirst(), KEY_FIELD_NAME); + } + + @AfterClass + public static void cleanup() + { + list = null; + dp = null; + c = null; + u = null; + + deleteTestContainer(); + } + + private static void deleteTestContainer() + { + var project = ContainerManager.getForPath(PROJECT_NAME); + + if (project != null) + ContainerManager.deleteAll(project, TestContext.get().getUser()); + } + + @Test + public void testListServiceInOwnFolder() + { + Map lists = ListService.get().getLists(c); + assertTrue("Test List not found in own container", lists.containsKey(LIST_NAME)); + ListItem li = lists.get(LIST_NAME).getListItem(PARENT_LI_KEY, u, c); + assertEquals("Item not found in own container", PARENT_LIST_ITEM, li.getProperty(dp)); + } + + @Test + public void testListServiceInWorkbook() throws Exception + { + Container workbook1 = createWorkbook("Unit Test Workbook 1"); + Container workbook2 = createWorkbook("Unit Test Workbook 2"); + Map lists = ListService.get().getLists(workbook1); + assertTrue("Test List not found in workbook", lists.containsKey(LIST_NAME)); + + ListDefinition childList1 = ListService.get().getLists(workbook1).get(LIST_NAME); + ListDefinition childList2 = ListService.get().getLists(workbook2).get(LIST_NAME); + + assertEquals("Lists available to each child container are not the same", childList1.toString(), childList2.toString()); + int childKey1 = addListItem(workbook1, childList1, CHILD1_LIST_ITEM, null); + int childKey2 = addListItem(workbook2, childList2, CHILD2_LIST_ITEM, null); + + assertNull("Parent item should not be visible in child container", childList1.getListItem(PARENT_LI_KEY, u, workbook1)); + assertNull("Sibling child item should not be visible in another child container", childList1.getListItem(childKey2, u, workbook1)); + assertEquals("Parent container should be able to see child container item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, c).getProperty(dp)); + assertEquals("Child container should be able to see its own list item", CHILD1_LIST_ITEM, childList1.getListItem(childKey1, u, workbook1).getProperty(dp)); + } + + /** + * GitHub Issue #1013: Rows added to a list in a subfolder must be cleaned up when that subfolder + * is deleted. Previously, those rows were left orphaned in the database. + */ + @Test + public void testSubfolderDeletionCleansUpListData() throws Exception + { + Container subfolder1 = createSubfolder("subfolder1"); + Container subfolder2 = createSubfolder("subfolder2"); + + File attachment1 = FileUtil.createTempFile("subfolder1", ".txt"); + attachment1.deleteOnExit(); + File attachment2 = FileUtil.createTempFile("subfolder2", ".txt"); + attachment2.deleteOnExit(); + + int key1 = addListItem(subfolder1, list, "subfolder1Item", attachment1); + int key2 = addListItem(subfolder2, list, "subfolder2Item", attachment2); + + // Precondition: parent can see both subfolder items (setUp inserts key 1; subfolders get keys 2 and 3) + assertNotNull("Subfolder1 item should be visible from subfolder1 before deletion", list.getListItem(key1, u, subfolder1)); + assertNotNull("Subfolder2 item should be visible from subfolder2 before deletion", list.getListItem(key2, u, subfolder2)); + + TableInfo table = list.getTable(u, c, ContainerFilter.getUnsafeEverythingFilter()); + assertNotNull("Expected list table to resolve", table); + + // Preverify rows in the database + long rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); + assertEquals("Expected only two rows (parent, subfolder1, subfolder2)", 3, rowCount); + + // Delete subfolder1 — should delete its list rows and attachments, + // not touch subfolder2 or the parent or the list definition + ListService.get().deleteLists(subfolder1, u, null); + + // Verify row is deleted from underlying table + rowCount = new TableSelector(table, Set.of("Container")).getRowCount(); + assertEquals("Expected only two rows (parent, subfolder2)", 2, rowCount); + + // Items belonging to other containers should be unaffected + assertNotNull("Parent item should survive subfolder1 deletion", list.getListItem(PARENT_LI_KEY, u, c)); + assertNotNull("Subfolder2 item should survive subfolder1 deletion", list.getListItem(key2, u, subfolder2)); + } + + private Container createSubfolder(String folderName) + { + return ContainerManager.createContainer(c, folderName, u); + } + + private Container createWorkbook(String title) + { + return ContainerManager.createContainer(c, null, title, null, WorkbookContainerType.NAME, u); + } + } +} From 0667fbe61828a57220585a3d938e611eab0c14de Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 17 Apr 2026 09:50:38 -0700 Subject: [PATCH 3/3] crlf --- .../api/audit/AbstractAuditTypeProvider.java | 794 +++++++++--------- 1 file changed, 397 insertions(+), 397 deletions(-) diff --git a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java index 5e811a43d4d..9d573cc1536 100644 --- a/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java +++ b/api/src/org/labkey/api/audit/AbstractAuditTypeProvider.java @@ -1,397 +1,397 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.audit; - -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.labkey.api.audit.data.DataMapColumn; -import org.labkey.api.audit.data.DataMapDiffColumn; -import org.labkey.api.audit.query.AbstractAuditDomainKind; -import org.labkey.api.audit.query.DefaultAuditTypeTable; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.AbstractTableInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.MultiChoice; -import org.labkey.api.data.MutableColumnInfo; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.dataiterator.DataIterator; -import org.labkey.api.dataiterator.ExistingRecordDataIterator; -import org.labkey.api.exp.ChangePropertyDescriptorException; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.query.AliasedColumn; -import org.labkey.api.query.DetailsURL; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.UserSchema; -import org.labkey.api.security.User; -import org.labkey.api.util.DateUtil; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.view.ActionURL; - -import java.sql.Time; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.labkey.api.audit.query.AbstractAuditDomainKind.AUDIT_RECORD_DATA_MAP_CONCEPT_URI; -import static org.labkey.api.audit.query.AbstractAuditDomainKind.NEW_RECORD_PROP_NAME; -import static org.labkey.api.audit.query.AbstractAuditDomainKind.OLD_RECORD_PROP_NAME; - -public abstract class AbstractAuditTypeProvider implements AuditTypeProvider -{ - public static final String QUERY_SCHEMA_NAME = "auditLog"; - public static final String SCHEMA_NAME = "audit"; - - public static final String COLUMN_NAME_ROW_ID = "RowId"; - public static final String COLUMN_NAME_CONTAINER = "Container"; - public static final String COLUMN_NAME_COMMENT = "Comment"; - public static final String COLUMN_NAME_USER_COMMENT = "UserComment"; - public static final String COLUMN_NAME_EVENT_TYPE = "EventType"; - public static final String COLUMN_NAME_CREATED = "Created"; - public static final String COLUMN_NAME_CREATED_BY = "CreatedBy"; - public static final String COLUMN_NAME_IMPERSONATED_BY = "ImpersonatedBy"; - public static final String COLUMN_NAME_PROJECT_ID = "ProjectId"; - public static final String COLUMN_NAME_TRANSACTION_ID = "TransactionID"; - public static final String COLUMN_NAME_DATA_CHANGES = "DataChanges"; - - private final AbstractAuditDomainKind _domainKind; - - public AbstractAuditTypeProvider(@NotNull AbstractAuditDomainKind domainKind) - { - // TODO: consolidate domain kind initialization to this constructor and stop overriding getDomainKind() - _domainKind = domainKind; - // Register the DomainKind - PropertyService.get().registerDomainKind(getDomainKind()); - } - - protected final AbstractAuditDomainKind getDomainKind() - { - if (_domainKind == null) - throw new IllegalStateException(String.format("The audit type : \"%s\" has a null domain kind", getLabel())); - - return _domainKind; - } - - // Expose the domain kind to AbstractAuditDomainKind$TestCase without touching every subclass - public AbstractAuditDomainKind getAuditDomainKind() - { - return getDomainKind(); - } - - @Override - public final void initializeProvider(User user) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - domainKind.validate(); - - Domain domain = getDomain(true); - - // if the domain doesn't exist, create it - if (domain == null) - { - try - { - String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); - domain = PropertyService.get().createDomain(getDomainContainer(), domainURI, domainKind.getKindName()); - for (PropertyDescriptor pd : domainKind.getProperties()) - { - domain.addPropertyOfPropertyDescriptor(pd); - } - domain.save(user); - domain = getDomain(true); - } - catch (ChangePropertyDescriptorException e) - { - throw new RuntimeException(e); - } - } - - // adjust potential domain kind changes - ensureProperties(user, domain); - } - - // NOTE: Changing the name of an existing PropertyDescriptor will lose data! - private void ensureProperties(User user, Domain domain) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - if (domain != null && domainKind != null) - { - // Create a map of desired properties - Map props = new CaseInsensitiveHashMap<>(); - for (PropertyDescriptor pd : domainKind.getProperties()) - props.put(pd.getName(), pd); - - // Create a map of existing properties - Map current = new CaseInsensitiveHashMap<>(); - for (DomainProperty dp : domain.getProperties()) - { - current.put(dp.getName(), dp); - } - - Set toAdd = new LinkedHashSet<>(); - for (PropertyDescriptor pd : props.values()) - if (!current.containsKey(pd.getName())) - toAdd.add(pd); - - Set toUpdate = new LinkedHashSet<>(); - boolean changed = false; - - for (DomainProperty dp : current.values()) - { - if (props.containsKey(dp.getName())) - toUpdate.add(dp); - else - { - dp.delete(); - changed = true; - } - } - - for (PropertyDescriptor pd : toAdd) - { - domain.addPropertyOfPropertyDescriptor(pd); - } - - try (DbScope.Transaction transaction = domainKind.getScope().ensureTransaction()) - { - // CONSIDER: Avoid always updating the existing properties -- only update changed props. - for (DomainProperty dp : toUpdate) - { - PropertyDescriptor desired = props.get(dp.getName()); - assert desired != null; - - if (differ(desired, dp, domain.getContainer())) - { - changed = true; - copyTo(dp, desired, domain.getContainer()); - } - } - - changed = changed || !toAdd.isEmpty(); - if (changed) - { - domain.save(user); - } - - assert domain.getStorageTableName() != null; - assert domain.getDomainKind() != null; - assert domain.getDomainKind().getClass().equals(domainKind.getClass()); - - StorageProvisioner.get().ensureTableIndices(domain); - transaction.commit(); - } - catch (ChangePropertyDescriptorException e) - { - throw new RuntimeException(e); - } - } - } - - // Issue 26311: We want to trigger a save if the scale has changed - // CONSIDER: check for other differences here as well. - private boolean differ(PropertyDescriptor pd, DomainProperty dp, Container c) - { - return dp.getScale() != pd.getScale() || !dp.getRangeURI().equals(pd.getRangeURI()); - } - - private void copyTo(DomainProperty dp, PropertyDescriptor pd, Container c) - { - dp.setRangeURI(pd.getRangeURI()); - dp.setLabel(pd.getLabel()); - dp.setRequired(pd.isRequired()); - dp.setHidden(pd.isHidden()); - dp.setMvEnabled(pd.isMvEnabled()); - dp.setScale(pd.getScale()); - if (pd.getDefaultValueType() != null) - dp.setDefaultValueTypeEnum(DefaultValueType.valueOf(pd.getDefaultValueType())); - } - - @Override - public final Domain getDomain() - { - return getDomain(false); - } - - @Override - public final Domain getDomain(boolean forUpdate) - { - AbstractAuditDomainKind domainKind = getDomainKind(); - - String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); - - return PropertyService.get().getDomain(getDomainContainer(), domainURI, forUpdate); - } - - - protected DbSchema getSchema() - { - return DbSchema.get(SCHEMA_NAME, DbSchemaType.Provisioned); - } - - - public TableInfo createStorageTableInfo() - { - Domain domain = getDomain(); - if (null == domain) - throw new NullPointerException("Could not find domain for " + getEventName()); - return StorageProvisioner.createTableInfo(domain); - } - - @Override - public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) - { - return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()); - } - - public List getDefaultVisibleColumns() - { - return null; - } - - public static Container getDomainContainer() - { - return ContainerManager.getSharedContainer(); - } - - protected void appendValueMapColumns(AbstractTableInfo table) - { - appendValueMapColumns(table, null); - } - - protected void appendValueMapColumns(AbstractTableInfo table, String eventName) - { - appendValueMapColumns(table, eventName, false); - } - - protected void appendValueMapColumns(AbstractTableInfo table, String eventName, boolean noUrl) - { - MutableColumnInfo oldCol = table.getMutableColumn(FieldKey.fromString(OLD_RECORD_PROP_NAME)); - MutableColumnInfo newCol = table.getMutableColumn(FieldKey.fromString(NEW_RECORD_PROP_NAME)); - - if (oldCol != null) - { - var added = table.addColumn(new AliasedColumn(table, "OldValues", oldCol)); - added.setDisplayColumnFactory(DataMapColumn::new); - added.setLabel(AbstractAuditDomainKind.OLD_RECORD_PROP_CAPTION); - added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); - oldCol.setHidden(true); - } - - if (newCol != null) - { - var added = table.addColumn(new AliasedColumn(table, "NewValues", newCol)); - added.setDisplayColumnFactory(DataMapColumn::new); - added.setLabel(AbstractAuditDomainKind.NEW_RECORD_PROP_CAPTION); - added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); - newCol.setHidden(true); - } - - // add a column to show the differences between old and new values - if (oldCol != null && newCol != null) - table.addColumn(new DataMapDiffColumn(table, COLUMN_NAME_DATA_CHANGES, oldCol, newCol)); - - if (!noUrl) - { - String urlStr = "audit-detailedAuditChanges.view?auditRowId=${rowId}"; - if (!StringUtils.isEmpty(eventName)) - urlStr = urlStr + "&auditEventType=" + eventName; - DetailsURL url = DetailsURL.fromString(urlStr); - url.setStrictContainerContextEval(true); - table.setDetailsURL(url); - } - - } - - @Override - public ActionURL getAuditUrl() - { - return AuditLogService.get().getAuditUrl(); - } - - public static Map decodeFromDataMap(String properties) - { - try - { - if (properties != null) - { - return PageFlowUtil.mapFromQueryString(properties); - } - return Collections.emptyMap(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - public static String encodeForDataMap(Map properties) - { - if (properties == null) return null; - - Map stringMap = new LinkedHashMap<>(); - for (Map.Entry entry : properties.entrySet()) - { - // see AuditHandler.getRecordForInsert(), rather than create a new map just skip values here - if (entry.getKey().equals(DataIterator.ROWNUMBER_COLUMNNAME) || - entry.getKey().equals(ExistingRecordDataIterator.EXISTING_RECORD_COLUMN_NAME) || - entry.getKey().equals(ExperimentService.ALIASCOLUMNALIAS)) - continue; - Object value = entry.getValue(); - if (value instanceof Time time) - { - String formatted = DateUtil.formatIsoLongTime(time); - stringMap.put(entry.getKey(), formatted); - } - else if (value instanceof Date date) - { - // Issue 35002 - normalize Date values to avoid Timestamp/Date toString differences - // Issue 36472 - use iso format to show date-time values - String formatted = DateUtil.toISO(date); - stringMap.put(entry.getKey(), formatted); - } - else if (value instanceof java.sql.Array arr) - { - // GitHub Issue 1073: Updating a List MVTC field shows array in audit for values with quotes - var arrayVal = MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, arr); - stringMap.put(entry.getKey(), PageFlowUtil.joinValuesToStringForExport(arrayVal)); - } - else - stringMap.put(entry.getKey(), value == null ? null : value.toString()); - } - return PageFlowUtil.toQueryString(stringMap.entrySet()); - } - - public int moveEvents(Container targetContainer, String idColumnName, Collection ids) - { - return Table.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.audit; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.audit.data.DataMapColumn; +import org.labkey.api.audit.data.DataMapDiffColumn; +import org.labkey.api.audit.query.AbstractAuditDomainKind; +import org.labkey.api.audit.query.DefaultAuditTypeTable; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.AbstractTableInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.MultiChoice; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.dataiterator.DataIterator; +import org.labkey.api.dataiterator.ExistingRecordDataIterator; +import org.labkey.api.exp.ChangePropertyDescriptorException; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.query.AliasedColumn; +import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.util.DateUtil; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.view.ActionURL; + +import java.sql.Time; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.labkey.api.audit.query.AbstractAuditDomainKind.AUDIT_RECORD_DATA_MAP_CONCEPT_URI; +import static org.labkey.api.audit.query.AbstractAuditDomainKind.NEW_RECORD_PROP_NAME; +import static org.labkey.api.audit.query.AbstractAuditDomainKind.OLD_RECORD_PROP_NAME; + +public abstract class AbstractAuditTypeProvider implements AuditTypeProvider +{ + public static final String QUERY_SCHEMA_NAME = "auditLog"; + public static final String SCHEMA_NAME = "audit"; + + public static final String COLUMN_NAME_ROW_ID = "RowId"; + public static final String COLUMN_NAME_CONTAINER = "Container"; + public static final String COLUMN_NAME_COMMENT = "Comment"; + public static final String COLUMN_NAME_USER_COMMENT = "UserComment"; + public static final String COLUMN_NAME_EVENT_TYPE = "EventType"; + public static final String COLUMN_NAME_CREATED = "Created"; + public static final String COLUMN_NAME_CREATED_BY = "CreatedBy"; + public static final String COLUMN_NAME_IMPERSONATED_BY = "ImpersonatedBy"; + public static final String COLUMN_NAME_PROJECT_ID = "ProjectId"; + public static final String COLUMN_NAME_TRANSACTION_ID = "TransactionID"; + public static final String COLUMN_NAME_DATA_CHANGES = "DataChanges"; + + private final AbstractAuditDomainKind _domainKind; + + public AbstractAuditTypeProvider(@NotNull AbstractAuditDomainKind domainKind) + { + // TODO: consolidate domain kind initialization to this constructor and stop overriding getDomainKind() + _domainKind = domainKind; + // Register the DomainKind + PropertyService.get().registerDomainKind(getDomainKind()); + } + + protected final AbstractAuditDomainKind getDomainKind() + { + if (_domainKind == null) + throw new IllegalStateException(String.format("The audit type : \"%s\" has a null domain kind", getLabel())); + + return _domainKind; + } + + // Expose the domain kind to AbstractAuditDomainKind$TestCase without touching every subclass + public AbstractAuditDomainKind getAuditDomainKind() + { + return getDomainKind(); + } + + @Override + public final void initializeProvider(User user) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + domainKind.validate(); + + Domain domain = getDomain(true); + + // if the domain doesn't exist, create it + if (domain == null) + { + try + { + String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); + domain = PropertyService.get().createDomain(getDomainContainer(), domainURI, domainKind.getKindName()); + for (PropertyDescriptor pd : domainKind.getProperties()) + { + domain.addPropertyOfPropertyDescriptor(pd); + } + domain.save(user); + domain = getDomain(true); + } + catch (ChangePropertyDescriptorException e) + { + throw new RuntimeException(e); + } + } + + // adjust potential domain kind changes + ensureProperties(user, domain); + } + + // NOTE: Changing the name of an existing PropertyDescriptor will lose data! + private void ensureProperties(User user, Domain domain) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + if (domain != null && domainKind != null) + { + // Create a map of desired properties + Map props = new CaseInsensitiveHashMap<>(); + for (PropertyDescriptor pd : domainKind.getProperties()) + props.put(pd.getName(), pd); + + // Create a map of existing properties + Map current = new CaseInsensitiveHashMap<>(); + for (DomainProperty dp : domain.getProperties()) + { + current.put(dp.getName(), dp); + } + + Set toAdd = new LinkedHashSet<>(); + for (PropertyDescriptor pd : props.values()) + if (!current.containsKey(pd.getName())) + toAdd.add(pd); + + Set toUpdate = new LinkedHashSet<>(); + boolean changed = false; + + for (DomainProperty dp : current.values()) + { + if (props.containsKey(dp.getName())) + toUpdate.add(dp); + else + { + dp.delete(); + changed = true; + } + } + + for (PropertyDescriptor pd : toAdd) + { + domain.addPropertyOfPropertyDescriptor(pd); + } + + try (DbScope.Transaction transaction = domainKind.getScope().ensureTransaction()) + { + // CONSIDER: Avoid always updating the existing properties -- only update changed props. + for (DomainProperty dp : toUpdate) + { + PropertyDescriptor desired = props.get(dp.getName()); + assert desired != null; + + if (differ(desired, dp, domain.getContainer())) + { + changed = true; + copyTo(dp, desired, domain.getContainer()); + } + } + + changed = changed || !toAdd.isEmpty(); + if (changed) + { + domain.save(user); + } + + assert domain.getStorageTableName() != null; + assert domain.getDomainKind() != null; + assert domain.getDomainKind().getClass().equals(domainKind.getClass()); + + StorageProvisioner.get().ensureTableIndices(domain); + transaction.commit(); + } + catch (ChangePropertyDescriptorException e) + { + throw new RuntimeException(e); + } + } + } + + // Issue 26311: We want to trigger a save if the scale has changed + // CONSIDER: check for other differences here as well. + private boolean differ(PropertyDescriptor pd, DomainProperty dp, Container c) + { + return dp.getScale() != pd.getScale() || !dp.getRangeURI().equals(pd.getRangeURI()); + } + + private void copyTo(DomainProperty dp, PropertyDescriptor pd, Container c) + { + dp.setRangeURI(pd.getRangeURI()); + dp.setLabel(pd.getLabel()); + dp.setRequired(pd.isRequired()); + dp.setHidden(pd.isHidden()); + dp.setMvEnabled(pd.isMvEnabled()); + dp.setScale(pd.getScale()); + if (pd.getDefaultValueType() != null) + dp.setDefaultValueTypeEnum(DefaultValueType.valueOf(pd.getDefaultValueType())); + } + + @Override + public final Domain getDomain() + { + return getDomain(false); + } + + @Override + public final Domain getDomain(boolean forUpdate) + { + AbstractAuditDomainKind domainKind = getDomainKind(); + + String domainURI = domainKind.generateDomainURI(QUERY_SCHEMA_NAME, getEventName(), getDomainContainer(), null); + + return PropertyService.get().getDomain(getDomainContainer(), domainURI, forUpdate); + } + + + protected DbSchema getSchema() + { + return DbSchema.get(SCHEMA_NAME, DbSchemaType.Provisioned); + } + + + public TableInfo createStorageTableInfo() + { + Domain domain = getDomain(); + if (null == domain) + throw new NullPointerException("Could not find domain for " + getEventName()); + return StorageProvisioner.createTableInfo(domain); + } + + @Override + public TableInfo createTableInfo(UserSchema userSchema, ContainerFilter cf) + { + return new DefaultAuditTypeTable(this, createStorageTableInfo(), userSchema, cf, getDefaultVisibleColumns()); + } + + public List getDefaultVisibleColumns() + { + return null; + } + + public static Container getDomainContainer() + { + return ContainerManager.getSharedContainer(); + } + + protected void appendValueMapColumns(AbstractTableInfo table) + { + appendValueMapColumns(table, null); + } + + protected void appendValueMapColumns(AbstractTableInfo table, String eventName) + { + appendValueMapColumns(table, eventName, false); + } + + protected void appendValueMapColumns(AbstractTableInfo table, String eventName, boolean noUrl) + { + MutableColumnInfo oldCol = table.getMutableColumn(FieldKey.fromString(OLD_RECORD_PROP_NAME)); + MutableColumnInfo newCol = table.getMutableColumn(FieldKey.fromString(NEW_RECORD_PROP_NAME)); + + if (oldCol != null) + { + var added = table.addColumn(new AliasedColumn(table, "OldValues", oldCol)); + added.setDisplayColumnFactory(DataMapColumn::new); + added.setLabel(AbstractAuditDomainKind.OLD_RECORD_PROP_CAPTION); + added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); + oldCol.setHidden(true); + } + + if (newCol != null) + { + var added = table.addColumn(new AliasedColumn(table, "NewValues", newCol)); + added.setDisplayColumnFactory(DataMapColumn::new); + added.setLabel(AbstractAuditDomainKind.NEW_RECORD_PROP_CAPTION); + added.setConceptURI(AUDIT_RECORD_DATA_MAP_CONCEPT_URI); + newCol.setHidden(true); + } + + // add a column to show the differences between old and new values + if (oldCol != null && newCol != null) + table.addColumn(new DataMapDiffColumn(table, COLUMN_NAME_DATA_CHANGES, oldCol, newCol)); + + if (!noUrl) + { + String urlStr = "audit-detailedAuditChanges.view?auditRowId=${rowId}"; + if (!StringUtils.isEmpty(eventName)) + urlStr = urlStr + "&auditEventType=" + eventName; + DetailsURL url = DetailsURL.fromString(urlStr); + url.setStrictContainerContextEval(true); + table.setDetailsURL(url); + } + + } + + @Override + public ActionURL getAuditUrl() + { + return AuditLogService.get().getAuditUrl(); + } + + public static Map decodeFromDataMap(String properties) + { + try + { + if (properties != null) + { + return PageFlowUtil.mapFromQueryString(properties); + } + return Collections.emptyMap(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + public static String encodeForDataMap(Map properties) + { + if (properties == null) return null; + + Map stringMap = new LinkedHashMap<>(); + for (Map.Entry entry : properties.entrySet()) + { + // see AuditHandler.getRecordForInsert(), rather than create a new map just skip values here + if (entry.getKey().equals(DataIterator.ROWNUMBER_COLUMNNAME) || + entry.getKey().equals(ExistingRecordDataIterator.EXISTING_RECORD_COLUMN_NAME) || + entry.getKey().equals(ExperimentService.ALIASCOLUMNALIAS)) + continue; + Object value = entry.getValue(); + if (value instanceof Time time) + { + String formatted = DateUtil.formatIsoLongTime(time); + stringMap.put(entry.getKey(), formatted); + } + else if (value instanceof Date date) + { + // Issue 35002 - normalize Date values to avoid Timestamp/Date toString differences + // Issue 36472 - use iso format to show date-time values + String formatted = DateUtil.toISO(date); + stringMap.put(entry.getKey(), formatted); + } + else if (value instanceof java.sql.Array arr) + { + // GitHub Issue 1073: Updating a List MVTC field shows array in audit for values with quotes + var arrayVal = MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, arr); + stringMap.put(entry.getKey(), PageFlowUtil.joinValuesToStringForExport(arrayVal)); + } + else + stringMap.put(entry.getKey(), value == null ? null : value.toString()); + } + return PageFlowUtil.toQueryString(stringMap.entrySet()); + } + + public int moveEvents(Container targetContainer, String idColumnName, Collection ids) + { + return Table.updateContainer(createStorageTableInfo(), idColumnName, ids, targetContainer, null, false); + } +}