Skip to content

correct metadata "modified" flags and related logic #17172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shuwenwei wants to merge 2 commits into
base: master
Choose a base branch
from
+75 −161
Open

correct metadata "modified" flags and related logic #17172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
53d7c03
fix setModified
shuwenwei Feb 5, 2026
bdb81cc
add break
shuwenwei Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 27 additions & 42 deletions .../main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,11 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
context.isDebug(),
context);
if (timeSeriesMetadata != null) {
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
long t2 = System.nanoTime();
List<ModEntry> pathModifications =
context.getPathModifications(
resource, seriesPath.getDeviceId(), seriesPath.getMeasurement());
timeSeriesMetadata.setModified(
timeSeriesMetadata.isModified() || !pathModifications.isEmpty());
timeSeriesMetadata.setModified(!pathModifications.isEmpty());
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications));
int modificationCount = pathModifications.size();
Expand All @@ -141,13 +139,14 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
timeSeriesMetadata =
(TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter);
if (timeSeriesMetadata != null) {
SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType());
timeSeriesMetadata.setChunkMetadataLoader(
new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter));
}
}

if (timeSeriesMetadata != null) {
SchemaUtils.changeTimeseriesMetadataModified(
timeSeriesMetadata, seriesPath.getSeriesType());
if (timeSeriesMetadata.getStatistics().getStartTime()
> timeSeriesMetadata.getStatistics().getEndTime()) {
return null;
Expand Down Expand Up @@ -209,15 +208,12 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
alignedTimeSeriesMetadata =
loadAlignedTimeSeriesMetadataFromDisk(
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows);
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);
} else { // if the tsfile is unclosed, we just get it directly from TsFileResource
loadFromMem = true;
alignedTimeSeriesMetadata =
(AbstractAlignedTimeSeriesMetadata)
resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter);
if (alignedTimeSeriesMetadata != null) {
SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList);

alignedTimeSeriesMetadata.setChunkMetadataLoader(
new MemAlignedChunkMetadataLoader(
resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows));
Expand All @@ -226,6 +222,8 @@ public static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadata(
}

if (alignedTimeSeriesMetadata != null) {
SchemaUtils.changeAlignedTimeseriesMetadataModified(
alignedTimeSeriesMetadata, targetDataTypeList);
if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
> alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
return null;
Expand Down Expand Up @@ -320,25 +318,18 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
int i = 0;
for (String valueMeasurement : valueMeasurementList) {
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(
resource.getTsFileID(), deviceId, valueMeasurement),
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
context);
exist = (exist || (valueColumn != null));
if (valueColumn != null) {
SchemaUtils.changeAlignedMetadataModified(
valueColumn, alignedPath.getSchemaList().get(i).getType());
}
valueTimeSeriesMetadataList.add(valueColumn);
i++;
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
Expand Down Expand Up @@ -458,36 +449,34 @@ public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata tim
* IOException will be thrown
*/
public static List<IPageReader> loadPageReaderList(
IChunkMetadata chunkMetaData,
Filter globalTimeFilter,
boolean isAligned,
List<TSDataType> targetDataTypeList)
IChunkMetadata chunkMetaData, Filter globalTimeFilter, List<TSDataType> targetDataTypeList)
throws IOException {
checkArgument(chunkMetaData != null, "Can't init null chunkMeta");

IChunkReader chunkReader;
boolean isModified = false;
if (isAligned) {
boolean cannotUseStatisticsAfterAlter = false;
if (chunkMetaData instanceof AbstractAlignedChunkMetadata) {
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) chunkMetaData;
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
if (!SchemaUtils.isUsingSameStatistics(
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
targetDataTypeList.get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) {
isModified = true;
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
if (alignedChunkMetadata.isModified()) {
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) {
if (!SchemaUtils.canUseStatisticsAfterAlter(
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
targetDataTypeList.get(i))) {
cannotUseStatisticsAfterAlter = true;
break;
}
}
}
}
IChunkLoader chunkLoader = alignedChunkMetadata.getChunkLoader();
chunkReader = chunkLoader.getChunkReader(alignedChunkMetadata, globalTimeFilter);
} else {
if (!SchemaUtils.isUsingSameStatistics(chunkMetaData.getDataType(), targetDataTypeList.get(0))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(0))) {
isModified = true;
chunkMetaData.setModified(true);
if (chunkMetaData.isModified()
&& !SchemaUtils.canUseStatisticsAfterAlter(
chunkMetaData.getDataType(), targetDataTypeList.get(0))) {
cannotUseStatisticsAfterAlter = true;
}
IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
try {
Expand All @@ -499,14 +488,10 @@ public static List<IPageReader> loadPageReaderList(
}
}

return isModified
? chunkReader.loadPageReaderList().stream()
.peek(
iPageReader -> {
iPageReader.setModified(true);
})
.collect(Collectors.toList())
: chunkReader.loadPageReaderList();
if (cannotUseStatisticsAfterAlter) {
chunkReader.loadPageReaderList().forEach(iPageReader -> iPageReader.setModified(true));
}
return chunkReader.loadPageReaderList();
}

/**
Expand Down
33 changes: 6 additions & 27 deletions ...c/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,15 +453,6 @@ private void initFirstChunkMetadata() throws IOException {
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()));
unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(
orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false);
if (isAligned) {
SchemaUtils.changeAlignedMetadataModified(
(AbstractAlignedChunkMetadata) firstChunkMetadata,
firstChunkMetadata.getDataType(),
getTsDataTypeList());
} else {
SchemaUtils.changeMetadataModified(
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
}
if (firstChunkMetadata.equals(cachedChunkMetadata.peek())) {
firstChunkMetadata = cachedChunkMetadata.poll();
break;
Expand Down Expand Up @@ -489,15 +480,6 @@ private void unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata(

if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) {
firstChunkMetadata = cachedChunkMetadata.poll();
if (isAligned) {
SchemaUtils.changeAlignedMetadataModified(
(AbstractAlignedChunkMetadata) firstChunkMetadata,
firstChunkMetadata.getDataType(),
getTsDataTypeList());
} else {
SchemaUtils.changeMetadataModified(
firstChunkMetadata, firstChunkMetadata.getDataType(), dataType);
}
}
}

Expand All @@ -510,19 +492,16 @@ protected void unpackOneTimeSeriesMetadata(ITimeSeriesMetadata timeSeriesMetadat
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) chunkMetadata;
for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) {
if ((alignedChunkMetadata.getValueChunkMetadataList().get(i) != null)
&& !SchemaUtils.isUsingSameStatistics(
if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null
&& !SchemaUtils.canUseStatisticsAfterAlter(
alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(),
getTsDataTypeList().get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(i))) {
getTsDataTypeList().get(i))) {
alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true);
}
}
chunkMetadata = alignedChunkMetadata;
} else if (chunkMetadata instanceof ChunkMetadata) {
if (!SchemaUtils.isUsingSameStatistics(
chunkMetadata.getDataType(), getTsDataTypeList().get(0))
&& !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(0))) {
if (!SchemaUtils.canUseStatisticsAfterAlter(
chunkMetadata.getDataType(), getTsDataTypeList().get(0))) {
chunkMetadata.setModified(true);
}
}
Expand Down Expand Up @@ -704,7 +683,7 @@ private void unpackOneChunkMetaData(IChunkMetadata chunkMetaData) throws IOExcep
}
List<IPageReader> pageReaderList =
FileLoaderUtils.loadPageReaderList(
chunkMetaData, scanOptions.getGlobalTimeFilter(), isAligned, getTsDataTypeList());
chunkMetaData, scanOptions.getGlobalTimeFilter(), getTsDataTypeList());

// init TsBlockBuilder for each page reader
pageReaderList.forEach(p -> p.initTsBlockBuilder(getTsDataTypeList()));
Expand Down
132 changes: 41 additions & 91 deletions iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;

import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
Expand Down Expand Up @@ -314,120 +317,67 @@ public static boolean isUsingSameColumn(TSDataType originalDataType, TSDataType
dataTypeColumnClassMap.get(originalDataType), dataTypeColumnClassMap.get(dataType));
}

public static boolean isUsingSameStatistics(TSDataType originalDataType, TSDataType dataType) {
if (originalDataType == dataType) {
return true;
}
if (!dataTypeColumnStatisticsClassMap.containsKey(originalDataType)
|| !dataTypeColumnStatisticsClassMap.containsKey(dataType)) {
return false;
}
return Objects.equals(
dataTypeColumnStatisticsClassMap.get(originalDataType),
dataTypeColumnStatisticsClassMap.get(dataType));
}

public static boolean canUseStatisticsAfterAlter(TSDataType dataType) {
return !canNotUseStatisticAfterAlterClassSet.contains(dataType);
}

public static void changeMetadataModified(
TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) {
public static void changeTimeseriesMetadataModified(
@Nullable TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) {
if (timeseriesMetadata == null) {
return;
}
if (!SchemaUtils.isUsingSameStatistics(timeseriesMetadata.getTsDataType(), targetDataType)
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) {
timeseriesMetadata.setModified(true);
List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList();
if (chunkMetadataList != null) {
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
if (chunkMetadata != null) {
chunkMetadata.setModified(true);
}
}
}
if (!SchemaUtils.canUseStatisticsAfterAlter(
timeseriesMetadata.getTsDataType(), targetDataType)) {
markAllChunkMetadataModified(timeseriesMetadata);
}
}

public static void changeAlignedMetadataModified(
AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata,
public static void changeAlignedTimeseriesMetadataModified(
@Nullable AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata,
List<TSDataType> targetDataTypeList) {
if (alignedTimeSeriesMetadata == null) {
return;
}

int i = 0;
for (TimeseriesMetadata timeseriesMetadata :
alignedTimeSeriesMetadata.getValueTimeseriesMetadataList()) {
if ((timeseriesMetadata != null)
&& !SchemaUtils.isUsingSameStatistics(
timeseriesMetadata.getTsDataType(), targetDataTypeList.get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) {
timeseriesMetadata.setModified(true);
alignedTimeSeriesMetadata.setModified(true);
List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList();
if (chunkMetadataList != null) {
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
if (chunkMetadata != null) {
chunkMetadata.setModified(true);
}
}
}
for (int i = 0; i < alignedTimeSeriesMetadata.getValueTimeseriesMetadataList().size(); i++) {
TimeseriesMetadata valueTimeseriesMetadata =
alignedTimeSeriesMetadata.getValueTimeseriesMetadataList().get(i);
if (valueTimeseriesMetadata != null
&& !SchemaUtils.canUseStatisticsAfterAlter(
valueTimeseriesMetadata.getTsDataType(), targetDataTypeList.get(i))) {
markAllChunkMetadataModified(alignedTimeSeriesMetadata.getTimeseriesMetadata());
return;
}
i++;
}
}

public static void changeAlignedMetadataModified(
TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) {
if (timeseriesMetadata == null) {
return;
}

if (!SchemaUtils.isUsingSameStatistics(timeseriesMetadata.getTsDataType(), targetDataType)
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) {
timeseriesMetadata.setModified(true);
List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList();
if (chunkMetadataList != null) {
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
if (chunkMetadata != null) {
chunkMetadata.setModified(true);
}
}
private static void markAllChunkMetadataModified(@NotNull TimeseriesMetadata timeseriesMetadata) {
timeseriesMetadata.setModified(true);
for (IChunkMetadata iChunkMetadata : timeseriesMetadata.getChunkMetadataList()) {
if (iChunkMetadata != null) {
iChunkMetadata.setModified(true);
Copy link
Member Author

@shuwenwei shuwenwei Feb 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image Image This won't work

}
}
}

public static void changeMetadataModified(
IChunkMetadata chunkMetadata, TSDataType sourceDataType, TSDataType targetDataType) {
if (chunkMetadata == null) {
return;
}
if (!SchemaUtils.isUsingSameStatistics(sourceDataType, targetDataType)
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) {
chunkMetadata.setModified(true);
public static boolean canUseStatisticsAfterAlter(
TSDataType originalDataType, TSDataType targetDataType) {
if (isUsingSameStatistics(originalDataType, targetDataType)) {
return true;
}
return canUseStatisticsAfterAlter(targetDataType);
}

public static void changeAlignedMetadataModified(
AbstractAlignedChunkMetadata chunkMetadata,
TSDataType sourceDataType,
List<TSDataType> targetDataTypeList) {
if (chunkMetadata == null) {
return;
private static boolean isUsingSameStatistics(TSDataType originalDataType, TSDataType dataType) {
if (originalDataType == dataType) {
return true;
}
int i = 0;
for (IChunkMetadata iChunkMetadata : chunkMetadata.getValueChunkMetadataList()) {
if ((iChunkMetadata != null)
&& !SchemaUtils.isUsingSameStatistics(
iChunkMetadata.getDataType(), targetDataTypeList.get(i))
&& !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) {
iChunkMetadata.setModified(true);
chunkMetadata.setModified(true);
}
i++;
if (!dataTypeColumnStatisticsClassMap.containsKey(originalDataType)
|| !dataTypeColumnStatisticsClassMap.containsKey(dataType)) {
return false;
}
return Objects.equals(
dataTypeColumnStatisticsClassMap.get(originalDataType),
dataTypeColumnStatisticsClassMap.get(dataType));
}

private static boolean canUseStatisticsAfterAlter(TSDataType dataType) {
return !canNotUseStatisticAfterAlterClassSet.contains(dataType);
}

public static void rewriteAlignedChunkMetadataStatistics(
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@
<thrift.version>0.14.1</thrift.version>
<xz.version>1.9</xz.version>
<zstd-jni.version>1.5.6-3</zstd-jni.version>
<tsfile.version>2.2.1-260203-SNAPSHOT</tsfile.version>
<tsfile.version>2.2.1-260205-SNAPSHOT</tsfile.version>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
Expand Down
Loading