From 06b6ab8033f03654cfce101047bdd6570edd3c29 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 24 Mar 2026 15:53:44 +0000 Subject: [PATCH 1/7] HBASE-30102 Add metric to account for region data classified as cold by the Time Based Priority logic Change-Id: I5601a37300a3f5d10fe4886ba988f2d25e66b546 --- .../apache/hadoop/hbase/RegionMetrics.java | 6 + .../hadoop/hbase/RegionMetricsBuilder.java | 26 ++++- .../MetricsRegionServerSource.java | 5 + .../regionserver/MetricsRegionSourceImpl.java | 4 + .../regionserver/MetricsRegionWrapper.java | 5 + .../TestMetricsRegionSourceImpl.java | 5 + .../main/protobuf/server/ClusterStatus.proto | 8 +- .../hadoop/hbase/io/hfile/BlockCacheUtil.java | 2 +- .../hadoop/hbase/io/hfile/HFileInfo.java | 5 + .../hbase/io/hfile/bucket/BucketCache.java | 33 ++++-- .../io/hfile/bucket/BucketProtoUtils.java | 4 +- .../regionserver/DataTieringManager.java | 67 ++++++++++- .../hbase/regionserver/HRegionServer.java | 30 ++--- .../hadoop/hbase/regionserver/HStore.java | 4 + .../MetricsRegionWrapperImpl.java | 12 ++ .../regionserver/regionListStoreStats.jsp | 2 + .../master/TestRegionsRecoveryChore.java | 6 + .../MetricsRegionWrapperStub.java | 5 + .../regionserver/TestDataTieringManager.java | 106 ++++++++++++++++++ 19 files changed, 300 insertions(+), 35 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java index b029d0288564..a4551c8a3409 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java @@ -144,4 +144,10 @@ default String getNameAsString() { /** Returns current prefetch ratio of this region on this server */ float getCurrentRegionCachedRatio(); + + /** + * Returns the ratio of tiered cold data size to total region (HFiles) size on this server, in the + * range [0,1]. + */ + float getCurrentRegionColdDataRatio(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java index d3361693079a..2486cd89652b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java @@ -81,7 +81,8 @@ public static RegionMetrics toRegionMetrics(ClusterStatusProtos.RegionLoad regio .setUncompressedStoreFileSize( new Size(regionLoadPB.getStoreUncompressedSizeMB(), Size.Unit.MEGABYTE)) .setRegionSizeMB(new Size(regionLoadPB.getRegionSizeMB(), Size.Unit.MEGABYTE)) - .setCurrentRegionCachedRatio(regionLoadPB.getCurrentRegionCachedRatio()).build(); + .setCurrentRegionCachedRatio(regionLoadPB.getCurrentRegionCachedRatio()) + .setCurrentRegionColdDataRatio(regionLoadPB.getCurrentRegionColdDataRatio()).build(); } private static List @@ -122,7 +123,8 @@ public static ClusterStatusProtos.RegionLoad toRegionLoad(RegionMetrics regionMe .setStoreUncompressedSizeMB( (int) regionMetrics.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE)) .setRegionSizeMB((int) regionMetrics.getRegionSizeMB().get(Size.Unit.MEGABYTE)) - .setCurrentRegionCachedRatio(regionMetrics.getCurrentRegionCachedRatio()).build(); + .setCurrentRegionCachedRatio(regionMetrics.getCurrentRegionCachedRatio()) + .setCurrentRegionColdDataRatio(regionMetrics.getCurrentRegionColdDataRatio()).build(); } public static RegionMetricsBuilder newBuilder(byte[] name) { @@ -158,6 +160,7 @@ public static RegionMetricsBuilder newBuilder(byte[] name) { private CompactionState compactionState; private Size regionSizeMB = Size.ZERO; private float currentRegionCachedRatio; + private float currentRegionColdDataRatio; private RegionMetricsBuilder(byte[] name) { this.name = name; @@ -303,6 +306,11 @@ public RegionMetricsBuilder setCurrentRegionCachedRatio(float value) { return this; } + public RegionMetricsBuilder setCurrentRegionColdDataRatio(float value) { + this.currentRegionColdDataRatio = value; + return this; + } + public RegionMetrics build() { return new RegionMetricsImpl(name, storeCount, storeFileCount, storeRefCount, maxCompactedStoreFileRefCount, compactingCellCount, compactedCellCount, storeFileSize, @@ -310,7 +318,8 @@ public RegionMetrics build() { uncompressedStoreFileSize, writeRequestCount, readRequestCount, cpRequestCount, filteredReadRequestCount, completedSequenceId, storeSequenceIds, dataLocality, lastMajorCompactionTimestamp, dataLocalityForSsd, blocksLocalWeight, blocksLocalWithSsdWeight, - blocksTotalWeight, compactionState, regionSizeMB, currentRegionCachedRatio); + blocksTotalWeight, compactionState, regionSizeMB, currentRegionCachedRatio, + currentRegionColdDataRatio); } private static class RegionMetricsImpl implements RegionMetrics { @@ -343,6 +352,7 @@ private static class RegionMetricsImpl implements RegionMetrics { private final CompactionState compactionState; private final Size regionSizeMB; private final float currentRegionCachedRatio; + private final float currentRegionColdDataRatio; RegionMetricsImpl(byte[] name, int storeCount, int storeFileCount, int storeRefCount, int maxCompactedStoreFileRefCount, final long compactingCellCount, long compactedCellCount, @@ -352,7 +362,8 @@ private static class RegionMetricsImpl implements RegionMetrics { long filteredReadRequestCount, long completedSequenceId, Map storeSequenceIds, float dataLocality, long lastMajorCompactionTimestamp, float dataLocalityForSsd, long blocksLocalWeight, long blocksLocalWithSsdWeight, long blocksTotalWeight, - CompactionState compactionState, Size regionSizeMB, float currentRegionCachedRatio) { + CompactionState compactionState, Size regionSizeMB, float currentRegionCachedRatio, + float currentRegionColdDataRatio) { this.name = Preconditions.checkNotNull(name); this.storeCount = storeCount; this.storeFileCount = storeFileCount; @@ -382,6 +393,7 @@ private static class RegionMetricsImpl implements RegionMetrics { this.compactionState = compactionState; this.regionSizeMB = regionSizeMB; this.currentRegionCachedRatio = currentRegionCachedRatio; + this.currentRegionColdDataRatio = currentRegionColdDataRatio; } @Override @@ -529,6 +541,11 @@ public float getCurrentRegionCachedRatio() { return currentRegionCachedRatio; } + @Override + public float getCurrentRegionColdDataRatio() { + return currentRegionColdDataRatio; + } + @Override public String toString() { StringBuilder sb = @@ -571,6 +588,7 @@ public String toString() { Strings.appendKeyValue(sb, "compactionState", compactionState); Strings.appendKeyValue(sb, "regionSizeMB", regionSizeMB); Strings.appendKeyValue(sb, "currentRegionCachedRatio", currentRegionCachedRatio); + Strings.appendKeyValue(sb, "currentRegionColdDataRatio", currentRegionColdDataRatio); return sb.toString(); } } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index ffcbb1fa5d27..96d90f341b0b 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -654,6 +654,11 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String CURRENT_REGION_CACHE_RATIO = "currentRegionCacheRatio"; String CURRENT_REGION_CACHE_RATIO_DESC = "The percentage of caching completed for this region."; + String CURRENT_REGION_COLD_DATA_RATIO = "currentRegionColdDataRatio"; + + String CURRENT_REGION_COLD_DATA_RATIO_DESC = "The percentage of data in this region that " + + "is marked as cold by the configured time based priority logic. "; + String EXCLUDE_DATA_NODES_COUNT = "excludedDataNodesCount"; String EXCLUDE_DATA_NODES_COUNT_DESC = "Count of slow/connect error DataNodes excluded during WAL write operation"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java index f9cafa124949..be87513b1dbc 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java @@ -245,6 +245,10 @@ void snapshot(MetricsRecordBuilder mrb, boolean ignored) { Interns.info(regionNamePrefix + MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO, MetricsRegionServerSource.CURRENT_REGION_CACHE_RATIO_DESC), this.regionWrapper.getCurrentRegionCacheRatio()); + mrb.addGauge( + Interns.info(regionNamePrefix + MetricsRegionServerSource.CURRENT_REGION_COLD_DATA_RATIO, + MetricsRegionServerSource.CURRENT_REGION_COLD_DATA_RATIO_DESC), + this.regionWrapper.getCurrentRegionColdDataRatio()); mrb.addCounter( Interns.info(regionNamePrefix + MetricsRegionSource.COMPACTIONS_COMPLETED_COUNT, MetricsRegionSource.COMPACTIONS_COMPLETED_DESC), diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index 66cbd47a66ad..e2d79abab48c 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -80,6 +80,11 @@ public interface MetricsRegionWrapper { */ float getCurrentRegionCacheRatio(); + /** + * Gets the current cold date % ratio for this region. + */ + float getCurrentRegionColdDataRatio(); + /** * Get the total number of read requests that have been issued against this region */ diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java index 5d2f1f9e0519..c04898439450 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java @@ -130,6 +130,11 @@ public float getCurrentRegionCacheRatio() { return 0; } + @Override + public float getCurrentRegionColdDataRatio() { + return 0; + } + @Override public long getReadRequestCount() { return 0; diff --git a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto index 58fd3c8d2a5b..afff971687d9 100644 --- a/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto +++ b/hbase-protocol-shaded/src/main/protobuf/server/ClusterStatus.proto @@ -183,6 +183,12 @@ message RegionLoad { /** Current region cache ratio on this server */ optional float current_region_cached_ratio = 29; + + /** + * Ratio of tiered cold data size to total region size (HFiles) on this server, + * in the range [0,1]. See DataTieringManager region cold data tracking. + */ + optional float current_region_cold_data_ratio = 30; } message UserLoad { @@ -316,7 +322,7 @@ message ServerLoad { * The metrics for write requests on this region server */ optional uint64 write_requests_count = 14; - + /** * The active monitored tasks */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index e39cb21a422e..f48da92f515d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -295,7 +295,7 @@ public static HFileBlock getBlockForCaching(CacheConfig cacheConf, HFileBlock bl public static HFileContext cloneContext(HFileContext context) { HFileContext newContext = new HFileContextBuilder().withBlockSize(context.getBlocksize()) .withBytesPerCheckSum(0).withChecksumType(ChecksumType.NULL) // no checksums in cached data - .withCompression(context.getCompression()) + .withCompression(context.getCompression()).withHFileName(context.getHFileName()) .withDataBlockEncoding(context.getDataBlockEncoding()) .withHBaseCheckSum(context.isUseHBaseChecksum()).withCompressTags(context.isCompressTags()) .withIncludesMvcc(context.isIncludesMvcc()).withIncludesTags(context.isIncludesTags()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java index b3da98f13434..b258caa31332 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileInfo.java @@ -77,6 +77,8 @@ public class HFileInfo implements SortedMap { static final byte[] KEY_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX + "KEY_OF_BIGGEST_CELL"); static final byte[] LEN_OF_BIGGEST_CELL = Bytes.toBytes(RESERVED_PREFIX + "LEN_OF_BIGGEST_CELL"); public static final byte[] MAX_TAGS_LEN = Bytes.toBytes(RESERVED_PREFIX + "MAX_TAGS_LEN"); + public static final byte[] FILE_SIZE = Bytes.toBytes(RESERVED_PREFIX + "FILE_SIZE"); + public static final byte[] FILE_PATH = Bytes.toBytes(RESERVED_PREFIX + "FILE_PATH"); private final SortedMap map = new TreeMap<>(Bytes.BYTES_COMPARATOR); /** @@ -397,6 +399,9 @@ public void initMetaAndIndex(HFile.Reader reader) throws IOException { } // close the block reader context.getInputStreamWrapper().unbuffer(); + + put(FILE_SIZE, Bytes.toBytes(context.getFileSize())); + put(FILE_PATH, Bytes.toBytes(context.getFilePath().toString())); } catch (Throwable t) { IOUtils.closeQuietly(context.getInputStreamWrapper(), e -> LOG.warn("failed to close input stream wrapper", e)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 4839494ca623..8dfa66c76664 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -784,10 +784,9 @@ private void updateRegionCachedSize(BlockCacheKey key, long cachedSize) { String regionName = key.getRegionName(); regionCachedSize.merge(regionName, cachedSize, (previousSize, newBlockSize) -> previousSize + newBlockSize); - LOG.trace("Updating region cached size for region: {}", regionName); // If all the blocks for a region are evicted from the cache, // remove the entry for that region from regionCachedSize map. - if (regionCachedSize.get(regionName) <= 0) { + if (regionCachedSize.getOrDefault(regionName, 0L) <= 0) { regionCachedSize.remove(regionName); } } @@ -1645,13 +1644,7 @@ private void updateRegionSizeMapWhileRetrievingFromFile() { dumpPrefetchList(); } regionCachedSize.clear(); - fullyCachedFiles.forEach((hFileName, hFileSize) -> { - // Get the region name for each file - String regionEncodedName = hFileSize.getFirst(); - long cachedFileSize = hFileSize.getSecond(); - regionCachedSize.merge(regionEncodedName, cachedFileSize, - (oldpf, fileSize) -> oldpf + fileSize); - }); + backingMap.forEach((k, v) -> updateRegionCachedSize(k, v.getLength())); } private void dumpPrefetchList() { @@ -1723,7 +1716,10 @@ private void updateCacheIndex(BucketCacheProtos.BackingMap chunk, java.util.Map deserializer) throws IOException { Pair, NavigableSet> pair2 = BucketProtoUtils.fromPB(deserializer, chunk, this::createRecycler); - backingMap.putAll(pair2.getFirst()); + pair2.getFirst().forEach((k, v) -> { + backingMap.put(k, v); + updateRegionCachedSize(k, v.getLength()); + }); blocksByHFile.addAll(pair2.getSecond()); } @@ -1774,6 +1770,7 @@ private void retrieveChunkedBackingMap(FileInputStream in) throws IOException { backingMap.clear(); blocksByHFile.clear(); + regionCachedSize.clear(); // Read the backing map entries in batches. int numChunks = 0; @@ -1787,7 +1784,6 @@ private void retrieveChunkedBackingMap(FileInputStream in) throws IOException { verifyFileIntegrity(cacheEntry); verifyCapacityAndClasses(cacheEntry.getCacheCapacity(), cacheEntry.getIoClass(), cacheEntry.getMapClass()); - updateRegionSizeMapWhileRetrievingFromFile(); } /** @@ -1949,6 +1945,10 @@ public int evictBlocksRangeByHfileName(String hfileName, long initOffset, long e // split references, we might be evicting just half of the blocks LOG.debug("found {} blocks for file {}, starting offset: {}, end offset: {}", keySet.size(), hfileName, initOffset, endOffset); + return evictBlockSet(keySet); + } + + private int evictBlockSet(Set keySet) { int numEvicted = 0; for (BlockCacheKey key : keySet) { if (evictBlock(key)) { @@ -2490,7 +2490,16 @@ public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf String fileName = hFileInfo.getHFileContext().getHFileName(); DataTieringManager dataTieringManager = DataTieringManager.getInstance(); if (dataTieringManager != null && !dataTieringManager.isHotData(hFileInfo, conf)) { - LOG.debug("Data tiering is enabled for file: '{}' and it is not hot data", fileName); + LOG.debug("Custom tiering is enabled for file: '{}' and it is not hot data", fileName); + // If custom tiering has been just enabled for a file that was cached, we now need + // to evict it. + Set keySet = + getAllCacheKeysForFile(hFileInfo.getHFileContext().getHFileName(), 0, Long.MAX_VALUE); + int evictedBlocks = evictBlockSet(keySet); + if (evictedBlocks > 0) { + LOG.info("Evicted {} blocks for file {} as it is now considered cold by DataTieringManager", + evictedBlocks, fileName); + } return Optional.of(false); } // if we don't have the file in fullyCachedFiles, we should cache it diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java index b87e0e0dd62a..e3536e8dce4c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java @@ -101,8 +101,8 @@ private static void addEntryToBuilder(Map.Entry entr private static BucketCacheProtos.BlockCacheKey toPB(BlockCacheKey key) { return BucketCacheProtos.BlockCacheKey.newBuilder().setHfilename(key.getHfileName()) - .setOffset(key.getOffset()).setPrimaryReplicaBlock(key.isPrimary()) - .setBlockType(toPB(key.getBlockType())).build(); + .setFamilyName(key.getCfName()).setRegionName(key.getRegionName()).setOffset(key.getOffset()) + .setPrimaryReplicaBlock(key.isPrimary()).setBlockType(toPB(key.getBlockType())).build(); } private static BucketCacheProtos.BlockType toPB(BlockType blockType) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java index 2c92d9238dcc..53766189a564 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DataTieringManager.java @@ -17,17 +17,25 @@ */ package org.apache.hadoop.hbase.regionserver; +import static org.apache.hadoop.hbase.io.hfile.HFileInfo.FILE_PATH; +import static org.apache.hadoop.hbase.io.hfile.HFileInfo.FILE_SIZE; + +import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,6 +64,11 @@ public class DataTieringManager { private static DataTieringManager instance; private final Map onlineRegions; + // Accounts for the total size of cold data in each region, together with a list of cold files in + // that region. + private final Map, Long>> regionColdDataSize = + new ConcurrentHashMap<>(); + private DataTieringManager(Map onlineRegions) { this.onlineRegions = onlineRegions; } @@ -203,7 +216,34 @@ public boolean isHotData(HFileInfo hFileInfo, Configuration configuration) { if (isWithinGracePeriod(maxTimestamp, configuration)) { return true; } - return hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); + LOG.debug("Max TS: {} for file {}. Cutoff Age TS: {}", maxTimestamp, + hFileInfo.getHFileContext().getHFileName(), getDataTieringHotDataAge(configuration)); + boolean isHot = hotDataValidator(maxTimestamp, getDataTieringHotDataAge(configuration)); + if (!isHot) { + Path path = new Path(Bytes.toString(hFileInfo.get(FILE_PATH))); + String regionName = path.getParent().getParent().getName(); + regionColdDataSize.compute(regionName, (k, v) -> { + if (v == null) { + List files = new ArrayList<>(); + files.add(hFileInfo.getHFileContext().getHFileName()); + LOG.debug("computing file {} with size {} as cold data for region {}", + hFileInfo.getHFileContext().getHFileName(), Bytes.toLong(hFileInfo.get(FILE_SIZE)), + regionName); + return new Pair<>(files, Bytes.toLong(hFileInfo.get(FILE_SIZE))); + } else { + if (!v.getFirst().contains(hFileInfo.getHFileContext().getHFileName())) { + v.getFirst().add(hFileInfo.getHFileContext().getHFileName()); + v.setSecond(v.getSecond() + Bytes.toLong(hFileInfo.get(FILE_SIZE))); + LOG.debug( + "adding file {} with size {} as cold data for region {}. Total cold data size for the region is {}", + hFileInfo.getHFileContext().getHFileName(), Bytes.toLong(hFileInfo.get(FILE_SIZE)), + regionName, v.getSecond()); + } + return v; + } + }); + } + return isHot; } // DataTieringType.NONE or other types are considered hot by default return true; @@ -347,4 +387,29 @@ private static boolean isDataTieringFeatureEnabled(Configuration conf) { public static void resetForTestingOnly() { instance = null; } + + public Map, Long>> getRegionColdDataSize() { + return regionColdDataSize; + } + + /** + * Updates regionColdData size for the region containing the passed compactedFiles. + */ + public void updateRegionColdDataSize(String encodedRegionName, + Collection compactedFiles, Collection newFiles) { + regionColdDataSize.computeIfPresent(encodedRegionName, (k, v) -> { + for (HStoreFile file : compactedFiles) { + if (v.getFirst().contains(file.getPath().getName())) { + v.getFirst().remove(file.getPath().getName()); + v.setSecond(v.getSecond() - Bytes.toLong(file.getMetadataValue(FILE_SIZE))); + } + } + for (HStoreFile file : newFiles) { + // call isHotData to account for the new file size in regionColdDataSize, if the new file is + // considered cold data as per data-tiering logic. + isHotData(file.getFileInfo().getHFileInfo(), file.getFileInfo().getConf()); + } + return v; + }); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index eea82ca511eb..fd608a2f80cc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -65,7 +65,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Consumer; import java.util.stream.Collectors; import javax.management.MalformedObjectNameException; import javax.servlet.http.HttpServlet; @@ -110,9 +109,7 @@ import org.apache.hadoop.hbase.http.InfoServer; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheFactory; -import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.io.util.MemorySizeUtil; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.DecommissionedHostRejectedException; @@ -1239,7 +1236,6 @@ private ClusterStatusProtos.ServerLoad buildServerLoad(long reportStartTime, lon }); }); }); - serverLoad.setReportStartTime(reportStartTime); serverLoad.setReportEndTime(reportEndTime); if (this.infoServer != null) { @@ -1539,15 +1535,6 @@ private static int roundSize(long sizeInByte, int sizeUnit) { } } - private void computeIfPersistentBucketCache(Consumer computation) { - if (blockCache instanceof CombinedBlockCache) { - BlockCache l2 = ((CombinedBlockCache) blockCache).getSecondLevelCache(); - if (l2 instanceof BucketCache && ((BucketCache) l2).isCachePersistent()) { - computation.accept((BucketCache) l2); - } - } - } - /** * @param r Region to get RegionLoad for. * @param regionLoadBldr the RegionLoad.Builder, can be null @@ -1613,6 +1600,16 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, } }); }); + final MutableFloat currentRegionColdDataRatio = new MutableFloat(0.0f); + if (DataTieringManager.getInstance() != null) { + Pair, Long> coldEntry = + DataTieringManager.getInstance().getRegionColdDataSize().get(regionEncodedName); + if (coldEntry != null) { + int coldSizeMB = roundSize(coldEntry.getSecond(), unitMB); + currentRegionColdDataRatio + .setValue(regionSizeMB == 0 ? 0.0f : (float) coldSizeMB / regionSizeMB); + } + } HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution(); float dataLocality = hdfsBd.getBlockLocalityIndex(serverName.getHostname()); @@ -1644,7 +1641,8 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, .setBlocksLocalWithSsdWeight(blocksLocalWithSsdWeight).setBlocksTotalWeight(blocksTotalWeight) .setCompactionState(ProtobufUtil.createCompactionStateForRegionLoad(r.getCompactionState())) .setLastMajorCompactionTs(r.getOldestHfileTs(true)).setRegionSizeMB(regionSizeMB) - .setCurrentRegionCachedRatio(currentRegionCachedRatio.floatValue()); + .setCurrentRegionCachedRatio(currentRegionCachedRatio.floatValue()) + .setCurrentRegionColdDataRatio(currentRegionColdDataRatio.floatValue()); r.setCompleteSequenceId(regionLoadBldr); return regionLoadBldr.build(); } @@ -3144,6 +3142,10 @@ public HRegion getRegion(final String encodedRegionName) { @Override public boolean removeRegion(final HRegion r, ServerName destination) { HRegion toReturn = this.onlineRegions.remove(r.getRegionInfo().getEncodedName()); + if (DataTieringManager.getInstance() != null) { + DataTieringManager.getInstance().getRegionColdDataSize() + .remove(r.getRegionInfo().getEncodedName()); + } metricsRegionServerImpl.requestsCountCache.remove(r.getRegionInfo().getEncodedName()); if (destination != null) { long closeSeqNum = r.getMaxFlushedSeqId(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index fde89d122e28..ead0dd4800f1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1279,6 +1279,10 @@ void replaceStoreFiles(Collection compactedFiles, Collection { synchronized (filesCompacting) { filesCompacting.removeAll(compactedFiles); + if (DataTieringManager.getInstance() != null) { + DataTieringManager.getInstance().updateRegionColdDataSize( + region.getRegionInfo().getEncodedName(), compactedFiles, result); + } } }); // These may be null when the RS is shutting down. The space quota Chores will fix the Region diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java index 81ed1849fca4..dfcbf3d10312 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.metrics2.MetricsExecutor; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; @@ -68,6 +69,8 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable private float currentRegionCacheRatio; private final String tableDescriptorHash; + private float currentRegionColdDataRatio; + public MetricsRegionWrapperImpl(HRegion region) { this.region = region; this.tableDescriptorHash = determineTableDescriptorHash(); @@ -142,6 +145,10 @@ public float getCurrentRegionCacheRatio() { return currentRegionCacheRatio; } + public float getCurrentRegionColdDataRatio() { + return currentRegionColdDataRatio; + } + @Override public long getStoreRefCount() { return storeRefCount; @@ -349,6 +356,11 @@ public void run() { region.getRegionInfo().getEncodedName(), regionCachedAmount.getValue(), tempStoreFileSize); currentRegionCacheRatio = regionCachedAmount.floatValue() / tempStoreFileSize; + if (DataTieringManager.getInstance() != null) { + currentRegionColdDataRatio = DataTieringManager.getInstance().getRegionColdDataSize() + .getOrDefault(region.getRegionInfo().getEncodedName(), new Pair<>(null, 0L)).getSecond() + / (float) tempStoreFileSize; + } } numStoreFiles = tempNumStoreFiles; storeRefCount = tempStoreRefCount; diff --git a/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp b/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp index b663b74536e9..45934d552349 100644 --- a/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/regionserver/regionListStoreStats.jsp @@ -45,6 +45,7 @@ Data Locality Len Of Biggest Cell % Cached + % Cold Data @@ -111,6 +112,7 @@ <%= load.getDataLocality() %> <%= String.format("%,1d", lenOfBiggestCellInRegion) %> <%= StringUtils.formatPercent(load.getCurrentRegionCachedRatio(), 2) %> + <%= StringUtils.formatPercent(load.getCurrentRegionColdDataRatio(), 2) %> <% } %> <% } %> diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java index 06b66d118bf3..eeba839ac111 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java @@ -399,6 +399,7 @@ public List getTasks() { public Map getRegionCachedInfo() { return new HashMap<>(); } + }; return serverMetrics; } @@ -550,6 +551,11 @@ public Size getRegionSizeMB() { public float getCurrentRegionCachedRatio() { return 0.0f; } + + @Override + public float getCurrentRegionColdDataRatio() { + return 0.0f; + } }; return regionMetrics; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java index 9c9a5e9dc9c5..ed3621748fa9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java @@ -97,6 +97,11 @@ public float getCurrentRegionCacheRatio() { return 0; } + @Override + public float getCurrentRegionColdDataRatio() { + return 0; + } + @Override public long getReadRequestCount() { return 105; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 26269acd1783..8041a59a735f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -61,6 +62,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker; import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory; @@ -349,6 +351,110 @@ public void testPickColdDataFiles() throws IOException { assert (coldDataFiles.containsKey(hStoreFiles.get(3).getFileInfo().getActiveFileName())); } + /** + * {@link DataTieringManager#isHotData(HFileInfo, org.apache.hadoop.conf.Configuration)} should + * record cold files in {@link DataTieringManager#getRegionColdDataSize()} (path and aggregate + * size). + */ + @Test + public void testRegionColdDataSizeRecordColdHFile() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile coldFile = hStoreFiles.get(3); + String region = coldFile.getPath().getParent().getParent().getName(); + assertFalse("fixture file should be cold for TIME_RANGE tiering", dataTieringManager + .isHotData(coldFile.getFileInfo().getHFileInfo(), coldFile.getFileInfo().getConf())); + + Map, Long>> coldByRegion = dataTieringManager.getRegionColdDataSize(); + assertTrue(coldByRegion.containsKey(region)); + Pair, Long> entry = coldByRegion.get(region); + long expected = Bytes.toLong(coldFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE)); + assertEquals(expected, (long) entry.getSecond()); + assertEquals(1, entry.getFirst().size()); + assertTrue(entry.getFirst().contains(coldFile.getPath().getName())); + } + + @Test + public void testRegionColdDataSizeSkipsHotHFile() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile hotFile = hStoreFiles.get(0); + assertTrue(dataTieringManager.isHotData(hotFile.getFileInfo().getHFileInfo(), + hotFile.getFileInfo().getConf())); + assertTrue(dataTieringManager.getRegionColdDataSize().isEmpty()); + } + + @Test + public void testRegionColdDataSizeSkipsNoTieringHFile() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile file = hStoreFiles.get(1); + assertTrue(dataTieringManager.isHotData(file.getFileInfo().getHFileInfo(), + file.getFileInfo().getConf())); + String encoded = file.getPath().getParent().getParent().getName(); + assertFalse(dataTieringManager.getRegionColdDataSize().containsKey(encoded)); + } + + @Test + public void testRegionColdDataSizeForSameHFile() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile coldFile = hStoreFiles.get(3); + long expected = Bytes.toLong(coldFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE)); + dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(), + coldFile.getFileInfo().getConf()); + dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(), + coldFile.getFileInfo().getConf()); + + String region = coldFile.getPath().getParent().getParent().getName(); + Pair, Long> entry = dataTieringManager.getRegionColdDataSize().get(region); + assertNotNull(entry); + assertEquals(expected, (long) entry.getSecond()); + assertEquals(1, entry.getFirst().size()); + } + + @Test + public void testUpdateRegionColdDataSizeNoopWhenRegionNotTracked() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + HStoreFile coldFile = hStoreFiles.get(3); + dataTieringManager.updateRegionColdDataSize("not-a-real-encoded-region", + Collections.singletonList(coldFile), Collections.emptyList()); + assertTrue(dataTieringManager.getRegionColdDataSize().isEmpty()); + } + + @Test + public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewHot() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile coldFile = hStoreFiles.get(3); + String regionName = coldFile.getPath().getParent().getParent().getName(); + dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(), + coldFile.getFileInfo().getConf()); + + HRegion region = testOnlineRegions.get(regionName); + assertNotNull(region); + HStore hStore = region.getStore(Bytes.toBytes("cf2")); + HStoreFile newFile = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), System.currentTimeMillis(), region.getRegionFileSystem()); + newFile.initReader(); + hStore.refreshStoreFiles(); + + dataTieringManager.updateRegionColdDataSize(regionName, Collections.singletonList(coldFile), + Collections.singletonList(newFile)); + + Pair, Long> after = dataTieringManager.getRegionColdDataSize().get(regionName); + assertNotNull(after); + assertTrue("Cold compacted file should be removed from tracking", + after.getFirst().isEmpty() || !after.getFirst().contains(coldFile.getPath().getName())); + assertEquals(0L, (long) after.getSecond()); + } + /* * Verify that two cold blocks(both) are evicted when bucket reaches its capacity. The hot file * remains in the cache. From 3b4bd11ff37b0d83ba0bd07aa25b9ebb22e1fbe5 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Mon, 27 Apr 2026 11:32:24 +0100 Subject: [PATCH 2/7] UT fix Change-Id: I8eb236beaf7976ccd02349aa81277ca84925e7e6 --- .../hbase/io/hfile/bucket/BucketProtoUtils.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java index e3536e8dce4c..95808a7a8554 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketProtoUtils.java @@ -100,9 +100,16 @@ private static void addEntryToBuilder(Map.Entry entr } private static BucketCacheProtos.BlockCacheKey toPB(BlockCacheKey key) { - return BucketCacheProtos.BlockCacheKey.newBuilder().setHfilename(key.getHfileName()) - .setFamilyName(key.getCfName()).setRegionName(key.getRegionName()).setOffset(key.getOffset()) - .setPrimaryReplicaBlock(key.isPrimary()).setBlockType(toPB(key.getBlockType())).build(); + BucketCacheProtos.BlockCacheKey.Builder builder = BucketCacheProtos.BlockCacheKey.newBuilder() + .setHfilename(key.getHfileName()).setOffset(key.getOffset()) + .setPrimaryReplicaBlock(key.isPrimary()).setBlockType(toPB(key.getBlockType())); + if (key.getCfName() != null) { + builder.setFamilyName(key.getCfName()); + } + if (key.getRegionName() != null) { + builder.setRegionName(key.getRegionName()); + } + return builder.build(); } private static BucketCacheProtos.BlockType toPB(BlockType blockType) { From 14fb76d7762b7cb9abe5d98b4d61e88c0ad7ee0f Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Tue, 28 Apr 2026 13:52:11 +0100 Subject: [PATCH 3/7] addressing review comments Change-Id: I392517f882e7c5a8c6063b16f525f6467956a3bb --- .../hadoop/hbase/io/hfile/bucket/BucketCache.java | 3 ++- .../hadoop/hbase/regionserver/HRegionServer.java | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 8dfa66c76664..bf05285bea31 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -2497,7 +2497,8 @@ public Optional shouldCacheFile(HFileInfo hFileInfo, Configuration conf getAllCacheKeysForFile(hFileInfo.getHFileContext().getHFileName(), 0, Long.MAX_VALUE); int evictedBlocks = evictBlockSet(keySet); if (evictedBlocks > 0) { - LOG.info("Evicted {} blocks for file {} as it is now considered cold by DataTieringManager", + LOG.debug( + "Evicted {} blocks for file {} as it is now considered cold by DataTieringManager", evictedBlocks, fileName); } return Optional.of(false); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index fd608a2f80cc..5358a118328f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1602,13 +1602,13 @@ RegionLoad createRegionLoad(final HRegion r, RegionLoad.Builder regionLoadBldr, }); final MutableFloat currentRegionColdDataRatio = new MutableFloat(0.0f); if (DataTieringManager.getInstance() != null) { - Pair, Long> coldEntry = - DataTieringManager.getInstance().getRegionColdDataSize().get(regionEncodedName); - if (coldEntry != null) { - int coldSizeMB = roundSize(coldEntry.getSecond(), unitMB); - currentRegionColdDataRatio - .setValue(regionSizeMB == 0 ? 0.0f : (float) coldSizeMB / regionSizeMB); - } + DataTieringManager.getInstance().getRegionColdDataSize().computeIfPresent(regionEncodedName, + (k, v) -> { + int coldSizeMB = roundSize(v.getSecond(), unitMB); + currentRegionColdDataRatio + .setValue(regionSizeMB == 0 ? 0.0f : (float) coldSizeMB / regionSizeMB); + return v; + }); } HDFSBlocksDistribution hdfsBd = r.getHDFSBlocksDistribution(); From 91ac5d52c5ed61e95faf4b9d2e9a19be5dd18a4d Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 29 Apr 2026 12:35:59 +0100 Subject: [PATCH 4/7] Apply suggestion from @petersomogyi Co-authored-by: Peter Somogyi --- .../apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java index e2d79abab48c..9b407ccb1186 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java @@ -81,7 +81,7 @@ public interface MetricsRegionWrapper { float getCurrentRegionCacheRatio(); /** - * Gets the current cold date % ratio for this region. + * Gets the current cold data % ratio for this region. */ float getCurrentRegionColdDataRatio(); From 19456d37a00d41697fd31f5a82b0c62e31065f2b Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 29 Apr 2026 12:36:14 +0100 Subject: [PATCH 5/7] Apply suggestion from @petersomogyi Co-authored-by: Peter Somogyi --- .../hadoop/hbase/regionserver/MetricsRegionServerSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 96d90f341b0b..1087ca3c60a0 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -657,7 +657,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String CURRENT_REGION_COLD_DATA_RATIO = "currentRegionColdDataRatio"; String CURRENT_REGION_COLD_DATA_RATIO_DESC = "The percentage of data in this region that " - + "is marked as cold by the configured time based priority logic. "; + + "is marked as cold by the configured time based priority logic."; String EXCLUDE_DATA_NODES_COUNT = "excludedDataNodesCount"; String EXCLUDE_DATA_NODES_COUNT_DESC = From 1e5e23ba1f54f912e5cb0434f06da0bf3ed10a4a Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Wed, 29 Apr 2026 20:34:46 +0100 Subject: [PATCH 6/7] adding extra UT to simulate compacted new cold file Change-Id: I2194da9f2d1e596ae76a0fa244a521a698ff13f9 --- .../regionserver/TestDataTieringManager.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index 8041a59a735f..a3aa53e9ec7e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -455,6 +455,48 @@ public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewHot() throws assertEquals(0L, (long) after.getSecond()); } + /** + * Like {@link #testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewHot}, but the replacement + * store file is still cold under TIME_RANGE rules so {@link DataTieringManager} should keep the + * region entry and record the new file's size. + */ + @Test + public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewCold() throws IOException { + initializeTestEnvironment(); + dataTieringManager.getRegionColdDataSize().clear(); + + HStoreFile coldFile = hStoreFiles.get(3); + String regionName = coldFile.getPath().getParent().getParent().getName(); + dataTieringManager.isHotData(coldFile.getFileInfo().getHFileInfo(), + coldFile.getFileInfo().getConf()); + + HRegion region = testOnlineRegions.get(regionName); + assertNotNull(region); + HStore hStore = region.getStore(Bytes.toBytes("cf2")); + // Region2 hot-age is 2.5 * DAY; use 4 * DAY so the new file stays cold. + long coldTimestamp = System.currentTimeMillis() - 4 * DAY; + HStoreFile newFile = createHStoreFile(hStore.getStoreContext().getFamilyStoreDirectoryPath(), + hStore.getReadOnlyConfiguration(), coldTimestamp, region.getRegionFileSystem()); + newFile.initReader(); + hStore.refreshStoreFiles(); + + assertFalse("new store file must be cold for this scenario", + dataTieringManager.isHotData(newFile.getFileInfo().getHFileInfo(), newFile.getFileInfo() + .getConf())); + + dataTieringManager.updateRegionColdDataSize(regionName, Collections.singletonList(coldFile), + Collections.singletonList(newFile)); + + Pair, Long> after = dataTieringManager.getRegionColdDataSize().get(regionName); + assertNotNull(after); + assertFalse("compacted cold file should no longer be tracked", + after.getFirst().contains(coldFile.getPath().getName())); + assertEquals(1, after.getFirst().size()); + assertTrue(after.getFirst().contains(newFile.getPath().getName())); + long expectedNew = Bytes.toLong(newFile.getFileInfo().getHFileInfo().get(HFileInfo.FILE_SIZE)); + assertEquals(expectedNew, (long) after.getSecond()); + } + /* * Verify that two cold blocks(both) are evicted when bucket reaches its capacity. The hot file * remains in the cache. From 860d2a69e3fb5d5d02dbaa6e1d390e0c930fd727 Mon Sep 17 00:00:00 2001 From: Wellington Ramos Chevreuil Date: Thu, 30 Apr 2026 10:05:00 +0100 Subject: [PATCH 7/7] spotless Change-Id: I171bd2169c18ca47795d239af60ca2414410d060 --- .../hadoop/hbase/regionserver/TestDataTieringManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java index a3aa53e9ec7e..7ef7ce0884bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDataTieringManager.java @@ -480,9 +480,8 @@ public void testUpdateRegionColdDataSizeRemovesCompactedColdAddsNewCold() throws newFile.initReader(); hStore.refreshStoreFiles(); - assertFalse("new store file must be cold for this scenario", - dataTieringManager.isHotData(newFile.getFileInfo().getHFileInfo(), newFile.getFileInfo() - .getConf())); + assertFalse("new store file must be cold for this scenario", dataTieringManager + .isHotData(newFile.getFileInfo().getHFileInfo(), newFile.getFileInfo().getConf())); dataTieringManager.updateRegionColdDataSize(regionName, Collections.singletonList(coldFile), Collections.singletonList(newFile));