elastic · felixbarny · Nov 7, 2024 · Nov 8, 2024
diff --git a/docs/changelog/116438.yaml b/docs/changelog/116438.yaml
@@ -0,0 +1,5 @@
+pr: 116438
+summary: Add mapped fields count to index stats
+area: Stats
+type: enhancement
+issues: []
diff --git a/.../src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java b/.../src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java
@@ -494,7 +494,7 @@ private static ShardStats getShardStats(IndexMetadata indexMeta, int shardIndex,
         stats.docs = new DocsStats(100, 0, randomByteSizeValue().getBytes());
         stats.store = new StoreStats();
         stats.indexing = new IndexingStats(new IndexingStats.Stats(1, 1, 1, 1, 1, 1, 1, 1, false, 1, targetWriteLoad, 1));
-        return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0);
+        return new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0, 0);
     }
 
     static void putComposableIndexTemplate(String id, List<String> patterns, @Nullable Settings settings) throws IOException {

diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -547,9 +547,11 @@ public void testSimpleStats() throws Exception {
         assertThat(stats.getIndex("test1").getPrimaries().getMerge(), notNullValue());
         assertThat(stats.getIndex("test1").getPrimaries().getFlush(), notNullValue());
         assertThat(stats.getIndex("test1").getPrimaries().getRefresh(), notNullValue());
+        assertThat(stats.getIndex("test1").getMappedFieldsCount(), equalTo(2L)); // field + field.keyword
 
         assertThat(stats.getIndex("test2").getPrimaries().getDocs().getCount(), equalTo(1L));
         assertThat(stats.getIndex("test2").getTotal().getDocs().getCount(), equalTo(test2ExpectedWrites));
+        assertThat(stats.getIndex("test2").getMappedFieldsCount(), equalTo(2L)); // field + field.keyword
 
         // make sure that number of requests in progress is 0
         assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexCurrent(), equalTo(0L));

diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -188,6 +188,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0);
     public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0);
     public static final TransportVersion LOGSDB_TELEMETRY_STATS = def(8_785_00_0);
+    public static final TransportVersion INDEX_STATS_MAPPED_FIELDS = def(8_786_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

diff --git a/...c/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/...c/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -276,7 +276,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
                             seqNoStats,
                             retentionLeaseStats,
                             indexShard.isSearchIdle(),
-                            indexShard.searchIdleTime()
+                            indexShard.searchIdleTime(),
+                            indexShard.mapperService().mappingLookup().getTotalFieldsCount()
                         )
                     );
                 }

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java
@@ -31,6 +31,8 @@ public class IndexStats implements Iterable<IndexShardStats> {
 
     private final ShardStats shards[];
 
+    private Long mappedFieldsCount;
+
     public IndexStats(
         String index,
         String uuid,
@@ -124,6 +126,19 @@ public CommonStats getPrimaries() {
         return stats;
     }
 
+    public Long getMappedFieldsCount() {
+        if (this.mappedFieldsCount != null) {
+            return mappedFieldsCount;
+        }
+        for (ShardStats shard : shards) {
+            if (shard.getShardRouting().primary()) {
+                mappedFieldsCount = shard.getMappedFieldsCount();
+                break;
+            }
+        }
+        return mappedFieldsCount;
+    }
+
     public static class IndexStatsBuilder {
         private final String indexName;
         private final String uuid;

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java
@@ -203,6 +203,9 @@ protected Iterator<ToXContent> customXContentChunks(ToXContent.Params params) {
                             if (indexStats.getState() != null) {
                                 builder.field("status", indexStats.getState().toString().toLowerCase(Locale.ROOT));
                             }
+                            if (indexStats.getMappedFieldsCount() != null) {
+                                builder.field("mapped_fields_count", indexStats.getMappedFieldsCount());
+                            }
                             builder.startObject("primaries");
                             indexStats.getPrimaries().toXContent(builder, p);
                             builder.endObject();

diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
@@ -47,6 +47,7 @@ public class ShardStats implements Writeable, ToXContentFragment {
     private final boolean isSearchIdle;
 
     private final long searchIdleTime;
+    private final long mappedFieldsCount;
 
     public ShardStats(StreamInput in) throws IOException {
         assert Transports.assertNotTransportThread("O(#shards) work must always fork to an appropriate executor");
@@ -69,6 +70,11 @@ public ShardStats(StreamInput in) throws IOException {
             isSearchIdle = false;
             searchIdleTime = 0;
         }
+        if (in.getTransportVersion().onOrAfter(TransportVersions.INDEX_STATS_MAPPED_FIELDS)) {
+            mappedFieldsCount = in.readVLong();
+        } else {
+            mappedFieldsCount = -1;
+        }
     }
 
     public ShardStats(
@@ -81,7 +87,8 @@ public ShardStats(
         String statePath,
         boolean isCustomDataPath,
         boolean isSearchIdle,
-        long searchIdleTime
+        long searchIdleTime,
+        long mappedFieldsCount
     ) {
         this.shardRouting = shardRouting;
         this.commonStats = commonStats;
@@ -93,6 +100,7 @@ public ShardStats(
         this.isCustomDataPath = isCustomDataPath;
         this.isSearchIdle = isSearchIdle;
         this.searchIdleTime = searchIdleTime;
+        this.mappedFieldsCount = mappedFieldsCount;
     }
 
     public ShardStats(
@@ -103,7 +111,8 @@ public ShardStats(
         final SeqNoStats seqNoStats,
         final RetentionLeaseStats retentionLeaseStats,
         boolean isSearchIdle,
-        long searchIdleTime
+        long searchIdleTime,
+        long mappedFieldsCount
     ) {
         this(
             shardRouting,
@@ -115,7 +124,8 @@ public ShardStats(
             shardPath.getRootStatePath().toString(),
             shardPath.isCustomDataPath(),
             isSearchIdle,
-            searchIdleTime
+            searchIdleTime,
+            mappedFieldsCount
         );
     }
 
@@ -133,7 +143,8 @@ public boolean equals(Object o) {
             && Objects.equals(seqNoStats, that.seqNoStats)
             && Objects.equals(retentionLeaseStats, that.retentionLeaseStats)
             && Objects.equals(isSearchIdle, that.isSearchIdle)
-            && Objects.equals(searchIdleTime, that.searchIdleTime);
+            && Objects.equals(searchIdleTime, that.searchIdleTime)
+            && Objects.equals(mappedFieldsCount, that.mappedFieldsCount);
     }
 
     @Override
@@ -148,7 +159,8 @@ public int hashCode() {
             seqNoStats,
             retentionLeaseStats,
             isSearchIdle,
-            searchIdleTime
+            searchIdleTime,
+            mappedFieldsCount
         );
     }
 
@@ -202,6 +214,10 @@ public long getSearchIdleTime() {
         return searchIdleTime;
     }
 
+    public long getMappedFieldsCount() {
+        return mappedFieldsCount;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         shardRouting.writeTo(out);
@@ -220,6 +236,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(isSearchIdle);
             out.writeVLong(searchIdleTime);
         }
+        if (out.getTransportVersion().onOrAfter(TransportVersions.INDEX_STATS_MAPPED_FIELDS)) {
+            out.writeVLong(mappedFieldsCount);
+        }
     }
 
     @Override

diff --git a/...c/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/...c/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -132,7 +132,8 @@ protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRou
                 seqNoStats,
                 retentionLeaseStats,
                 indexShard.isSearchIdle(),
-                indexShard.searchIdleTime()
+                indexShard.searchIdleTime(),
+                indexShard.mapperService().mappingLookup().getTotalFieldsCount()
             );
         });
     }

diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -535,7 +535,8 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index
                     seqNoStats,
                     retentionLeaseStats,
                     indexShard.isSearchIdle(),
-                    indexShard.searchIdleTime()
+                    indexShard.searchIdleTime(),
+                    indexShard.mapperService().mappingLookup().getTotalFieldsCount()
                 ) }
         );
     }

diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -671,7 +671,7 @@ private static ShardStats createShardStats(ShardId shardId) {
             .resolve(shardRouting.shardId().getIndex().getUUID())
             .resolve(String.valueOf(shardRouting.shardId().id()));
         ShardPath shardPath = new ShardPath(false, path, path, shardRouting.shardId());
-        return new ShardStats(shardRouting, shardPath, createShardLevelCommonStats(), null, null, null, false, 0);
+        return new ShardStats(shardRouting, shardPath, createShardLevelCommonStats(), null, null, null, false, 0, 0);
     }
 
     public static NodeStats createNodeStats() {

diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java
@@ -120,6 +120,7 @@ public void testCreation() {
             null,
             null,
             false,
+            0,
             0
         );
         ClusterStatsNodeResponse nodeResponse = new ClusterStatsNodeResponse(

diff --git a/...st/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/...st/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -677,7 +677,9 @@ public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadat
                 stats.warmer = new WarmerStats();
                 stats.denseVectorStats = new DenseVectorStats();
                 stats.sparseVectorStats = new SparseVectorStats();
-                shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0));
+                shardStats.add(
+                    new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0, 0)
+                );
             }
         }
         return IndicesStatsTests.newIndicesStatsResponse(

diff --git a/...src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java b/...src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java
@@ -75,7 +75,7 @@ public void testGetIndices() {
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardId));
                 ShardPath shardPath = new ShardPath(false, path, path, shId);
                 ShardRouting routing = createShardRouting(shId, (shardId == 0));
-                shards.add(new ShardStats(routing, shardPath, null, null, null, null, false, 0));
+                shards.add(new ShardStats(routing, shardPath, null, null, null, null, false, 0, 0));
                 AtomicLong primaryShardsCounter = expectedIndexToPrimaryShardsCount.computeIfAbsent(
                     index.getName(),
                     k -> new AtomicLong(0L)
@@ -124,7 +124,7 @@ public void testChunkedEncodingPerIndex() {
             Path path = createTempDir().resolve("indices").resolve(shId.getIndex().getUUID()).resolve(String.valueOf(shId.id()));
             ShardPath shardPath = new ShardPath(false, path, path, shId);
             ShardRouting routing = createShardRouting(shId, (shId.id() == 0));
-            stats.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null, false, 0));
+            stats.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null, false, 0, 0));
         }
         final IndicesStatsResponse indicesStatsResponse = new IndicesStatsResponse(
             stats.toArray(new ShardStats[0]),

diff --git a/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -122,8 +122,28 @@ public void testFillShardLevelInfo() {
         CommonStats commonStats2 = new CommonStats();
         commonStats2.store = new StoreStats(1000, 999, 0L);
         ShardStats[] stats = new ShardStats[] {
-            new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0, null, null, null, false, 0),
-            new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1, null, null, null, false, 0),
+            new ShardStats(
+                test_0,
+                new ShardPath(false, test0Path, test0Path, test_0.shardId()),
+                commonStats0,
+                null,
+                null,
+                null,
+                false,
+                0,
+                0
+            ),
+            new ShardStats(
+                test_1,
+                new ShardPath(false, test1Path, test1Path, test_1.shardId()),
+                commonStats1,
+                null,
+                null,
+                null,
+                false,
+                0,
+                0
+            ),
             new ShardStats(
                 test_1,
                 new ShardPath(false, test1Path, test1Path, test_1.shardId()),
@@ -132,6 +152,7 @@ public void testFillShardLevelInfo() {
                 null,
                 null,
                 false,
+                0,
                 0
             ) };
         Map<String, Long> shardSizes = new HashMap<>();

diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java
@@ -115,6 +115,6 @@ private ShardStats createShardStats(
             .add(
                 new IndexingStats.Stats(0, 0, 0, 0, 0, 0, 0, 0, false, 0, totalIndexingTimeSinceShardStartedInNanos, totalActiveTimeInNanos)
             );
-        return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false, false, 0);
+        return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false, false, 0, 0);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -1622,7 +1622,8 @@ public void testShardStats() throws IOException {
             shard.seqNoStats(),
             shard.getRetentionLeaseStats(),
             shard.isSearchIdle(),
-            shard.searchIdleTime()
+            shard.searchIdleTime(),
+            0
         );
         assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
         assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());

diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestShardsActionTests.java
@@ -61,6 +61,7 @@ public void testBuildTable() {
                 null,
                 null,
                 false,
+                0,
                 0
             );
             shardStatsMap.put(shardRouting, shardStats);

diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -124,7 +124,8 @@ ShardStats[] adjustShardStats(ShardStats[] shardsStats) {
                 shardStats.getStatePath(),
                 shardStats.isCustomDataPath(),
                 shardStats.isSearchIdle(),
-                shardStats.getSearchIdleTime()
+                shardStats.getSearchIdleTime(),
+                shardStats.getMappedFieldsCount()
             );
         }).toArray(ShardStats[]::new);
     }

diff --git a/...ugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java b/...ugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java
@@ -90,7 +90,7 @@ private static ShardStats shardStat(long byteCount, long docCount, ShardRouting
         CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL);
         commonStats.getStore().add(storeStats);
         commonStats.getDocs().add(docsStats);
-        return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0);
+        return new ShardStats(routing, fakeShardPath, commonStats, null, null, null, false, 0, 0);
     }
 
     static IndexMetadata indexMetadata(String indexName, int numberOfShards, int numberOfReplicas, String... dataTierPrefs) {

diff --git a/...ugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java b/...ugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForNoFollowersStepTests.java
@@ -184,7 +184,7 @@ public void testNoShardStats() {
             .numberOfReplicas(randomIntBetween(1, 10))
             .build();
 
-        ShardStats sStats = new ShardStats(null, mockShardPath(), null, null, null, null, false, 0);
+        ShardStats sStats = new ShardStats(null, mockShardPath(), null, null, null, null, false, 0, 0);
         ShardStats[] shardStats = new ShardStats[1];
         shardStats[0] = sStats;
 
@@ -293,7 +293,7 @@ private IndexStats randomIndexStats(boolean isLeaderIndex, int numOfShards) {
     }
 
     private ShardStats randomShardStats(boolean isLeaderIndex) {
-        return new ShardStats(null, mockShardPath(), null, null, null, randomRetentionLeaseStats(isLeaderIndex), false, 0);
+        return new ShardStats(null, mockShardPath(), null, null, null, randomRetentionLeaseStats(isLeaderIndex), false, 0, 0);
     }
 
     private RetentionLeaseStats randomRetentionLeaseStats(boolean isLeaderIndex) {

diff --git a/.../org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/.../org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java
@@ -55,10 +55,10 @@ public void setUp() throws Exception {
                 null,
                 new ShardStats[] {
                     // Primaries
-                    new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null, false, 0),
-                    new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null, false, 0),
+                    new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null, false, 0, 0),
+                    new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null, null, false, 0, 0),
                     // Replica
-                    new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null, null, false, 0) }
+                    new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null, null, false, 0, 0) }
             )
         );
     }

diff --git a/...ava/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java b/...ava/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java
@@ -394,7 +394,7 @@ private static ShardStats[] createShardStats(Map<String, long[]> checkpoints) {
                 Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
 
                 shardStats.add(
-                    new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, seqNoStats, null, false, 0)
+                    new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, seqNoStats, null, false, 0, 0)
                 );
             }
 

diff --git a/...t/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java b/...t/java/org/elasticsearch/xpack/transform/checkpoint/TransformsCheckpointServiceTests.java
@@ -283,6 +283,7 @@ private static ShardStats[] createRandomShardStats(
                                 invalidSeqNoStats,
                                 null,
                                 false,
+                                0,
                                 0
                             )
                         );
@@ -296,6 +297,7 @@ private static ShardStats[] createRandomShardStats(
                                 validSeqNoStats,
                                 null,
                                 false,
+                                0,
                                 0
                             )
                         );
-Original file line number
+Diff line change
@@ Expand Up / @@ -120,6 +120,7 @@ public void testCreation() { @@
                 null,
                 null,
                 false,
+,
             );
             ClusterStatsNodeResponse nodeResponse = new ClusterStatsNodeResponse(
@@ Expand Down @@