Skip to content

Calculate routing num shards correctly during reshard #125601

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
May 8, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
671c348
commit
ankikuma Mar 13, 2025
7ae62e7
indexMetadata comments
ankikuma Mar 18, 2025
31281e5
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Mar 18, 2025
ff98f8e
SplitIndexIT
ankikuma Mar 25, 2025
37d61fd
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Mar 25, 2025
b29a437
IndexMetadata.java
ankikuma Mar 25, 2025
eacb67a
[CI] Auto commit changes from spotless
elasticsearchmachine Mar 25, 2025
0a15de3
commit
ankikuma Apr 7, 2025
86f66bc
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Apr 7, 2025
41a70ba
merge conflicts in elasticsearch
ankikuma Apr 8, 2025
44c9c4f
merge conflicts in elasticsearch
ankikuma Apr 8, 2025
c8382c2
merge conflicts in elasticsearch
ankikuma Apr 8, 2025
e77399d
commit elasticsearch
ankikuma Apr 18, 2025
2946f37
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Apr 18, 2025
c21b9f5
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Apr 18, 2025
33a8b45
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma Apr 21, 2025
a2de378
address review comments and add test
ankikuma May 6, 2025
8d10925
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma May 6, 2025
fa25099
reshardAddShards
ankikuma May 7, 2025
4886ff3
Merge remote-tracking branch 'upstream/main' into 03132025/ReshardRou…
ankikuma May 7, 2025
47ab570
minor change
ankikuma May 7, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1998,34 +1998,31 @@ public Builder numberOfShards(int numberOfShards) {

/**
* Builder to create IndexMetadata that has an increased shard count (used for re-shard).
* The new shard count must be a multiple of the original shardcount.
* The new shard count must be a multiple of the original shardcount as well as a factor
* of routingNumShards.
* We do not support shrinking the shard count.
* @param shardCount updated shardCount
*
* TODO: Check if this.version needs to be incremented
* @param targetShardCount target shard count after resharding
*/
public Builder reshardAddShards(int shardCount) {
// Assert routingNumShards is null ?
// Assert numberOfShards > 0
if (shardCount % numberOfShards() != 0) {
public Builder reshardAddShards(int targetShardCount) {
final int sourceNumShards = numberOfShards();
if (targetShardCount % sourceNumShards != 0) {
throw new IllegalArgumentException(
"New shard count ["
+ shardCount
+ targetShardCount
+ "] should be a multiple"
+ " of current shard count ["
+ numberOfShards()
+ sourceNumShards
+ "] for ["
+ index
+ "]"
);
}
IndexVersion indexVersionCreated = indexCreatedVersion(settings);
settings = Settings.builder().put(settings).put(SETTING_NUMBER_OF_SHARDS, shardCount).build();
var newPrimaryTerms = new long[shardCount];
settings = Settings.builder().put(settings).put(SETTING_NUMBER_OF_SHARDS, targetShardCount).build();
var newPrimaryTerms = new long[targetShardCount];
Arrays.fill(newPrimaryTerms, this.primaryTerms.length, newPrimaryTerms.length, SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
System.arraycopy(primaryTerms, 0, newPrimaryTerms, 0, this.primaryTerms.length);
primaryTerms = newPrimaryTerms;
routingNumShards = MetadataCreateIndexService.calculateNumRoutingShards(shardCount, indexVersionCreated);
routingNumShards = MetadataCreateIndexService.getIndexNumberOfRoutingShards(settings, sourceNumShards, this.routingNumShards);
return this;
}

Expand Down Expand Up @@ -3034,7 +3031,7 @@ public static ShardId selectCloneShard(int shardId, IndexMetadata sourceIndexMet
return new ShardId(sourceIndexMetadata.getIndex(), shardId);
}

private static void assertSplitMetadata(int numSourceShards, int numTargetShards, IndexMetadata sourceIndexMetadata) {
public static void assertSplitMetadata(int numSourceShards, int numTargetShards, IndexMetadata sourceIndexMetadata) {
if (numSourceShards > numTargetShards) {
throw new IllegalArgumentException(
"the number of source shards ["
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1282,10 +1282,23 @@ private static void validateSoftDeleteSettings(Settings indexSettings) {
* it will return the value configured for that index.
*/
static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable IndexMetadata sourceMetadata) {
final int routingNumShards = getIndexNumberOfRoutingShards(
indexSettings,
sourceMetadata == null ? 1 : sourceMetadata.getNumberOfShards(),
sourceMetadata == null ? 0 : sourceMetadata.getRoutingNumShards()
);
return routingNumShards;
}

/**
* Calculates the number of routing shards based on the configured value in indexSettings or if recovering from another index
* it will return the value configured for that index.
*/
static int getIndexNumberOfRoutingShards(Settings indexSettings, final int sourceNumShards, final int sourceRoutingNumShards) {
final int numTargetShards = INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettings);
final IndexVersion indexVersionCreated = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(indexSettings);
final int routingNumShards;
if (sourceMetadata == null || sourceMetadata.getNumberOfShards() == 1) {
if (sourceNumShards == 1) {
// in this case we either have no index to recover from or
// we have a source index with 1 shard and without an explicit split factor
// or one that is valid in that case we can split into whatever and auto-generate a new factor.
Expand All @@ -1299,7 +1312,7 @@ static int getIndexNumberOfRoutingShards(Settings indexSettings, @Nullable Index
} else {
assert IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettings) == false
: "index.number_of_routing_shards should not be present on the target index on resize";
routingNumShards = sourceMetadata.getRoutingNumShards();
routingNumShards = sourceRoutingNumShards;
}
return routingNumShards;
}
Expand Down