Skip to content

New per-project only settings can be defined and used by components #127280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Apr 30, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
7a5d446
New per-project only settings can be defined and used by components
alexey-ivanov-es Apr 23, 2025
6eedbb4
Update docs/changelog/127252.yaml
alexey-ivanov-es Apr 23, 2025
a7ab20d
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 23, 2025
0c45219
[CI] Auto commit changes from spotless
elasticsearchmachine Apr 23, 2025
20732ae
Fix tests
alexey-ivanov-es Apr 23, 2025
671ae16
Remove comments
alexey-ivanov-es Apr 23, 2025
efebd32
[CI] Auto commit changes from spotless
elasticsearchmachine Apr 23, 2025
1d42716
Update docs/changelog/127280.yaml
alexey-ivanov-es Apr 23, 2025
0d60756
Delete old changelog file
alexey-ivanov-es Apr 23, 2025
94b9fcd
Address review comments
alexey-ivanov-es Apr 23, 2025
a8a31c3
Move ProjectSettingsUpdater to serverless
alexey-ivanov-es Apr 24, 2025
1d6ae73
Delete docs/changelog/127280.yaml
alexey-ivanov-es Apr 24, 2025
eebe304
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 24, 2025
e040c9d
Fix tests
alexey-ivanov-es Apr 24, 2025
0234193
Fix a bug in the cluster state application
alexey-ivanov-es Apr 28, 2025
90f3c75
Make abstract
alexey-ivanov-es Apr 28, 2025
910fd79
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 28, 2025
dfc13a3
Make ingest.geoip.downloader.enabled project-scoped
alexey-ivanov-es Apr 29, 2025
53d12e5
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 29, 2025
589105e
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 30, 2025
59adba1
Merge branch 'main' into per-project-settings
alexey-ivanov-es Apr 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Move ProjectSettingsUpdater to serverless
  • Loading branch information
alexey-ivanov-es committed Apr 24, 2025
commit a8a31c36ebc7e1a103a66cede9f58e74d4b6e0a4
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/

package org.elasticsearch.common.settings;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.core.Tuple;

import java.util.Map;

import static org.elasticsearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX;

public class BaseSettingsUpdater {
protected final AbstractScopedSettings scopedSettings;

public BaseSettingsUpdater(AbstractScopedSettings scopedSettings) {
this.scopedSettings = scopedSettings;
}

private static void logUnknownSetting(final String settingType, final Map.Entry<String, String> e, final Logger logger) {
logger.warn("ignoring existing unknown {} setting: [{}] with value [{}]; archiving", settingType, e.getKey(), e.getValue());
}

private static void logInvalidSetting(
final String settingType,
final Map.Entry<String, String> e,
final IllegalArgumentException ex,
final Logger logger
) {
logger.warn(
(Supplier<?>) () -> "ignoring existing invalid "
+ settingType
+ " setting: ["
+ e.getKey()
+ "] with value ["
+ e.getValue()
+ "]; archiving",
ex
);
}

/**
* Partitions the settings into those that are known and valid versus those that are unknown or invalid. The resulting tuple contains
* the known and valid settings in the first component and the unknown or invalid settings in the second component. Note that archived
* settings contained in the settings to partition are included in the first component.
*
* @param settings the settings to partition
* @param settingsType a string to identify the settings (for logging)
* @param logger a logger to sending warnings to
* @return the partitioned settings
*/
protected final Tuple<Settings, Settings> partitionKnownAndValidSettings(
final Settings settings,
final String settingsType,
final Logger logger
) {
final Settings existingArchivedSettings = settings.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX));
final Settings settingsExcludingExistingArchivedSettings = settings.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX) == false);
final Settings settingsWithUnknownOrInvalidArchived = scopedSettings.archiveUnknownOrInvalidSettings(
settingsExcludingExistingArchivedSettings,
e -> BaseSettingsUpdater.logUnknownSetting(settingsType, e, logger),
(e, ex) -> BaseSettingsUpdater.logInvalidSetting(settingsType, e, ex, logger)
);
return Tuple.tuple(
Settings.builder()
.put(settingsWithUnknownOrInvalidArchived.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX) == false))
.put(existingArchivedSettings)
.build(),
settingsWithUnknownOrInvalidArchived.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX))
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,23 @@
package org.elasticsearch.common.settings;

import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlocks;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.core.Tuple;

import java.util.Map;

import static org.elasticsearch.cluster.ClusterState.builder;
import static org.elasticsearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX;

/**
* Updates transient and persistent cluster state settings if there are any changes
* due to the update.
*/
public final class SettingsUpdater {
public final class SettingsUpdater extends BaseSettingsUpdater {
final Settings.Builder transientUpdates = Settings.builder();
final Settings.Builder persistentUpdates = Settings.builder();
private final AbstractScopedSettings scopedSettings;

public SettingsUpdater(AbstractScopedSettings scopedSettings) {
this.scopedSettings = scopedSettings;
public SettingsUpdater(ClusterSettings scopedSettings) {
super(scopedSettings);
}

public synchronized Settings getTransientUpdates() {
Expand Down Expand Up @@ -125,90 +119,4 @@ public synchronized ClusterState updateSettings(

return clusterState;
}

public synchronized ProjectMetadata updateProjectSettings(
final ProjectMetadata projectMetadata,
final Settings settingsToApply,
final Logger logger
) {
final Tuple<Settings, Settings> partitionedSettings = partitionKnownAndValidSettings(projectMetadata.settings(), "project", logger);
final Settings knownAndValidPersistentSettings = partitionedSettings.v1();
final Settings unknownOrInvalidSettings = partitionedSettings.v2();
Settings.Builder builder = Settings.builder().put(knownAndValidPersistentSettings);

boolean changed = scopedSettings.updateSettings(
settingsToApply,
builder,
persistentUpdates,
"project[" + projectMetadata.id() + "]"
);
if (changed == false) {
return projectMetadata;
}

Settings finalSettings = builder.build();
// validate that settings and their values are correct
scopedSettings.validate(finalSettings, true);

Settings resultSettings = Settings.builder().put(finalSettings).put(unknownOrInvalidSettings).build();
ProjectMetadata.Builder result = ProjectMetadata.builder(projectMetadata).settings(resultSettings);
// validate that SettingsUpdaters can be applied without errors
scopedSettings.validateUpdate(resultSettings);

return result.build();
}

/**
* Partitions the settings into those that are known and valid versus those that are unknown or invalid. The resulting tuple contains
* the known and valid settings in the first component and the unknown or invalid settings in the second component. Note that archived
* settings contained in the settings to partition are included in the first component.
*
* @param settings the settings to partition
* @param settingsType a string to identify the settings (for logging)
* @param logger a logger to sending warnings to
* @return the partitioned settings
*/
private Tuple<Settings, Settings> partitionKnownAndValidSettings(
final Settings settings,
final String settingsType,
final Logger logger
) {
final Settings existingArchivedSettings = settings.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX));
final Settings settingsExcludingExistingArchivedSettings = settings.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX) == false);
final Settings settingsWithUnknownOrInvalidArchived = scopedSettings.archiveUnknownOrInvalidSettings(
settingsExcludingExistingArchivedSettings,
e -> logUnknownSetting(settingsType, e, logger),
(e, ex) -> logInvalidSetting(settingsType, e, ex, logger)
);
return Tuple.tuple(
Settings.builder()
.put(settingsWithUnknownOrInvalidArchived.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX) == false))
.put(existingArchivedSettings)
.build(),
settingsWithUnknownOrInvalidArchived.filter(k -> k.startsWith(ARCHIVED_SETTINGS_PREFIX))
);
}

private static void logUnknownSetting(final String settingType, final Map.Entry<String, String> e, final Logger logger) {
logger.warn("ignoring existing unknown {} setting: [{}] with value [{}]; archiving", settingType, e.getKey(), e.getValue());
}

private static void logInvalidSetting(
final String settingType,
final Map.Entry<String, String> e,
final IllegalArgumentException ex,
final Logger logger
) {
logger.warn(
(Supplier<?>) () -> "ignoring existing invalid "
+ settingType
+ " setting: ["
+ e.getKey()
+ "] with value ["
+ e.getValue()
+ "]; archiving",
ex
);
}

}
Loading
Loading