Skip to content

Commit 37737b7

Browse files
committed
Improve security migration resilience by handling version conflicts
1 parent 4e68ecc commit 37737b7

File tree

13 files changed

+306
-29
lines changed

13 files changed

+306
-29
lines changed

server/src/main/java/org/elasticsearch/index/IndexVersions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ private static Version parseUnchecked(String version) {
192192

193193
public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1);
194194
public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1);
195+
public static final IndexVersion SECURITY_MIGRATIONS_METADATA = def(9_044_0_00, Version.LUCENE_10_3_1);
195196

196197
/*
197198
* STOP! READ THIS FIRST! No, really,

x-pack/plugin/security/build.gradle

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@ dependencies {
4242
internalClusterTestImplementation(testArtifact(project(xpackModule('core'))))
4343
api 'com.unboundid:unboundid-ldapsdk:6.0.3'
4444

45+
internalClusterTestImplementation project(path: ':modules:lang-painless')
46+
internalClusterTestImplementation project(path: ':modules:lang-painless:spi')
47+
4548
// the following are all SAML dependencies - might as well download the whole internet
4649
api "org.opensaml:opensaml-core:${versions.opensaml}"
4750
api "org.opensaml:opensaml-saml-api:${versions.opensaml}"

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/support/CleanupRoleMappingDuplicatesMigrationIT.java

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,6 @@ public void testMigrationFallbackNamePreCondition() throws Exception {
241241
waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
242242
// First migration is on a new index, so should skip all migrations. If we reset, it should re-trigger and run all migrations
243243
resetMigration();
244-
// Wait for the first migration to finish
245-
waitForMigrationCompletion(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION - 1);
246244

247245
// Make sure migration didn't run yet (blocked by the fallback name)
248246
assertMigrationLessThan(SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION);
@@ -315,10 +313,7 @@ public void testNewIndexSkipMigration() {
315313
ensureGreen();
316314
deleteSecurityIndex(); // hack to force a new security index to be created
317315
ensureGreen();
318-
CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(
319-
masterNode,
320-
SecurityMigrations.CLEANUP_ROLE_MAPPING_DUPLICATES_MIGRATION_VERSION
321-
);
316+
CountDownLatch awaitMigrations = awaitMigrationVersionUpdates(masterNode, SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey());
322317
// Create a native role mapping to create security index and trigger migration
323318
createNativeRoleMapping("everyone_kibana_alone");
324319
// Make sure no migration ran (set to current version without applying prior migrations)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.security.support;
9+
10+
import org.elasticsearch.action.ActionListener;
11+
import org.elasticsearch.action.search.SearchRequest;
12+
import org.elasticsearch.action.search.SearchResponse;
13+
import org.elasticsearch.action.support.WriteRequest;
14+
import org.elasticsearch.cluster.ClusterState;
15+
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.cluster.service.ClusterService;
17+
import org.elasticsearch.core.TimeValue;
18+
import org.elasticsearch.index.query.QueryBuilders;
19+
import org.elasticsearch.painless.PainlessPlugin;
20+
import org.elasticsearch.plugins.Plugin;
21+
import org.elasticsearch.search.SearchHit;
22+
import org.elasticsearch.test.ESIntegTestCase;
23+
import org.elasticsearch.test.ESTestCase;
24+
import org.elasticsearch.test.SecurityIntegTestCase;
25+
import org.elasticsearch.xcontent.ToXContent;
26+
import org.elasticsearch.xcontent.XContentBuilder;
27+
import org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction;
28+
import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
29+
import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
30+
import org.junit.Before;
31+
32+
import java.io.IOException;
33+
import java.util.Collection;
34+
import java.util.List;
35+
import java.util.Map;
36+
import java.util.concurrent.ExecutorService;
37+
import java.util.concurrent.Executors;
38+
import java.util.concurrent.atomic.AtomicBoolean;
39+
import java.util.concurrent.atomic.AtomicLong;
40+
import java.util.stream.Collectors;
41+
import java.util.stream.Stream;
42+
43+
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
44+
import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_DATA_KEY;
45+
import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY;
46+
import static org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ROLE_TYPE;
47+
import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7;
48+
import static org.elasticsearch.xpack.security.support.SecurityMigrations.ROLE_METADATA_FLATTENED_MIGRATION_VERSION;
49+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
50+
51+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
52+
public class MetadataFlattenedMigrationIT extends SecurityIntegTestCase {
53+
54+
private final AtomicLong versionCounter = new AtomicLong(1);
55+
56+
@Before
57+
public void resetVersion() {
58+
versionCounter.set(1);
59+
}
60+
61+
public void testMigrationSuccessful() throws Exception {
62+
internalCluster().setBootstrapMasterNodeIndex(0);
63+
internalCluster().startNode();
64+
ensureGreen();
65+
createRoles();
66+
67+
resetMigration();
68+
waitForMigrationCompletion(ROLE_METADATA_FLATTENED_MIGRATION_VERSION);
69+
assertAllRolesHaveMetadataFlattened();
70+
}
71+
72+
public void testMigrationWithConcurrentUpdates() throws Exception {
73+
internalCluster().setBootstrapMasterNodeIndex(0);
74+
internalCluster().startNode();
75+
ensureGreen();
76+
77+
waitForMigrationCompletion(ROLE_METADATA_FLATTENED_MIGRATION_VERSION);
78+
var roles = createRoles();
79+
final var nativeRoleStore = internalCluster().getInstance(NativeRolesStore.class);
80+
81+
try (ExecutorService executor = Executors.newSingleThreadExecutor()) {
82+
final AtomicBoolean runUpdateRolesBackground = new AtomicBoolean(true);
83+
executor.submit(() -> {
84+
while (runUpdateRolesBackground.get()) {
85+
// Only update half the list so the other half can be verified as migrated
86+
RoleDescriptor roleToUpdate = randomFrom(roles.subList(0, roles.size() / 2));
87+
88+
RoleDescriptor updatedRole = new RoleDescriptor(
89+
roleToUpdate.getName(),
90+
new String[] { "monitor" },
91+
null,
92+
null,
93+
null,
94+
null,
95+
Map.of("test", "value", "timestamp", System.currentTimeMillis(), "random", randomAlphaOfLength(10)),
96+
null
97+
);
98+
nativeRoleStore.putRole(
99+
WriteRequest.RefreshPolicy.IMMEDIATE,
100+
updatedRole,
101+
ActionListener.wrap(resp -> {}, ESTestCase::fail)
102+
);
103+
try {
104+
Thread.sleep(10);
105+
} catch (InterruptedException e) {
106+
throw new RuntimeException(e);
107+
}
108+
}
109+
});
110+
111+
resetMigration();
112+
try {
113+
waitForMigrationCompletion(ROLE_METADATA_FLATTENED_MIGRATION_VERSION);
114+
} finally {
115+
runUpdateRolesBackground.set(false);
116+
executor.shutdown();
117+
}
118+
}
119+
assertAllRolesHaveMetadataFlattened();
120+
}
121+
122+
private void resetMigration() {
123+
client().execute(
124+
UpdateIndexMigrationVersionAction.INSTANCE,
125+
new UpdateIndexMigrationVersionAction.Request(
126+
TimeValue.MAX_VALUE,
127+
ROLE_METADATA_FLATTENED_MIGRATION_VERSION - 1,
128+
INTERNAL_SECURITY_MAIN_INDEX_7
129+
)
130+
).actionGet();
131+
}
132+
133+
private List<RoleDescriptor> createRoles() throws IOException {
134+
var roles = randomList(
135+
25,
136+
50,
137+
() -> new RoleDescriptor(
138+
randomAlphaOfLength(20),
139+
null,
140+
null,
141+
null,
142+
null,
143+
null,
144+
Map.of("test", "value", "timestamp", System.currentTimeMillis(), "random", randomAlphaOfLength(10)),
145+
Map.of()
146+
)
147+
);
148+
for (RoleDescriptor role : roles) {
149+
indexRoleDirectly(role);
150+
}
151+
indicesAdmin().prepareRefresh(INTERNAL_SECURITY_MAIN_INDEX_7).get();
152+
return roles;
153+
}
154+
155+
private void indexRoleDirectly(RoleDescriptor role) throws IOException {
156+
XContentBuilder builder = buildRoleDocument(role);
157+
prepareIndex(INTERNAL_SECURITY_MAIN_INDEX_7).setId(ROLE_TYPE + "-" + role.getName())
158+
.setSource(builder)
159+
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
160+
.get();
161+
}
162+
163+
private XContentBuilder buildRoleDocument(RoleDescriptor role) throws IOException {
164+
XContentBuilder builder = jsonBuilder().startObject();
165+
// metadata_flattened is populated by the native role store, so write directly to index to simulate pre-migration state
166+
role.innerToXContent(builder, ToXContent.EMPTY_PARAMS, true);
167+
builder.endObject();
168+
return builder;
169+
}
170+
171+
private void assertMigrationVersionAtLeast(int expectedVersion) {
172+
assertThat(getCurrentMigrationVersion(), greaterThanOrEqualTo(expectedVersion));
173+
}
174+
175+
private int getCurrentMigrationVersion(ClusterState state) {
176+
IndexMetadata indexMetadata = state.metadata().getProject().index(INTERNAL_SECURITY_MAIN_INDEX_7);
177+
if (indexMetadata == null || indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) == null) {
178+
return 0;
179+
}
180+
return Integer.parseInt(indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY).get(MIGRATION_VERSION_CUSTOM_DATA_KEY));
181+
}
182+
183+
private int getCurrentMigrationVersion() {
184+
ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
185+
return getCurrentMigrationVersion(clusterService.state());
186+
}
187+
188+
private void waitForMigrationCompletion(int version) throws Exception {
189+
assertBusy(() -> assertMigrationVersionAtLeast(version));
190+
}
191+
192+
private void assertAllRolesHaveMetadataFlattened() {
193+
SearchRequest searchRequest = new SearchRequest(INTERNAL_SECURITY_MAIN_INDEX_7);
194+
searchRequest.source().query(QueryBuilders.termQuery("type", "role")).size(1000);
195+
SearchResponse response = client().search(searchRequest).actionGet();
196+
for (SearchHit hit : response.getHits().getHits()) {
197+
@SuppressWarnings("unchecked")
198+
Map<String, Object> metadata = (Map<String, Object>) hit.getSourceAsMap().get("metadata_flattened");
199+
// Only check non-reserved roles
200+
if (metadata.get("_reserved") == null) {
201+
assertEquals("value", metadata.get("test"));
202+
}
203+
}
204+
response.decRef();
205+
}
206+
207+
@Override
208+
protected Collection<Class<? extends Plugin>> nodePlugins() {
209+
return Stream.concat(super.nodePlugins().stream(), Stream.of(PainlessPlugin.class)).collect(Collectors.toList());
210+
}
211+
}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,12 @@
5050
import org.elasticsearch.index.IndexVersion;
5151
import org.elasticsearch.indices.IndexClosedException;
5252
import org.elasticsearch.indices.SystemIndexDescriptor;
53+
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
5354
import org.elasticsearch.rest.RestStatus;
5455
import org.elasticsearch.threadpool.Scheduler;
5556
import org.elasticsearch.xcontent.XContentType;
5657
import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
58+
import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
5759
import org.elasticsearch.xpack.security.SecurityFeatures;
5860
import org.elasticsearch.xpack.security.action.rolemapping.ReservedRoleMappingAction;
5961

@@ -158,6 +160,7 @@ private IndexState unavailableState(ProjectId projectId, ProjectStatus status) {
158160
false,
159161
false,
160162
null,
163+
false,
161164
null,
162165
null,
163166
null,
@@ -180,6 +183,7 @@ public class IndexState {
180183
public final boolean mappingUpToDate;
181184
public final boolean createdOnLatestVersion;
182185
public final RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus;
186+
public final boolean securityMigrationRunning;
183187
public final Integer migrationsVersion;
184188
// Min mapping version supported by the descriptors in the cluster
185189
public final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion;
@@ -201,6 +205,7 @@ public IndexState(
201205
boolean mappingUpToDate,
202206
boolean createdOnLatestVersion,
203207
RoleMappingsCleanupMigrationStatus roleMappingsCleanupMigrationStatus,
208+
boolean securityMigrationRunning,
204209
Integer migrationsVersion,
205210
SystemIndexDescriptor.MappingsVersion minClusterMappingVersion,
206211
Integer indexMappingVersion,
@@ -220,6 +225,7 @@ public IndexState(
220225
this.migrationsVersion = migrationsVersion;
221226
this.createdOnLatestVersion = createdOnLatestVersion;
222227
this.roleMappingsCleanupMigrationStatus = roleMappingsCleanupMigrationStatus;
228+
this.securityMigrationRunning = securityMigrationRunning;
223229
this.minClusterMappingVersion = minClusterMappingVersion;
224230
this.indexMappingVersion = indexMappingVersion;
225231
this.concreteIndexName = concreteIndexName;
@@ -247,6 +253,7 @@ public boolean equals(Object o) {
247253
&& mappingUpToDate == other.mappingUpToDate
248254
&& createdOnLatestVersion == other.createdOnLatestVersion
249255
&& roleMappingsCleanupMigrationStatus == other.roleMappingsCleanupMigrationStatus
256+
&& securityMigrationRunning == other.securityMigrationRunning
250257
&& Objects.equals(indexMappingVersion, other.indexMappingVersion)
251258
&& Objects.equals(migrationsVersion, other.migrationsVersion)
252259
&& Objects.equals(minClusterMappingVersion, other.minClusterMappingVersion)
@@ -268,6 +275,7 @@ public int hashCode() {
268275
mappingUpToDate,
269276
createdOnLatestVersion,
270277
roleMappingsCleanupMigrationStatus,
278+
securityMigrationRunning,
271279
migrationsVersion,
272280
minClusterMappingVersion,
273281
indexMappingVersion,
@@ -370,6 +378,8 @@ public String toString() {
370378
+ createdOnLatestVersion
371379
+ ", roleMappingsCleanupMigrationStatus="
372380
+ roleMappingsCleanupMigrationStatus
381+
+ ", securityMigrationRunning="
382+
+ securityMigrationRunning
373383
+ ", migrationsVersion="
374384
+ migrationsVersion
375385
+ ", minClusterMappingVersion="
@@ -821,6 +831,9 @@ private IndexState updateProjectState(ProjectState project) {
821831
project,
822832
migrationsVersion
823833
);
834+
var persistentTaskCustomMetadata = PersistentTasksCustomMetadata.get(project.metadata());
835+
final boolean securityMigrationRunning = persistentTaskCustomMetadata != null
836+
&& persistentTaskCustomMetadata.getTask(SecurityMigrationTaskParams.TASK_NAME) != null;
824837
final boolean mappingIsUpToDate = indexMetadata == null || checkIndexMappingUpToDate(project);
825838
final SystemIndexDescriptor.MappingsVersion minClusterMappingVersion = getMinSecurityIndexMappingVersion(project);
826839
final int indexMappingVersion = loadIndexMappingVersion(systemIndexDescriptor.getAliasName(), project.metadata());
@@ -853,6 +866,7 @@ private IndexState updateProjectState(ProjectState project) {
853866
mappingIsUpToDate,
854867
createdOnLatestVersion,
855868
roleMappingsCleanupMigrationStatus,
869+
securityMigrationRunning,
856870
migrationsVersion,
857871
minClusterMappingVersion,
858872
indexMappingVersion,

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrationExecutor.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ public SecurityMigrationExecutor(
5454
@Override
5555
protected void nodeOperation(AllocatedPersistentTask task, SecurityMigrationTaskParams params, PersistentTaskState state) {
5656
ActionListener<Void> listener = ActionListener.wrap((res) -> task.markAsCompleted(), (exception) -> {
57-
logger.warn("Security migration failed: " + exception);
57+
logger.warn("Security migration failed", exception);
5858
task.markAsFailed(exception);
5959
});
6060

0 commit comments

Comments
 (0)