From 6804e876be33273dccd91fcdff1d85dd2c3943d5 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 15:05:24 +0100 Subject: [PATCH 1/9] mmaprototype: avoid shadowing loadSummary is also a type. Linter complained in CI all out of a sudden :shrug: --- pkg/kv/kvserver/allocator/mmaprototype/load.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/load.go b/pkg/kv/kvserver/allocator/mmaprototype/load.go index c64881a5e354..f0ff2fa341c4 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/load.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/load.go @@ -492,7 +492,7 @@ func loadSummaryForDimension( meanLoad LoadValue, meanUtil float64, ) (summary loadSummary) { - loadSummary := loadLow + summ := loadLow if dim == WriteBandwidth && capacity == UnknownCapacity { // Ignore smaller than 1MiB differences in write bandwidth. This 1MiB // value is somewhat arbitrary, but is based on EBS gp3 having a default @@ -546,13 +546,13 @@ func loadSummaryForDimension( meanFractionNoChange = 0.05 ) if fractionAbove > meanFractionSlow { - loadSummary = overloadSlow + summ = overloadSlow } else if fractionAbove < meanFractionLow { - loadSummary = loadLow + summ = loadLow } else if fractionAbove >= meanFractionNoChange { - loadSummary = loadNoChange + summ = loadNoChange } else { - loadSummary = loadNormal + summ = loadNormal } if capacity != UnknownCapacity && meanUtil*1.1 < fractionUsed { // Further tune the summary based on utilization. @@ -575,9 +575,9 @@ func loadSummaryForDimension( if meanUtil*1.75 < fractionUsed { return min(summaryUpperBound, overloadSlow) } - return min(summaryUpperBound, max(loadSummary, loadNoChange)) + return min(summaryUpperBound, max(summ, loadNoChange)) } - return min(summaryUpperBound, loadSummary) + return min(summaryUpperBound, summ) } func highDiskSpaceUtilization(load LoadValue, capacity LoadValue) bool { From 386c03d22443662a97cad1c7bf64f99249b8f483 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 11:02:11 +0100 Subject: [PATCH 2/9] mmaprototype: remove failureDetectionSummary We'll replace it with a store-level construct in follow-up commits. --- .../allocator/mmaprototype/allocator.go | 4 -- .../allocator/mmaprototype/allocator_state.go | 18 +------- .../allocator/mmaprototype/cluster_state.go | 45 ------------------- .../mmaprototype/cluster_state_test.go | 17 +------ .../kvserver/allocator/mmaprototype/load.go | 5 +-- .../testdata/cluster_state/multiple_ranges | 8 ++-- .../testdata/cluster_state/rebalance_replica | 8 ++-- .../testdata/cluster_state/remove_replica | 8 ++-- .../testdata/cluster_state/removed_ranges | 8 ++-- .../testdata/cluster_state/transfer_lease | 16 +++---- 10 files changed, 29 insertions(+), 108 deletions(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator.go index 76b1d4eea9bf..02c032b9d4b7 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator.go @@ -43,10 +43,6 @@ type Allocator interface { // stores. RemoveNodeAndStores(nodeID roachpb.NodeID) error - // UpdateFailureDetectionSummary tells the allocator about the current - // failure detection state for a node. A node starts in the fdOK state. - UpdateFailureDetectionSummary(nodeID roachpb.NodeID, fd failureDetectionSummary) error - // ProcessStoreLoadMsg provides frequent the state of every store and its // associated node in the cluster. ProcessStoreLoadMsg(ctx context.Context, msg *StoreLoadMsg) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go index 30705cb93dab..3ba26816c9da 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go @@ -493,10 +493,6 @@ func (a *allocatorState) rebalanceStores( continue } candSls := a.cs.computeLoadSummary(ctx, cand.storeID, &means.storeLoad, &means.nodeLoad) - if sls.fd != fdOK { - log.KvDistribution.VInfof(ctx, 2, "skipping store s%d: failure detection status not OK", cand.storeID) - continue - } candsSet.candidates = append(candsSet.candidates, candidateInfo{ StoreID: cand.storeID, storeLoadSummary: candSls, @@ -608,7 +604,7 @@ func (a *allocatorState) rebalanceStores( // If the node is cpu overloaded, or the store/node is not fdOK, exclude // the other stores on this node from receiving replicas shed by this // store. - excludeStoresOnNode := store.nls > overloadSlow || store.fd != fdOK + excludeStoresOnNode := store.nls > overloadSlow storesToExclude = storesToExclude[:0] if excludeStoresOnNode { nodeID := ss.NodeID @@ -819,15 +815,6 @@ func (a *allocatorState) RemoveNodeAndStores(nodeID roachpb.NodeID) error { panic("unimplemented") } -// UpdateFailureDetectionSummary implements the Allocator interface. -func (a *allocatorState) UpdateFailureDetectionSummary( - nodeID roachpb.NodeID, fd failureDetectionSummary, -) error { - a.mu.Lock() - defer a.mu.Unlock() - panic("unimplemented") -} - // ProcessStoreLeaseholderMsg implements the Allocator interface. func (a *allocatorState) ProcessStoreLoadMsg(ctx context.Context, msg *StoreLoadMsg) { a.mu.Lock() @@ -1381,9 +1368,6 @@ func (a *allocatorState) computeCandidatesForRange( } ss := a.cs.stores[storeID] csls := a.cs.meansMemo.getStoreLoadSummary(ctx, means, storeID, ss.loadSeqNum) - if csls.fd != fdOK { - continue - } cset.candidates = append(cset.candidates, candidateInfo{ StoreID: storeID, storeLoadSummary: csls, diff --git a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go index 20f1d7029456..d78e9a8b5566 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go @@ -829,47 +829,10 @@ func newStoreState() *storeState { return ss } -// failureDetectionSummary is provided by an external entity and never -// computed inside the allocator. -type failureDetectionSummary uint8 - -// All state transitions are permitted by the allocator. For example, fdDead -// => fdOk is allowed since the allocator can simply stop shedding replicas -// and then start adding replicas (if underloaded). -const ( - fdOK failureDetectionSummary = iota - // Don't add replicas or leases. - fdSuspect - // Move leases away. Don't add replicas or leases. - fdDrain - // Node is dead, so move leases and replicas away from it. - fdDead -) - -func (fds failureDetectionSummary) String() string { - return redact.StringWithoutMarkers(fds) -} - -// SafeFormat implements the redact.SafeFormatter interface. -func (fds failureDetectionSummary) SafeFormat(w redact.SafePrinter, _ rune) { - switch fds { - case fdOK: - w.Print("ok") - case fdSuspect: - w.Print("suspect") - case fdDrain: - w.Print("drain") - case fdDead: - w.Print("dead") - } -} - type nodeState struct { stores []roachpb.StoreID NodeLoad adjustedCPU LoadValue - - fdSummary failureDetectionSummary } func newNodeState(nodeID roachpb.NodeID) *nodeState { @@ -1905,13 +1868,6 @@ func (cs *clusterState) setStoreMembership(storeID roachpb.StoreID, state storeM } } -func (cs *clusterState) updateFailureDetectionSummary( - nodeID roachpb.NodeID, fd failureDetectionSummary, -) { - ns := cs.nodes[nodeID] - ns.fdSummary = fd -} - //====================================================================== // clusterState accessors: // @@ -2193,7 +2149,6 @@ func computeLoadSummary( nls: nls, dimSummary: dimSummary, highDiskSpaceUtilization: highDiskSpaceUtil, - fd: ns.fdSummary, maxFractionPendingIncrease: ss.maxFractionPendingIncrease, maxFractionPendingDecrease: ss.maxFractionPendingDecrease, loadSeqNum: ss.loadSeqNum, diff --git a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go index 5163bb0f5e20..b048593c437d 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go @@ -301,8 +301,8 @@ func TestClusterState(t *testing.T) { var buf strings.Builder for _, nodeID := range nodeList { ns := cs.nodes[roachpb.NodeID(nodeID)] - fmt.Fprintf(&buf, "node-id=%s failure-summary=%s locality-tiers=%s\n", - ns.NodeID, ns.fdSummary, cs.stores[ns.stores[0]].StoreAttributesAndLocality.locality()) + fmt.Fprintf(&buf, "node-id=%s locality-tiers=%s\n", + ns.NodeID, cs.stores[ns.stores[0]].StoreAttributesAndLocality.locality()) for _, storeID := range ns.stores { ss := cs.stores[storeID] fmt.Fprintf(&buf, " store-id=%v membership=%v attrs=%s locality-code=%s\n", @@ -386,19 +386,6 @@ func TestClusterState(t *testing.T) { printPostingList(&buf, removedStores) return buf.String() - case "update-failure-detection": - nodeID := dd.ScanArg[roachpb.NodeID](t, d, "node-id") - failureDetectionString := dd.ScanArg[string](t, d, "summary") - var fd failureDetectionSummary - for i := fdOK; i < fdDead+1; i++ { - if i.String() == failureDetectionString { - fd = i - break - } - } - cs.updateFailureDetectionSummary(nodeID, fd) - return printNodeListMeta() - case "store-load-msg": msg := parseStoreLoadMsg(t, d.Input) cs.processStoreLoadMsg(context.Background(), &msg) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/load.go b/pkg/kv/kvserver/allocator/mmaprototype/load.go index f0ff2fa341c4..3700a86724b4 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/load.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/load.go @@ -197,7 +197,6 @@ type storeLoadSummary struct { nls loadSummary dimSummary [NumLoadDimensions]loadSummary highDiskSpaceUtilization bool - fd failureDetectionSummary maxFractionPendingIncrease, maxFractionPendingDecrease float64 loadSeqNum uint64 @@ -208,9 +207,9 @@ func (sls storeLoadSummary) String() string { } func (sls storeLoadSummary) SafeFormat(w redact.SafePrinter, _ rune) { - w.Printf("(store=%v worst=%v cpu=%v writes=%v bytes=%v node=%v high_disk=%v fd=%v, frac_pending=%.2f,%.2f(%t))", + w.Printf("(store=%v worst=%v cpu=%v writes=%v bytes=%v node=%v high_disk=%v frac_pending=%.2f,%.2f(%t))", sls.sls, sls.worstDim, sls.dimSummary[CPURate], sls.dimSummary[WriteBandwidth], sls.dimSummary[ByteSize], - sls.nls, sls.highDiskSpaceUtilization, sls.fd, sls.maxFractionPendingIncrease, + sls.nls, sls.highDiskSpaceUtilization, sls.maxFractionPendingIncrease, sls.maxFractionPendingDecrease, sls.maxFractionPendingIncrease < epsilon && sls.maxFractionPendingDecrease < epsilon) } diff --git a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/multiple_ranges b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/multiple_ranges index 216e0526daa1..346050c75317 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/multiple_ranges +++ b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/multiple_ranges @@ -2,10 +2,10 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: # Both stores are more constrained wrt CPURate. store-load-msg diff --git a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/rebalance_replica b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/rebalance_replica index b691506e6ed0..55203ea8a691 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/rebalance_replica +++ b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/rebalance_replica @@ -2,10 +2,10 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: store-load-msg store-id=1 node-id=1 load=[80,80,80] capacity=[100,100,100] secondary-load=0 load-time=0s diff --git a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/remove_replica b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/remove_replica index c677b308c7ad..40c26673df7c 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/remove_replica +++ b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/remove_replica @@ -2,10 +2,10 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: store-load-msg store-id=2 node-id=2 load=[20,80,80] capacity=[100,100,100] secondary-load=0 load-time=0s diff --git a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/removed_ranges b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/removed_ranges index 2cd0f0a57bcf..3658bdbb29df 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/removed_ranges +++ b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/removed_ranges @@ -5,10 +5,10 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: store-leaseholder-msg store-id=1 diff --git a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/transfer_lease b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/transfer_lease index bc2167e8363f..5cc84f5dd438 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/transfer_lease +++ b/pkg/kv/kvserver/allocator/mmaprototype/testdata/cluster_state/transfer_lease @@ -2,10 +2,10 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: store-load-msg store-id=1 node-id=1 load=[80,80,80] capacity=[100,100,100] secondary-load=1 load-time=0s @@ -86,7 +86,7 @@ set-store store-id=1 node-id=1 attrs=purple locality-tiers=region=us-west-1,zone=us-west-1a store-id=2 node-id=2 attrs=yellow locality-tiers=region=us-east-1,zone=us-east-1a ---- -node-id=1 failure-summary=ok locality-tiers=region=us-west-1,zone=us-west-1a,node=1 - store-id=1 membership=full attrs=purple locality-code=1:2:3: -node-id=2 failure-summary=ok locality-tiers=region=us-east-1,zone=us-east-1a,node=2 - store-id=2 membership=full attrs=yellow locality-code=4:5:6: +node-id=1 locality-tiers=region=us-west-1,zone=us-west-1a,node=1 + store-id=1 attrs=purple locality-code=1:2:3: +node-id=2 locality-tiers=region=us-east-1,zone=us-east-1a,node=2 + store-id=2 attrs=yellow locality-code=4:5:6: From d6a097c29c51f122e53b614a4a9684c5309afd5c Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 11:06:17 +0100 Subject: [PATCH 3/9] mmaprototype: remove storeMembership We'll replace it with a store-level construct in follow-up commits. --- .../allocator/mmaprototype/cluster_state.go | 36 ----------------- .../mmaprototype/cluster_state_test.go | 39 +++---------------- 2 files changed, 5 insertions(+), 70 deletions(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go index d78e9a8b5566..78bbd0ed74ef 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state.go @@ -597,37 +597,9 @@ type pendingReplicaChange struct { enactedAtTime time.Time } -// TODO(kvoli): This will eventually be used to represent the state of a node's -// membership. This corresponds to non-decommissioning, decommissioning and -// decommissioned. Fill in commentary and use. -type storeMembership int8 - -const ( - storeMembershipMember storeMembership = iota - storeMembershipRemoving - storeMembershipRemoved -) - -func (s storeMembership) String() string { - return redact.StringWithoutMarkers(s) -} - -// SafeFormat implements the redact.SafeFormatter interface. -func (s storeMembership) SafeFormat(w redact.SafePrinter, _ rune) { - switch s { - case storeMembershipMember: - w.Print("full") - case storeMembershipRemoving: - w.Print("removing") - case storeMembershipRemoved: - w.Print("removed") - } -} - // storeState maintains the complete state about a store as known to the // allocator. type storeState struct { - storeMembership storeLoad StoreAttributesAndLocality adjusted struct { @@ -1860,14 +1832,6 @@ func (cs *clusterState) setStore(sal StoreAttributesAndLocality) { } } -func (cs *clusterState) setStoreMembership(storeID roachpb.StoreID, state storeMembership) { - if ss, ok := cs.stores[storeID]; ok { - ss.storeMembership = state - } else { - panic(fmt.Sprintf("store %d not found in cluster state", storeID)) - } -} - //====================================================================== // clusterState accessors: // diff --git a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go index b048593c437d..27a398b5bdbf 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go @@ -207,7 +207,7 @@ func printPendingChangesTest(changes []*pendingReplicaChange) string { return buf.String() } -func testingGetStoreList(t *testing.T, cs *clusterState) (member, removed storeIDPostingList) { +func testingGetStoreList(t *testing.T, cs *clusterState) storeIDPostingList { var clusterStoreList, nodeStoreList storeIDPostingList // Ensure that the storeIDs in the cluster store map and the stores listed // under each node are the same. @@ -222,15 +222,7 @@ func testingGetStoreList(t *testing.T, cs *clusterState) (member, removed storeI require.True(t, clusterStoreList.isEqual(nodeStoreList), "expected store lists to be equal %v != %v", clusterStoreList, nodeStoreList) - for storeID, ss := range cs.stores { - switch ss.storeMembership { - case storeMembershipMember, storeMembershipRemoving: - member.insert(storeID) - case storeMembershipRemoved: - removed.insert(storeID) - } - } - return member, removed + return clusterStoreList } func testingGetPendingChanges(t *testing.T, cs *clusterState) []*pendingReplicaChange { @@ -305,8 +297,8 @@ func TestClusterState(t *testing.T) { ns.NodeID, cs.stores[ns.stores[0]].StoreAttributesAndLocality.locality()) for _, storeID := range ns.stores { ss := cs.stores[storeID] - fmt.Fprintf(&buf, " store-id=%v membership=%v attrs=%s locality-code=%s\n", - ss.StoreID, ss.storeMembership, ss.StoreAttrs, ss.localityTiers.str) + fmt.Fprintf(&buf, " store-id=%v attrs=%s locality-code=%s\n", + ss.StoreID, ss.StoreAttrs, ss.localityTiers.str) } } return buf.String() @@ -336,7 +328,7 @@ func TestClusterState(t *testing.T) { case "get-load-info": var buf strings.Builder - memberStores, _ := testingGetStoreList(t, cs) + memberStores := testingGetStoreList(t, cs) for _, storeID := range memberStores { ss := cs.stores[storeID] ns := cs.nodes[ss.NodeID] @@ -365,27 +357,6 @@ func TestClusterState(t *testing.T) { } return printNodeListMeta() - case "set-store-membership": - storeID := dd.ScanArg[roachpb.StoreID](t, d, "store-id") - var storeMembershipVal storeMembership - switch str := dd.ScanArg[string](t, d, "membership"); str { - case "member": - storeMembershipVal = storeMembershipMember - case "removing": - storeMembershipVal = storeMembershipRemoving - case "removed": - storeMembershipVal = storeMembershipRemoved - } - cs.setStoreMembership(storeID, storeMembershipVal) - - var buf strings.Builder - nonRemovedStores, removedStores := testingGetStoreList(t, cs) - buf.WriteString("member store-ids: ") - printPostingList(&buf, nonRemovedStores) - buf.WriteString("\nremoved store-ids: ") - printPostingList(&buf, removedStores) - return buf.String() - case "store-load-msg": msg := parseStoreLoadMsg(t, d.Input) cs.processStoreLoadMsg(context.Background(), &msg) From 5e1f3f157d9b792ccbe5df14250ac794f17e9012 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 11:26:25 +0100 Subject: [PATCH 4/9] mmaprototype: move and rename storeIDPostingList -> storeSet --- .../allocator/mmaprototype/BUILD.bazel | 1 + .../allocator/mmaprototype/allocator_state.go | 8 +- .../mmaprototype/cluster_state_test.go | 4 +- .../allocator/mmaprototype/constraint.go | 156 +---------------- .../mmaprototype/constraint_matcher.go | 32 ++-- .../mmaprototype/constraint_matcher_test.go | 8 +- .../allocator/mmaprototype/constraint_test.go | 6 +- .../kvserver/allocator/mmaprototype/load.go | 2 +- .../allocator/mmaprototype/memo_helper.go | 2 +- .../allocator/mmaprototype/store_set.go | 157 ++++++++++++++++++ 10 files changed, 193 insertions(+), 183 deletions(-) create mode 100644 pkg/kv/kvserver/allocator/mmaprototype/store_set.go diff --git a/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel b/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel index 9832aaf9ca8b..31d3d1d5c8b8 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel +++ b/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "memo_helper.go", "messages.go", "rebalance_advisor.go", + "store_set.go", "top_k_replicas.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/mmaprototype", diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go index 3ba26816c9da..57b3766719a5 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go @@ -353,8 +353,8 @@ func (a *allocatorState) rebalanceStores( var changes []PendingRangeChange var disj [1]constraintsConj - var storesToExclude storeIDPostingList - var storesToExcludeForRange storeIDPostingList + var storesToExclude storeSet + var storesToExcludeForRange storeSet scratchNodes := map[roachpb.NodeID]*NodeLoad{} scratchStores := map[roachpb.StoreID]struct{}{} // The caller has a fixed concurrency limit it can move ranges at, when it @@ -461,7 +461,7 @@ func (a *allocatorState) rebalanceStores( store.StoreID, rangeID)) } cands, _ := rstate.constraints.candidatesToMoveLease() - var candsPL storeIDPostingList + var candsPL storeSet for _, cand := range cands { candsPL.insert(cand.storeID) } @@ -1347,7 +1347,7 @@ func (a *allocatorState) ensureAnalyzedConstraints(rstate *rangeState) bool { func (a *allocatorState) computeCandidatesForRange( ctx context.Context, expr constraintsDisj, - storesToExclude storeIDPostingList, + storesToExclude storeSet, loadSheddingStore roachpb.StoreID, ) (_ candidateSet, sheddingSLS storeLoadSummary) { means := a.cs.meansMemo.getMeans(expr) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go index 27a398b5bdbf..fb4b622444b0 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/cluster_state_test.go @@ -207,8 +207,8 @@ func printPendingChangesTest(changes []*pendingReplicaChange) string { return buf.String() } -func testingGetStoreList(t *testing.T, cs *clusterState) storeIDPostingList { - var clusterStoreList, nodeStoreList storeIDPostingList +func testingGetStoreList(t *testing.T, cs *clusterState) storeSet { + var clusterStoreList, nodeStoreList storeSet // Ensure that the storeIDs in the cluster store map and the stores listed // under each node are the same. for storeID := range cs.stores { diff --git a/pkg/kv/kvserver/allocator/mmaprototype/constraint.go b/pkg/kv/kvserver/allocator/mmaprototype/constraint.go index c9acc0d49f56..5ffc9e628af4 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/constraint.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/constraint.go @@ -24,7 +24,7 @@ import ( // and constraints conjunctions. The primary ones are normalizedSpanConfig and // rangeAnalyzedConstraints. // -// Other misc pieces: storeIDPostingList represents a set of stores, and is +// Other misc pieces: storeSet represents a set of stores, and is // used here and will be used elsewhere for set operations. // localityTierInterner is used for interning the tiers to avoid string // comparisons, used for diversity computation. It will also be used @@ -1408,8 +1408,8 @@ func (rac *rangeAnalyzedConstraints) candidatesToConvertFromVoterToNonVoter() ( if constraintSetNeeded && !voterConstraintSetNeeded { return constraintSet, nil } - cset := makeStoreIDPostingList(constraintSet) - cset.intersect(makeStoreIDPostingList(voterConstraintSet)) + cset := makeStoreSet(constraintSet) + cset.intersect(makeStoreSet(voterConstraintSet)) return cset, nil } @@ -1945,152 +1945,6 @@ func (l localityTiers) diversityScore(other localityTiers) float64 { return 0 } -// Ordered and de-duped list of storeIDs. Represents a set of stores. Used for -// fast set operations for constraint satisfaction. -type storeIDPostingList []roachpb.StoreID - -func makeStoreIDPostingList(a []roachpb.StoreID) storeIDPostingList { - slices.Sort(a) - return a -} - -func (s *storeIDPostingList) union(b storeIDPostingList) { - a := *s - n := len(a) - m := len(b) - for i, j := 0, 0; j < m; { - if i < n && a[i] < b[j] { - i++ - continue - } - // i >= n || a[i] >= b[j] - if i >= n || a[i] > b[j] { - a = append(a, b[j]) - j++ - continue - } - // a[i] == b[j] - i++ - j++ - } - if len(a) > n { - slices.Sort(a) - *s = a - } -} - -func (s *storeIDPostingList) intersect(b storeIDPostingList) { - // TODO(sumeer): For larger lists, probe using smaller list. - a := *s - n := len(a) - m := len(b) - k := 0 - for i, j := 0, 0; i < n && j < m; { - if a[i] < b[j] { - i++ - } else if a[i] > b[j] { - j++ - } else { - a[k] = a[i] - i++ - j++ - k++ - } - } - *s = a[:k] -} - -func (s *storeIDPostingList) isEqual(b storeIDPostingList) bool { - a := *s - n := len(a) - m := len(b) - if n != m { - return false - } - for i := range b { - if a[i] != b[i] { - return false - } - } - return true -} - -// Returns true iff found (and successfully removed). -func (s *storeIDPostingList) remove(storeID roachpb.StoreID) bool { - a := *s - n := len(a) - found := false - for i := range a { - if a[i] == storeID { - // INVARIANT: i < n, so i <= n-1 and i+1 <= n. - copy(a[i:n-1], a[i+1:n]) - found = true - break - } - } - if !found { - return false - } - *s = a[:n-1] - return true -} - -// Returns true iff the storeID was not already in the set. -func (s *storeIDPostingList) insert(storeID roachpb.StoreID) bool { - a := *s - n := len(a) - var pos int - for pos = 0; pos < n; pos++ { - if storeID < a[pos] { - break - } else if storeID == a[pos] { - return false - } - } - var b storeIDPostingList - if cap(a) > n { - b = a[:n+1] - } else { - m := 2 * cap(a) - const minLength = 10 - if m < minLength { - m = minLength - } - b = make([]roachpb.StoreID, n+1, m) - // Insert at pos, so pos-1 is the last element before the insertion. - if pos > 0 { - copy(b[:pos], a[:pos]) - } - } - copy(b[pos+1:n+1], a[pos:n]) - b[pos] = storeID - *s = b - return true -} - -func (s *storeIDPostingList) contains(storeID roachpb.StoreID) bool { - _, found := slices.BinarySearch(*s, storeID) - return found -} - -const ( - // offset64 is the initial hash value, and is taken from fnv.go - offset64 = 14695981039346656037 - - // prime64 is a large-ish prime number used in hashing and taken from fnv.go. - prime64 = 1099511628211 -) - -// FNV-1a hash algorithm. -func (s *storeIDPostingList) hash() uint64 { - h := uint64(offset64) - for _, storeID := range *s { - h ^= uint64(storeID) - h *= prime64 - } - return h -} - const notMatchedLeasePreferencIndex = math.MaxInt32 // matchedLeasePreferenceIndex returns the index of the lease preference that @@ -2124,7 +1978,7 @@ var _ = analyzeConstraintsBuf{} var _ = storeAndLocality{} var _ = localityTierInterner{} var _ = localityTiers{} -var _ = storeIDPostingList{} +var _ = storeSet{} var _ = constraintsDisj{}.hash var _ = constraintsDisj{}.isEqual @@ -2160,7 +2014,7 @@ func init() { var lt localityTiers var _ = lt.diversityScore - var pl storeIDPostingList + var pl storeSet var _ = pl.union var _ = pl.intersect var _ = pl.isEqual diff --git a/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher.go b/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher.go index 83b512cb39b9..4c157193e6d1 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher.go @@ -43,7 +43,7 @@ type matchedConstraints struct { } type matchedSet struct { - storeIDPostingList + storeSet } func newConstraintMatcher(interner *stringInterner) *constraintMatcher { @@ -166,21 +166,21 @@ func (cm *constraintMatcher) getMatchedSetForConstraint(c internedConstraint) *m // constrainStoresForConjunction populates storeSet with the stores matching // the given conjunction of constraints. // -// TODO(sumeer): make storeIDPostingList a struct and use a sync.Pool. +// TODO(sumeer): make storeSet a struct and use a sync.Pool. func (cm *constraintMatcher) constrainStoresForConjunction( - constraints []internedConstraint, storeSet *storeIDPostingList, + constraints []internedConstraint, storeSet *storeSet, ) { *storeSet = (*storeSet)[:0] if len(constraints) == 0 { - *storeSet = append(*storeSet, cm.allStores.storeIDPostingList...) + *storeSet = append(*storeSet, cm.allStores.storeSet...) return } for i := range constraints { matchedSet := cm.getMatchedSetForConstraint(constraints[i]) if i == 0 { - *storeSet = append(*storeSet, matchedSet.storeIDPostingList...) + *storeSet = append(*storeSet, matchedSet.storeSet...) } else { - storeSet.intersect(matchedSet.storeIDPostingList) + storeSet.intersect(matchedSet.storeSet) } if len(*storeSet) == 0 { return @@ -212,26 +212,24 @@ func (cm *constraintMatcher) storeMatches( // constrainStoresForExpr populates storeSet with the stores matching the // given expression. -func (cm *constraintMatcher) constrainStoresForExpr( - expr constraintsDisj, storeSet *storeIDPostingList, -) { +func (cm *constraintMatcher) constrainStoresForExpr(expr constraintsDisj, set *storeSet) { if len(expr) == 0 { - *storeSet = append(*storeSet, cm.allStores.storeIDPostingList...) + *set = append(*set, cm.allStores.storeSet...) return } - // Optimize for a single conjunction, by using storeSet directly in the call + // Optimize for a single conjunction, by using set directly in the call // to constrainStoresForConjunction. - var scratch storeIDPostingList - scratchPtr := storeSet + var scratch storeSet + scratchPtr := set for i := range expr { cm.constrainStoresForConjunction(expr[i], scratchPtr) if len(*scratchPtr) == 0 { continue } - if scratchPtr != storeSet { - storeSet.union(*scratchPtr) + if scratchPtr != set { + set.union(*scratchPtr) } else { - // The storeSet contains the first non-empty set. Collect the remaining + // The set contains the first non-empty set. Collect the remaining // sets in scratch. scratchPtr = &scratch } @@ -251,7 +249,7 @@ func (cm *constraintMatcher) checkConsistency() error { } } for c, pl := range cm.constraints { - for _, storeID := range pl.storeIDPostingList { + for _, storeID := range pl.storeSet { store, ok := cm.stores[storeID] if !ok { return errors.AssertionFailedf("constraint set mentions unknown storeID %d", storeID) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher_test.go b/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher_test.go index 3798c53a6723..79352082b6b8 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher_test.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/constraint_matcher_test.go @@ -77,15 +77,15 @@ func TestConstraintMatcher(t *testing.T) { Value: interner.toString(c.value), } sepStr := "" - if len(pl.storeIDPostingList) > 0 { + if len(pl.storeSet) > 0 { sepStr = " " } fmt.Fprintf(b, "%s:%s", rc.String(), sepStr) - printPostingList(b, pl.storeIDPostingList) + printPostingList(b, pl.storeSet) fmt.Fprintf(b, "\n") } fmt.Fprintf(b, "all-stores: ") - printPostingList(b, cm.allStores.storeIDPostingList) + printPostingList(b, cm.allStores.storeSet) fmt.Fprintf(b, "\n") err := cm.checkConsistency() require.NoError(t, err) @@ -129,7 +129,7 @@ func TestConstraintMatcher(t *testing.T) { disj = append(disj, interner.internConstraintsConj(cc)) } } - var pl storeIDPostingList + var pl storeSet if len(disj) <= 1 { if randutil.FastUint32()%2 == 0 { var conj []internedConstraint diff --git a/pkg/kv/kvserver/allocator/mmaprototype/constraint_test.go b/pkg/kv/kvserver/allocator/mmaprototype/constraint_test.go index 6ed5083af78b..a418dc00242a 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/constraint_test.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/constraint_test.go @@ -150,7 +150,7 @@ func TestNormalizedSpanConfig(t *testing.T) { }) } -func printPostingList(b *strings.Builder, pl storeIDPostingList) { +func printPostingList(b *strings.Builder, pl storeSet) { for i := range pl { prefix := "" if i > 0 { @@ -161,7 +161,7 @@ func printPostingList(b *strings.Builder, pl storeIDPostingList) { } func TestStoreIDPostingList(t *testing.T) { - pls := map[string]storeIDPostingList{} + pls := map[string]storeSet{} forceAllocation := rand.Intn(2) == 1 datadriven.RunTest(t, "testdata/posting_list", @@ -178,7 +178,7 @@ func TestStoreIDPostingList(t *testing.T) { storeIDs = append(storeIDs, roachpb.StoreID(storeID)) } } - pl := makeStoreIDPostingList(storeIDs) + pl := makeStoreSet(storeIDs) if forceAllocation { pl = pl[:len(pl):len(pl)] } diff --git a/pkg/kv/kvserver/allocator/mmaprototype/load.go b/pkg/kv/kvserver/allocator/mmaprototype/load.go index 3700a86724b4..58f2ba9cbfaa 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/load.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/load.go @@ -228,7 +228,7 @@ func (sls storeLoadSummary) SafeFormat(w redact.SafePrinter, _ rune) { type meansForStoreSet struct { constraintsDisj meansLoad - stores storeIDPostingList + stores storeSet storeSummaries map[roachpb.StoreID]storeLoadSummary } diff --git a/pkg/kv/kvserver/allocator/mmaprototype/memo_helper.go b/pkg/kv/kvserver/allocator/mmaprototype/memo_helper.go index 2e2df169f76a..0dc15283bc8e 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/memo_helper.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/memo_helper.go @@ -7,7 +7,7 @@ package mmaprototype // clearableMemoMap is a generic map suited to the needs of various memos used // in the allocator. The key is any type that implements the mapKey interface -// (e.g. storeIDPostingList). There is no removal from the map since the memo +// (e.g. storeSet). There is no removal from the map since the memo // is built up during an allocator round and then cleared for the next round. // The key-value pairs are stored in a mapEntry, which the caller is // responsible for initializing. The mapEntrySlice and its mapEntries are diff --git a/pkg/kv/kvserver/allocator/mmaprototype/store_set.go b/pkg/kv/kvserver/allocator/mmaprototype/store_set.go new file mode 100644 index 000000000000..44f4631de1f0 --- /dev/null +++ b/pkg/kv/kvserver/allocator/mmaprototype/store_set.go @@ -0,0 +1,157 @@ +// Copyright 2025 The Cockroach Authors. +// +// Use of this software is governed by the CockroachDB Software License +// included in the /LICENSE file. + +package mmaprototype + +import ( + "slices" + + "github.com/cockroachdb/cockroach/pkg/roachpb" +) + +// Ordered and de-duped list of storeIDs. Represents a set of stores. Used for +// fast set operations for constraint satisfaction. +type storeSet []roachpb.StoreID + +func makeStoreSet(a []roachpb.StoreID) storeSet { + slices.Sort(a) + return a +} + +func (s *storeSet) union(b storeSet) { + a := *s + n := len(a) + m := len(b) + for i, j := 0, 0; j < m; { + if i < n && a[i] < b[j] { + i++ + continue + } + // i >= n || a[i] >= b[j] + if i >= n || a[i] > b[j] { + a = append(a, b[j]) + j++ + continue + } + // a[i] == b[j] + i++ + j++ + } + if len(a) > n { + slices.Sort(a) + *s = a + } +} + +func (s *storeSet) intersect(b storeSet) { + // TODO(sumeer): For larger lists, probe using smaller list. + a := *s + n := len(a) + m := len(b) + k := 0 + for i, j := 0, 0; i < n && j < m; { + if a[i] < b[j] { + i++ + } else if a[i] > b[j] { + j++ + } else { + a[k] = a[i] + i++ + j++ + k++ + } + } + *s = a[:k] +} + +func (s *storeSet) isEqual(b storeSet) bool { + a := *s + n := len(a) + m := len(b) + if n != m { + return false + } + for i := range b { + if a[i] != b[i] { + return false + } + } + return true +} + +// Returns true iff found (and successfully removed). +func (s *storeSet) remove(storeID roachpb.StoreID) bool { + a := *s + n := len(a) + found := false + for i := range a { + if a[i] == storeID { + // INVARIANT: i < n, so i <= n-1 and i+1 <= n. + copy(a[i:n-1], a[i+1:n]) + found = true + break + } + } + if !found { + return false + } + *s = a[:n-1] + return true +} + +// Returns true iff the storeID was not already in the set. +func (s *storeSet) insert(storeID roachpb.StoreID) bool { + a := *s + n := len(a) + var pos int + for pos = 0; pos < n; pos++ { + if storeID < a[pos] { + break + } else if storeID == a[pos] { + return false + } + } + var b storeSet + if cap(a) > n { + b = a[:n+1] + } else { + m := 2 * cap(a) + const minLength = 10 + if m < minLength { + m = minLength + } + b = make([]roachpb.StoreID, n+1, m) + // Insert at pos, so pos-1 is the last element before the insertion. + if pos > 0 { + copy(b[:pos], a[:pos]) + } + } + copy(b[pos+1:n+1], a[pos:n]) + b[pos] = storeID + *s = b + return true +} + +func (s *storeSet) contains(storeID roachpb.StoreID) bool { + _, found := slices.BinarySearch(*s, storeID) + return found +} + +const ( // offset64 is the initial hash value, and is taken from fnv.go + offset64 = 14695981039346656037 + + // prime64 is a large-ish prime number used in hashing and taken from fnv.go. + prime64 = 1099511628211 +) + +// FNV-1a hash algorithm. +func (s *storeSet) hash() uint64 { + h := uint64(offset64) + for _, storeID := range *s { + h ^= uint64(storeID) + h *= prime64 + } + return h +} From b9994d024987723d714edc38cb971fda45078893 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 11:27:52 +0100 Subject: [PATCH 5/9] mmaprototype: fix a typo --- pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go | 2 +- pkg/kv/kvserver/allocator/mmaprototype/constraint.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go index 57b3766719a5..6f3e4301fc78 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go @@ -1209,7 +1209,7 @@ func sortTargetCandidateSetAndPick( // ones that have notMatchedLeasePreferenceIndex. j = 0 for _, cand := range cands.candidates { - if cand.leasePreferenceIndex == notMatchedLeasePreferencIndex { + if cand.leasePreferenceIndex == notMatchedLeasePreferenceIndex { break } j++ diff --git a/pkg/kv/kvserver/allocator/mmaprototype/constraint.go b/pkg/kv/kvserver/allocator/mmaprototype/constraint.go index 5ffc9e628af4..9eda7d628db7 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/constraint.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/constraint.go @@ -1945,10 +1945,10 @@ func (l localityTiers) diversityScore(other localityTiers) float64 { return 0 } -const notMatchedLeasePreferencIndex = math.MaxInt32 +const notMatchedLeasePreferenceIndex = math.MaxInt32 // matchedLeasePreferenceIndex returns the index of the lease preference that -// matches, else notMatchedLeasePreferencIndex +// matches, else notMatchedLeasePreferenceIndex func matchedLeasePreferenceIndex( storeID roachpb.StoreID, leasePreferences []internedLeasePreference, @@ -1962,7 +1962,7 @@ func matchedLeasePreferenceIndex( return int32(j) } } - return notMatchedLeasePreferencIndex + return notMatchedLeasePreferenceIndex } // Avoid unused lint errors. From 69cd6e4d6a5c0dc0993c118dc1e675f7dd0f4f9a Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 11:29:24 +0100 Subject: [PATCH 6/9] mmaprototype: move storeLoadSummary to its own file --- .../allocator/mmaprototype/BUILD.bazel | 1 + .../kvserver/allocator/mmaprototype/load.go | 23 -------------- .../mmaprototype/store_load_summary.go | 31 +++++++++++++++++++ 3 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go diff --git a/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel b/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel index 31d3d1d5c8b8..46a7dacf123b 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel +++ b/pkg/kv/kvserver/allocator/mmaprototype/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "memo_helper.go", "messages.go", "rebalance_advisor.go", + "store_load_summary.go", "store_set.go", "top_k_replicas.go", ], diff --git a/pkg/kv/kvserver/allocator/mmaprototype/load.go b/pkg/kv/kvserver/allocator/mmaprototype/load.go index 58f2ba9cbfaa..35b8e1b4a564 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/load.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/load.go @@ -191,29 +191,6 @@ type meanNodeLoad struct { utilCPU float64 } -type storeLoadSummary struct { - worstDim LoadDimension // for logging only - sls loadSummary - nls loadSummary - dimSummary [NumLoadDimensions]loadSummary - highDiskSpaceUtilization bool - maxFractionPendingIncrease, maxFractionPendingDecrease float64 - - loadSeqNum uint64 -} - -func (sls storeLoadSummary) String() string { - return redact.StringWithoutMarkers(sls) -} - -func (sls storeLoadSummary) SafeFormat(w redact.SafePrinter, _ rune) { - w.Printf("(store=%v worst=%v cpu=%v writes=%v bytes=%v node=%v high_disk=%v frac_pending=%.2f,%.2f(%t))", - sls.sls, sls.worstDim, sls.dimSummary[CPURate], sls.dimSummary[WriteBandwidth], sls.dimSummary[ByteSize], - sls.nls, sls.highDiskSpaceUtilization, sls.maxFractionPendingIncrease, - sls.maxFractionPendingDecrease, - sls.maxFractionPendingIncrease < epsilon && sls.maxFractionPendingDecrease < epsilon) -} - // The allocator often needs mean load information for a set of stores. This // set is implied by a constraintsDisj. We also want to know the set of stores // that satisfy that contraintsDisj. meansForStoreSet encapsulates all of this diff --git a/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go new file mode 100644 index 000000000000..9b5eeaeb2ec4 --- /dev/null +++ b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go @@ -0,0 +1,31 @@ +// Copyright 2025 The Cockroach Authors. +// +// Use of this software is governed by the CockroachDB Software License +// included in the /LICENSE file. + +package mmaprototype + +import "github.com/cockroachdb/redact" + +type storeLoadSummary struct { + worstDim LoadDimension // for logging only + sls loadSummary + nls loadSummary + dimSummary [NumLoadDimensions]loadSummary + highDiskSpaceUtilization bool + maxFractionPendingIncrease, maxFractionPendingDecrease float64 + + loadSeqNum uint64 +} + +func (sls storeLoadSummary) String() string { + return redact.StringWithoutMarkers(sls) +} + +func (sls storeLoadSummary) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("(store=%v worst=%v cpu=%v writes=%v bytes=%v node=%v high_disk=%v frac_pending=%.2f,%.2f(%t))", + sls.sls, sls.worstDim, sls.dimSummary[CPURate], sls.dimSummary[WriteBandwidth], sls.dimSummary[ByteSize], + sls.nls, sls.highDiskSpaceUtilization, sls.maxFractionPendingIncrease, + sls.maxFractionPendingDecrease, + sls.maxFractionPendingIncrease < epsilon && sls.maxFractionPendingDecrease < epsilon) +} From c0ea6d4bf8e487a4aaebec07d77f2aa2b7615362 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Wed, 5 Nov 2025 13:28:36 +0100 Subject: [PATCH 7/9] mmaprototype: add comment to storeLoadSummary --- pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go index 9b5eeaeb2ec4..fe54469c6325 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go @@ -7,6 +7,10 @@ package mmaprototype import "github.com/cockroachdb/redact" +// A storeLoadSummary is a classification of a store's load relative to the mean +// load across a set of permissible stores (often, all that satisfy the +// constraints for a given range). Sources and targets are primarily picked +// based on the store- and node level load summaries contained in this struct. type storeLoadSummary struct { worstDim LoadDimension // for logging only sls loadSummary From 5e46b67e2dda8fbe45011856c1a3a56f6d4bdb47 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Thu, 6 Nov 2025 13:46:08 +0100 Subject: [PATCH 8/9] mmaprototype: remove RemoveNodeAndStores It is unused and unclear what it would be used for. Easy to re-add should a need emerge. --- pkg/kv/kvserver/allocator/mmaprototype/allocator.go | 4 ---- pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go | 7 ------- 2 files changed, 11 deletions(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator.go index 02c032b9d4b7..c862606ce733 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator.go @@ -39,10 +39,6 @@ type Allocator interface { // about the nodes in the cluster is a side effect of this method. SetStore(store StoreAttributesAndLocality) - // RemoveNodeAndStores tells the allocator to remove the NodeID and all its - // stores. - RemoveNodeAndStores(nodeID roachpb.NodeID) error - // ProcessStoreLoadMsg provides frequent the state of every store and its // associated node in the cluster. ProcessStoreLoadMsg(ctx context.Context, msg *StoreLoadMsg) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go index 6f3e4301fc78..7a1c28d8288f 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/allocator_state.go @@ -808,13 +808,6 @@ func (a *allocatorState) SetStore(store StoreAttributesAndLocality) { a.cs.setStore(store) } -// RemoveNodeAndStores implements the Allocator interface. -func (a *allocatorState) RemoveNodeAndStores(nodeID roachpb.NodeID) error { - a.mu.Lock() - defer a.mu.Unlock() - panic("unimplemented") -} - // ProcessStoreLeaseholderMsg implements the Allocator interface. func (a *allocatorState) ProcessStoreLoadMsg(ctx context.Context, msg *StoreLoadMsg) { a.mu.Lock() From 4822f99913b31c806b9049baea74e0998008e543 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Thu, 6 Nov 2025 13:47:00 +0100 Subject: [PATCH 9/9] mmaprototype: fix typo --- pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go index fe54469c6325..10b6c32c7487 100644 --- a/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go +++ b/pkg/kv/kvserver/allocator/mmaprototype/store_load_summary.go @@ -10,7 +10,7 @@ import "github.com/cockroachdb/redact" // A storeLoadSummary is a classification of a store's load relative to the mean // load across a set of permissible stores (often, all that satisfy the // constraints for a given range). Sources and targets are primarily picked -// based on the store- and node level load summaries contained in this struct. +// based on the store and node level load summaries contained in this struct. type storeLoadSummary struct { worstDim LoadDimension // for logging only sls loadSummary