allocatorimpl: consult with mma for lease count convergence

Previously, the replicate queue was updated to check with MMA before considering
a change as a candidate when LBRebalancingMultiMetricAndCount is set. This
commit applies the same logic to the lease queue, filtering out candidates
deemed undesirable.

Author: wenyihu6

pkg/kv/kvserver/allocator/allocatorimpl/allocator.go

@@ -2543,9 +2543,17 @@ func (a *Allocator) TransferLeaseTarget(
 			return validTargets[a.randGen.Intn(len(validTargets))]
 		}
 
+		targetStores := make([]roachpb.StoreID, 0, len(sl.Stores))
+		for _, s := range sl.Stores {
+			targetStores = append(targetStores, s.StoreID)
+		}
+		handle := a.as.BuildMMARebalanceAdvisor(source.StoreID, targetStores)
 		var bestOption roachpb.ReplicaDescriptor
 		candidates := make([]roachpb.ReplicaDescriptor, 0, len(validTargets))
 		bestOptionLeaseCount := int32(math.MaxInt32)
+		// Similar to replicate queue, lease queue only filters out overloaded
+		// stores at the final target selection step. See comments on top of
+		// allocatorSync.BuildMMARebalanceAdvisor for more details.
 		for _, repl := range validTargets {
 			if leaseRepl.StoreID() == repl.StoreID {
 				continue
@@ -2555,7 +2563,13 @@ func (a *Allocator) TransferLeaseTarget(
 				continue
 			}
 			if float64(storeDesc.Capacity.LeaseCount) < candidateLeasesMean-0.5 {
-				candidates = append(candidates, repl)
+				// Only include the candidate if it is not in conflict with mma's goals.
+				// Note that even if all candidates are excluded, the len(candidates) ==
+				// 0 branch below will still return the replica with the lowest lease
+				// count if we are required to shed the lease (excludeLeaseRepl==true).
+				if !a.as.IsInConflictWithMMA(ctx, repl.StoreID, handle, true /*cpuOnly*/) {
+					candidates = append(candidates, repl)
+				}
 			}
 			if storeDesc.Capacity.LeaseCount < bestOptionLeaseCount {
 				bestOption = repl

pkg/kv/kvserver/allocator/allocatorimpl/allocator_test.go

@@ -2205,6 +2205,183 @@ func TestAllocatorTransferLeaseTargetIOOverloadCheck(t *testing.T) {
 
 }
 
+// TestAllocatorTransferLeaseTargetMMAConflict tests that the MMA conflict
+// checking logic in TransferLeaseTarget works correctly. It should check with
+// mma but still allow transfer if the transfer is needed to shed due to IO
+// overload or lease preference violation.
+func TestAllocatorTransferLeaseTargetMMAConflict(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+	ctx := context.Background()
+
+	existing := replicas(1, 2, 3)
+
+	leasePreferences := func() []roachpb.LeasePreference {
+		return []roachpb.LeasePreference{
+			{Constraints: []roachpb.Constraint{{Key: "region", Value: "us-east", Type: roachpb.Constraint_REQUIRED}}},
+		}
+	}
+
+	// Helper function to create default store descriptors
+	createDefaultStores := func() []*roachpb.StoreDescriptor {
+		var stores []*roachpb.StoreDescriptor
+		for i := 1; i <= 3; i++ {
+			region := "us-east"
+			if i == 1 {
+				region = "us-west" // Store 1 violates us-east preference
+			}
+
+			leaseCount := int32(50)
+			if i == 1 {
+				leaseCount = 100 // Store 1 has more leases for convergence testing
+			}
+
+			stores = append(stores, &roachpb.StoreDescriptor{
+				StoreID: roachpb.StoreID(i),
+				Node: roachpb.NodeDescriptor{
+					NodeID: roachpb.NodeID(i),
+					Locality: roachpb.Locality{
+						Tiers: []roachpb.Tier{
+							{Key: "region", Value: region},
+						},
+					},
+				},
+				Capacity: roachpb.StoreCapacity{
+					LeaseCount:     leaseCount,
+					IOThresholdMax: TestingIOThresholdWithScore(0.1), // Default low IO score
+				},
+			})
+		}
+		return stores
+	}
+
+	testCases := []struct {
+		name               string
+		leaseholder        roachpb.StoreID
+		mmaReturnsConflict bool
+		expected           roachpb.StoreID
+		conf               *roachpb.SpanConfig
+		enforcement        IOOverloadEnforcementLevel
+		setupStores        func() []*roachpb.StoreDescriptor // Function to create stores for this test case
+	}{
+		{
+			name:               "normal lease count convergence respects MMA conflict",
+			leaseholder:        1,
+			mmaReturnsConflict: true,
+			expected:           0, // Should be blocked by MMA conflict
+			conf:               emptySpanConfig(),
+			setupStores:        createDefaultStores,
+		},
+		{
+			name:               "normal lease count convergence proceeds when MMA allows",
+			leaseholder:        1,
+			mmaReturnsConflict: false,
+			// 2, 3 are both valid targets, 2 is picked since this is a random
+			// deterministic choice.
+			expected:    2,
+			conf:        emptySpanConfig(),
+			setupStores: createDefaultStores,
+		},
+		{
+			name:               "lease preferences bypass MMA conflict",
+			leaseholder:        1, // Store 1 violates preference (us-west)
+			mmaReturnsConflict: true,
+			// Pick store 2 regardless of MMA conflict.
+			expected:    2,
+			conf:        &roachpb.SpanConfig{LeasePreferences: leasePreferences()},
+			setupStores: createDefaultStores,
+		},
+		{
+			name:               "lease preferences respect MMA conflict when no violation",
+			leaseholder:        2, // Store 2 satisfies preference (us-east)
+			mmaReturnsConflict: true,
+			// Should be blocked by MMA conflict since no preference violation.
+			expected:    0,
+			conf:        &roachpb.SpanConfig{LeasePreferences: leasePreferences()},
+			setupStores: createDefaultStores,
+		},
+		{
+			name:               "lease preferences work when MMA allows",
+			leaseholder:        1, // Store 1 violates preference (us-west)
+			mmaReturnsConflict: false,
+			// Should move to store 2 (first preferred store).
+			expected:    2,
+			conf:        &roachpb.SpanConfig{LeasePreferences: leasePreferences()},
+			setupStores: createDefaultStores,
+		},
+		{
+			name:               "IO overload bypasses MMA conflict when needs to be shed",
+			leaseholder:        1,
+			mmaReturnsConflict: true,
+			// Pick store 2 even though it's in conflict with MMA.
+			expected:    2,
+			conf:        emptySpanConfig(),
+			enforcement: IOOverloadThresholdShed,
+			setupStores: func() []*roachpb.StoreDescriptor {
+				stores := createDefaultStores()
+				stores[0].Capacity.IOThresholdMax = TestingIOThresholdWithScore(0.5) // Store 1 is IO overloaded
+				return stores
+			},
+		},
+		{
+			name:               "transfer allowed by IO overload and mma",
+			leaseholder:        1,
+			mmaReturnsConflict: false,
+			// Should move to store 3. 2 is blocked by
+			// IOOverloadThresholdBlockTransfers. 3 is allowed by check and mma.
+			expected:    3,
+			conf:        emptySpanConfig(),
+			enforcement: IOOverloadThresholdBlockTransfers,
+			setupStores: func() []*roachpb.StoreDescriptor {
+				stores := createDefaultStores()
+				stores[1].Capacity.IOThresholdMax = TestingIOThresholdWithScore(0.5) // Store 2 is IO overloaded
+				return stores
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Create allocator with custom MMA knobs for this test case
+			stopper, g, sp, a, _ := CreateTestAllocatorWithKnobs(ctx, 10, true /* deterministic */, nil, &mmaintegration.TestingKnobs{
+				OverrideIsInConflictWithMMA: func(cand roachpb.StoreID) bool {
+					return tc.mmaReturnsConflict
+				},
+			})
+			defer stopper.Stop(ctx)
+
+			// Set up stores using the test case's setup function
+			stores := tc.setupStores()
+			sg := gossiputil.NewStoreGossiper(g)
+			sg.GossipStores(stores, t)
+
+			// Set up IO overload enforcement if specified
+			if tc.enforcement != 0 {
+				LeaseIOOverloadThresholdEnforcement.Override(ctx, &a.st.SV, tc.enforcement)
+			}
+
+			EnableLoadBasedLeaseRebalancing.Override(ctx, &a.st.SV, false)
+
+			target := a.TransferLeaseTarget(
+				ctx,
+				sp,
+				&roachpb.RangeDescriptor{},
+				tc.conf,
+				existing,
+				&mockRepl{
+					replicationFactor: 3,
+					storeID:           tc.leaseholder,
+				},
+				allocator.RangeUsageInfo{}, /* stats */
+				allocator.TransferLeaseOptions{
+					CheckCandidateFullness: true,
+				},
+			)
+			require.Equal(t, tc.expected, target.StoreID)
+		})
+	}
+}
+
 func TestAllocatorTransferLeaseToReplicasNeedingSnapshot(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_able_to_shed_leases.txt

@@ -41,17 +41,17 @@ write_bytes_per_second#1: last:  [s1=25989, s2=25993, s3=29967, s4=31991, s5=359
 write_bytes_per_second#1: thrash_pct: [s1=161%, s2=82%, s3=144%, s4=176%, s5=158%]  (sum=722%)
 artifacts[mma-only]: 9a1122b07e8a1e71
 ==========================
-cpu#1: last:  [s1=3199860194, s2=2799497761, s3=3199674637, s4=2800510026, s5=2999404241] (stddev=178780081.94, mean=2999789371.80, sum=14998946859)
-cpu#1: thrash_pct: [s1=124%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=422%)
+cpu#1: last:  [s1=3200015723, s2=2799409624, s3=3199923468, s4=2800455801, s5=2999667444] (stddev=178902277.04, mean=2999894412.00, sum=14999472060)
+cpu#1: thrash_pct: [s1=109%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=408%)
 cpu_util#1: last:  [s1=0.36, s2=0.31, s3=0.36, s4=0.31, s5=0.33] (stddev=0.02, mean=0.33, sum=2)
-cpu_util#1: thrash_pct: [s1=124%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=422%)
+cpu_util#1: thrash_pct: [s1=109%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=408%)
 leases#1: first: [s1=25, s2=0, s3=0, s4=0, s5=0] (stddev=10.00, mean=5.00, sum=25)
-leases#1: last:  [s1=6, s2=2, s3=5, s4=5, s5=7] (stddev=1.67, mean=5.00, sum=25)
+leases#1: last:  [s1=7, s2=3, s3=3, s4=5, s5=7] (stddev=1.79, mean=5.00, sum=25)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 replicas#1: first: [s1=25, s2=13, s3=13, s4=12, s5=12] (stddev=5.02, mean=15.00, sum=75)
 replicas#1: last:  [s1=16, s2=14, s3=16, s4=14, s5=15] (stddev=0.89, mean=15.00, sum=75)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
-write_bytes_per_second#1: last:  [s1=31998, s2=27994, s3=31996, s4=28005, s5=29994] (stddev=1787.74, mean=29997.40, sum=149987)
-write_bytes_per_second#1: thrash_pct: [s1=124%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=422%)
-artifacts[mma-count]: e81950c3e42f43d
+write_bytes_per_second#1: last:  [s1=32000, s2=27994, s3=31999, s4=28004, s5=29996] (stddev=1789.08, mean=29998.60, sum=149993)
+write_bytes_per_second#1: thrash_pct: [s1=109%, s2=62%, s3=49%, s4=107%, s5=79%]  (sum=408%)
+artifacts[mma-count]: afcb108e54d06e3c
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_write_uniform_cpu.txt

@@ -47,20 +47,20 @@ write_bytes_per_second#1: last:  [s1=5987983, s2=5994822, s3=5992147, s4=6010824
 write_bytes_per_second#1: thrash_pct: [s1=95%, s2=73%, s3=98%, s4=123%, s5=83%, s6=70%, s7=56%, s8=49%, s9=97%, s10=77%]  (sum=821%)
 artifacts[mma-only]: a187f7fcdc7520a5
 ==========================
-cpu#1: last:  [s1=497905231, s2=499033106, s3=501825000, s4=500466725, s5=498817870, s6=497161276, s7=500212885, s8=499757059, s9=500662500, s10=500093532] (stddev=1312513.07, mean=499593518.40, sum=4995935184)
-cpu#1: thrash_pct: [s1=216%, s2=351%, s3=34%, s4=83%, s5=276%, s6=173%, s7=275%, s8=269%, s9=36%, s10=273%]  (sum=1985%)
-cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.17, s4=0.17, s5=0.17, s6=0.17, s7=0.17, s8=0.17, s9=0.17, s10=0.17] (stddev=0.00, mean=0.17, sum=2)
-cpu_util#1: thrash_pct: [s1=216%, s2=351%, s3=34%, s4=83%, s5=276%, s6=173%, s7=275%, s8=269%, s9=36%, s10=273%]  (sum=1985%)
+cpu#1: last:  [s1=507916789, s2=498501253, s3=500579751, s4=499842814, s5=501335541, s6=494153659, s7=496397433, s8=499418762, s9=499983556, s10=504531465] (stddev=3665905.75, mean=500266102.30, sum=5002661023)
+cpu#1: thrash_pct: [s1=492%, s2=358%, s3=429%, s4=435%, s5=479%, s6=229%, s7=232%, s8=428%, s9=435%, s10=417%]  (sum=3933%)
+cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.17, s4=0.17, s5=0.17, s6=0.16, s7=0.17, s8=0.17, s9=0.17, s10=0.17] (stddev=0.00, mean=0.17, sum=2)
+cpu_util#1: thrash_pct: [s1=492%, s2=358%, s3=429%, s4=435%, s5=479%, s6=229%, s7=232%, s8=428%, s9=435%, s10=417%]  (sum=3933%)
 disk_fraction_used#1: first: [s1=0.00, s2=0.00, s3=0.00, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00, s10=0.00] (stddev=0.00, mean=0.00, sum=0)
-disk_fraction_used#1: last:  [s1=0.03, s2=0.03, s3=0.03, s4=0.03, s5=0.03, s6=0.03, s7=0.03, s8=0.03, s9=0.03, s10=0.03] (stddev=0.00, mean=0.03, sum=0)
-disk_fraction_used#1: thrash_pct: [s1=35%, s2=131%, s3=61%, s4=65%, s5=50%, s6=69%, s7=59%, s8=132%, s9=40%, s10=64%]  (sum=707%)
+disk_fraction_used#1: last:  [s1=0.04, s2=0.04, s3=0.03, s4=0.03, s5=0.03, s6=0.02, s7=0.04, s8=0.03, s9=0.03, s10=0.04] (stddev=0.01, mean=0.03, sum=0)
+disk_fraction_used#1: thrash_pct: [s1=440%, s2=282%, s3=270%, s4=437%, s5=536%, s6=284%, s7=139%, s8=248%, s9=473%, s10=235%]  (sum=3344%)
 leases#1: first: [s1=19, s2=11, s3=6, s4=3, s5=3, s6=3, s7=4, s8=3, s9=4, s10=4] (stddev=4.92, mean=6.00, sum=60)
-leases#1: last:  [s1=9, s2=6, s3=5, s4=6, s5=4, s6=5, s7=6, s8=6, s9=5, s10=8] (stddev=1.41, mean=6.00, sum=60)
-leases#1: thrash_pct: [s1=57%, s2=130%, s3=64%, s4=53%, s5=76%, s6=77%, s7=115%, s8=116%, s9=51%, s10=154%]  (sum=894%)
+leases#1: last:  [s1=6, s2=8, s3=5, s4=7, s5=9, s6=6, s7=7, s8=3, s9=6, s10=3] (stddev=1.84, mean=6.00, sum=60)
+leases#1: thrash_pct: [s1=245%, s2=181%, s3=201%, s4=304%, s5=284%, s6=114%, s7=148%, s8=189%, s9=302%, s10=201%]  (sum=2169%)
 replicas#1: first: [s1=39, s2=33, s3=23, s4=16, s5=13, s6=12, s7=11, s8=11, s9=11, s10=11] (stddev=9.76, mean=18.00, sum=180)
-replicas#1: last:  [s1=20, s2=18, s3=20, s4=18, s5=18, s6=18, s7=17, s8=17, s9=17, s10=17] (stddev=1.10, mean=18.00, sum=180)
-replicas#1: thrash_pct: [s1=87%, s2=274%, s3=116%, s4=159%, s5=96%, s6=111%, s7=190%, s8=254%, s9=82%, s10=182%]  (sum=1551%)
-write_bytes_per_second#1: last:  [s1=6011172, s2=5993736, s3=5993997, s4=5993701, s5=5994247, s6=6010778, s7=5991339, s8=6011998, s9=5993515, s10=5994635] (stddev=8166.94, mean=5998911.80, sum=59989118)
-write_bytes_per_second#1: thrash_pct: [s1=122%, s2=299%, s3=169%, s4=214%, s5=140%, s6=179%, s7=155%, s8=217%, s9=140%, s10=140%]  (sum=1775%)
-artifacts[mma-count]: d9bbbb822e478251
+replicas#1: last:  [s1=18, s2=19, s3=18, s4=18, s5=18, s6=14, s7=18, s8=18, s9=18, s10=21] (stddev=1.61, mean=18.00, sum=180)
+replicas#1: thrash_pct: [s1=621%, s2=423%, s3=396%, s4=730%, s5=682%, s6=259%, s7=283%, s8=426%, s9=769%, s10=321%]  (sum=4911%)
+write_bytes_per_second#1: last:  [s1=6665505, s2=6679332, s3=4655369, s4=5343865, s5=5999973, s6=3993692, s7=7993314, s8=5328834, s9=6010782, s10=7317305] (stddev=1154442.08, mean=5998797.10, sum=59987971)
+write_bytes_per_second#1: thrash_pct: [s1=592%, s2=450%, s3=421%, s4=587%, s5=529%, s6=409%, s7=333%, s8=413%, s9=607%, s10=371%]  (sum=4712%)
+artifacts[mma-count]: 5f5d90a8fe809362
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/skewed_cpu_even_ranges_mma.txt

@@ -72,17 +72,17 @@ write_bytes_per_second#1: last:  [s1=3751, s2=3916, s3=4019, s4=6903, s5=6911, s
 write_bytes_per_second#1: thrash_pct: [s1=565%, s2=570%, s3=599%, s4=134%, s5=162%, s6=134%, s7=162%, s8=159%, s9=173%]  (sum=2659%)
 artifacts[mma-only]: 8b414911c677f8f9
 ==========================
-cpu#1: last:  [s1=564321656, s2=570703549, s3=567256633, s4=561501935, s5=560527044, s6=555833144, s7=574286961, s8=558080746, s9=557632305] (stddev=5946449.12, mean=563349330.33, sum=5070143973)
-cpu#1: thrash_pct: [s1=67%, s2=123%, s3=155%, s4=40%, s5=102%, s6=49%, s7=72%, s8=26%, s9=56%]  (sum=689%)
-cpu_util#1: last:  [s1=0.11, s2=0.11, s3=0.11, s4=0.11, s5=0.11, s6=0.11, s7=0.11, s8=0.11, s9=0.11] (stddev=0.00, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=67%, s2=123%, s3=155%, s4=40%, s5=102%, s6=49%, s7=72%, s8=26%, s9=56%]  (sum=689%)
+cpu#1: last:  [s1=705611295, s2=851106563, s3=567039636, s4=429874660, s5=555084030, s6=565901683, s7=415672487, s8=555517775, s9=414198713] (stddev=135421522.68, mean=562222982.44, sum=5060006842)
+cpu#1: thrash_pct: [s1=127%, s2=175%, s3=165%, s4=68%, s5=99%, s6=64%, s7=76%, s8=48%, s9=78%]  (sum=900%)
+cpu_util#1: last:  [s1=0.14, s2=0.17, s3=0.11, s4=0.09, s5=0.11, s6=0.11, s7=0.08, s8=0.11, s9=0.08] (stddev=0.03, mean=0.11, sum=1)
+cpu_util#1: thrash_pct: [s1=127%, s2=175%, s3=165%, s4=68%, s5=99%, s6=64%, s7=76%, s8=48%, s9=78%]  (sum=900%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
-leases#1: last:  [s1=8, s2=7, s3=7, s4=16, s5=14, s6=11, s7=15, s8=13, s9=17] (stddev=3.68, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=72%, s2=94%, s3=123%, s4=94%, s5=184%, s6=108%, s7=106%, s8=74%, s9=127%]  (sum=981%)
+leases#1: last:  [s1=10, s2=8, s3=7, s4=13, s5=12, s6=14, s7=16, s8=13, s9=15] (stddev=2.91, mean=12.00, sum=108)
+leases#1: thrash_pct: [s1=113%, s2=129%, s3=129%, s4=106%, s5=148%, s6=97%, s7=111%, s8=68%, s9=109%]  (sum=1011%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
-replicas#1: last:  [s1=35, s2=36, s3=34, s4=38, s5=33, s6=36, s7=38, s8=38, s9=36] (stddev=1.70, mean=36.00, sum=324)
-replicas#1: thrash_pct: [s1=348%, s2=213%, s3=216%, s4=589%, s5=1204%, s6=840%, s7=643%, s8=643%, s9=987%]  (sum=5682%)
-write_bytes_per_second#1: last:  [s1=5514, s2=5506, s3=5089, s4=6396, s5=5508, s6=6117, s7=6365, s8=6580, s9=6210] (stddev=492.12, mean=5920.56, sum=53285)
-write_bytes_per_second#1: thrash_pct: [s1=1214%, s2=1091%, s3=1145%, s4=1119%, s5=1628%, s6=1261%, s7=1349%, s8=1334%, s9=1459%]  (sum=11600%)
-artifacts[mma-count]: 6790eb7f413fc1f4
+replicas#1: last:  [s1=37, s2=33, s3=33, s4=38, s5=36, s6=38, s7=38, s8=35, s9=36] (stddev=1.89, mean=36.00, sum=324)
+replicas#1: thrash_pct: [s1=295%, s2=124%, s3=257%, s4=483%, s5=813%, s6=549%, s7=536%, s8=361%, s9=520%]  (sum=3938%)
+write_bytes_per_second#1: last:  [s1=5845, s2=4977, s3=5056, s4=6368, s5=6060, s6=6557, s7=6546, s8=6035, s9=6137] (stddev=548.69, mean=5953.44, sum=53581)
+write_bytes_per_second#1: thrash_pct: [s1=1367%, s2=1468%, s3=1377%, s4=1191%, s5=1312%, s6=1216%, s7=1216%, s8=990%, s9=1263%]  (sum=11400%)
+artifacts[mma-count]: 521e92db29757c36
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/skewed_cpu_skewed_write.txt

@@ -57,19 +57,19 @@ write_bytes_per_second#1: last:  [s1=8367672, s2=10580757, s3=10581322, s4=10531
 write_bytes_per_second#1: thrash_pct: [s1=1%, s2=5%, s3=7%, s4=2%, s5=115%, s6=110%]  (sum=241%)
 artifacts[mma-only]: 31964600a8a5db0a
 ==========================
-cpu#1: last:  [s1=830551694, s2=843605039, s3=831306454, s4=831088101, s5=832966580, s6=830458451] (stddev=4668942.17, mean=833329386.50, sum=4999976319)
-cpu#1: thrash_pct: [s1=15%, s2=83%, s3=74%, s4=39%, s5=46%, s6=51%]  (sum=309%)
+cpu#1: last:  [s1=832135023, s2=829718568, s3=829738010, s4=845587865, s5=833948228, s6=829560377] (stddev=5658868.56, mean=833448011.83, sum=5000688071)
+cpu#1: thrash_pct: [s1=59%, s2=84%, s3=90%, s4=46%, s5=47%, s6=44%]  (sum=370%)
 cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.17, s4=0.17, s5=0.17, s6=0.17] (stddev=0.00, mean=0.17, sum=1)
-cpu_util#1: thrash_pct: [s1=15%, s2=83%, s3=74%, s4=39%, s5=46%, s6=51%]  (sum=309%)
+cpu_util#1: thrash_pct: [s1=59%, s2=84%, s3=90%, s4=46%, s5=47%, s6=44%]  (sum=370%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0] (stddev=16.97, mean=12.00, sum=72)
-leases#1: last:  [s1=11, s2=16, s3=11, s4=15, s5=11, s6=8] (stddev=2.71, mean=12.00, sum=72)
-leases#1: thrash_pct: [s1=53%, s2=75%, s3=79%, s4=76%, s5=79%, s6=89%]  (sum=451%)
+leases#1: last:  [s1=10, s2=13, s3=16, s4=13, s5=11, s6=9] (stddev=2.31, mean=12.00, sum=72)
+leases#1: thrash_pct: [s1=83%, s2=85%, s3=80%, s4=77%, s5=90%, s6=84%]  (sum=500%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36] (stddev=0.00, mean=36.00, sum=216)
-replicas#1: last:  [s1=38, s2=35, s3=35, s4=37, s5=35, s6=36] (stddev=1.15, mean=36.00, sum=216)
-replicas#1: thrash_pct: [s1=336%, s2=388%, s3=388%, s4=401%, s5=415%, s6=293%]  (sum=2221%)
-write_bytes_per_second#1: last:  [s1=10028292, s2=10525501, s3=9475047, s4=10022859, s5=9422060, s6=10524839] (stddev=440174.25, mean=9999766.33, sum=59998598)
-write_bytes_per_second#1: thrash_pct: [s1=62%, s2=88%, s3=98%, s4=192%, s5=204%, s6=186%]  (sum=830%)
-artifacts[mma-count]: b42b5562c69d2ae7
+replicas#1: last:  [s1=36, s2=35, s3=37, s4=34, s5=37, s6=37] (stddev=1.15, mean=36.00, sum=216)
+replicas#1: thrash_pct: [s1=815%, s2=525%, s3=571%, s4=680%, s5=632%, s6=402%]  (sum=3624%)
+write_bytes_per_second#1: last:  [s1=8867846, s2=10027943, s3=10524259, s4=10023228, s5=9973422, s6=10582360] (stddev=562404.06, mean=9999843.00, sum=59999058)
+write_bytes_per_second#1: thrash_pct: [s1=160%, s2=129%, s3=99%, s4=244%, s5=250%, s6=228%]  (sum=1111%)
+artifacts[mma-count]: d266bb6748d76c02
 ==========================
 Cluster Set Up
 	n1(AU_EAST,AU_EAST_1,5vcpu): {s1:(256GiB)}