Source file src/github.com/launchdarkly/go-server-sdk-evaluation/v2/evaluator_bucketing_test.go

Documentation: github.com/launchdarkly/go-server-sdk-evaluation/v2

     1  package evaluation
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"strconv"
     7  	"testing"
     8  
     9  	"github.com/launchdarkly/go-server-sdk-evaluation/v2/ldbuilders"
    10  	"github.com/launchdarkly/go-server-sdk-evaluation/v2/ldmodel"
    11  
    12  	"github.com/launchdarkly/go-sdk-common/v3/ldattr"
    13  	"github.com/launchdarkly/go-sdk-common/v3/ldcontext"
    14  	"github.com/launchdarkly/go-sdk-common/v3/ldvalue"
    15  
    16  	"github.com/stretchr/testify/assert"
    17  	"github.com/stretchr/testify/require"
    18  )
    19  
    20  // See evaluator_bucketing_testdata_test.go for the values used in parameterized tests here.
    21  
    22  var noSeed = ldvalue.OptionalInt{}
    23  
    24  func makeEvalScope(context ldcontext.Context, evalOptions ...EvaluatorOption) *evaluationScope {
    25  	evaluator := NewEvaluatorWithOptions(basicDataProvider(), evalOptions...).(*evaluator)
    26  	return &evaluationScope{context: context, owner: evaluator}
    27  }
    28  
    29  func makeUserContextWithSecondaryKey(t *testing.T, key, secondary string) ldcontext.Context {
    30  	// It is deliberately not possible to set a secondary key via the regular context builder API.
    31  	// The secondary attribute can only be present when a context is parsed from user JSON in the
    32  	// old schema.
    33  	userJSON := ldvalue.ObjectBuild().SetString("key", key).SetString("secondary", secondary).Build().JSONString()
    34  	var context ldcontext.Context
    35  	err := json.Unmarshal([]byte(userJSON), &context)
    36  	require.NoError(t, err)
    37  	return context
    38  }
    39  
    40  func findBucketValueInVariationList(bucketValue float32, buckets []ldmodel.WeightedVariation) int {
    41  	// This partially replicates logic in variationOrRolloutResult-- that's deliberate since we
    42  	// want to make sure that logic doesn't change unintentionally
    43  	bucketValueInt := int(bucketValue * 100000)
    44  	cumulativeWeight := 0
    45  	for i, bucket := range buckets {
    46  		cumulativeWeight += bucket.Weight
    47  		if bucketValueInt < cumulativeWeight {
    48  			return i
    49  		}
    50  	}
    51  	return len(buckets) - 1
    52  }
    53  
    54  func TestRolloutBucketing(t *testing.T) {
    55  	buckets := []ldmodel.WeightedVariation{
    56  		// The variation indices here are deliberately out of order so we can be sure it's really using that
    57  		// field, rather than assuming it is the same as the index of the WeightedVariation array element.
    58  		{Variation: 3, Weight: 20000}, // [0,20000)
    59  		{Variation: 2, Weight: 20000}, // [20000,40000)
    60  		{Variation: 1, Weight: 20000}, // [40000,60000)
    61  		{Variation: 0, Weight: 40000}, // [60000,100000]
    62  	}
    63  
    64  	for _, defaultContextKind := range []bool{true, false} {
    65  		// The purpose of running everything twice with defaultContext=true or false is to prove that
    66  		// comnputeBucketValue is always checking the desired context kind whenever it gets attributes
    67  		// from the context.
    68  
    69  		t.Run(fmt.Sprintf("defaultContextKind=%t", defaultContextKind), func(t *testing.T) {
    70  			baseRollout := ldmodel.Rollout{Variations: buckets}
    71  			contextKind := ldcontext.DefaultKind
    72  
    73  			checkResult := func(t *testing.T, p bucketingTestParams, context ldcontext.Context, rollout ldmodel.Rollout) {
    74  				// For each of these test cases, we're doing two tests. First, we test the lower-level method
    75  				// computeBucketValue, which tells the actual bucket value. Then, we test variationOrRolloutResult--
    76  				// which also calls computeBucketValue, but we are verifying that variationOrRolloutResult then
    77  				// applies the right logic to pick the result variation.
    78  
    79  				if !defaultContextKind {
    80  					context = ldcontext.NewMulti(
    81  						ldcontext.NewWithKind("irrelevantKind", "irrelevantKey"),
    82  						ldcontext.NewBuilderFromContext(context).Kind(contextKind).Build(),
    83  					)
    84  					rollout.ContextKind = contextKind
    85  				}
    86  
    87  				bucketValue, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed,
    88  					rollout.ContextKind, p.flagOrSegmentKey, rollout.BucketBy, p.salt)
    89  				assert.NoError(t, err)
    90  				assert.Equal(t, bucketingFailureReason(0), failReason)
    91  				assert.InEpsilon(t, p.expectedBucketValue, bucketValue, 0.0000001)
    92  
    93  				variationIndex, inExperiment, err := makeEvalScope(context).variationOrRolloutResult(
    94  					ldmodel.VariationOrRollout{Rollout: rollout},
    95  					p.flagOrSegmentKey,
    96  					p.salt,
    97  				)
    98  				assert.NoError(t, err)
    99  				expectedBucket := findBucketValueInVariationList(p.expectedBucketValue, buckets)
   100  				assert.Equal(t, buckets[expectedBucket].Variation, variationIndex)
   101  				assert.False(t, inExperiment)
   102  			}
   103  
   104  			t.Run("by key", func(t *testing.T) {
   105  				for _, p := range makeBucketingTestParams() {
   106  					t.Run(p.description(), func(t *testing.T) {
   107  						context := ldcontext.New(p.contextValue)
   108  						checkResult(t, p, context, baseRollout)
   109  					})
   110  				}
   111  			})
   112  
   113  			t.Run("by custom string attribute", func(t *testing.T) {
   114  				rollout := baseRollout
   115  				rollout.BucketBy = ldattr.NewLiteralRef("attr1")
   116  
   117  				for _, p := range makeBucketingTestParams() {
   118  					t.Run(p.description(), func(t *testing.T) {
   119  						context := ldcontext.NewBuilder(p.contextValue).SetString("attr1", p.contextValue).Build()
   120  						checkResult(t, p, context, rollout)
   121  					})
   122  				}
   123  			})
   124  
   125  			t.Run("by custom int attribute", func(t *testing.T) {
   126  				rollout := baseRollout
   127  				rollout.BucketBy = ldattr.NewLiteralRef("attr1")
   128  
   129  				for _, p := range makeBucketingTestParamsWithNumericStringValues() {
   130  					t.Run(p.description(), func(t *testing.T) {
   131  						n, err := strconv.Atoi(p.contextValue)
   132  						require.NoError(t, err)
   133  						context := ldcontext.NewBuilder(p.contextValue).SetInt("attr1", n).Build()
   134  						checkResult(t, p, context, rollout)
   135  					})
   136  				}
   137  			})
   138  
   139  			t.Run("secondary key changes result if secondary is explicitly enabled", func(t *testing.T) {
   140  				for _, p := range makeBucketingTestParams() {
   141  					t.Run(p.description(), func(t *testing.T) {
   142  						context1 := ldcontext.New(p.contextValue)
   143  						context2 := makeUserContextWithSecondaryKey(t, p.contextValue, "some-secondary-key")
   144  
   145  						evalScope1 := makeEvalScope(context1, EvaluatorOptionEnableSecondaryKey(true))
   146  						bucketValue1, failReason, err := evalScope1.computeBucketValue(false, noSeed,
   147  							"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   148  						assert.NoError(t, err)
   149  						assert.Equal(t, bucketingFailureReason(0), failReason)
   150  						assert.InEpsilon(t, p.expectedBucketValue, bucketValue1, 0.0000001)
   151  
   152  						evalScope2 := makeEvalScope(context2, EvaluatorOptionEnableSecondaryKey(true))
   153  						bucketValue2, failReason, err := evalScope2.computeBucketValue(false, noSeed,
   154  							"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   155  						assert.NoError(t, err)
   156  						assert.Equal(t, bucketingFailureReason(0), failReason)
   157  						assert.NotEqual(t, bucketValue1, bucketValue2)
   158  					})
   159  				}
   160  			})
   161  
   162  			t.Run("secondary key does not change result if secondary is not explicitly enabled", func(t *testing.T) {
   163  				for _, p := range makeBucketingTestParams() {
   164  					t.Run(p.description(), func(t *testing.T) {
   165  						context1 := ldcontext.New(p.contextValue)
   166  						context2 := makeUserContextWithSecondaryKey(t, p.contextValue, "some-secondary-key")
   167  
   168  						evalScope1 := makeEvalScope(context1)
   169  						bucketValue1, failReason, err := evalScope1.computeBucketValue(false, noSeed,
   170  							"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   171  						assert.NoError(t, err)
   172  						assert.Equal(t, bucketingFailureReason(0), failReason)
   173  						assert.InEpsilon(t, p.expectedBucketValue, bucketValue1, 0.0000001)
   174  
   175  						evalScope2 := makeEvalScope(context2)
   176  						bucketValue2, failReason, err := evalScope2.computeBucketValue(false, noSeed,
   177  							"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   178  						assert.NoError(t, err)
   179  						assert.Equal(t, bucketingFailureReason(0), failReason)
   180  						assert.Equal(t, bucketValue1, bucketValue2)
   181  					})
   182  				}
   183  			})
   184  		})
   185  	}
   186  }
   187  
   188  func TestExperimentBucketing(t *testing.T) {
   189  	// seed here carefully chosen so users fall into different buckets
   190  	buckets := []ldmodel.WeightedVariation{
   191  		ldbuilders.Bucket(1, 10000),
   192  		ldbuilders.Bucket(0, 20000),
   193  		ldbuilders.BucketUntracked(0, 70000),
   194  	}
   195  	baseExperiment := ldmodel.Rollout{Kind: ldmodel.RolloutKindExperiment, Variations: buckets}
   196  
   197  	// We won't check every permutation that was covered in TestRolloutBucketing - mostly areas where
   198  	// the behavior of experiments is expected to be different from the behavior of rollouts.
   199  
   200  	checkResult := func(t *testing.T, p bucketingTestParams, context ldcontext.Context, experiment ldmodel.Rollout) {
   201  		// Here we enable the secondary key behavior just to verify that it will still *not*
   202  		// be used in an experiment
   203  		evalScope := makeEvalScope(context, EvaluatorOptionEnableSecondaryKey(true))
   204  
   205  		bucketValue, failReason, err := evalScope.computeBucketValue(true, p.seed,
   206  			experiment.ContextKind, p.flagOrSegmentKey, experiment.BucketBy, p.salt)
   207  		assert.NoError(t, err)
   208  		assert.Equal(t, bucketingFailureReason(0), failReason)
   209  		assert.InEpsilon(t, p.expectedBucketValue, bucketValue, 0.0000001)
   210  
   211  		experiment.Seed = p.seed
   212  		variationIndex, inExperiment, err := evalScope.variationOrRolloutResult(
   213  			ldmodel.VariationOrRollout{Rollout: experiment},
   214  			p.flagOrSegmentKey,
   215  			p.salt,
   216  		)
   217  		assert.NoError(t, err)
   218  		expectedBucket := findBucketValueInVariationList(p.expectedBucketValue, buckets)
   219  		assert.Equal(t, buckets[expectedBucket].Variation, variationIndex)
   220  		assert.Equal(t, !buckets[expectedBucket].Untracked, inExperiment)
   221  	}
   222  
   223  	t.Run("by key", func(t *testing.T) {
   224  		for _, p := range makeBucketingTestParamsForExperiments() {
   225  			t.Run(p.description(), func(t *testing.T) {
   226  				context := ldcontext.New(p.contextValue)
   227  				checkResult(t, p, context, baseExperiment)
   228  			})
   229  		}
   230  	})
   231  
   232  	t.Run("changing hashKey and salt has no effect when seed is specified", func(t *testing.T) {
   233  		for _, p := range makeBucketingTestParamsForExperiments() {
   234  			if !p.seed.IsDefined() {
   235  				continue
   236  			}
   237  			t.Run(p.description(), func(t *testing.T) {
   238  				context := ldcontext.New(p.contextValue)
   239  
   240  				modifiedParams1 := p
   241  				modifiedParams1.flagOrSegmentKey += "xxx"
   242  				checkResult(t, modifiedParams1, context, baseExperiment) // did not change expectedBucketValue, still passes
   243  
   244  				modifiedParams2 := p
   245  				modifiedParams2.salt += "yyy"
   246  				checkResult(t, modifiedParams2, context, baseExperiment) // did not change expectedBucketValue, still passes
   247  			})
   248  		}
   249  	})
   250  
   251  	t.Run("changing seed produces different bucket value", func(t *testing.T) {
   252  		for _, p := range makeBucketingTestParamsForExperiments() {
   253  			t.Run(p.description(), func(t *testing.T) {
   254  				context := ldcontext.New(p.contextValue)
   255  
   256  				bucketValue1, failReason, err := makeEvalScope(context).computeBucketValue(true, p.seed,
   257  					"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   258  				assert.NoError(t, err)
   259  				assert.Equal(t, bucketingFailureReason(0), failReason)
   260  
   261  				var modifiedSeed ldvalue.OptionalInt
   262  				if p.seed.IsDefined() {
   263  					modifiedSeed = ldvalue.NewOptionalInt(p.seed.IntValue() + 1)
   264  				} else {
   265  					modifiedSeed = ldvalue.NewOptionalInt(999)
   266  				}
   267  				bucketValue2, failReason, err := makeEvalScope(context).computeBucketValue(true, modifiedSeed,
   268  					"", p.flagOrSegmentKey, ldattr.Ref{}, p.salt)
   269  				assert.NoError(t, err)
   270  				assert.Equal(t, bucketingFailureReason(0), failReason)
   271  
   272  				assert.NotEqual(t, bucketValue1, bucketValue2)
   273  			})
   274  		}
   275  	})
   276  
   277  	t.Run("when context kind is not found, first bucket is chosen but inExperiment is false", func(t *testing.T) {
   278  		context := ldcontext.NewWithKind("rightkind", "key")
   279  		experiment := baseExperiment
   280  
   281  		variationIndex, inExperiment, err := makeEvalScope(context).variationOrRolloutResult(
   282  			ldmodel.VariationOrRollout{Rollout: experiment},
   283  			"flagkey",
   284  			"salt",
   285  		)
   286  		assert.NoError(t, err)
   287  		assert.Equal(t, baseExperiment.Variations[0].Variation, variationIndex)
   288  		assert.False(t, inExperiment)
   289  	})
   290  
   291  	t.Run("secondary key is ignored, even if enabled in the evaluator", func(t *testing.T) {
   292  		for _, p := range makeBucketingTestParamsForExperiments() {
   293  			t.Run(p.description(), func(t *testing.T) {
   294  				context := makeUserContextWithSecondaryKey(t, p.contextValue, "shouldbeignored")
   295  				checkResult(t, p, context, baseExperiment) // did not change expectedBucketValue, still passes
   296  			})
   297  		}
   298  	})
   299  
   300  	t.Run("bucketBy is ignored", func(t *testing.T) {
   301  		for _, p := range makeBucketingTestParamsForExperiments() {
   302  			t.Run(p.description(), func(t *testing.T) {
   303  				context := ldcontext.NewBuilder(p.contextValue).SetString("attr1", p.contextValue+"xyz").Build()
   304  				experiment := baseExperiment
   305  				experiment.BucketBy = ldattr.NewLiteralRef("attr1")
   306  
   307  				checkResult(t, p, context, experiment) // did not change expectedBucketValue, still passes
   308  			})
   309  		}
   310  	})
   311  }
   312  
   313  func TestVariationOrRolloutResultErrorConditions(t *testing.T) {
   314  	context := ldcontext.New("key")
   315  
   316  	for _, p := range []struct {
   317  		rollout              ldmodel.Rollout
   318  		expectedErrorMessage string
   319  	}{
   320  		{
   321  			rollout:              ldmodel.Rollout{},
   322  			expectedErrorMessage: "rollout or experiment with no variations",
   323  		},
   324  		{
   325  			rollout:              ldmodel.Rollout{Kind: ldmodel.RolloutKindExperiment},
   326  			expectedErrorMessage: "rollout or experiment with no variations",
   327  		},
   328  		{
   329  			rollout: ldmodel.Rollout{
   330  				BucketBy:   ldattr.NewRef("///"),
   331  				Variations: []ldmodel.WeightedVariation{{Variation: 0, Weight: 100000}},
   332  			},
   333  			expectedErrorMessage: "attribute reference",
   334  		},
   335  	} {
   336  		t.Run(p.expectedErrorMessage, func(t *testing.T) {
   337  			vr := ldmodel.VariationOrRollout{Rollout: p.rollout}
   338  			_, _, err := makeEvalScope(context).variationOrRolloutResult(vr, "hashKey", "salt")
   339  			if assert.Error(t, err) {
   340  				assert.Contains(t, err.Error(), p.expectedErrorMessage)
   341  			}
   342  		})
   343  	}
   344  }
   345  
   346  func TestComputeBucketValueInvalidConditions(t *testing.T) {
   347  	flagKey, salt := "flagKey", "saltyA" // irrelevant to these tests
   348  
   349  	t.Run("single-kind context does not match desired kind", func(t *testing.T) {
   350  		context := ldcontext.New("key")
   351  		desiredKind := ldcontext.Kind("org")
   352  		bucket, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed, desiredKind, flagKey, ldattr.Ref{}, "saltyA")
   353  		assert.NoError(t, err)
   354  		assert.Equal(t, bucketingFailureContextLacksDesiredKind, failReason)
   355  		assert.Equal(t, float32(0), bucket)
   356  	})
   357  
   358  	t.Run("multi-kind context does not match desired kind", func(t *testing.T) {
   359  		context := ldcontext.NewMulti(ldcontext.New("irrelevantKey1"), ldcontext.NewWithKind("irrelevantKind", "irrelevantKey2"))
   360  		desiredKind := ldcontext.Kind("org")
   361  		bucket, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed, desiredKind, flagKey, ldattr.Ref{}, "saltyA")
   362  		assert.NoError(t, err)
   363  		assert.Equal(t, bucketingFailureContextLacksDesiredKind, failReason)
   364  		assert.Equal(t, float32(0), bucket)
   365  	})
   366  
   367  	t.Run("bucket by nonexistent attribute", func(t *testing.T) {
   368  		context := ldcontext.New("key")
   369  		bucket, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed, "", flagKey, ldattr.NewLiteralRef("unknownAttr"), salt)
   370  		assert.NoError(t, err)
   371  		assert.Equal(t, bucketingFailureAttributeNotFound, failReason)
   372  		assert.Equal(t, float32(0), bucket)
   373  	})
   374  
   375  	t.Run("bucket by non-integer numeric attribute", func(t *testing.T) {
   376  		context := ldcontext.NewBuilder("key").SetFloat64("floatAttr", 999.999).Build()
   377  		bucket, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed, "", flagKey, ldattr.NewLiteralRef("floatAttr"), salt)
   378  		assert.NoError(t, err)
   379  		assert.Equal(t, bucketingFailureAttributeValueWrongType, failReason)
   380  		assert.Equal(t, float32(0), bucket)
   381  	})
   382  
   383  	t.Run("bucket by invalid attribute reference", func(t *testing.T) {
   384  		context := ldcontext.New("key")
   385  		badAttr := ldattr.NewRef("///")
   386  		_, failReason, err := makeEvalScope(context).computeBucketValue(false, noSeed, "", flagKey, badAttr, salt)
   387  		assert.Error(t, err) // Unlike the other invalid conditions, we treat this one as a malformed flag error
   388  		assert.Equal(t, bucketingFailureInvalidAttrRef, failReason)
   389  	})
   390  }
   391  
   392  func TestBucketValueBeyondLastBucketIsPinnedToLastBucket(t *testing.T) {
   393  	vr := ldbuilders.Rollout(ldbuilders.Bucket(0, 5000), ldbuilders.Bucket(1, 5000))
   394  	user := ldcontext.NewBuilder("userKeyD").SetInt("intAttr", 99999).Build()
   395  	variationIndex, inExperiment, err := makeEvalScope(user).variationOrRolloutResult(vr, "hashKey", "saltyA")
   396  	assert.NoError(t, err)
   397  	assert.Equal(t, 1, variationIndex)
   398  	assert.False(t, inExperiment)
   399  }
   400  
   401  func TestBucketValueBeyondLastBucketIsPinnedToLastBucketForExperiment(t *testing.T) {
   402  	vr := ldbuilders.Experiment(ldvalue.NewOptionalInt(42), ldbuilders.Bucket(0, 5000), ldbuilders.Bucket(1, 5000))
   403  	user := ldcontext.NewBuilder("userKeyD").SetInt("intAttr", 99999).Build()
   404  	variationIndex, inExperiment, err := makeEvalScope(user).variationOrRolloutResult(vr, "hashKey", "saltyA")
   405  	assert.NoError(t, err)
   406  	assert.Equal(t, 1, variationIndex)
   407  	assert.True(t, inExperiment)
   408  }
   409  

View as plain text