ldclient_evaluation_benchmark_test.go

Documentation: github.com/launchdarkly/go-server-sdk/v6

     1  package ldclient
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"testing"
     7  
     8  	"github.com/launchdarkly/go-sdk-common/v3/ldattr"
     9  	"github.com/launchdarkly/go-sdk-common/v3/ldcontext"
    10  	"github.com/launchdarkly/go-sdk-common/v3/lduser"
    11  	"github.com/launchdarkly/go-sdk-common/v3/ldvalue"
    12  	ldevents "github.com/launchdarkly/go-sdk-events/v2"
    13  	"github.com/launchdarkly/go-server-sdk-evaluation/v2/ldbuilders"
    14  	"github.com/launchdarkly/go-server-sdk-evaluation/v2/ldmodel"
    15  	"github.com/launchdarkly/go-server-sdk/v6/internal/datakinds"
    16  	"github.com/launchdarkly/go-server-sdk/v6/internal/sharedtest"
    17  	"github.com/launchdarkly/go-server-sdk/v6/ldcomponents"
    18  	"github.com/launchdarkly/go-server-sdk/v6/subsystems"
    19  )
    20  
    21  // These benchmarks cover the LDClient evaluation flow, including looking up the target flag, applying all
    22  // relevant targets and rules, and producing analytics event data (but then discarding the event data,
    23  // so the event processor logic is not included).
    24  //
    25  // This was adapted from a user-contributed PR: https://github.com/launchdarkly/go-server-sdk/pull/28
    26  
    27  // Note about heap allocations:
    28  //
    29  // Benchmarks whose names end in "NoAlloc" are expected _not_ to cause any heap allocations (not counting
    30  // setup work done before ResetTimer()). This is enforced by the Makefile's benchmarks target. As long as
    31  // events are disabled, it should be possible to do most kinds of flag evaluations without causing any
    32  // heap allocations.
    33  //
    34  // See notes about heap allocations in CONTRIBUTING.md.
    35  
    36  type evalBenchmarkEnv struct {
    37  	client           *LDClient
    38  	evalUser         ldcontext.Context
    39  	targetFeatureKey string
    40  	targetUsers      []ldcontext.Context
    41  }
    42  
    43  func newEvalBenchmarkEnv() *evalBenchmarkEnv {
    44  	return &evalBenchmarkEnv{}
    45  }
    46  
    47  func (env *evalBenchmarkEnv) setUp(withEventGeneration bool, bc evalBenchmarkCase, variations []ldvalue.Value) {
    48  	// Set up the client.
    49  	env.client = makeTestClientWithConfig(func(c *Config) {
    50  		if withEventGeneration {
    51  			// In this mode, we use a stub EventProcessor implementation that immediately discards
    52  			// every event, but the SDK will still generate the events before passing them to the stub,
    53  			// so we are still measuring the overhead of that.
    54  			c.Events = benchmarkStubEventProcessorFactory{}
    55  		} else {
    56  			// Completely disable all event functionality, so we are only testing the evaluation logic
    57  			// (plus retrieval of the flag from the in-memory store). The SDK only behaves this way if
    58  			// Events is set to the specific factory returned by NoEvents().
    59  			c.Events = ldcomponents.NoEvents()
    60  		}
    61  	})
    62  
    63  	// Set up the feature flag store. Note that we're using a regular in-memory data store, so the
    64  	// benchmarks will include the overhead of calling Get on the store.
    65  	testFlags := makeEvalBenchmarkFlags(bc, variations)
    66  	for _, ff := range testFlags {
    67  		env.client.store.Upsert(datakinds.Features, ff.Key, sharedtest.FlagDescriptor(*ff))
    68  	}
    69  
    70  	env.evalUser = makeEvalBenchmarkUser(bc)
    71  
    72  	// Target a feature key in the middle of the list in case a linear search is being used.
    73  	targetFeatureKeyIndex := 0
    74  	if bc.numFlags > 0 {
    75  		targetFeatureKeyIndex = bc.numFlags / 2
    76  	}
    77  	env.targetFeatureKey = fmt.Sprintf("flag-%d", targetFeatureKeyIndex)
    78  
    79  	// Create users to match all of the user keys in the flag's target list. These will be used
    80  	// only in BenchmarkUsersFoundInTargets; with all the other benchmarks, we are deliberately
    81  	// using a user key that is *not* found in the targets.
    82  	env.targetUsers = make([]ldcontext.Context, bc.numTargets)
    83  	for i := 0; i < bc.numTargets; i++ {
    84  		env.targetUsers[i] = lduser.NewUser(makeEvalBenchmarkTargetUserKey(i))
    85  	}
    86  }
    87  
    88  func (env *evalBenchmarkEnv) tearDown() {
    89  	// Prepare for the next benchmark case.
    90  	env.client.Close()
    91  	env.client = nil
    92  	env.targetFeatureKey = ""
    93  }
    94  
    95  type benchmarkStubEventProcessorFactory struct{}
    96  
    97  func (f benchmarkStubEventProcessorFactory) Build(context subsystems.ClientContext) (ldevents.EventProcessor, error) {
    98  	return ldcomponents.NoEvents().Build(context)
    99  }
   100  
   101  func makeEvalBenchmarkUser(bc evalBenchmarkCase) ldcontext.Context {
   102  	if bc.shouldMatch {
   103  		builder := lduser.NewUserBuilder("user-match")
   104  		switch bc.operator {
   105  		case ldmodel.OperatorGreaterThan:
   106  			builder.Custom("numAttr", ldvalue.Int(10000))
   107  		case ldmodel.OperatorContains:
   108  			builder.Name("name-0")
   109  		case ldmodel.OperatorMatches:
   110  			builder.Custom("stringAttr", ldvalue.String("stringAttr-0"))
   111  		case ldmodel.OperatorAfter:
   112  			builder.Custom("dateAttr", ldvalue.String("2999-12-31T00:00:00.000-00:00"))
   113  		case ldmodel.OperatorIn:
   114  			builder.Custom("stringAttr", ldvalue.String("stringAttr-0"))
   115  		}
   116  		return builder.Build()
   117  	}
   118  	// default is that the user will not be matched by any clause or target
   119  	return lduser.NewUserBuilder("user-nomatch").
   120  		Name("name-nomatch").
   121  		Custom("stringAttr", ldvalue.String("stringAttr-nomatch")).
   122  		Custom("numAttr", ldvalue.Int(0)).
   123  		Custom("dateAttr", ldvalue.String("1980-01-01T00:00:00.000-00:00")).
   124  		Build()
   125  }
   126  
   127  type evalBenchmarkCase struct {
   128  	numUsers      int
   129  	numFlags      int
   130  	numVariations int
   131  	numTargets    int
   132  	numRules      int
   133  	numClauses    int
   134  	prereqsWidth  int
   135  	prereqsDepth  int
   136  	operator      ldmodel.Operator
   137  	shouldMatch   bool
   138  }
   139  
   140  var ruleEvalBenchmarkCases = []evalBenchmarkCase{
   141  	// simple
   142  	{
   143  		numUsers:      1000,
   144  		numFlags:      1000,
   145  		numVariations: 2,
   146  		numTargets:    1,
   147  	},
   148  
   149  	// realistic
   150  	{
   151  		numUsers:      10000,
   152  		numFlags:      10000,
   153  		numVariations: 2,
   154  		numTargets:    1,
   155  	},
   156  	{
   157  		numUsers:      10000,
   158  		numFlags:      10000,
   159  		numVariations: 2,
   160  		numTargets:    10,
   161  	},
   162  	{
   163  		numUsers:      10000,
   164  		numFlags:      1000,
   165  		numVariations: 2,
   166  		numRules:      1,
   167  		numClauses:    1,
   168  	},
   169  	{
   170  		numUsers:      10000,
   171  		numFlags:      1000,
   172  		numVariations: 2,
   173  		numRules:      1,
   174  		numClauses:    3,
   175  	},
   176  	{
   177  		numUsers:      10000,
   178  		numFlags:      1000,
   179  		numVariations: 2,
   180  		numRules:      5,
   181  		numClauses:    3,
   182  	},
   183  
   184  	// prereqs
   185  	{
   186  		numUsers:      10000,
   187  		numFlags:      1000,
   188  		numVariations: 2,
   189  		numRules:      1,
   190  		numClauses:    1,
   191  		prereqsWidth:  5,
   192  		prereqsDepth:  1,
   193  	},
   194  	{
   195  		numUsers:      10000,
   196  		numFlags:      1000,
   197  		numVariations: 2,
   198  		numRules:      1,
   199  		numClauses:    1,
   200  		prereqsWidth:  1,
   201  		prereqsDepth:  5,
   202  	},
   203  	{
   204  		numUsers:      10000,
   205  		numFlags:      1000,
   206  		numVariations: 2,
   207  		numTargets:    1,
   208  		prereqsWidth:  2,
   209  		prereqsDepth:  2,
   210  	},
   211  	{
   212  		numUsers:      10000,
   213  		numFlags:      1000,
   214  		numVariations: 2,
   215  		numRules:      1,
   216  		numClauses:    1,
   217  		prereqsWidth:  5,
   218  		prereqsDepth:  5,
   219  	},
   220  
   221  	// operations - if not specified, the default is OperatorIn
   222  	{
   223  		numUsers:      10000,
   224  		numFlags:      1000,
   225  		numVariations: 2,
   226  		numRules:      1,
   227  		numClauses:    1,
   228  		operator:      ldmodel.OperatorGreaterThan,
   229  	},
   230  	{
   231  		numUsers:      10000,
   232  		numFlags:      1000,
   233  		numVariations: 2,
   234  		numRules:      1,
   235  		numClauses:    1,
   236  		operator:      ldmodel.OperatorContains,
   237  	},
   238  	{
   239  		numUsers:      10000,
   240  		numFlags:      1000,
   241  		numVariations: 2,
   242  		numRules:      1,
   243  		numClauses:    1,
   244  		operator:      ldmodel.OperatorMatches,
   245  	},
   246  }
   247  
   248  var targetMatchBenchmarkCases = []evalBenchmarkCase{
   249  	{
   250  		numUsers:      1000,
   251  		numFlags:      1000,
   252  		numVariations: 2,
   253  		numTargets:    10,
   254  	},
   255  	{
   256  		numUsers:      1000,
   257  		numFlags:      1000,
   258  		numVariations: 2,
   259  		numTargets:    100,
   260  	},
   261  	{
   262  		numUsers:      1000,
   263  		numFlags:      1000,
   264  		numVariations: 2,
   265  		numTargets:    1000,
   266  	},
   267  }
   268  
   269  var ruleMatchBenchmarkCases = []evalBenchmarkCase{
   270  	// These cases are deliberately simple because the benchmark is meant to focus on the evaluation of
   271  	// one specific type of matching operation. The user will match the first clause in the first rule.
   272  	{
   273  		numFlags:      1,
   274  		numRules:      1,
   275  		numClauses:    1,
   276  		numVariations: 2,
   277  		operator:      ldmodel.OperatorIn,
   278  		shouldMatch:   true,
   279  	},
   280  	{
   281  		numFlags:      1,
   282  		numRules:      1,
   283  		numClauses:    1,
   284  		numVariations: 2,
   285  		operator:      ldmodel.OperatorContains,
   286  		shouldMatch:   true,
   287  	},
   288  	{
   289  		numFlags:      1,
   290  		numRules:      1,
   291  		numClauses:    1,
   292  		numVariations: 2,
   293  		operator:      ldmodel.OperatorGreaterThan,
   294  		shouldMatch:   true,
   295  	},
   296  	{
   297  		numFlags:      1,
   298  		numRules:      1,
   299  		numClauses:    1,
   300  		numVariations: 2,
   301  		operator:      ldmodel.OperatorAfter,
   302  		shouldMatch:   true,
   303  	},
   304  	{
   305  		numFlags:      1,
   306  		numRules:      1,
   307  		numClauses:    1,
   308  		numVariations: 2,
   309  		operator:      ldmodel.OperatorMatches,
   310  		shouldMatch:   true,
   311  	},
   312  }
   313  
   314  var (
   315  	// Always record the result of an operation to prevent the compiler eliminating the function call.
   316  	//
   317  	// Always store the result to a package level variable so the compiler cannot eliminate the benchmark itself.
   318  	boolResult   bool
   319  	intResult    int
   320  	stringResult string
   321  	jsonResult   ldvalue.Value
   322  )
   323  
   324  func benchmarkEval(
   325  	b *testing.B,
   326  	withEventGeneration bool,
   327  	makeVariation func(int) ldvalue.Value,
   328  	cases []evalBenchmarkCase,
   329  	action func(*evalBenchmarkEnv),
   330  ) {
   331  	env := newEvalBenchmarkEnv()
   332  	for _, bc := range cases {
   333  		variations := make([]ldvalue.Value, bc.numVariations)
   334  		for i := 0; i < bc.numVariations; i++ {
   335  			variations[i] = makeVariation(i)
   336  		}
   337  		env.setUp(withEventGeneration, bc, variations)
   338  
   339  		b.Run(fmt.Sprintf("%+v", bc), func(b *testing.B) {
   340  			for i := 0; i < b.N; i++ {
   341  				action(env)
   342  			}
   343  		})
   344  		env.tearDown()
   345  	}
   346  }
   347  
   348  // This benchmark executes only a single basic evaluation case. It is mainly useful in very
   349  // detailed profiling and allocation tracing where you don't want a huge log file.
   350  func BenchmarkSingleVariation(b *testing.B) {
   351  	singleCase := []evalBenchmarkCase{ruleEvalBenchmarkCases[0]}
   352  	benchmarkEval(b, false, makeBoolVariation, singleCase, func(env *evalBenchmarkEnv) {
   353  		boolResult, _ = env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   354  	})
   355  }
   356  
   357  func BenchmarkSingleVariationWithEvents(b *testing.B) {
   358  	singleCase := []evalBenchmarkCase{ruleEvalBenchmarkCases[0]}
   359  	benchmarkEval(b, true, makeBoolVariation, singleCase, func(env *evalBenchmarkEnv) {
   360  		boolResult, _ = env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   361  	})
   362  }
   363  
   364  func BenchmarkBoolVariationNoAlloc(b *testing.B) {
   365  	benchmarkEval(b, false, makeBoolVariation, ruleEvalBenchmarkCases, func(env *evalBenchmarkEnv) {
   366  		boolResult, _ = env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   367  	})
   368  }
   369  
   370  // The ___WithEvents version of the benchmark enables the LDClient code path that creates an evaluation
   371  // event instance, even though the event will not be sent anywhere, so we can measure the overhead of
   372  // that step. It is not repeated for BenchmarkIntVariation, etc., because the data type of the
   373  // variation makes no difference in how events are generated.
   374  func BenchmarkBoolVariationWithEvents(b *testing.B) {
   375  	benchmarkEval(b, true, makeBoolVariation, ruleEvalBenchmarkCases, func(env *evalBenchmarkEnv) {
   376  		boolResult, _ = env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   377  	})
   378  }
   379  
   380  func BenchmarkIntVariationNoAlloc(b *testing.B) {
   381  	benchmarkEval(b, false, makeIntVariation, ruleEvalBenchmarkCases, func(env *evalBenchmarkEnv) {
   382  		intResult, _ = env.client.IntVariation(env.targetFeatureKey, env.evalUser, 0)
   383  	})
   384  }
   385  
   386  func BenchmarkStringVariationNoAlloc(b *testing.B) {
   387  	benchmarkEval(b, false, makeStringVariation, ruleEvalBenchmarkCases, func(env *evalBenchmarkEnv) {
   388  		stringResult, _ = env.client.StringVariation(env.targetFeatureKey, env.evalUser, "variation-0")
   389  	})
   390  }
   391  
   392  func BenchmarkJSONVariationNoAlloc(b *testing.B) {
   393  	defaultValAsRawJSON := ldvalue.Raw(json.RawMessage(`{"result":{"value":[0]}}`))
   394  	benchmarkEval(b, false, makeJSONVariation, ruleEvalBenchmarkCases, func(env *evalBenchmarkEnv) {
   395  		jsonResult, _ = env.client.JSONVariation(env.targetFeatureKey, env.evalUser, defaultValAsRawJSON)
   396  	})
   397  }
   398  
   399  func BenchmarkUsersFoundInTargetsNoAlloc(b *testing.B) {
   400  	benchmarkEval(b, false, makeBoolVariation,
   401  		targetMatchBenchmarkCases,
   402  		func(env *evalBenchmarkEnv) {
   403  			for _, user := range env.targetUsers {
   404  				r, _ := env.client.BoolVariation(env.targetFeatureKey, user, false)
   405  				boolResult = r
   406  			}
   407  		})
   408  }
   409  
   410  func BenchmarkUserNotFoundInTargetsNoAlloc(b *testing.B) {
   411  	benchmarkEval(b, false, makeBoolVariation,
   412  		targetMatchBenchmarkCases,
   413  		func(env *evalBenchmarkEnv) {
   414  			for range env.targetUsers {
   415  				r, _ := env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   416  				boolResult = r
   417  			}
   418  		})
   419  }
   420  
   421  func BenchmarkUserMatchesRuleNoAlloc(b *testing.B) {
   422  	benchmarkEval(b, false, makeBoolVariation,
   423  		ruleMatchBenchmarkCases,
   424  		func(env *evalBenchmarkEnv) {
   425  			boolResult, _ = env.client.BoolVariation(env.targetFeatureKey, env.evalUser, false)
   426  		})
   427  }
   428  
   429  // Input data creation
   430  
   431  // Except for when we're running BenchmarkUserMatchesRule, the flag rules and clauses we create here are
   432  // intended *not* to match the user, so the more of them we create, the more we are testing the overhead
   433  // of iterating through and evaluating all the clauses.
   434  
   435  func makeBoolVariation(i int) ldvalue.Value {
   436  	return ldvalue.Bool(i%2 == 0)
   437  }
   438  
   439  func makeIntVariation(i int) ldvalue.Value {
   440  	return ldvalue.Int(i)
   441  }
   442  
   443  func makeStringVariation(i int) ldvalue.Value {
   444  	return ldvalue.String(fmt.Sprintf("variation-%d", i))
   445  }
   446  
   447  func makeJSONVariation(i int) ldvalue.Value {
   448  	return ldvalue.ObjectBuild().Set(
   449  		"result",
   450  		ldvalue.ObjectBuild().Set("value", ldvalue.ArrayOf(ldvalue.Int(i))).Build(),
   451  	).Build()
   452  }
   453  
   454  func makeEvalBenchmarkClauses(numClauses int, op ldmodel.Operator) []ldmodel.Clause {
   455  	clauses := make([]ldmodel.Clause, 0, numClauses)
   456  	for i := 0; i < numClauses; i++ {
   457  		clause := ldmodel.Clause{Op: op}
   458  		switch op {
   459  		case ldmodel.OperatorGreaterThan:
   460  			clause.Attribute = ldattr.NewLiteralRef("numAttr")
   461  			clause.Values = []ldvalue.Value{ldvalue.Int(i)}
   462  		case ldmodel.OperatorContains:
   463  			clause.Attribute = ldattr.NewLiteralRef("name")
   464  			clause.Values = []ldvalue.Value{
   465  				ldvalue.String(fmt.Sprintf("name-%d", i)),
   466  				ldvalue.String(fmt.Sprintf("name-%d", i+1)),
   467  				ldvalue.String(fmt.Sprintf("name-%d", i+2)),
   468  			}
   469  		case ldmodel.OperatorMatches:
   470  			clause.Attribute = ldattr.NewLiteralRef("stringAttr")
   471  			clause.Values = []ldvalue.Value{
   472  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i)),
   473  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i+1)),
   474  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i+2)),
   475  			}
   476  		case ldmodel.OperatorAfter:
   477  			clause.Attribute = ldattr.NewLiteralRef("dateAttr")
   478  			clause.Values = []ldvalue.Value{
   479  				ldvalue.String(fmt.Sprintf("%d-01-01T00:00:00.000-00:00", 2000+i)),
   480  				ldvalue.String(fmt.Sprintf("%d-01-01T00:00:00.000-00:00", 2001+i)),
   481  				ldvalue.String(fmt.Sprintf("%d-01-01T00:00:00.000-00:00", 2002+i)),
   482  			}
   483  		default:
   484  			clause.Op = ldmodel.OperatorIn
   485  			clause.Attribute = ldattr.NewLiteralRef("stringAttr")
   486  			clause.Values = []ldvalue.Value{
   487  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i)),
   488  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i+1)),
   489  				ldvalue.String(fmt.Sprintf("stringAttr-%d", i+2)),
   490  			}
   491  		}
   492  		clauses = append(clauses, clause)
   493  	}
   494  	return clauses
   495  }
   496  
   497  func makeEvalBenchmarkTargetUserKey(i int) string {
   498  	return fmt.Sprintf("user-%d", i)
   499  }
   500  
   501  func makeEvalBenchmarkFlags(bc evalBenchmarkCase, variations []ldvalue.Value) []*ldmodel.FeatureFlag {
   502  	testFlags := make([]*ldmodel.FeatureFlag, 0, bc.numFlags)
   503  	for i := 0; i < bc.numFlags; i++ {
   504  		flag := ldbuilders.NewFlagBuilder(fmt.Sprintf("flag-%d", i)).
   505  			Version(1).
   506  			On(true).
   507  			Variations(variations...).
   508  			FallthroughVariation(1)
   509  		for j := 0; j < bc.numVariations; j++ {
   510  			values := make([]string, bc.numTargets)
   511  			for k := 0; k < bc.numTargets; k++ {
   512  				values[k] = makeEvalBenchmarkTargetUserKey(k)
   513  			}
   514  			flag.AddTarget(j, values...)
   515  		}
   516  		for j := 0; j < bc.numRules; j++ {
   517  			flag.AddRule(ldbuilders.NewRuleBuilder().
   518  				ID(fmt.Sprintf("%d-%d", i, j)).
   519  				Clauses(makeEvalBenchmarkClauses(bc.numClauses, bc.operator)...).
   520  				Variation(0))
   521  		}
   522  		f := flag.Build()
   523  		testFlags = append(testFlags, &f)
   524  	}
   525  
   526  	if bc.prereqsWidth > 0 && bc.prereqsDepth > 0 {
   527  		assignPrereqTree(testFlags, bc.prereqsWidth, bc.prereqsDepth)
   528  	}
   529  
   530  	return testFlags
   531  }
   532  
   533  // assignPrereqTree assigns prerequisites to each of the given feature flags such that each flag
   534  // has at most `width` children and `depth` ancestors. If the depth of the prerequisite "tree"
   535  // exceeds `depth`, a new tree is assigned starting with the next feature flag the root node.
   536  func assignPrereqTree(flags []*ldmodel.FeatureFlag, width, depth int) {
   537  	var parentLevel []*ldmodel.FeatureFlag
   538  	levelIndex := 0
   539  
   540  	i := 0
   541  	for i < len(flags) {
   542  		if levelIndex > depth {
   543  			levelIndex = 0
   544  			parentLevel = []*ldmodel.FeatureFlag{flags[i]}
   545  		}
   546  		if levelIndex == 0 {
   547  			levelIndex++
   548  			i++
   549  			continue
   550  		}
   551  
   552  		var childLevel []*ldmodel.FeatureFlag
   553  		for _, parent := range parentLevel {
   554  			for w := 0; w < width && i+w < len(flags); w++ {
   555  				child := flags[i+w]
   556  				child.Prerequisites = []ldmodel.Prerequisite{{Key: parent.Key, Variation: 0}}
   557  				childLevel = append(childLevel, child)
   558  			}
   559  			i += width
   560  		}
   561  		parentLevel = childLevel
   562  		levelIndex++
   563  	}
   564  }
   565
View as plain text