Source file src/github.com/tetratelabs/wazero/internal/engine/compiler/engine.go

Documentation: github.com/tetratelabs/wazero/internal/engine/compiler

     1  package compiler
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"reflect"
     8  	"runtime"
     9  	"sort"
    10  	"sync"
    11  	"unsafe"
    12  
    13  	"github.com/tetratelabs/wazero/api"
    14  	"github.com/tetratelabs/wazero/experimental"
    15  	"github.com/tetratelabs/wazero/internal/asm"
    16  	"github.com/tetratelabs/wazero/internal/bitpack"
    17  	"github.com/tetratelabs/wazero/internal/filecache"
    18  	"github.com/tetratelabs/wazero/internal/internalapi"
    19  	"github.com/tetratelabs/wazero/internal/platform"
    20  	"github.com/tetratelabs/wazero/internal/version"
    21  	"github.com/tetratelabs/wazero/internal/wasm"
    22  	"github.com/tetratelabs/wazero/internal/wasmdebug"
    23  	"github.com/tetratelabs/wazero/internal/wasmruntime"
    24  	"github.com/tetratelabs/wazero/internal/wazeroir"
    25  )
    26  
    27  // NOTE: The offset of many of the struct fields defined here are referenced from
    28  // assembly using the constants below such as moduleEngineFunctionsOffset.
    29  // If changing a struct, update the constant and associated tests as needed.
    30  type (
    31  	// engine is a Compiler implementation of wasm.Engine
    32  	engine struct {
    33  		enabledFeatures api.CoreFeatures
    34  		codes           map[wasm.ModuleID]*compiledModule // guarded by mutex.
    35  		fileCache       filecache.Cache
    36  		mux             sync.RWMutex
    37  		// setFinalizer defaults to runtime.SetFinalizer, but overridable for tests.
    38  		setFinalizer  func(obj interface{}, finalizer interface{})
    39  		wazeroVersion string
    40  	}
    41  
    42  	// moduleEngine implements wasm.ModuleEngine
    43  	moduleEngine struct {
    44  		// See note at top of file before modifying this struct.
    45  
    46  		// functions are the functions in a module instances.
    47  		// The index is module instance-scoped. We intentionally avoid using map
    48  		// as the underlying memory region is accessed by assembly directly by using
    49  		// codesElement0Address.
    50  		functions []function
    51  
    52  		// Keep a reference to the compiled module to prevent the GC from reclaiming
    53  		// it while the code may still be needed.
    54  		module *compiledModule
    55  	}
    56  
    57  	// callEngine holds context per moduleEngine.Call, and shared across all the
    58  	// function calls originating from the same moduleEngine.Call execution.
    59  	//
    60  	// This implements api.Function.
    61  	callEngine struct {
    62  		internalapi.WazeroOnlyType
    63  
    64  		// See note at top of file before modifying this struct.
    65  
    66  		// These contexts are read and written by compiled code.
    67  		// Note: structs are embedded to reduce the costs to access fields inside them. Also, this eases field offset
    68  		// calculation.
    69  		moduleContext
    70  		stackContext
    71  		exitContext
    72  		archContext
    73  
    74  		// The following fields are not accessed by compiled code directly.
    75  
    76  		// stack is the go-allocated stack for holding values and call frames.
    77  		// Note: We never edit len or cap in compiled code, so we won't get screwed when GC comes in.
    78  		//
    79  		// At any point of execution, say currently executing function F2 which was called by F1, then
    80  		// the stack should look like like:
    81  		//
    82  		// 	[..., arg0, arg1, ..., argN, _, _, _, v1, v2, v3, ....
    83  		//	      ^                     {       }
    84  		//	      |                F1's callFrame
    85  		//	      |
    86  		//  stackBasePointer
    87  		//
    88  		// where
    89  		//  - callFrame is the F1's callFrame which called F2. It contains F1's return address, F1's base pointer, and F1's *function.
    90  		//  - stackBasePointer is the stack base pointer stored at (callEngine stackContext.stackBasePointerInBytes)
    91  		//  - arg0, ..., argN are the function parameters, and v1, v2, v3,... are the local variables
    92  		//    including the non-function param locals as well as the temporary variable produced by instructions (e.g i32.const).
    93  		//
    94  		// If the F2 makes a function call to F3 which takes two arguments, then the stack will become:
    95  		//
    96  		// 	[..., arg0, arg1, ..., argN, _, _, _, v1, v2, v3, _, _, _
    97  		//	                            {       }     ^      {       }
    98  		//	                       F1's callFrame     | F2's callFrame
    99  		//	                                          |
   100  		//                                     stackBasePointer
   101  		// where
   102  		// 	- F2's callFrame is pushed above the v2 and v3 (arguments for F3).
   103  		//  - The previous stackBasePointer (pointed at arg0) was saved inside the F2's callFrame.
   104  		//
   105  		// Then, if F3 returns one result, say w1, then the result will look like:
   106  		//
   107  		// 	[..., arg0, arg1, ..., argN, _, _, _, v1, w1, ...
   108  		//	      ^                     {       }
   109  		//	      |                F1's callFrame
   110  		//	      |
   111  		//  stackBasePointer
   112  		//
   113  		// where
   114  		// 	- stackBasePointer was reverted to the position at arg0
   115  		//  - The result from F3 was pushed above v1
   116  		//
   117  		// If the number of parameters is smaller than that of return values, then the empty slots are reserved
   118  		// below the callFrame to store the results on teh return.
   119  		// For example, if F3 takes no parameter but returns N(>0) results, then the stack
   120  		// after making a call against F3 will look like:
   121  		//
   122  		// 	[..., arg0, arg1, ..., argN, _, _, _, v1, v2, v3, res_1, _, res_N, _, _, _
   123  		//	                            {       }            ^                {       }
   124  		//	                       F1's callFrame            |           F2's callFrame
   125  		//	                                                 |
   126  		//                                            stackBasePointer
   127  		// where res_1, ..., res_N are the reserved slots below the call frame. In general,
   128  		// the number of reserved slots equals max(0, len(results)-len(params).
   129  		//
   130  		// This reserved slots are necessary to save the result values onto the stack while not destroying
   131  		// the callFrame value on function returns.
   132  		stack []uint64
   133  
   134  		// initialFn is the initial function for this call engine.
   135  		initialFn *function
   136  
   137  		// Keep a reference to the compiled module to prevent the GC from reclaiming
   138  		// it while the code may still be needed.
   139  		module *compiledModule
   140  
   141  		// stackIterator provides a way to iterate over the stack for Listeners.
   142  		// It is setup and valid only during a call to a Listener hook.
   143  		stackIterator stackIterator
   144  	}
   145  
   146  	// moduleContext holds the per-function call specific module information.
   147  	// This is subject to be manipulated from compiled native code whenever we make function calls.
   148  	moduleContext struct {
   149  		// See note at top of file before modifying this struct.
   150  
   151  		// fn holds the currently executed *function.
   152  		fn *function
   153  
   154  		// moduleInstance is the address of module instance from which we initialize
   155  		// the following fields. This is set whenever we enter a function or return from function calls.
   156  		//
   157  		// On the entry to the native code, this must be initialized to zero to let native code preamble know
   158  		// that this is the initial function call (which leads to moduleContext initialization pass).
   159  		moduleInstance *wasm.ModuleInstance //lint:ignore U1000 This is only used by Compiler code.
   160  
   161  		// globalElement0Address is the address of the first element in the global slice,
   162  		// i.e. &ModuleInstance.Globals[0] as uintptr.
   163  		globalElement0Address uintptr
   164  		// memoryElement0Address is the address of the first element in the global slice,
   165  		// i.e. &ModuleInstance.Memory.Buffer[0] as uintptr.
   166  		memoryElement0Address uintptr
   167  		// memorySliceLen is the length of the memory buffer, i.e. len(ModuleInstance.Memory.Buffer).
   168  		memorySliceLen uint64
   169  		// memoryInstance holds the memory instance for this module instance.
   170  		memoryInstance *wasm.MemoryInstance
   171  		// tableElement0Address is the address of the first item in the tables slice,
   172  		// i.e. &ModuleInstance.Tables[0] as uintptr.
   173  		tablesElement0Address uintptr
   174  
   175  		// functionsElement0Address is &moduleContext.functions[0] as uintptr.
   176  		functionsElement0Address uintptr
   177  
   178  		// typeIDsElement0Address holds the &ModuleInstance.TypeIDs[0] as uintptr.
   179  		typeIDsElement0Address uintptr
   180  
   181  		// dataInstancesElement0Address holds the &ModuleInstance.DataInstances[0] as uintptr.
   182  		dataInstancesElement0Address uintptr
   183  
   184  		// elementInstancesElement0Address holds the &ModuleInstance.ElementInstances[0] as uintptr.
   185  		elementInstancesElement0Address uintptr
   186  	}
   187  
   188  	// stackContext stores the data to access engine.stack.
   189  	stackContext struct {
   190  		// See note at top of file before modifying this struct.
   191  
   192  		// stackPointer on .stack field which is accessed by stack[stackBasePointer+stackBasePointerInBytes*8].
   193  		//
   194  		// Note: stackPointer is not used in assembly since the native code knows exact position of
   195  		// each variable in the value stack from the info from compilation.
   196  		// Therefore, only updated when native code exit from the Compiler world and go back to the Go function.
   197  		stackPointer uint64
   198  
   199  		// stackBasePointerInBytes is updated whenever we make function calls.
   200  		// Background: Functions might be compiled as if they use the stack from the bottom.
   201  		// However, in reality, they have to use it from the middle of the stack depending on
   202  		// when these function calls are made. So instead of accessing stack via stackPointer alone,
   203  		// functions are compiled, so they access the stack via [stackBasePointer](fixed for entire function) + [stackPointer].
   204  		// More precisely, stackBasePointer is set to [callee's stack pointer] + [callee's stack base pointer] - [caller's params].
   205  		// This way, compiled functions can be independent of the timing of functions calls made against them.
   206  		stackBasePointerInBytes uint64
   207  
   208  		// stackElement0Address is &engine.stack[0] as uintptr.
   209  		// Note: this is updated when growing the stack in builtinFunctionGrowStack.
   210  		stackElement0Address uintptr
   211  
   212  		// stackLenInBytes is len(engine.stack[0]) * 8 (bytes).
   213  		// Note: this is updated when growing the stack in builtinFunctionGrowStack.
   214  		stackLenInBytes uint64
   215  	}
   216  
   217  	// exitContext will be manipulated whenever compiled native code returns into the Go function.
   218  	exitContext struct {
   219  		// See note at top of file before modifying this struct.
   220  
   221  		// Where we store the status code of Compiler execution.
   222  		statusCode nativeCallStatusCode
   223  
   224  		// Set when statusCode == compilerStatusCallBuiltInFunction
   225  		// Indicating the function call index.
   226  		builtinFunctionCallIndex wasm.Index
   227  
   228  		// returnAddress is the return address which the engine jumps into
   229  		// after executing a builtin function or host function.
   230  		returnAddress uintptr
   231  
   232  		// callerModuleInstance holds the caller's wasm.ModuleInstance, and is only valid if currently executing a host function.
   233  		callerModuleInstance *wasm.ModuleInstance
   234  	}
   235  
   236  	// callFrame holds the information to which the caller function can return.
   237  	// This is mixed in callEngine.stack with other Wasm values just like any other
   238  	// native program (where the stack is the system stack though), and we retrieve the struct
   239  	// with unsafe pointer casts.
   240  	callFrame struct {
   241  		// See note at top of file before modifying this struct.
   242  
   243  		// returnAddress is the return address to which the engine jumps when the callee function returns.
   244  		returnAddress uintptr
   245  		// returnStackBasePointerInBytes is the stack base pointer to set on stackContext.stackBasePointerInBytes
   246  		// when the callee function returns.
   247  		returnStackBasePointerInBytes uint64
   248  		// function is the caller *function, and is used to retrieve the stack trace.
   249  		// Note: should be possible to revive *function from returnAddress, but might be costly.
   250  		function *function
   251  	}
   252  
   253  	// Function corresponds to function instance in Wasm, and is created from `code`.
   254  	function struct {
   255  		// See note at top of file before modifying this struct.
   256  
   257  		// codeInitialAddress is the pre-calculated pointer pointing to the initial byte of .codeSegment slice.
   258  		// That mean codeInitialAddress always equals uintptr(unsafe.Pointer(&.codeSegment[0]))
   259  		// and we cache the value (uintptr(unsafe.Pointer(&.codeSegment[0]))) to this field,
   260  		// so we don't need to repeat the calculation on each function call.
   261  		codeInitialAddress uintptr
   262  		// moduleInstance holds the address of source.ModuleInstance.
   263  		moduleInstance *wasm.ModuleInstance
   264  		// typeID is the corresponding wasm.FunctionTypeID for funcType.
   265  		typeID wasm.FunctionTypeID
   266  		// funcType is the function type for this function. Created during compilation.
   267  		funcType *wasm.FunctionType
   268  		// parent holds code from which this is created.
   269  		parent *compiledFunction
   270  	}
   271  
   272  	compiledModule struct {
   273  		// The data that need to be accessed by compiledFunction.parent are
   274  		// separated in an embedded field because we use finalizers to manage
   275  		// the lifecycle of compiledModule instances and having cyclic pointers
   276  		// prevents the Go runtime from calling them, which results in memory
   277  		// leaks since the memory mapped code segments cannot be released.
   278  		//
   279  		// The indirection guarantees that the finalizer set on compiledModule
   280  		// instances can run when all references are gone, and the Go GC can
   281  		// manage to reclaim the compiledCode when all compiledFunction objects
   282  		// referencing it have been freed.
   283  		*compiledCode
   284  		functions []compiledFunction
   285  
   286  		ensureTermination bool
   287  	}
   288  
   289  	compiledCode struct {
   290  		source     *wasm.Module
   291  		executable asm.CodeSegment
   292  	}
   293  
   294  	// compiledFunction corresponds to a function in a module (not instantiated one). This holds the machine code
   295  	// compiled by wazero compiler.
   296  	compiledFunction struct {
   297  		// codeSegment is holding the compiled native code as a byte slice.
   298  		executableOffset uintptr
   299  		// See the doc for codeStaticData type.
   300  		// stackPointerCeil is the max of the stack pointer this function can reach. Lazily applied via maybeGrowStack.
   301  		stackPointerCeil uint64
   302  
   303  		index           wasm.Index
   304  		goFunc          interface{}
   305  		listener        experimental.FunctionListener
   306  		parent          *compiledCode
   307  		sourceOffsetMap sourceOffsetMap
   308  	}
   309  
   310  	// sourceOffsetMap holds the information to retrieve the original offset in
   311  	// the Wasm binary from the offset in the native binary.
   312  	//
   313  	// The fields are implemented as bit-packed arrays of 64 bits integers to
   314  	// reduce the memory footprint. Indexing into such arrays is not as fast as
   315  	// indexing into a simple slice, but the source offset map is intended to be
   316  	// used for debugging, lookups into the arrays should not appear on code
   317  	// paths that are critical to the application performance.
   318  	//
   319  	// The bitpack.OffsetArray fields may be nil, use bitpack.OffsetArrayLen to
   320  	// determine whether they are empty prior to indexing into the arrays to
   321  	// avoid panics caused by accessing nil pointers.
   322  	sourceOffsetMap struct {
   323  		// See note at top of file before modifying this struct.
   324  
   325  		// irOperationOffsetsInNativeBinary is index-correlated with irOperationSourceOffsetsInWasmBinary,
   326  		// and maps each index (corresponding to each IR Operation) to the offset in the compiled native code.
   327  		irOperationOffsetsInNativeBinary bitpack.OffsetArray
   328  		// irOperationSourceOffsetsInWasmBinary is index-correlated with irOperationOffsetsInNativeBinary.
   329  		// See wazeroir.CompilationResult irOperationOffsetsInNativeBinary.
   330  		irOperationSourceOffsetsInWasmBinary bitpack.OffsetArray
   331  	}
   332  
   333  	// functionListenerInvocation captures arguments needed to perform function
   334  	// listener invocations when unwinding the call stack.
   335  	functionListenerInvocation struct {
   336  		experimental.FunctionListener
   337  		def api.FunctionDefinition
   338  	}
   339  )
   340  
   341  // Native code reads/writes Go's structs with the following constants.
   342  // See TestVerifyOffsetValue for how to derive these values.
   343  const (
   344  	// Offsets for moduleEngine.functions
   345  	moduleEngineFunctionsOffset = 0
   346  
   347  	// Offsets for callEngine moduleContext.
   348  	callEngineModuleContextFnOffset                              = 0
   349  	callEngineModuleContextModuleInstanceOffset                  = 8
   350  	callEngineModuleContextGlobalElement0AddressOffset           = 16
   351  	callEngineModuleContextMemoryElement0AddressOffset           = 24
   352  	callEngineModuleContextMemorySliceLenOffset                  = 32
   353  	callEngineModuleContextMemoryInstanceOffset                  = 40
   354  	callEngineModuleContextTablesElement0AddressOffset           = 48
   355  	callEngineModuleContextFunctionsElement0AddressOffset        = 56
   356  	callEngineModuleContextTypeIDsElement0AddressOffset          = 64
   357  	callEngineModuleContextDataInstancesElement0AddressOffset    = 72
   358  	callEngineModuleContextElementInstancesElement0AddressOffset = 80
   359  
   360  	// Offsets for callEngine stackContext.
   361  	callEngineStackContextStackPointerOffset            = 88
   362  	callEngineStackContextStackBasePointerInBytesOffset = 96
   363  	callEngineStackContextStackElement0AddressOffset    = 104
   364  	callEngineStackContextStackLenInBytesOffset         = 112
   365  
   366  	// Offsets for callEngine exitContext.
   367  	callEngineExitContextNativeCallStatusCodeOffset     = 120
   368  	callEngineExitContextBuiltinFunctionCallIndexOffset = 124
   369  	callEngineExitContextReturnAddressOffset            = 128
   370  	callEngineExitContextCallerModuleInstanceOffset     = 136
   371  
   372  	// Offsets for function.
   373  	functionCodeInitialAddressOffset = 0
   374  	functionModuleInstanceOffset     = 8
   375  	functionTypeIDOffset             = 16
   376  	functionSize                     = 40
   377  
   378  	// Offsets for wasm.ModuleInstance.
   379  	moduleInstanceGlobalsOffset          = 24
   380  	moduleInstanceMemoryOffset           = 48
   381  	moduleInstanceTablesOffset           = 56
   382  	moduleInstanceEngineOffset           = 80
   383  	moduleInstanceTypeIDsOffset          = 96
   384  	moduleInstanceDataInstancesOffset    = 120
   385  	moduleInstanceElementInstancesOffset = 144
   386  
   387  	// Offsets for wasm.TableInstance.
   388  	tableInstanceTableOffset    = 0
   389  	tableInstanceTableLenOffset = 8
   390  
   391  	// Offsets for wasm.MemoryInstance.
   392  	memoryInstanceBufferOffset    = 0
   393  	memoryInstanceBufferLenOffset = 8
   394  
   395  	// Offsets for wasm.GlobalInstance.
   396  	globalInstanceValueOffset = 8
   397  
   398  	// Offsets for Go's interface.
   399  	// https://research.swtch.com/interfaces
   400  	// https://github.com/golang/go/blob/release-branch.go1.20/src/runtime/runtime2.go#L207-L210
   401  	interfaceDataOffset = 8
   402  
   403  	// Consts for wasm.DataInstance.
   404  	dataInstanceStructSize = 24
   405  
   406  	// Consts for wasm.ElementInstance.
   407  	elementInstanceStructSize = 24
   408  
   409  	// pointerSizeLog2 satisfies: 1 << pointerSizeLog2 = sizeOf(uintptr)
   410  	pointerSizeLog2 = 3
   411  
   412  	// callFrameDataSizeInUint64 is the size of callFrame struct per 8 bytes (= size of uint64).
   413  	callFrameDataSizeInUint64 = 24 / 8
   414  )
   415  
   416  // nativeCallStatusCode represents the result of `nativecall`.
   417  // This is set by the native code.
   418  type nativeCallStatusCode uint32
   419  
   420  const (
   421  	// nativeCallStatusCodeReturned means the nativecall reaches the end of function, and returns successfully.
   422  	nativeCallStatusCodeReturned nativeCallStatusCode = iota
   423  	// nativeCallStatusCodeCallGoHostFunction means the nativecall returns to make a host function call.
   424  	nativeCallStatusCodeCallGoHostFunction
   425  	// nativeCallStatusCodeCallBuiltInFunction means the nativecall returns to make a builtin function call.
   426  	nativeCallStatusCodeCallBuiltInFunction
   427  	// nativeCallStatusCodeUnreachable means the function invocation reaches "unreachable" instruction.
   428  	nativeCallStatusCodeUnreachable
   429  	// nativeCallStatusCodeInvalidFloatToIntConversion means an invalid conversion of integer to floats happened.
   430  	nativeCallStatusCodeInvalidFloatToIntConversion
   431  	// nativeCallStatusCodeMemoryOutOfBounds means an out-of-bounds memory access happened.
   432  	nativeCallStatusCodeMemoryOutOfBounds
   433  	// nativeCallStatusCodeInvalidTableAccess means either offset to the table was out of bounds of table, or
   434  	// the target element in the table was uninitialized during call_indirect instruction.
   435  	nativeCallStatusCodeInvalidTableAccess
   436  	// nativeCallStatusCodeTypeMismatchOnIndirectCall means the type check failed during call_indirect.
   437  	nativeCallStatusCodeTypeMismatchOnIndirectCall
   438  	nativeCallStatusIntegerOverflow
   439  	nativeCallStatusIntegerDivisionByZero
   440  	nativeCallStatusModuleClosed
   441  )
   442  
   443  // causePanic causes a panic with the corresponding error to the nativeCallStatusCode.
   444  func (s nativeCallStatusCode) causePanic() {
   445  	var err error
   446  	switch s {
   447  	case nativeCallStatusIntegerOverflow:
   448  		err = wasmruntime.ErrRuntimeIntegerOverflow
   449  	case nativeCallStatusIntegerDivisionByZero:
   450  		err = wasmruntime.ErrRuntimeIntegerDivideByZero
   451  	case nativeCallStatusCodeInvalidFloatToIntConversion:
   452  		err = wasmruntime.ErrRuntimeInvalidConversionToInteger
   453  	case nativeCallStatusCodeUnreachable:
   454  		err = wasmruntime.ErrRuntimeUnreachable
   455  	case nativeCallStatusCodeMemoryOutOfBounds:
   456  		err = wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess
   457  	case nativeCallStatusCodeInvalidTableAccess:
   458  		err = wasmruntime.ErrRuntimeInvalidTableAccess
   459  	case nativeCallStatusCodeTypeMismatchOnIndirectCall:
   460  		err = wasmruntime.ErrRuntimeIndirectCallTypeMismatch
   461  	}
   462  	panic(err)
   463  }
   464  
   465  func (s nativeCallStatusCode) String() (ret string) {
   466  	switch s {
   467  	case nativeCallStatusCodeReturned:
   468  		ret = "returned"
   469  	case nativeCallStatusCodeCallGoHostFunction:
   470  		ret = "call_host_function"
   471  	case nativeCallStatusCodeCallBuiltInFunction:
   472  		ret = "call_builtin_function"
   473  	case nativeCallStatusCodeUnreachable:
   474  		ret = "unreachable"
   475  	case nativeCallStatusCodeInvalidFloatToIntConversion:
   476  		ret = "invalid float to int conversion"
   477  	case nativeCallStatusCodeMemoryOutOfBounds:
   478  		ret = "memory out of bounds"
   479  	case nativeCallStatusCodeInvalidTableAccess:
   480  		ret = "invalid table access"
   481  	case nativeCallStatusCodeTypeMismatchOnIndirectCall:
   482  		ret = "type mismatch on indirect call"
   483  	case nativeCallStatusIntegerOverflow:
   484  		ret = "integer overflow"
   485  	case nativeCallStatusIntegerDivisionByZero:
   486  		ret = "integer division by zero"
   487  	case nativeCallStatusModuleClosed:
   488  		ret = "module closed"
   489  	default:
   490  		panic("BUG")
   491  	}
   492  	return
   493  }
   494  
   495  // releaseCompiledModule is a runtime.SetFinalizer function that munmaps the compiledModule.executable.
   496  func releaseCompiledModule(cm *compiledModule) {
   497  	if err := cm.executable.Unmap(); err != nil {
   498  		// munmap failure cannot recover, and happen asynchronously on the
   499  		// finalizer thread. While finalizer functions can return errors,
   500  		// they are ignored.
   501  		panic(fmt.Errorf("compiler: failed to munmap code segment: %w", err))
   502  	}
   503  }
   504  
   505  // CompiledModuleCount implements the same method as documented on wasm.Engine.
   506  func (e *engine) CompiledModuleCount() uint32 {
   507  	return uint32(len(e.codes))
   508  }
   509  
   510  // DeleteCompiledModule implements the same method as documented on wasm.Engine.
   511  func (e *engine) DeleteCompiledModule(module *wasm.Module) {
   512  	e.deleteCompiledModule(module)
   513  }
   514  
   515  // Close implements the same method as documented on wasm.Engine.
   516  func (e *engine) Close() (err error) {
   517  	e.mux.Lock()
   518  	defer e.mux.Unlock()
   519  	// Releasing the references to compiled codes including the memory-mapped machine codes.
   520  	e.codes = nil
   521  	return
   522  }
   523  
   524  // CompileModule implements the same method as documented on wasm.Engine.
   525  func (e *engine) CompileModule(_ context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) error {
   526  	if _, ok, err := e.getCompiledModule(module, listeners); ok { // cache hit!
   527  		return nil
   528  	} else if err != nil {
   529  		return err
   530  	}
   531  
   532  	irCompiler, err := wazeroir.NewCompiler(e.enabledFeatures, callFrameDataSizeInUint64, module, ensureTermination)
   533  	if err != nil {
   534  		return err
   535  	}
   536  
   537  	var withGoFunc bool
   538  	localFuncs, importedFuncs := len(module.FunctionSection), module.ImportFunctionCount
   539  	cm := &compiledModule{
   540  		compiledCode: &compiledCode{
   541  			source: module,
   542  		},
   543  		functions:         make([]compiledFunction, localFuncs),
   544  		ensureTermination: ensureTermination,
   545  	}
   546  
   547  	if localFuncs == 0 {
   548  		return e.addCompiledModule(module, cm, withGoFunc)
   549  	}
   550  
   551  	// As this uses mmap, we need to munmap on the compiled machine code when it's GCed.
   552  	e.setFinalizer(cm, releaseCompiledModule)
   553  	ln := len(listeners)
   554  	cmp := newCompiler()
   555  	asmNodes := new(asmNodes)
   556  	offsets := new(offsets)
   557  
   558  	// The executable code is allocated in memory mappings held by the
   559  	// CodeSegment, which gros on demand when it exhausts its capacity.
   560  	var executable asm.CodeSegment
   561  	defer func() {
   562  		// At the end of the function, the executable is set on the compiled
   563  		// module and the local variable cleared; until then, the function owns
   564  		// the memory mapping and is reponsible for clearing it if it returns
   565  		// due to an error. Note that an error at this stage is not recoverable
   566  		// so we panic if we fail to unmap the memory segment.
   567  		if err := executable.Unmap(); err != nil {
   568  			panic(fmt.Errorf("compiler: failed to munmap code segment: %w", err))
   569  		}
   570  	}()
   571  
   572  	for i := range module.CodeSection {
   573  		typ := &module.TypeSection[module.FunctionSection[i]]
   574  		buf := executable.NextCodeSection()
   575  		funcIndex := wasm.Index(i)
   576  		compiledFn := &cm.functions[i]
   577  		compiledFn.executableOffset = executable.Size()
   578  		compiledFn.parent = cm.compiledCode
   579  		compiledFn.index = importedFuncs + funcIndex
   580  		if i < ln {
   581  			compiledFn.listener = listeners[i]
   582  		}
   583  
   584  		if codeSeg := &module.CodeSection[i]; codeSeg.GoFunc != nil {
   585  			cmp.Init(typ, nil, compiledFn.listener != nil)
   586  			withGoFunc = true
   587  			if err = compileGoDefinedHostFunction(buf, cmp); err != nil {
   588  				def := module.FunctionDefinition(compiledFn.index)
   589  				return fmt.Errorf("error compiling host go func[%s]: %w", def.DebugName(), err)
   590  			}
   591  			compiledFn.goFunc = codeSeg.GoFunc
   592  		} else {
   593  			ir, err := irCompiler.Next()
   594  			if err != nil {
   595  				return fmt.Errorf("failed to lower func[%d]: %v", i, err)
   596  			}
   597  			cmp.Init(typ, ir, compiledFn.listener != nil)
   598  
   599  			compiledFn.stackPointerCeil, compiledFn.sourceOffsetMap, err = compileWasmFunction(buf, cmp, ir, asmNodes, offsets)
   600  			if err != nil {
   601  				def := module.FunctionDefinition(compiledFn.index)
   602  				return fmt.Errorf("error compiling wasm func[%s]: %w", def.DebugName(), err)
   603  			}
   604  		}
   605  	}
   606  
   607  	if runtime.GOARCH == "arm64" {
   608  		// On arm64, we cannot give all of rwx at the same time, so we change it to exec.
   609  		if err := platform.MprotectRX(executable.Bytes()); err != nil {
   610  			return err
   611  		}
   612  	}
   613  	cm.executable, executable = executable, asm.CodeSegment{}
   614  	return e.addCompiledModule(module, cm, withGoFunc)
   615  }
   616  
   617  // NewModuleEngine implements the same method as documented on wasm.Engine.
   618  func (e *engine) NewModuleEngine(module *wasm.Module, instance *wasm.ModuleInstance) (wasm.ModuleEngine, error) {
   619  	me := &moduleEngine{
   620  		functions: make([]function, len(module.FunctionSection)+int(module.ImportFunctionCount)),
   621  	}
   622  
   623  	// Note: imported functions are resolved in moduleEngine.ResolveImportedFunction.
   624  
   625  	cm, ok, err := e.getCompiledModule(module,
   626  		// listeners arg is not needed here since NewModuleEngine is called after CompileModule which
   627  		// ensures the association of listener with *code.
   628  		nil)
   629  	if !ok {
   630  		return nil, errors.New("source module must be compiled before instantiation")
   631  	} else if err != nil {
   632  		return nil, err
   633  	}
   634  
   635  	for i := range cm.functions {
   636  		c := &cm.functions[i]
   637  		offset := int(module.ImportFunctionCount) + i
   638  		typeIndex := module.FunctionSection[i]
   639  		me.functions[offset] = function{
   640  			codeInitialAddress: cm.executable.Addr() + c.executableOffset,
   641  			moduleInstance:     instance,
   642  			typeID:             instance.TypeIDs[typeIndex],
   643  			funcType:           &module.TypeSection[typeIndex],
   644  			parent:             c,
   645  		}
   646  	}
   647  
   648  	me.module = cm
   649  	return me, nil
   650  }
   651  
   652  // ResolveImportedFunction implements wasm.ModuleEngine.
   653  func (e *moduleEngine) ResolveImportedFunction(index, indexInImportedModule wasm.Index, importedModuleEngine wasm.ModuleEngine) {
   654  	imported := importedModuleEngine.(*moduleEngine)
   655  	// Copies the content from the import target moduleEngine.
   656  	e.functions[index] = imported.functions[indexInImportedModule]
   657  }
   658  
   659  // GetGlobalValue implements the same method as documented on wasm.ModuleEngine.
   660  func (e *moduleEngine) GetGlobalValue(wasm.Index) (lo, hi uint64) {
   661  	panic("BUG: GetGlobalValue should never be called on compiler mode")
   662  }
   663  
   664  // OwnsGlobals implements the same method as documented on wasm.ModuleEngine.
   665  func (e *moduleEngine) OwnsGlobals() bool { return false }
   666  
   667  // ResolveImportedMemory implements wasm.ModuleEngine.
   668  func (e *moduleEngine) ResolveImportedMemory(wasm.ModuleEngine) {}
   669  
   670  // FunctionInstanceReference implements the same method as documented on wasm.ModuleEngine.
   671  func (e *moduleEngine) FunctionInstanceReference(funcIndex wasm.Index) wasm.Reference {
   672  	return uintptr(unsafe.Pointer(&e.functions[funcIndex]))
   673  }
   674  
   675  // DoneInstantiation implements wasm.ModuleEngine.
   676  func (e *moduleEngine) DoneInstantiation() {}
   677  
   678  // NewFunction implements wasm.ModuleEngine.
   679  func (e *moduleEngine) NewFunction(index wasm.Index) api.Function {
   680  	return e.newFunction(&e.functions[index])
   681  }
   682  
   683  func (e *moduleEngine) newFunction(f *function) api.Function {
   684  	initStackSize := initialStackSize
   685  	if initialStackSize < f.parent.stackPointerCeil {
   686  		initStackSize = f.parent.stackPointerCeil * 2
   687  	}
   688  	return e.newCallEngine(initStackSize, f)
   689  }
   690  
   691  // LookupFunction implements the same method as documented on wasm.ModuleEngine.
   692  func (e *moduleEngine) LookupFunction(t *wasm.TableInstance, typeId wasm.FunctionTypeID, tableOffset wasm.Index) (*wasm.ModuleInstance, wasm.Index) {
   693  	if tableOffset >= uint32(len(t.References)) || t.Type != wasm.RefTypeFuncref {
   694  		panic(wasmruntime.ErrRuntimeInvalidTableAccess)
   695  	}
   696  	rawPtr := t.References[tableOffset]
   697  	if rawPtr == 0 {
   698  		panic(wasmruntime.ErrRuntimeInvalidTableAccess)
   699  	}
   700  
   701  	tf := functionFromUintptr(rawPtr)
   702  	if tf.typeID != typeId {
   703  		panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch)
   704  	}
   705  	return tf.moduleInstance, tf.parent.index
   706  }
   707  
   708  // functionFromUintptr resurrects the original *function from the given uintptr
   709  // which comes from either funcref table or OpcodeRefFunc instruction.
   710  func functionFromUintptr(ptr uintptr) *function {
   711  	// Wraps ptrs as the double pointer in order to avoid the unsafe access as detected by race detector.
   712  	//
   713  	// For example, if we have (*function)(unsafe.Pointer(ptr)) instead, then the race detector's "checkptr"
   714  	// subroutine wanrs as "checkptr: pointer arithmetic result points to invalid allocation"
   715  	// https://github.com/golang/go/blob/1ce7fcf139417d618c2730010ede2afb41664211/src/runtime/checkptr.go#L69
   716  	var wrapped *uintptr = &ptr
   717  	return *(**function)(unsafe.Pointer(wrapped))
   718  }
   719  
   720  // Definition implements the same method as documented on wasm.ModuleEngine.
   721  func (ce *callEngine) Definition() api.FunctionDefinition {
   722  	return ce.initialFn.definition()
   723  }
   724  
   725  func (f *function) definition() api.FunctionDefinition {
   726  	compiled := f.parent
   727  	return compiled.parent.source.FunctionDefinition(compiled.index)
   728  }
   729  
   730  // Call implements the same method as documented on wasm.ModuleEngine.
   731  func (ce *callEngine) Call(ctx context.Context, params ...uint64) (results []uint64, err error) {
   732  	ft := ce.initialFn.funcType
   733  	if n := ft.ParamNumInUint64; n != len(params) {
   734  		return nil, fmt.Errorf("expected %d params, but passed %d", n, len(params))
   735  	}
   736  	return ce.call(ctx, params, nil)
   737  }
   738  
   739  // CallWithStack implements the same method as documented on wasm.ModuleEngine.
   740  func (ce *callEngine) CallWithStack(ctx context.Context, stack []uint64) error {
   741  	params, results, err := wasm.SplitCallStack(ce.initialFn.funcType, stack)
   742  	if err != nil {
   743  		return err
   744  	}
   745  	_, err = ce.call(ctx, params, results)
   746  	return err
   747  }
   748  
   749  func (ce *callEngine) call(ctx context.Context, params, results []uint64) (_ []uint64, err error) {
   750  	m := ce.initialFn.moduleInstance
   751  	if ce.module.ensureTermination {
   752  		select {
   753  		case <-ctx.Done():
   754  			// If the provided context is already done, close the call context
   755  			// and return the error.
   756  			m.CloseWithCtxErr(ctx)
   757  			return nil, m.FailIfClosed()
   758  		default:
   759  		}
   760  	}
   761  
   762  	// We ensure that this Call method never panics as
   763  	// this Call method is indirectly invoked by embedders via store.CallFunction,
   764  	// and we have to make sure that all the runtime errors, including the one happening inside
   765  	// host functions, will be captured as errors, not panics.
   766  	defer func() {
   767  		err = ce.deferredOnCall(ctx, m, recover())
   768  		if err == nil {
   769  			// If the module closed during the call, and the call didn't err for another reason, set an ExitError.
   770  			err = m.FailIfClosed()
   771  		}
   772  		// Ensure that the compiled module will never be GC'd before this method returns.
   773  		runtime.KeepAlive(ce.module)
   774  	}()
   775  
   776  	ft := ce.initialFn.funcType
   777  	ce.initializeStack(ft, params)
   778  
   779  	if ce.module.ensureTermination {
   780  		done := m.CloseModuleOnCanceledOrTimeout(ctx)
   781  		defer done()
   782  	}
   783  
   784  	ce.execWasmFunction(ctx, m)
   785  
   786  	// This returns a safe copy of the results, instead of a slice view. If we
   787  	// returned a re-slice, the caller could accidentally or purposefully
   788  	// corrupt the stack of subsequent calls.
   789  	if results == nil && ft.ResultNumInUint64 > 0 {
   790  		results = make([]uint64, ft.ResultNumInUint64)
   791  	}
   792  	copy(results, ce.stack)
   793  	return results, nil
   794  }
   795  
   796  // initializeStack initializes callEngine.stack before entering native code.
   797  //
   798  // The stack must look like, if len(params) < len(results):
   799  //
   800  //	[arg0, arg1, ..., argN, 0, 0, 0, ...
   801  //	                       {       } ^
   802  //	                       callFrame |
   803  //	                                 |
   804  //	                            stackPointer
   805  //
   806  // else:
   807  //
   808  //	[arg0, arg1, ..., argN, _, _, _,  0, 0, 0, ...
   809  //	                      |        | {       }  ^
   810  //	                      |reserved| callFrame  |
   811  //	                      |        |            |
   812  //	                      |-------->       stackPointer
   813  //	                 len(results)-len(params)
   814  //
   815  //		 where we reserve the slots below the callframe with the length len(results)-len(params).
   816  //
   817  // Note: callFrame {  } is zeroed to indicate that the initial "caller" is this callEngine, not the Wasm function.
   818  //
   819  // See callEngine.stack as well.
   820  func (ce *callEngine) initializeStack(tp *wasm.FunctionType, args []uint64) {
   821  	for _, v := range args {
   822  		ce.pushValue(v)
   823  	}
   824  
   825  	ce.stackPointer = uint64(callFrameOffset(tp))
   826  
   827  	for i := 0; i < callFrameDataSizeInUint64; i++ {
   828  		ce.stack[ce.stackPointer] = 0
   829  		ce.stackPointer++
   830  	}
   831  }
   832  
   833  // callFrameOffset returns the offset of the call frame from the stack base pointer.
   834  //
   835  // See the diagram in callEngine.stack.
   836  func callFrameOffset(funcType *wasm.FunctionType) (ret int) {
   837  	ret = funcType.ResultNumInUint64
   838  	if ret < funcType.ParamNumInUint64 {
   839  		ret = funcType.ParamNumInUint64
   840  	}
   841  	return
   842  }
   843  
   844  // deferredOnCall takes the recovered value `recovered`, and wraps it
   845  // with the call frame stack traces when not nil. This also resets
   846  // the state of callEngine so that it can be used for the subsequent calls.
   847  //
   848  // This is defined for testability.
   849  func (ce *callEngine) deferredOnCall(ctx context.Context, m *wasm.ModuleInstance, recovered interface{}) (err error) {
   850  	if recovered != nil {
   851  		builder := wasmdebug.NewErrorBuilder()
   852  
   853  		// Unwinds call frames from the values stack, starting from the
   854  		// current function `ce.fn`, and the current stack base pointer `ce.stackBasePointerInBytes`.
   855  		fn := ce.fn
   856  		pc := uint64(ce.returnAddress)
   857  		stackBasePointer := int(ce.stackBasePointerInBytes >> 3)
   858  		functionListeners := make([]functionListenerInvocation, 0, 16)
   859  
   860  		for {
   861  			def := fn.definition()
   862  
   863  			// sourceInfo holds the source code information corresponding to the frame.
   864  			// It is not empty only when the DWARF is enabled.
   865  			var sources []string
   866  			if p := fn.parent; p.parent.executable.Bytes() != nil {
   867  				if fn.parent.sourceOffsetMap.irOperationSourceOffsetsInWasmBinary != nil {
   868  					offset := fn.getSourceOffsetInWasmBinary(pc)
   869  					sources = p.parent.source.DWARFLines.Line(offset)
   870  				}
   871  			}
   872  			builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources)
   873  
   874  			if fn.parent.listener != nil {
   875  				functionListeners = append(functionListeners, functionListenerInvocation{
   876  					FunctionListener: fn.parent.listener,
   877  					def:              fn.definition(),
   878  				})
   879  			}
   880  
   881  			callFrameOffset := callFrameOffset(fn.funcType)
   882  			if stackBasePointer != 0 {
   883  				frame := *(*callFrame)(unsafe.Pointer(&ce.stack[stackBasePointer+callFrameOffset]))
   884  				fn = frame.function
   885  				pc = uint64(frame.returnAddress)
   886  				stackBasePointer = int(frame.returnStackBasePointerInBytes >> 3)
   887  			} else { // base == 0 means that this was the last call frame stacked.
   888  				break
   889  			}
   890  		}
   891  
   892  		err = builder.FromRecovered(recovered)
   893  		for i := range functionListeners {
   894  			functionListeners[i].Abort(ctx, m, functionListeners[i].def, err)
   895  		}
   896  	}
   897  
   898  	// Allows the reuse of CallEngine.
   899  	ce.stackBasePointerInBytes, ce.stackPointer, ce.moduleInstance = 0, 0, nil
   900  	ce.moduleContext.fn = ce.initialFn
   901  	return
   902  }
   903  
   904  // getSourceOffsetInWasmBinary returns the corresponding offset in the original Wasm binary's code section
   905  // for the given pc (which is an absolute address in the memory).
   906  // If needPreviousInstr equals true, this returns the previous instruction's offset for the given pc.
   907  func (f *function) getSourceOffsetInWasmBinary(pc uint64) uint64 {
   908  	srcMap := &f.parent.sourceOffsetMap
   909  	n := bitpack.OffsetArrayLen(srcMap.irOperationOffsetsInNativeBinary) + 1
   910  
   911  	// Calculate the offset in the compiled native binary.
   912  	pcOffsetInNativeBinary := pc - uint64(f.codeInitialAddress)
   913  
   914  	// Then, do the binary search on the list of offsets in the native binary
   915  	// for all the IR operations. This returns the index of the *next* IR
   916  	// operation of the one corresponding to the origin of this pc.
   917  	// See sort.Search.
   918  	//
   919  	// TODO: the underlying implementation of irOperationOffsetsInNativeBinary
   920  	// uses uses delta encoding an calls to the Index method might require a
   921  	// O(N)  scan of the underlying array, turning binary search into a
   922  	// O(N*log(N)) operation. If this code path ends up being a bottleneck,
   923  	// we could add a Search method on the bitpack.OffsetArray types to delegate
   924  	// the lookup to the underlying data structure, allowing for the selection
   925  	// of a more optimized version of the algorithm. If you do so, please add a
   926  	// benchmark to verify the impact on compute time.
   927  	index := sort.Search(n, func(i int) bool {
   928  		if i == n-1 {
   929  			return true
   930  		}
   931  		return srcMap.irOperationOffsetsInNativeBinary.Index(i) >= pcOffsetInNativeBinary
   932  	})
   933  	if index == 0 && bitpack.OffsetArrayLen(srcMap.irOperationSourceOffsetsInWasmBinary) > 0 {
   934  		// When pc is the beginning of the function, the next IR
   935  		// operation (returned by sort.Search) is the first of the
   936  		// offset map.
   937  		return srcMap.irOperationSourceOffsetsInWasmBinary.Index(0)
   938  	}
   939  
   940  	if index == n || index == 0 { // This case, somehow pc is not found in the source offset map.
   941  		return 0
   942  	} else {
   943  		return srcMap.irOperationSourceOffsetsInWasmBinary.Index(index - 1)
   944  	}
   945  }
   946  
   947  func NewEngine(_ context.Context, enabledFeatures api.CoreFeatures, fileCache filecache.Cache) wasm.Engine {
   948  	return newEngine(enabledFeatures, fileCache)
   949  }
   950  
   951  func newEngine(enabledFeatures api.CoreFeatures, fileCache filecache.Cache) *engine {
   952  	return &engine{
   953  		enabledFeatures: enabledFeatures,
   954  		codes:           map[wasm.ModuleID]*compiledModule{},
   955  		setFinalizer:    runtime.SetFinalizer,
   956  		fileCache:       fileCache,
   957  		wazeroVersion:   version.GetWazeroVersion(),
   958  	}
   959  }
   960  
   961  // Do not make this variable as constant, otherwise there would be
   962  // dangerous memory access from native code.
   963  //
   964  // Background: Go has a mechanism called "goroutine stack-shrink" where Go
   965  // runtime shrinks Goroutine's stack when it is GCing. Shrinking means that
   966  // all the contents on the goroutine stack will be relocated by runtime,
   967  // Therefore, the memory address of these contents change undeterministically.
   968  // Not only shrinks, but also Go runtime grows the goroutine stack at any point
   969  // of function call entries, which also might end up relocating contents.
   970  //
   971  // On the other hand, we hold pointers to the data region of value stack and
   972  // call-frame stack slices and use these raw pointers from native code.
   973  // Therefore, it is dangerous if these two stacks are allocated on stack
   974  // as these stack's address might be changed by Goroutine which we cannot
   975  // detect.
   976  //
   977  // By declaring these values as `var`, slices created via `make([]..., var)`
   978  // will never be allocated on stack [1]. This means accessing these slices via
   979  // raw pointers is safe: As of version 1.21, Go's garbage collector never relocates
   980  // heap-allocated objects (aka no compaction of memory [2]).
   981  //
   982  // On Go upgrades, re-validate heap-allocation via `go build -gcflags='-m' ./internal/engine/compiler/...`.
   983  //
   984  //	[1] https://github.com/golang/go/blob/c19c4c566c63818dfd059b352e52c4710eecf14d/src/cmd/compile/internal/escape/utils.go#L213-L215
   985  //	[2] https://github.com/golang/go/blob/c19c4c566c63818dfd059b352e52c4710eecf14d/src/runtime/mgc.go#L9
   986  //	[3] https://mayurwadekar2.medium.com/escape-analysis-in-golang-ee40a1c064c1
   987  //	[4] https://medium.com/@yulang.chu/go-stack-or-heap-2-slices-which-keep-in-stack-have-limitation-of-size-b3f3adfd6190
   988  var initialStackSize uint64 = 512
   989  
   990  func (e *moduleEngine) newCallEngine(stackSize uint64, fn *function) *callEngine {
   991  	ce := &callEngine{
   992  		stack:         make([]uint64, stackSize),
   993  		archContext:   newArchContext(),
   994  		initialFn:     fn,
   995  		moduleContext: moduleContext{fn: fn},
   996  		module:        e.module,
   997  	}
   998  
   999  	stackHeader := (*reflect.SliceHeader)(unsafe.Pointer(&ce.stack))
  1000  	ce.stackContext = stackContext{
  1001  		stackElement0Address: stackHeader.Data,
  1002  		stackLenInBytes:      uint64(stackHeader.Len) << 3,
  1003  	}
  1004  	return ce
  1005  }
  1006  
  1007  func (ce *callEngine) popValue() (ret uint64) {
  1008  	ce.stackContext.stackPointer--
  1009  	ret = ce.stack[ce.stackTopIndex()]
  1010  	return
  1011  }
  1012  
  1013  func (ce *callEngine) pushValue(v uint64) {
  1014  	ce.stack[ce.stackTopIndex()] = v
  1015  	ce.stackContext.stackPointer++
  1016  }
  1017  
  1018  func (ce *callEngine) stackTopIndex() uint64 {
  1019  	return ce.stackContext.stackPointer + (ce.stackContext.stackBasePointerInBytes >> 3)
  1020  }
  1021  
  1022  const (
  1023  	builtinFunctionIndexMemoryGrow wasm.Index = iota
  1024  	builtinFunctionIndexGrowStack
  1025  	builtinFunctionIndexTableGrow
  1026  	builtinFunctionIndexFunctionListenerBefore
  1027  	builtinFunctionIndexFunctionListenerAfter
  1028  	builtinFunctionIndexCheckExitCode
  1029  	// builtinFunctionIndexBreakPoint is internal (only for wazero developers). Disabled by default.
  1030  	builtinFunctionIndexBreakPoint
  1031  )
  1032  
  1033  func (ce *callEngine) execWasmFunction(ctx context.Context, m *wasm.ModuleInstance) {
  1034  	codeAddr := ce.initialFn.codeInitialAddress
  1035  	modAddr := ce.initialFn.moduleInstance
  1036  
  1037  entry:
  1038  	{
  1039  		// Call into the native code.
  1040  		nativecall(codeAddr, ce, modAddr)
  1041  
  1042  		// Check the status code from Compiler code.
  1043  		switch status := ce.exitContext.statusCode; status {
  1044  		case nativeCallStatusCodeReturned:
  1045  		case nativeCallStatusCodeCallGoHostFunction:
  1046  			calleeHostFunction := ce.moduleContext.fn
  1047  			base := int(ce.stackBasePointerInBytes >> 3)
  1048  
  1049  			// In the compiler engine, ce.stack has enough capacity for the
  1050  			// max of param or result length, so we don't need to grow when
  1051  			// there are more results than parameters.
  1052  			stackLen := calleeHostFunction.funcType.ParamNumInUint64
  1053  			if resultLen := calleeHostFunction.funcType.ResultNumInUint64; resultLen > stackLen {
  1054  				stackLen = resultLen
  1055  			}
  1056  			stack := ce.stack[base : base+stackLen]
  1057  
  1058  			fn := calleeHostFunction.parent.goFunc
  1059  			switch fn := fn.(type) {
  1060  			case api.GoModuleFunction:
  1061  				fn.Call(ctx, ce.callerModuleInstance, stack)
  1062  			case api.GoFunction:
  1063  				fn.Call(ctx, stack)
  1064  			}
  1065  
  1066  			codeAddr, modAddr = ce.returnAddress, ce.moduleInstance
  1067  			goto entry
  1068  		case nativeCallStatusCodeCallBuiltInFunction:
  1069  			caller := ce.moduleContext.fn
  1070  			switch ce.exitContext.builtinFunctionCallIndex {
  1071  			case builtinFunctionIndexMemoryGrow:
  1072  				ce.builtinFunctionMemoryGrow(caller.moduleInstance.MemoryInstance)
  1073  			case builtinFunctionIndexGrowStack:
  1074  				ce.builtinFunctionGrowStack(caller.parent.stackPointerCeil)
  1075  			case builtinFunctionIndexTableGrow:
  1076  				ce.builtinFunctionTableGrow(caller.moduleInstance.Tables)
  1077  			case builtinFunctionIndexFunctionListenerBefore:
  1078  				ce.builtinFunctionFunctionListenerBefore(ctx, m, caller)
  1079  			case builtinFunctionIndexFunctionListenerAfter:
  1080  				ce.builtinFunctionFunctionListenerAfter(ctx, m, caller)
  1081  			case builtinFunctionIndexCheckExitCode:
  1082  				// Note: this operation must be done in Go, not native code. The reason is that
  1083  				// native code cannot be preempted and that means it can block forever if there are not
  1084  				// enough OS threads (which we don't have control over).
  1085  				if err := m.FailIfClosed(); err != nil {
  1086  					panic(err)
  1087  				}
  1088  			}
  1089  			if false {
  1090  				if ce.exitContext.builtinFunctionCallIndex == builtinFunctionIndexBreakPoint {
  1091  					runtime.Breakpoint()
  1092  				}
  1093  			}
  1094  
  1095  			codeAddr, modAddr = ce.returnAddress, ce.moduleInstance
  1096  			goto entry
  1097  		default:
  1098  			status.causePanic()
  1099  		}
  1100  	}
  1101  }
  1102  
  1103  // callStackCeiling is the maximum WebAssembly call frame stack height. This allows wazero to raise
  1104  // wasm.ErrCallStackOverflow instead of overflowing the Go runtime.
  1105  //
  1106  // The default value should suffice for most use cases. Those wishing to change this can via `go build -ldflags`.
  1107  //
  1108  // TODO: allows to configure this via context?
  1109  var callStackCeiling = uint64(5000000) // in uint64 (8 bytes) == 40000000 bytes in total == 40mb.
  1110  
  1111  func (ce *callEngine) builtinFunctionGrowStack(stackPointerCeil uint64) {
  1112  	oldLen := uint64(len(ce.stack))
  1113  	if callStackCeiling < oldLen {
  1114  		panic(wasmruntime.ErrRuntimeStackOverflow)
  1115  	}
  1116  
  1117  	// Extends the stack's length to oldLen*2+stackPointerCeil.
  1118  	newLen := oldLen<<1 + (stackPointerCeil)
  1119  	newStack := make([]uint64, newLen)
  1120  	top := ce.stackTopIndex()
  1121  	copy(newStack[:top], ce.stack[:top])
  1122  	ce.stack = newStack
  1123  	stackHeader := (*reflect.SliceHeader)(unsafe.Pointer(&ce.stack))
  1124  	ce.stackContext.stackElement0Address = stackHeader.Data
  1125  	ce.stackContext.stackLenInBytes = newLen << 3
  1126  }
  1127  
  1128  func (ce *callEngine) builtinFunctionMemoryGrow(mem *wasm.MemoryInstance) {
  1129  	newPages := ce.popValue()
  1130  
  1131  	if res, ok := mem.Grow(uint32(newPages)); !ok {
  1132  		ce.pushValue(uint64(0xffffffff)) // = -1 in signed 32-bit integer.
  1133  	} else {
  1134  		ce.pushValue(uint64(res))
  1135  	}
  1136  
  1137  	// Update the moduleContext fields as they become stale after the update ^^.
  1138  	bufSliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&mem.Buffer))
  1139  	ce.moduleContext.memorySliceLen = uint64(bufSliceHeader.Len)
  1140  	ce.moduleContext.memoryElement0Address = bufSliceHeader.Data
  1141  }
  1142  
  1143  func (ce *callEngine) builtinFunctionTableGrow(tables []*wasm.TableInstance) {
  1144  	tableIndex := uint32(ce.popValue())
  1145  	table := tables[tableIndex] // verified not to be out of range by the func validation at compilation phase.
  1146  	num := ce.popValue()
  1147  	ref := ce.popValue()
  1148  	res := table.Grow(uint32(num), uintptr(ref))
  1149  	ce.pushValue(uint64(res))
  1150  }
  1151  
  1152  // stackIterator implements experimental.StackIterator.
  1153  type stackIterator struct {
  1154  	stack   []uint64
  1155  	fn      *function
  1156  	base    int
  1157  	pc      uint64
  1158  	started bool
  1159  }
  1160  
  1161  func (si *stackIterator) reset(stack []uint64, fn *function, base int, pc uint64) {
  1162  	si.stack = stack
  1163  	si.fn = fn
  1164  	si.base = base
  1165  	si.pc = pc
  1166  	si.started = false
  1167  }
  1168  
  1169  func (si *stackIterator) clear() {
  1170  	si.stack = nil
  1171  	si.fn = nil
  1172  	si.base = 0
  1173  	si.started = false
  1174  }
  1175  
  1176  // Next implements the same method as documented on experimental.StackIterator.
  1177  func (si *stackIterator) Next() bool {
  1178  	if !si.started {
  1179  		si.started = true
  1180  		return true
  1181  	}
  1182  
  1183  	if si.fn == nil || si.base == 0 {
  1184  		return false
  1185  	}
  1186  
  1187  	frame := si.base + callFrameOffset(si.fn.funcType)
  1188  	si.pc = si.stack[frame+0]
  1189  	si.base = int(si.stack[frame+1] >> 3)
  1190  	// *function lives in the third field of callFrame struct. This must be
  1191  	// aligned with the definition of callFrame struct.
  1192  	si.fn = *(**function)(unsafe.Pointer(&si.stack[frame+2]))
  1193  	return si.fn != nil
  1194  }
  1195  
  1196  // ProgramCounter implements the same method as documented on experimental.StackIterator.
  1197  func (si *stackIterator) ProgramCounter() experimental.ProgramCounter {
  1198  	return experimental.ProgramCounter(si.pc)
  1199  }
  1200  
  1201  // Function implements the same method as documented on experimental.StackIterator.
  1202  func (si *stackIterator) Function() experimental.InternalFunction {
  1203  	return internalFunction{si.fn}
  1204  }
  1205  
  1206  // internalFunction implements experimental.InternalFunction.
  1207  type internalFunction struct{ *function }
  1208  
  1209  // Definition implements the same method as documented on experimental.InternalFunction.
  1210  func (f internalFunction) Definition() api.FunctionDefinition {
  1211  	return f.definition()
  1212  }
  1213  
  1214  // SourceOffsetForPC implements the same method as documented on experimental.InternalFunction.
  1215  func (f internalFunction) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 {
  1216  	p := f.parent
  1217  	if bitpack.OffsetArrayLen(p.sourceOffsetMap.irOperationSourceOffsetsInWasmBinary) == 0 {
  1218  		return 0 // source not available
  1219  	}
  1220  	return f.getSourceOffsetInWasmBinary(uint64(pc))
  1221  }
  1222  
  1223  func (ce *callEngine) builtinFunctionFunctionListenerBefore(ctx context.Context, mod api.Module, fn *function) {
  1224  	base := int(ce.stackBasePointerInBytes >> 3)
  1225  	pc := uint64(ce.returnAddress)
  1226  	ce.stackIterator.reset(ce.stack, fn, base, pc)
  1227  
  1228  	params := ce.stack[base : base+fn.funcType.ParamNumInUint64]
  1229  	fn.parent.listener.Before(ctx, mod, fn.definition(), params, &ce.stackIterator)
  1230  
  1231  	ce.stackIterator.clear()
  1232  }
  1233  
  1234  func (ce *callEngine) builtinFunctionFunctionListenerAfter(ctx context.Context, mod api.Module, fn *function) {
  1235  	base := int(ce.stackBasePointerInBytes >> 3)
  1236  	fn.parent.listener.After(ctx, mod, fn.definition(), ce.stack[base:base+fn.funcType.ResultNumInUint64])
  1237  }
  1238  
  1239  func compileGoDefinedHostFunction(buf asm.Buffer, cmp compiler) error {
  1240  	if err := cmp.compileGoDefinedHostFunction(); err != nil {
  1241  		return err
  1242  	}
  1243  	_, err := cmp.compile(buf)
  1244  	return err
  1245  }
  1246  
  1247  type asmNodes struct {
  1248  	nodes []asm.Node
  1249  }
  1250  
  1251  type offsets struct {
  1252  	values []uint64
  1253  }
  1254  
  1255  func compileWasmFunction(buf asm.Buffer, cmp compiler, ir *wazeroir.CompilationResult, asmNodes *asmNodes, offsets *offsets) (spCeil uint64, sm sourceOffsetMap, err error) {
  1256  	if err = cmp.compilePreamble(); err != nil {
  1257  		err = fmt.Errorf("failed to emit preamble: %w", err)
  1258  		return
  1259  	}
  1260  
  1261  	needSourceOffsets := len(ir.IROperationSourceOffsetsInWasmBinary) > 0
  1262  	var irOpBegins []asm.Node
  1263  	if needSourceOffsets {
  1264  		irOpBegins = append(asmNodes.nodes[:0], make([]asm.Node, len(ir.Operations))...)
  1265  		defer func() { asmNodes.nodes = irOpBegins }()
  1266  	}
  1267  
  1268  	var skip bool
  1269  	for i := range ir.Operations {
  1270  		op := &ir.Operations[i]
  1271  		if needSourceOffsets {
  1272  			// If this compilation requires source offsets for DWARF based back trace,
  1273  			// we emit a NOP node at the beginning of each IR operation to get the
  1274  			// binary offset of the beginning of the corresponding compiled native code.
  1275  			irOpBegins[i] = cmp.compileNOP()
  1276  		}
  1277  
  1278  		// Compiler determines whether skip the entire label.
  1279  		// For example, if the label doesn't have any caller,
  1280  		// we don't need to generate native code at all as we never reach the region.
  1281  		if op.Kind == wazeroir.OperationKindLabel {
  1282  			skip = cmp.compileLabel(op)
  1283  		}
  1284  		if skip {
  1285  			continue
  1286  		}
  1287  
  1288  		if false {
  1289  			fmt.Printf("compiling op=%s: %s\n", op.Kind, cmp)
  1290  		}
  1291  		switch op.Kind {
  1292  		case wazeroir.OperationKindUnreachable:
  1293  			err = cmp.compileUnreachable()
  1294  		case wazeroir.OperationKindLabel:
  1295  		// label op is already handled ^^.
  1296  		case wazeroir.OperationKindBr:
  1297  			err = cmp.compileBr(op)
  1298  		case wazeroir.OperationKindBrIf:
  1299  			err = cmp.compileBrIf(op)
  1300  		case wazeroir.OperationKindBrTable:
  1301  			err = cmp.compileBrTable(op)
  1302  		case wazeroir.OperationKindCall:
  1303  			err = cmp.compileCall(op)
  1304  		case wazeroir.OperationKindCallIndirect:
  1305  			err = cmp.compileCallIndirect(op)
  1306  		case wazeroir.OperationKindDrop:
  1307  			err = cmp.compileDrop(op)
  1308  		case wazeroir.OperationKindSelect:
  1309  			err = cmp.compileSelect(op)
  1310  		case wazeroir.OperationKindPick:
  1311  			err = cmp.compilePick(op)
  1312  		case wazeroir.OperationKindSet:
  1313  			err = cmp.compileSet(op)
  1314  		case wazeroir.OperationKindGlobalGet:
  1315  			err = cmp.compileGlobalGet(op)
  1316  		case wazeroir.OperationKindGlobalSet:
  1317  			err = cmp.compileGlobalSet(op)
  1318  		case wazeroir.OperationKindLoad:
  1319  			err = cmp.compileLoad(op)
  1320  		case wazeroir.OperationKindLoad8:
  1321  			err = cmp.compileLoad8(op)
  1322  		case wazeroir.OperationKindLoad16:
  1323  			err = cmp.compileLoad16(op)
  1324  		case wazeroir.OperationKindLoad32:
  1325  			err = cmp.compileLoad32(op)
  1326  		case wazeroir.OperationKindStore:
  1327  			err = cmp.compileStore(op)
  1328  		case wazeroir.OperationKindStore8:
  1329  			err = cmp.compileStore8(op)
  1330  		case wazeroir.OperationKindStore16:
  1331  			err = cmp.compileStore16(op)
  1332  		case wazeroir.OperationKindStore32:
  1333  			err = cmp.compileStore32(op)
  1334  		case wazeroir.OperationKindMemorySize:
  1335  			err = cmp.compileMemorySize()
  1336  		case wazeroir.OperationKindMemoryGrow:
  1337  			err = cmp.compileMemoryGrow()
  1338  		case wazeroir.OperationKindConstI32:
  1339  			err = cmp.compileConstI32(op)
  1340  		case wazeroir.OperationKindConstI64:
  1341  			err = cmp.compileConstI64(op)
  1342  		case wazeroir.OperationKindConstF32:
  1343  			err = cmp.compileConstF32(op)
  1344  		case wazeroir.OperationKindConstF64:
  1345  			err = cmp.compileConstF64(op)
  1346  		case wazeroir.OperationKindEq:
  1347  			err = cmp.compileEq(op)
  1348  		case wazeroir.OperationKindNe:
  1349  			err = cmp.compileNe(op)
  1350  		case wazeroir.OperationKindEqz:
  1351  			err = cmp.compileEqz(op)
  1352  		case wazeroir.OperationKindLt:
  1353  			err = cmp.compileLt(op)
  1354  		case wazeroir.OperationKindGt:
  1355  			err = cmp.compileGt(op)
  1356  		case wazeroir.OperationKindLe:
  1357  			err = cmp.compileLe(op)
  1358  		case wazeroir.OperationKindGe:
  1359  			err = cmp.compileGe(op)
  1360  		case wazeroir.OperationKindAdd:
  1361  			err = cmp.compileAdd(op)
  1362  		case wazeroir.OperationKindSub:
  1363  			err = cmp.compileSub(op)
  1364  		case wazeroir.OperationKindMul:
  1365  			err = cmp.compileMul(op)
  1366  		case wazeroir.OperationKindClz:
  1367  			err = cmp.compileClz(op)
  1368  		case wazeroir.OperationKindCtz:
  1369  			err = cmp.compileCtz(op)
  1370  		case wazeroir.OperationKindPopcnt:
  1371  			err = cmp.compilePopcnt(op)
  1372  		case wazeroir.OperationKindDiv:
  1373  			err = cmp.compileDiv(op)
  1374  		case wazeroir.OperationKindRem:
  1375  			err = cmp.compileRem(op)
  1376  		case wazeroir.OperationKindAnd:
  1377  			err = cmp.compileAnd(op)
  1378  		case wazeroir.OperationKindOr:
  1379  			err = cmp.compileOr(op)
  1380  		case wazeroir.OperationKindXor:
  1381  			err = cmp.compileXor(op)
  1382  		case wazeroir.OperationKindShl:
  1383  			err = cmp.compileShl(op)
  1384  		case wazeroir.OperationKindShr:
  1385  			err = cmp.compileShr(op)
  1386  		case wazeroir.OperationKindRotl:
  1387  			err = cmp.compileRotl(op)
  1388  		case wazeroir.OperationKindRotr:
  1389  			err = cmp.compileRotr(op)
  1390  		case wazeroir.OperationKindAbs:
  1391  			err = cmp.compileAbs(op)
  1392  		case wazeroir.OperationKindNeg:
  1393  			err = cmp.compileNeg(op)
  1394  		case wazeroir.OperationKindCeil:
  1395  			err = cmp.compileCeil(op)
  1396  		case wazeroir.OperationKindFloor:
  1397  			err = cmp.compileFloor(op)
  1398  		case wazeroir.OperationKindTrunc:
  1399  			err = cmp.compileTrunc(op)
  1400  		case wazeroir.OperationKindNearest:
  1401  			err = cmp.compileNearest(op)
  1402  		case wazeroir.OperationKindSqrt:
  1403  			err = cmp.compileSqrt(op)
  1404  		case wazeroir.OperationKindMin:
  1405  			err = cmp.compileMin(op)
  1406  		case wazeroir.OperationKindMax:
  1407  			err = cmp.compileMax(op)
  1408  		case wazeroir.OperationKindCopysign:
  1409  			err = cmp.compileCopysign(op)
  1410  		case wazeroir.OperationKindI32WrapFromI64:
  1411  			err = cmp.compileI32WrapFromI64()
  1412  		case wazeroir.OperationKindITruncFromF:
  1413  			err = cmp.compileITruncFromF(op)
  1414  		case wazeroir.OperationKindFConvertFromI:
  1415  			err = cmp.compileFConvertFromI(op)
  1416  		case wazeroir.OperationKindF32DemoteFromF64:
  1417  			err = cmp.compileF32DemoteFromF64()
  1418  		case wazeroir.OperationKindF64PromoteFromF32:
  1419  			err = cmp.compileF64PromoteFromF32()
  1420  		case wazeroir.OperationKindI32ReinterpretFromF32:
  1421  			err = cmp.compileI32ReinterpretFromF32()
  1422  		case wazeroir.OperationKindI64ReinterpretFromF64:
  1423  			err = cmp.compileI64ReinterpretFromF64()
  1424  		case wazeroir.OperationKindF32ReinterpretFromI32:
  1425  			err = cmp.compileF32ReinterpretFromI32()
  1426  		case wazeroir.OperationKindF64ReinterpretFromI64:
  1427  			err = cmp.compileF64ReinterpretFromI64()
  1428  		case wazeroir.OperationKindExtend:
  1429  			err = cmp.compileExtend(op)
  1430  		case wazeroir.OperationKindSignExtend32From8:
  1431  			err = cmp.compileSignExtend32From8()
  1432  		case wazeroir.OperationKindSignExtend32From16:
  1433  			err = cmp.compileSignExtend32From16()
  1434  		case wazeroir.OperationKindSignExtend64From8:
  1435  			err = cmp.compileSignExtend64From8()
  1436  		case wazeroir.OperationKindSignExtend64From16:
  1437  			err = cmp.compileSignExtend64From16()
  1438  		case wazeroir.OperationKindSignExtend64From32:
  1439  			err = cmp.compileSignExtend64From32()
  1440  		case wazeroir.OperationKindMemoryInit:
  1441  			err = cmp.compileMemoryInit(op)
  1442  		case wazeroir.OperationKindDataDrop:
  1443  			err = cmp.compileDataDrop(op)
  1444  		case wazeroir.OperationKindMemoryCopy:
  1445  			err = cmp.compileMemoryCopy()
  1446  		case wazeroir.OperationKindMemoryFill:
  1447  			err = cmp.compileMemoryFill()
  1448  		case wazeroir.OperationKindTableInit:
  1449  			err = cmp.compileTableInit(op)
  1450  		case wazeroir.OperationKindElemDrop:
  1451  			err = cmp.compileElemDrop(op)
  1452  		case wazeroir.OperationKindTableCopy:
  1453  			err = cmp.compileTableCopy(op)
  1454  		case wazeroir.OperationKindRefFunc:
  1455  			err = cmp.compileRefFunc(op)
  1456  		case wazeroir.OperationKindTableGet:
  1457  			err = cmp.compileTableGet(op)
  1458  		case wazeroir.OperationKindTableSet:
  1459  			err = cmp.compileTableSet(op)
  1460  		case wazeroir.OperationKindTableGrow:
  1461  			err = cmp.compileTableGrow(op)
  1462  		case wazeroir.OperationKindTableSize:
  1463  			err = cmp.compileTableSize(op)
  1464  		case wazeroir.OperationKindTableFill:
  1465  			err = cmp.compileTableFill(op)
  1466  		case wazeroir.OperationKindV128Const:
  1467  			err = cmp.compileV128Const(op)
  1468  		case wazeroir.OperationKindV128Add:
  1469  			err = cmp.compileV128Add(op)
  1470  		case wazeroir.OperationKindV128Sub:
  1471  			err = cmp.compileV128Sub(op)
  1472  		case wazeroir.OperationKindV128Load:
  1473  			err = cmp.compileV128Load(op)
  1474  		case wazeroir.OperationKindV128LoadLane:
  1475  			err = cmp.compileV128LoadLane(op)
  1476  		case wazeroir.OperationKindV128Store:
  1477  			err = cmp.compileV128Store(op)
  1478  		case wazeroir.OperationKindV128StoreLane:
  1479  			err = cmp.compileV128StoreLane(op)
  1480  		case wazeroir.OperationKindV128ExtractLane:
  1481  			err = cmp.compileV128ExtractLane(op)
  1482  		case wazeroir.OperationKindV128ReplaceLane:
  1483  			err = cmp.compileV128ReplaceLane(op)
  1484  		case wazeroir.OperationKindV128Splat:
  1485  			err = cmp.compileV128Splat(op)
  1486  		case wazeroir.OperationKindV128Shuffle:
  1487  			err = cmp.compileV128Shuffle(op)
  1488  		case wazeroir.OperationKindV128Swizzle:
  1489  			err = cmp.compileV128Swizzle(op)
  1490  		case wazeroir.OperationKindV128AnyTrue:
  1491  			err = cmp.compileV128AnyTrue(op)
  1492  		case wazeroir.OperationKindV128AllTrue:
  1493  			err = cmp.compileV128AllTrue(op)
  1494  		case wazeroir.OperationKindV128BitMask:
  1495  			err = cmp.compileV128BitMask(op)
  1496  		case wazeroir.OperationKindV128And:
  1497  			err = cmp.compileV128And(op)
  1498  		case wazeroir.OperationKindV128Not:
  1499  			err = cmp.compileV128Not(op)
  1500  		case wazeroir.OperationKindV128Or:
  1501  			err = cmp.compileV128Or(op)
  1502  		case wazeroir.OperationKindV128Xor:
  1503  			err = cmp.compileV128Xor(op)
  1504  		case wazeroir.OperationKindV128Bitselect:
  1505  			err = cmp.compileV128Bitselect(op)
  1506  		case wazeroir.OperationKindV128AndNot:
  1507  			err = cmp.compileV128AndNot(op)
  1508  		case wazeroir.OperationKindV128Shl:
  1509  			err = cmp.compileV128Shl(op)
  1510  		case wazeroir.OperationKindV128Shr:
  1511  			err = cmp.compileV128Shr(op)
  1512  		case wazeroir.OperationKindV128Cmp:
  1513  			err = cmp.compileV128Cmp(op)
  1514  		case wazeroir.OperationKindV128AddSat:
  1515  			err = cmp.compileV128AddSat(op)
  1516  		case wazeroir.OperationKindV128SubSat:
  1517  			err = cmp.compileV128SubSat(op)
  1518  		case wazeroir.OperationKindV128Mul:
  1519  			err = cmp.compileV128Mul(op)
  1520  		case wazeroir.OperationKindV128Div:
  1521  			err = cmp.compileV128Div(op)
  1522  		case wazeroir.OperationKindV128Neg:
  1523  			err = cmp.compileV128Neg(op)
  1524  		case wazeroir.OperationKindV128Sqrt:
  1525  			err = cmp.compileV128Sqrt(op)
  1526  		case wazeroir.OperationKindV128Abs:
  1527  			err = cmp.compileV128Abs(op)
  1528  		case wazeroir.OperationKindV128Popcnt:
  1529  			err = cmp.compileV128Popcnt(op)
  1530  		case wazeroir.OperationKindV128Min:
  1531  			err = cmp.compileV128Min(op)
  1532  		case wazeroir.OperationKindV128Max:
  1533  			err = cmp.compileV128Max(op)
  1534  		case wazeroir.OperationKindV128AvgrU:
  1535  			err = cmp.compileV128AvgrU(op)
  1536  		case wazeroir.OperationKindV128Pmin:
  1537  			err = cmp.compileV128Pmin(op)
  1538  		case wazeroir.OperationKindV128Pmax:
  1539  			err = cmp.compileV128Pmax(op)
  1540  		case wazeroir.OperationKindV128Ceil:
  1541  			err = cmp.compileV128Ceil(op)
  1542  		case wazeroir.OperationKindV128Floor:
  1543  			err = cmp.compileV128Floor(op)
  1544  		case wazeroir.OperationKindV128Trunc:
  1545  			err = cmp.compileV128Trunc(op)
  1546  		case wazeroir.OperationKindV128Nearest:
  1547  			err = cmp.compileV128Nearest(op)
  1548  		case wazeroir.OperationKindV128Extend:
  1549  			err = cmp.compileV128Extend(op)
  1550  		case wazeroir.OperationKindV128ExtMul:
  1551  			err = cmp.compileV128ExtMul(op)
  1552  		case wazeroir.OperationKindV128Q15mulrSatS:
  1553  			err = cmp.compileV128Q15mulrSatS(op)
  1554  		case wazeroir.OperationKindV128ExtAddPairwise:
  1555  			err = cmp.compileV128ExtAddPairwise(op)
  1556  		case wazeroir.OperationKindV128FloatPromote:
  1557  			err = cmp.compileV128FloatPromote(op)
  1558  		case wazeroir.OperationKindV128FloatDemote:
  1559  			err = cmp.compileV128FloatDemote(op)
  1560  		case wazeroir.OperationKindV128FConvertFromI:
  1561  			err = cmp.compileV128FConvertFromI(op)
  1562  		case wazeroir.OperationKindV128Dot:
  1563  			err = cmp.compileV128Dot(op)
  1564  		case wazeroir.OperationKindV128Narrow:
  1565  			err = cmp.compileV128Narrow(op)
  1566  		case wazeroir.OperationKindV128ITruncSatFromF:
  1567  			err = cmp.compileV128ITruncSatFromF(op)
  1568  		case wazeroir.OperationKindBuiltinFunctionCheckExitCode:
  1569  			err = cmp.compileBuiltinFunctionCheckExitCode()
  1570  		default:
  1571  			err = errors.New("unsupported")
  1572  		}
  1573  		if err != nil {
  1574  			err = fmt.Errorf("operation %s: %w", op.Kind.String(), err)
  1575  			return
  1576  		}
  1577  	}
  1578  
  1579  	spCeil, err = cmp.compile(buf)
  1580  	if err != nil {
  1581  		err = fmt.Errorf("failed to compile: %w", err)
  1582  		return
  1583  	}
  1584  
  1585  	if needSourceOffsets {
  1586  		offsetInNativeBin := append(offsets.values[:0], make([]uint64, len(irOpBegins))...)
  1587  		offsets.values = offsetInNativeBin
  1588  		for i, nop := range irOpBegins {
  1589  			offsetInNativeBin[i] = nop.OffsetInBinary()
  1590  		}
  1591  		sm.irOperationOffsetsInNativeBinary = bitpack.NewOffsetArray(offsetInNativeBin)
  1592  		sm.irOperationSourceOffsetsInWasmBinary = bitpack.NewOffsetArray(ir.IROperationSourceOffsetsInWasmBinary)
  1593  	}
  1594  	return
  1595  }
  1596  

View as plain text