Unverified Commit 3ec6fe61 authored by meowsbits's avatar meowsbits Committed by GitHub

tests: fix state test error checking (#25702)

Some tests define an 'expectException' error but the tests runner does not check for conditions where this test value is filled (error expected) but in which no error is returned by the test runner.

An example of this scenario is GeneralStateTests/stTransactionTest/HighGasPrice.json, which expects a 'TR_NoFunds' error, but the test runner does not return any error.
Signed-off-by: 's avatarmeows <b5c6@protonmail.com>
parent 7227c9ef
...@@ -57,12 +57,12 @@ func TestState(t *testing.T) { ...@@ -57,12 +57,12 @@ func TestState(t *testing.T) {
// Broken tests: // Broken tests:
// Expected failures: // Expected failures:
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test") // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test")
// For Istanbul, older tests were moved into LegacyTests // For Istanbul, older tests were moved into LegacyTests
for _, dir := range []string{ for _, dir := range []string{
...@@ -78,10 +78,6 @@ func TestState(t *testing.T) { ...@@ -78,10 +78,6 @@ func TestState(t *testing.T) {
t.Run(key+"/trie", func(t *testing.T) { t.Run(key+"/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
_, _, err := test.Run(subtest, vmconfig, false) _, _, err := test.Run(subtest, vmconfig, false)
if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return st.checkFailure(t, err) return st.checkFailure(t, err)
}) })
}) })
...@@ -93,10 +89,6 @@ func TestState(t *testing.T) { ...@@ -93,10 +89,6 @@ func TestState(t *testing.T) {
return err return err
} }
} }
if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return st.checkFailure(t, err) return st.checkFailure(t, err)
}) })
}) })
......
...@@ -159,11 +159,39 @@ func (t *StateTest) Subtests() []StateSubtest { ...@@ -159,11 +159,39 @@ func (t *StateTest) Subtests() []StateSubtest {
return sub return sub
} }
// checkError checks if the error returned by the state transition matches any expected error.
// A failing expectation returns a wrapped version of the original error, if any,
// or a new error detailing the failing expectation.
// This function does not return or modify the original error, it only evaluates and returns expectations for the error.
func (t *StateTest) checkError(subtest StateSubtest, err error) error {
expectedError := t.json.Post[subtest.Fork][subtest.Index].ExpectException
if err == nil && expectedError == "" {
return nil
}
if err == nil && expectedError != "" {
return fmt.Errorf("expected error %q, got no error", expectedError)
}
if err != nil && expectedError == "" {
return fmt.Errorf("unexpected error: %w", err)
}
if err != nil && expectedError != "" {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return nil
}
// Run executes a specific subtest and verifies the post-state and logs // Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) {
snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter)
if checkedErr := t.checkError(subtest, err); checkedErr != nil {
return snaps, statedb, checkedErr
}
// The error has been checked; if it was unexpected, it's already returned.
if err != nil { if err != nil {
return snaps, statedb, err // Here, an error exists but it was expected.
// We do not check the post state or logs.
return snaps, statedb, nil
} }
post := t.json.Post[subtest.Fork][subtest.Index] post := t.json.Post[subtest.Fork][subtest.Index]
// N.B: We need to do this in a two-step process, because the first Commit takes care // N.B: We need to do this in a two-step process, because the first Commit takes care
...@@ -231,7 +259,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh ...@@ -231,7 +259,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
snapshot := statedb.Snapshot() snapshot := statedb.Snapshot()
gaspool := new(core.GasPool) gaspool := new(core.GasPool)
gaspool.AddGas(block.GasLimit()) gaspool.AddGas(block.GasLimit())
if _, err := core.ApplyMessage(evm, msg, gaspool); err != nil { _, err = core.ApplyMessage(evm, msg, gaspool)
if err != nil {
statedb.RevertToSnapshot(snapshot) statedb.RevertToSnapshot(snapshot)
} }
// Add 0-value mining reward. This only makes a difference in the cases // Add 0-value mining reward. This only makes a difference in the cases
...@@ -244,7 +273,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh ...@@ -244,7 +273,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
statedb.Commit(config.IsEIP158(block.Number())) statedb.Commit(config.IsEIP158(block.Number()))
// And _now_ get the state root // And _now_ get the state root
root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) root := statedb.IntermediateRoot(config.IsEIP158(block.Number()))
return snaps, statedb, root, nil return snaps, statedb, root, err
} }
func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment