Tool for applying test vectors from Ethereum on FEVM

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "testing/conformance",
     "testing/integration",
     "testing/calibration",
+    "testing/fevm_test_vectors",
     "ipld/*",
     "testing/integration/tests/*-actor",
     "testing/calibration/contract/*-actor"

testing/conformance/benches/bench_drivers.rs

@@ -107,15 +107,23 @@ pub fn bench_vector_file(
         // if we broke the test, it's not a valid optimization :P
         let testresult = match check_strength {
             CheckStrength::FullTest => {
-                run_variant(bs.clone(), vector, variant, engines, true, None, None).map_err(
-                    |e| anyhow::anyhow!("run_variant failed (probably a test parsing bug): {}", e),
-                )?
-            }
-            CheckStrength::OnlyCheckSuccess => {
-                run_variant(bs.clone(), vector, variant, engines, false, None, None).map_err(
+                run_variant(bs.clone(), vector, variant, engines, true, None, None, None).map_err(
                     |e| anyhow::anyhow!("run_variant failed (probably a test parsing bug): {}", e),
                 )?
             }
+            CheckStrength::OnlyCheckSuccess => run_variant(
+                bs.clone(),
+                vector,
+                variant,
+                engines,
+                false,
+                None,
+                None,
+                None,
+            )
+            .map_err(|e| {
+                anyhow::anyhow!("run_variant failed (probably a test parsing bug): {}", e)
+            })?,
             CheckStrength::NoChecks => VariantResult::Ok {
                 id: variant.id.clone(),
             },

testing/conformance/src/driver.rs

@@ -23,7 +23,7 @@ use libipld_core::ipld::Ipld;
 use regex::Regex;
 use walkdir::DirEntry;
 
-use crate::tracing::TestTraceFun;
+use crate::tracing::{CaptureApplyRetFun, TestTraceFun};
 use crate::vector::{MessageVector, Variant};
 use crate::vm::{TestKernel, TestMachine, TestStatsRef};
 
@@ -66,7 +66,12 @@ pub fn is_runnable(entry: &DirEntry) -> bool {
 }
 
 /// Compares the result of running a message with the expected result.
-fn check_msg_result(expected_rec: &Receipt, ret: &ApplyRet, label: impl Display) -> Result<()> {
+fn check_msg_result(
+    expected_rec: &Receipt,
+    ret: &ApplyRet,
+    label: impl Display,
+    skip_compare_gas_used: bool,
+) -> Result<()> {
     let error = ret
         .failure_info
         .as_ref()
@@ -94,14 +99,16 @@ fn check_msg_result(expected_rec: &Receipt, ret: &ApplyRet, label: impl Display)
         ));
     }
 
-    let (expected, actual) = (expected_rec.gas_used, actual_rec.gas_used);
-    if expected != actual {
-        return Err(anyhow!(
-            "gas used of msg {} did not match; expected: {}, got {}",
-            label,
-            expected,
-            actual
-        ));
+    if !skip_compare_gas_used {
+        let (expected, actual) = (expected_rec.gas_used, actual_rec.gas_used);
+        if expected != actual {
+            return Err(anyhow!(
+                "gas used of msg {} did not match; expected: {}, got {}",
+                label,
+                expected,
+                actual
+            ));
+        }
     }
 
     Ok(())
@@ -112,9 +119,9 @@ fn compare_actors(
     identifier: impl Display,
     actual: Option<ActorState>,
     expected: Option<ActorState>,
-) -> Result<()> {
+) -> Result<bool> {
     if actual == expected {
-        return Ok(());
+        return Ok(true);
     }
     log::error!(
         "{} actor state differs: {:?} != {:?}",
@@ -139,14 +146,24 @@ fn compare_actors(
         }
         _ => {}
     }
-    Ok(())
+    Ok(false)
 }
 
 /// Compares the state-root with the postcondition state-root in the test vector. If they don't
 /// match, it performs a basic actor & state-diff of the message senders and receivers in the test
 /// vector, along with all system actors.
 fn compare_state_roots(bs: &MemoryBlockstore, root: &Cid, vector: &MessageVector) -> Result<()> {
-    if root == &vector.postconditions.state_tree.root_cid {
+    let skip_compare_addresses = vector.skip_compare_addresses.clone();
+    let skip_compare_actor_ids = vector.skip_compare_actor_ids.clone();
+    let additional_compare_addresses = vector.additional_compare_addresses.clone();
+
+    let mut need_compare_root = false;
+    let mut compare_actors_success = true;
+    if matches!(skip_compare_addresses, None) && matches!(skip_compare_actor_ids, None) {
+        need_compare_root = true;
+    }
+
+    if need_compare_root && root == &vector.postconditions.state_tree.root_cid {
         return Ok(());
     }
 
@@ -160,35 +177,68 @@ fn compare_state_roots(bs: &MemoryBlockstore, root: &Cid, vector: &MessageVector
 
     for m in &vector.apply_messages {
         let msg: Message = from_slice(&m.bytes)?;
-        let actual_actor = actual_st.get_actor_by_address(&msg.from)?;
-        let expected_actor = expected_st.get_actor_by_address(&msg.from)?;
-        compare_actors(bs, "sender", actual_actor, expected_actor)?;
+        if matches!(skip_compare_addresses.clone(), Some(skip_addrs) if !skip_addrs.contains(&msg.from))
+        {
+            let actual_actor = actual_st.get_actor_by_address(&msg.from)?;
+            let expected_actor = expected_st.get_actor_by_address(&msg.from)?;
+            if !compare_actors(bs, "sender", actual_actor, expected_actor)? {
+                compare_actors_success = false;
+            }
+        }
 
-        let actual_actor = actual_st.get_actor_by_address(&msg.to)?;
-        let expected_actor = expected_st.get_actor_by_address(&msg.to)?;
-        compare_actors(bs, "receiver", actual_actor, expected_actor)?;
+        if matches!(skip_compare_addresses.clone(), Some(skip_addrs) if !skip_addrs.contains(&msg.to))
+        {
+            let actual_actor = actual_st.get_actor_by_address(&msg.to)?;
+            let expected_actor = expected_st.get_actor_by_address(&msg.to)?;
+            if !compare_actors(bs, "receiver", actual_actor, expected_actor)? {
+                compare_actors_success = false;
+            }
+        }
+    }
+
+    if let Some(addrs) = additional_compare_addresses {
+        for addr in addrs {
+            let actual_actor = actual_st.get_actor_by_address(&addr)?;
+            let expected_actor = expected_st.get_actor_by_address(&addr)?;
+            if !compare_actors(bs, &addr.to_string(), actual_actor, expected_actor)? {
+                compare_actors_success = false;
+            }
+        }
     }
 
     // All system actors
     for id in 0..100 {
+        if matches!(skip_compare_actor_ids.clone(), Some(skip_actor_ids) if skip_actor_ids.contains(&id))
+        {
+            continue;
+        }
         let expected_actor = match expected_st.get_actor(id) {
             Ok(act) => act,
             Err(_) => continue, // we don't expect it anyways.
         };
         let actual_actor = actual_st.get_actor(id)?;
-        compare_actors(
+        if !compare_actors(
             bs,
             format_args!("builtin {}", id),
             actual_actor,
             expected_actor,
-        )?;
+        )? {
+            compare_actors_success = false;
+        }
     }
 
-    Err(anyhow!(
-        "wrong post root cid; expected {}, but got {}",
-        &vector.postconditions.state_tree.root_cid,
-        root
-    ))
+    if need_compare_root {
+        return Err(anyhow!(
+            "wrong post root cid; expected {}, but got {}",
+            &vector.postconditions.state_tree.root_cid,
+            root
+        ));
+    } else {
+        if compare_actors_success {
+            return Ok(());
+        }
+        return Err(anyhow!("compare actors fail"));
+    }
 }
 
 /// Represents the result from running a vector.
@@ -209,6 +259,7 @@ pub fn run_variant(
     mut check_correctness: bool,
     stats: TestStatsRef,
     trace: Option<TestTraceFun>,
+    capture_apply_ret_fn: Option<CaptureApplyRetFun>,
 ) -> anyhow::Result<VariantResult> {
     let id = variant.id.clone();
 
@@ -243,6 +294,14 @@ pub fn run_variant(
     let mut exec: DefaultExecutor<TestKernel> = DefaultExecutor::new(engine, machine)?;
     let mut rets = Vec::new();
 
+    let capture_apply_ret_fn = if let Some(f) = capture_apply_ret_fn {
+        f
+    } else {
+        Box::new(move |_| {
+            return Ok(());
+        })
+    };
+
     // Apply all messages in the vector.
     for (i, m) in v.apply_messages.iter().enumerate() {
         let msg: Message = from_slice(&m.bytes)?;
@@ -260,10 +319,12 @@ pub fn run_variant(
             Err(e) => return Ok(VariantResult::Failed { id, reason: e }),
         };
 
+        capture_apply_ret_fn((i as i32, ret.clone()))?;
+
         if check_correctness {
             // Compare the actual receipt with the expected receipt.
             let expected_receipt = &v.postconditions.receipts[i];
-            if let Err(err) = check_msg_result(expected_receipt, &ret, i) {
+            if let Err(err) = check_msg_result(expected_receipt, &ret, i, v.skip_compare_gas_used) {
                 return Ok(VariantResult::Failed { id, reason: err });
             }
         }

testing/conformance/src/externs.rs

@@ -1,22 +1,25 @@
+use anyhow::anyhow;
 // Copyright 2021-2023 Protocol Labs
 // SPDX-License-Identifier: Apache-2.0, MIT
 use fvm::externs::{Chain, Consensus, Externs, Rand};
 use fvm_shared::clock::ChainEpoch;
 use fvm_shared::consensus::ConsensusFault;
 
 use crate::rand::ReplayingRand;
-use crate::vector::Randomness;
+use crate::vector::{Randomness, TipsetCid};
 
 /// The externs stub for testing. Forwards randomness requests to the randomness
 /// replayer, which replays randomness stored in the vector.
 pub struct TestExterns {
+    pub tipset_cids: Vec<TipsetCid>,
     rand: ReplayingRand,
 }
 
 impl TestExterns {
     /// Creates a new TestExterns from randomness contained in a vector.
     pub fn new(r: &Randomness) -> Self {
         TestExterns {
+            tipset_cids: Default::default(),
             rand: ReplayingRand::new(r.as_slice()),
         }
     }
@@ -57,6 +60,11 @@ impl Consensus for TestExterns {
 
 impl Chain for TestExterns {
     fn get_tipset_cid(&self, _epoch: ChainEpoch) -> anyhow::Result<cid::Cid> {
-        todo!()
+        for tipset in &self.tipset_cids {
+            if tipset.epoch == _epoch {
+                return Ok(tipset.cid);
+            }
+        }
+        Err(anyhow!("cannot find tipset cid, epoch {}", _epoch))
     }
 }

testing/conformance/src/tracing.rs

@@ -18,6 +18,8 @@ pub type TestTrace = (Duration, ApplyRet);
 /// from all messages in the tests, in the order of execution.
 pub type TestTraceFun = Box<dyn FnOnce(Vec<TestTrace>) -> IoResult<()>>;
 
+pub type CaptureApplyRetFun = Box<dyn Fn((i32, ApplyRet)) -> IoResult<()>>;
+
 /// Tombstone of a single message execution.
 #[derive(Serialize)]
 pub struct TestMessageTombstone {