[ocaml] generate bindings from Rust code

[mimoo]

The idea of this PR is to avoid writing the OCaml bindings by hand, and generate them automatically from the Rust code. (I also recorded myself going over the code a bit if that helps.)

In the end, you end up writing this kind of code (taken from MinaProtocol/mina#9525):

fn main() {
    println!("(* this file is generated automatically *)\n");
    let env = &mut Env::default();

    decl_fake_generic!(T1, 0);
    decl_fake_generic!(T2, 1);

    decl_module!(env, "Types", {
        decl_type!(env, CamlScalarChallenge::<T1> => "scalar_challenge");
        decl_type!(env, CamlRandomOracles::<T1> => "random_oracles");
        decl_type!(env, CamlProofEvaluations::<T1> => "proof_evaluations");
        decl_type!(env, CamlPolyComm::<T1> => "poly_comm");
        decl_type!(env, CamlOpeningProof::<T1, T2> => "opening_proof");
        decl_type!(env, CamlProverCommitments::<T1> => "prover_commitments");
        decl_type!(env, CamlProverProof<T1, T2> => "prover_proof");
    });

    decl_module!(env, "BigInt256", {
        decl_type!(env, CamlBigInteger256 => "t");
        decl_func!(env, caml_bigint_256_of_numeral => "of_numeral");
    });

    decl_module!(env, "Fp", {
        decl_type!(env, CamlFp => "t");
        decl_func!(env, caml_pasta_fp_size_in_bits => "size_in_bits");
        decl_func!(env, caml_pasta_fp_size);
        decl_func!(env, caml_pasta_fp_add);

to generate a single OCaml file containing:

(* this file is generated automatically *)

module Types = struct
  type 'CamlF scalar_challenge = 'CamlF

  type 'CamlF random_oracles = {
    beta : 'CamlF;
    gamma : 'CamlF;
    alpha_chal : 'CamlF scalar_challenge;
    alpha : 'CamlF;
    zeta : 'CamlF;
    v : 'CamlF;
    u : 'CamlF;
    zeta_chal : 'CamlF scalar_challenge;
    v_chal : 'CamlF scalar_challenge;
    u_chal : 'CamlF scalar_challenge;
  }

  type 'F proof_evaluations = {
    l : 'F array;
    r : 'F array;
    o : 'F array;
    z : 'F array;
    t : 'F array;
    f : 'F array;
    sigma1 : 'F array;
    sigma2 : 'F array;
  }

  type 'CamlG poly_comm = { unshifted : 'CamlG array; shifted : 'CamlG option }

  type ('G, 'F) opening_proof = {
    lr : 'G * 'G array;
    delta : 'G;
    z1 : 'F;
    z2 : 'F;
    sg : 'G;
  }

  type 'CamlG prover_commitments = {
    l_comm : 'CamlG poly_comm;
    r_comm : 'CamlG poly_comm;
    o_comm : 'CamlG poly_comm;
    z_comm : 'CamlG poly_comm;
    t_comm : 'CamlG poly_comm;
  }

  type ('CamlG, 'CamlF) prover_proof = {
    commitments : 'CamlG prover_commitments;
    proof : ('CamlG, 'CamlF) opening_proof;
    evals : 'CamlF proof_evaluations * 'CamlF proof_evaluations;
    public : 'CamlF array;
    prev_challenges : 'CamlF array * 'CamlG poly_comm array;
  }
end
module BigInt256 = struct
  type t

  external of_numeral : bytes -> int -> int -> t = "caml_bigint_256_of_numeral"
end

module Fp = struct
  type t

  external size_in_bits : unit -> int = "caml_pasta_fp_size_in_bits"

  external caml_pasta_fp_size : unit -> BigInt256.t = "caml_pasta_fp_size"

  external caml_pasta_fp_add : t -> t -> t = "caml_pasta_fp_add"
end

Essentially, it creates a number of macros to help derive the OCaml bindings.

• #[derive(OcamlGen)] derive macro for types. This implements the OCamlDesc and OCamlBinding traits.
• #[derive(OcamlCustomTypes)] derive macro for custom types. This also implements the OCamlDesc and OCamlBinding traits.
• #[ocaml_gen] attribute macro for functions, which creates a *_to_ocaml() function returning the generated OCaml binding

The traits it implements on types are the following ones:

/// OCamlBinding is the trait implemented by types to generate their OCaml bindings.
/// It is usually derived automatically via the [OcamlGen] macro,
/// or the [OCamlCustomType] macro for custom types.
/// For functions, refer to the [ocaml_gen] macro.
pub trait OCamlBinding {
    /// will generate the OCaml bindings for a type (called root type).
    /// It takes the current environment [Env],
    /// as well as an optional name (if you wish to rename the type in OCaml).
    fn ocaml_binding(env: &mut Env, rename: Option<&'static str>) -> String;
}

/// OCamlDesc is the trait implemented by types to facilitate generation of their OCaml bindings.
/// It is usually derived automatically via the [OcamlGen] macro,
/// or the [OCamlCustomType] macro for custom types.
pub trait OCamlDesc {
    /// describes the type in OCaml, given the current environment [Env]
    /// and the list of generic type parameters of the root type
    /// (the type that makes use of this type)
    fn ocaml_desc(env: &Env, generics: &[&str]) -> String;

    /// Returns a unique ID for the type. This ID will not change if concrete type parameters are used.
    fn unique_id() -> u128;
}

Essentially, the ocaml_binding() function is called to generate bindings for a type, and it'll call ocaml_desc() on all of its fields (recursively). The ocaml_desc() function is aware of the current environment (via the Env var), and can thus re-write types that have been renamed or relocated in some other module.

In order to avoid mixing types, a unique_id() function is also implemented, which returns a unique id per-type (regardless of the concrete type parameters used)

[mrmr1993]

Can you check the generated assembly code and confirm that we're not doing extra 'deep deconstruct-reconstruct' clones due to this? If we are, lets just make the type parameter represent 'f array instead of 'f and deal with this on the OCaml end.

[mimoo]

I think you're catching a potential performance regression from this PR: #121 but it looks like I made the same change for 15-wires in this PR also.

IIRC, I didn't get why the types were not clearly defined if they were always vectors.

(Also I'm a bit worried about handling Vec<F> as a generic type parameter within ocaml-gen.)

[mimoo]

had some trouble with CI, but all good now!

[mimoo]

rebased

[mrmr1993]

This lgtm in general. I'm keen to not break the existing code while the migration isn't complete yet; I've commented where you've removed derive statements that will do so.

[mimoo]

make CI happy