From daab1483719afa5f77469b32b56313087e8a5e00 Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Wed, 24 Apr 2024 18:06:10 +0300 Subject: [PATCH 01/11] Updated ICICLE setup and prove to use v2, need to test correctness vs CPU version --- backend/groth16/bn254/icicle/doc.go | 2 +- backend/groth16/bn254/icicle/icicle.go | 468 +++++++++++-------- backend/groth16/bn254/icicle/marshal_test.go | 2 +- backend/groth16/bn254/icicle/noicicle.go | 2 +- backend/groth16/bn254/icicle/provingkey.go | 14 +- go.mod | 6 +- go.sum | 10 +- std/math/emulated/element.go | 6 + 8 files changed, 290 insertions(+), 220 deletions(-) diff --git a/backend/groth16/bn254/icicle/doc.go b/backend/groth16/bn254/icicle/doc.go index a77a7fbde9..3a662b35da 100644 --- a/backend/groth16/bn254/icicle/doc.go +++ b/backend/groth16/bn254/icicle/doc.go @@ -1,2 +1,2 @@ // Package icicle_bn254 implements ICICLE acceleration for BN254 Groth16 backend. -package icicle_bn254 +package icicle diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 5b1b235d33..6be02c87cd 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -1,17 +1,16 @@ //go:build icicle -package icicle_bn254 +package icicle import ( "fmt" "math/big" "math/bits" "time" - "unsafe" curve "github.com/consensys/gnark-crypto/ecc/bn254" - "github.com/consensys/gnark-crypto/ecc/bn254/fp" "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/consensys/gnark-crypto/ecc/bn254/fp" "github.com/consensys/gnark-crypto/ecc/bn254/fr/hash_to_field" "github.com/consensys/gnark-crypto/ecc/bn254/fr/pedersen" "github.com/consensys/gnark/backend" @@ -23,7 +22,16 @@ import ( "github.com/consensys/gnark/constraint/solver" "github.com/consensys/gnark/internal/utils" "github.com/consensys/gnark/logger" - iciclegnark "github.com/ingonyama-zk/iciclegnark/curves/bn254" + + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" + icicle_cr "github.com/ingonyama-zk/icicle/v2/wrappers/golang/cuda_runtime" + icicle_bn254 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254" + icicle_g2 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/g2" + icicle_msm "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/msm" + icicle_ntt "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/ntt" + icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" + + fcs "github.com/consensys/gnark/frontend/cs" ) const HasIcicle = true @@ -33,20 +41,33 @@ func (pk *ProvingKey) setupDevicePointers() error { return nil } pk.deviceInfo = &deviceInfo{} - n := int(pk.Domain.Cardinality) - sizeBytes := n * fr.Bytes - /************************* Start Domain Device Setup ***************************/ - copyCosetInvDone := make(chan unsafe.Pointer, 1) - copyCosetDone := make(chan unsafe.Pointer, 1) - copyDenDone := make(chan unsafe.Pointer, 1) /************************* CosetTableInv ***************************/ - go iciclegnark.CopyToDevice(pk.Domain.CosetTableInv, sizeBytes, copyCosetInvDone) + copyCosetInvDone := make(chan bool, 1) + go func() { + cosetTableInv, err := pk.Domain.CosetTableInv() + if err != nil { + panic("Something went wrong") // TODO + } + cosetTableInvHost := (icicle_core.HostSlice[fr.Element])(cosetTableInv) + cosetTableInvHost.CopyToDevice(&pk.DomainDevice.CosetTableInv, true) + copyCosetInvDone <- true + }() /************************* CosetTable ***************************/ - go iciclegnark.CopyToDevice(pk.Domain.CosetTable, sizeBytes, copyCosetDone) + copyCosetDone := make(chan bool, 1) + go func() { + coestTable, err := pk.Domain.CosetTable() + if err != nil { + panic("SOmething went wrong") // TODO + } + cosetTableHost := (icicle_core.HostSlice[fr.Element])(coestTable) + cosetTableHost.CopyToDevice(&pk.DomainDevice.CosetTable, true) + copyCosetDone <- true + }() /************************* Den ***************************/ + n := int(pk.Domain.Cardinality) var denI, oneI fr.Element oneI.SetOne() denI.Exp(pk.Domain.FrMultiplicativeGen, big.NewInt(int64(pk.Domain.Cardinality))) @@ -62,73 +83,137 @@ func (pk *ProvingKey) setupDevicePointers() error { denIcicleArr = append(denIcicleArr, denI) } - go iciclegnark.CopyToDevice(denIcicleArr, sizeBytes, copyDenDone) + copyDenDone := make(chan bool, 1) + go func() { + denIcicleArrHost := (icicle_core.HostSlice[fr.Element])(denIcicleArr) + denIcicleArrHost.CopyToDevice(&pk.DenDevice, true) + copyDenDone <- true + }() /************************* Twiddles and Twiddles Inv ***************************/ - twiddlesInv_d_gen, twddles_err := iciclegnark.GenerateTwiddleFactors(n, true) - if twddles_err != nil { - return twddles_err + ctx, err := icicle_cr.GetDefaultDeviceContext() + if err != icicle_cr.CudaSuccess { + panic("Couldn't create device context") // TODO } - twiddles_d_gen, twddles_err := iciclegnark.GenerateTwiddleFactors(n, false) - if twddles_err != nil { - return twddles_err + rou := pk.Domain.Generator.Bits() + rouIcicle := icicle_bn254.ScalarField{} + limbs := icicle_core.ConvertUint64ArrToUint32Arr(rou[:]) + rouIcicle.FromLimbs(limbs) + e := icicle_ntt.InitDomain(rouIcicle, ctx, false) + if e.IcicleErrorCode != icicle_core.IcicleSuccess { + panic("Couldn't initialize domain") // TODO } /************************* End Domain Device Setup ***************************/ - pk.DomainDevice.Twiddles = twiddles_d_gen - pk.DomainDevice.TwiddlesInv = twiddlesInv_d_gen - - pk.DomainDevice.CosetTableInv = <-copyCosetInvDone - pk.DomainDevice.CosetTable = <-copyCosetDone - pk.DenDevice = <-copyDenDone + <-copyCosetInvDone + <-copyCosetDone + <-copyDenDone /************************* Start G1 Device Setup ***************************/ /************************* A ***************************/ - pointsBytesA := len(pk.G1.A) * fp.Bytes * 2 - copyADone := make(chan unsafe.Pointer, 1) - go iciclegnark.CopyPointsToDevice(pk.G1.A, pointsBytesA, copyADone) // Make a function for points - + copyADone := make(chan bool, 1) + go func() { + g1AHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.A) + g1AHost.CopyToDevice(&pk.G1Device.A, true) + copyADone <- true + }() /************************* B ***************************/ - pointsBytesB := len(pk.G1.B) * fp.Bytes * 2 - copyBDone := make(chan unsafe.Pointer, 1) - go iciclegnark.CopyPointsToDevice(pk.G1.B, pointsBytesB, copyBDone) // Make a function for points - - /************************* K ***************************/ - var pointsNoInfinity []curve.G1Affine - for i, gnarkPoint := range pk.G1.K { - if gnarkPoint.IsInfinity() { - pk.InfinityPointIndicesK = append(pk.InfinityPointIndicesK, i) - } else { - pointsNoInfinity = append(pointsNoInfinity, gnarkPoint) - } - } - - pointsBytesK := len(pointsNoInfinity) * fp.Bytes * 2 - copyKDone := make(chan unsafe.Pointer, 1) - go iciclegnark.CopyPointsToDevice(pointsNoInfinity, pointsBytesK, copyKDone) // Make a function for points - + copyBDone := make(chan bool, 1) + go func() { + g1BHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.B) + g1BHost.CopyToDevice(&pk.G1Device.B, true) + copyBDone <- true + }() + /************************* K ***************************/ + copyKDone := make(chan bool, 1) + go func() { + g1KHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.K) + g1KHost.CopyToDevice(&pk.G1Device.K, true) + copyKDone <- true + }() /************************* Z ***************************/ - pointsBytesZ := len(pk.G1.Z) * fp.Bytes * 2 - copyZDone := make(chan unsafe.Pointer, 1) - go iciclegnark.CopyPointsToDevice(pk.G1.Z, pointsBytesZ, copyZDone) // Make a function for points - + copyZDone := make(chan bool, 1) + go func() { + g1ZHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.Z) + g1ZHost.CopyToDevice(&pk.G1Device.Z, true) + copyZDone <- true + }() /************************* End G1 Device Setup ***************************/ - pk.G1Device.A = <-copyADone - pk.G1Device.B = <-copyBDone - pk.G1Device.K = <-copyKDone - pk.G1Device.Z = <-copyZDone - + <-copyADone + <-copyBDone + <-copyKDone + <-copyZDone /************************* Start G2 Device Setup ***************************/ - pointsBytesB2 := len(pk.G2.B) * fp.Bytes * 4 - copyG2BDone := make(chan unsafe.Pointer, 1) - go iciclegnark.CopyG2PointsToDevice(pk.G2.B, pointsBytesB2, copyG2BDone) // Make a function for points - pk.G2Device.B = <-copyG2BDone + copyG2BDone := make(chan bool, 1) + go func() { + g2BHost := (icicle_core.HostSlice[curve.G2Affine])(pk.G2.B) + g2BHost.CopyToDevice(&pk.G2Device.B, true) + copyG2BDone <- true + }() + <-copyG2BDone /************************* End G2 Device Setup ***************************/ return nil } +func projectiveToGnarkAffine(p icicle_bn254.Projective) *curve.G1Affine { + px, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(p.X.ToBytesLittleEndian())) + py, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(p.Y.ToBytesLittleEndian())) + pz, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(p.Z.ToBytesLittleEndian())) + + var x, y, zInv fp.Element + + zInv.Inverse(&pz) + x.Mul(&px, &zInv) + y.Mul(&py, &zInv) + + return &curve.G1Affine{X: x, Y: y} +} + +func g1ProjectiveToG1Jac(p icicle_bn254.Projective) curve.G1Jac { + var p1 curve.G1Jac + p1.FromAffine(projectiveToGnarkAffine(p)) + + return p1 +} + +func toGnarkE2(f icicle_g2.G2BaseField) curve.E2 { + bytes := f.ToBytesLittleEndian() + a0, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[:len(bytes)/2])) + a1, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[len(bytes)/2:])) + return curve.E2{ + A0: a0, + A1: a1, + } +} + +func g2ProjectiveToGnarkAffine(p icicle_g2.G2Projective) *curve.G2Affine { + x := toGnarkE2(p.X) + y := toGnarkE2(p.Y) + z := toGnarkE2(p.Z) + var zSquared curve.E2 + zSquared.Mul(&z, &z) + + var X curve.E2 + X.Mul(&x, &z) + + var Y curve.E2 + Y.Mul(&y, &zSquared) + + return &curve.G2Affine{ + X: x, + Y: y, + } +} + +func g2ProjectiveToG2Jac(p icicle_g2.G2Projective) curve.G2Jac{ + var p1 curve.G2Jac + p1.FromAffine(g2ProjectiveToGnarkAffine(p)) + + return p1 +} + // Prove generates the proof of knowledge of a r1cs with full witness (secret + public part). func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...backend.ProverOption) (*groth16_bn254.Proof, error) { opt, err := backend.NewProverConfig(opts...) @@ -156,42 +241,36 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b solverOpts := opt.SolverOpts[:len(opt.SolverOpts):len(opt.SolverOpts)] privateCommittedValues := make([][]fr.Element, len(commitmentInfo)) - for i := range commitmentInfo { - solverOpts = append(solverOpts, solver.OverrideHint(commitmentInfo[i].HintID, func(i int) solver.Hint { - return func(_ *big.Int, in []*big.Int, out []*big.Int) error { - privateCommittedValues[i] = make([]fr.Element, len(commitmentInfo[i].PrivateCommitted)) - hashed := in[:len(commitmentInfo[i].PublicAndCommitmentCommitted)] - committed := in[len(hashed):] - for j, inJ := range committed { - privateCommittedValues[i][j].SetBigInt(inJ) - } - - var err error - if proof.Commitments[i], err = pk.CommitmentKeys[i].Commit(privateCommittedValues[i]); err != nil { - return err - } - - opt.HashToFieldFn.Write(constraint.SerializeCommitment(proof.Commitments[i].Marshal(), hashed, (fr.Bits-1)/8+1)) - hashBts := opt.HashToFieldFn.Sum(nil) - opt.HashToFieldFn.Reset() - nbBuf := fr.Bytes - if opt.HashToFieldFn.Size() < fr.Bytes { - nbBuf = opt.HashToFieldFn.Size() - } - var res fr.Element - res.SetBytes(hashBts[:nbBuf]) - res.BigInt(out[0]) - return err - } - }(i))) - } - if r1cs.GkrInfo.Is() { - var gkrData cs.GkrSolvingData - solverOpts = append(solverOpts, - solver.OverrideHint(r1cs.GkrInfo.SolveHintID, cs.GkrSolveHint(r1cs.GkrInfo, &gkrData)), - solver.OverrideHint(r1cs.GkrInfo.ProveHintID, cs.GkrProveHint(r1cs.GkrInfo.HashName, &gkrData))) - } + // override hints + bsb22ID := solver.GetHintID(fcs.Bsb22CommitmentComputePlaceholder) + solverOpts = append(solverOpts, solver.OverrideHint(bsb22ID, func(_ *big.Int, in []*big.Int, out []*big.Int) error { + i := int(in[0].Int64()) + in = in[1:] + privateCommittedValues[i] = make([]fr.Element, len(commitmentInfo[i].PrivateCommitted)) + hashed := in[:len(commitmentInfo[i].PublicAndCommitmentCommitted)] + committed := in[+len(hashed):] + for j, inJ := range committed { + privateCommittedValues[i][j].SetBigInt(inJ) + } + + var err error + if proof.Commitments[i], err = pk.CommitmentKeys[i].Commit(privateCommittedValues[i]); err != nil { + return err + } + + opt.HashToFieldFn.Write(constraint.SerializeCommitment(proof.Commitments[i].Marshal(), hashed, (fr.Bits-1)/8+1)) + hashBts := opt.HashToFieldFn.Sum(nil) + opt.HashToFieldFn.Reset() + nbBuf := fr.Bytes + if opt.HashToFieldFn.Size() < fr.Bytes { + nbBuf = opt.HashToFieldFn.Size() + } + var res fr.Element + res.SetBytes(hashBts[:nbBuf]) + res.BigInt(out[0]) + return nil + })) _solution, err := r1cs.Solve(fullWitness, solverOpts...) if err != nil { @@ -213,7 +292,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b } // H (witness reduction / FFT part) - var h unsafe.Pointer + var h icicle_core.DeviceSlice chHDone := make(chan struct{}, 1) go func() { h = computeH(solution.A, solution.B, solution.C, pk) @@ -225,7 +304,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // we need to copy and filter the wireValues for each multi exp // as pk.G1.A, pk.G1.B and pk.G2.B may have (a significant) number of point at infinity - var wireValuesADevice, wireValuesBDevice iciclegnark.OnDeviceData + var wireValuesADevice, wireValuesBDevice icicle_core.DeviceSlice chWireValuesA, chWireValuesB := make(chan struct{}, 1), make(chan struct{}, 1) go func() { @@ -237,18 +316,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b wireValuesA[j] = wireValues[i] j++ } - wireValuesASize := len(wireValuesA) - scalarBytes := wireValuesASize * fr.Bytes // Copy scalars to the device and retain ptr to them - copyDone := make(chan unsafe.Pointer, 1) - iciclegnark.CopyToDevice(wireValuesA, scalarBytes, copyDone) - wireValuesADevicePtr := <-copyDone - - wireValuesADevice = iciclegnark.OnDeviceData{ - P: wireValuesADevicePtr, - Size: wireValuesASize, - } + wireValuesAHost := (icicle_core.HostSlice[fr.Element])(wireValuesA) + wireValuesAHost.CopyToDevice(&wireValuesADevice, true) close(chWireValuesA) }() @@ -261,18 +332,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b wireValuesB[j] = wireValues[i] j++ } - wireValuesBSize := len(wireValuesB) - scalarBytes := wireValuesBSize * fr.Bytes // Copy scalars to the device and retain ptr to them - copyDone := make(chan unsafe.Pointer, 1) - iciclegnark.CopyToDevice(wireValuesB, scalarBytes, copyDone) - wireValuesBDevicePtr := <-copyDone - - wireValuesBDevice = iciclegnark.OnDeviceData{ - P: wireValuesBDevicePtr, - Size: wireValuesBSize, - } + wireValuesBHost := (icicle_core.HostSlice[fr.Element])(wireValuesB) + wireValuesBHost.CopyToDevice(&wireValuesBDevice, true) close(chWireValuesB) }() @@ -299,9 +362,16 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b computeBS1 := func() error { <-chWireValuesB - if bs1, _, err = iciclegnark.MsmOnDevice(wireValuesBDevice.P, pk.G1Device.B, wireValuesBDevice.Size, true); err != nil { - return err - } + cfg := icicle_msm.GetDefaultMSMConfig() + cfg.ArePointsMontgomeryForm = true + cfg.AreScalarsMontgomeryForm = true + bs1Stream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &bs1Stream + res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + icicle_msm.Msm(wireValuesBDevice, pk.G1Device.B, &cfg, res) + + // TODO - convert res[0] back to G1Jac + bs1 = g1ProjectiveToG1Jac(res[0]) bs1.AddMixed(&pk.G1.Beta) bs1.AddMixed(&deltas[1]) @@ -312,9 +382,16 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b computeAR1 := func() error { <-chWireValuesA - if ar, _, err = iciclegnark.MsmOnDevice(wireValuesADevice.P, pk.G1Device.A, wireValuesADevice.Size, true); err != nil { - return err - } + cfg := icicle_msm.GetDefaultMSMConfig() + cfg.ArePointsMontgomeryForm = true + cfg.AreScalarsMontgomeryForm = true + arStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &arStream + res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + icicle_msm.Msm(wireValuesADevice, pk.G1Device.A, &cfg, res) + + // TODO - convert res[0] back to G1Jac + ar = g1ProjectiveToG1Jac(res[0]) ar.AddMixed(&pk.G1.Alpha) ar.AddMixed(&deltas[0]) @@ -325,39 +402,27 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b computeKRS := func() error { var krs, krs2, p1 curve.G1Jac - sizeH := int(pk.Domain.Cardinality - 1) // comes from the fact the deg(H)=(n-1)+(n-1)-n=n-2 - // check for small circuits as iciclegnark doesn't handle zero sizes well - if len(pk.G1.Z) > 0 { - if krs2, _, err = iciclegnark.MsmOnDevice(h, pk.G1Device.Z, sizeH, true); err != nil { - return err - } - } + cfg := icicle_msm.GetDefaultMSMConfig() + cfg.ArePointsMontgomeryForm = true + cfg.AreScalarsMontgomeryForm = true + krsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &krsStream + resKrs2 := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + icicle_msm.Msm(h, pk.G1Device.Z, &cfg, resKrs2) + // TODO - convert res[0] back to G1Jac + krs2 = g1ProjectiveToG1Jac(resKrs2[0]) // filter the wire values if needed // TODO Perf @Tabaie worst memory allocation offender toRemove := commitmentInfo.GetPrivateCommitted() toRemove = append(toRemove, commitmentInfo.CommitmentIndexes()) - scalars := filterHeap(wireValues[r1cs.GetNbPublicVariables():], r1cs.GetNbPublicVariables(), internal.ConcatAll(toRemove...)) - - // filter zero/infinity points since icicle doesn't handle them - // See https://github.com/ingonyama-zk/icicle/issues/169 for more info - for _, indexToRemove := range pk.InfinityPointIndicesK { - scalars = append(scalars[:indexToRemove], scalars[indexToRemove+1:]...) - } - - scalarBytes := len(scalars) * fr.Bytes - - copyDone := make(chan unsafe.Pointer, 1) - iciclegnark.CopyToDevice(scalars, scalarBytes, copyDone) - scalars_d := <-copyDone - - krs, _, err = iciclegnark.MsmOnDevice(scalars_d, pk.G1Device.K, len(scalars), true) - iciclegnark.FreeDevicePointer(scalars_d) - - if err != nil { - return err - } + _wireValues := filterHeap(wireValues[r1cs.GetNbPublicVariables():], r1cs.GetNbPublicVariables(), internal.ConcatAll(toRemove...)) + _wireValuesHost := (icicle_core.HostSlice[fr.Element])(_wireValues) + resKrs := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + icicle_msm.Msm(_wireValuesHost, pk.G1Device.K, &cfg, resKrs) + // TODO - convert res[0] back to G1Jac + krs2 = g1ProjectiveToG1Jac(resKrs[0]) krs.AddMixed(&deltas[2]) @@ -379,9 +444,16 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b var Bs, deltaS curve.G2Jac <-chWireValuesB - if Bs, _, err = iciclegnark.MsmG2OnDevice(wireValuesBDevice.P, pk.G2Device.B, wireValuesBDevice.Size, true); err != nil { - return err - } + + cfg := icicle_msm.GetDefaultMSMConfig() + cfg.ArePointsMontgomeryForm = true + cfg.AreScalarsMontgomeryForm = true + krsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &krsStream + res := make(icicle_core.HostSlice[icicle_g2.G2Projective], 1) + icicle_msm.Msm(wireValuesBDevice, pk.G2Device.B, &cfg, res) + // TODO - convert res[0] back to G1Jac + Bs = g2ProjectiveToG2Jac(res[0]) deltaS.FromAffine(&pk.G2.Delta) deltaS.ScalarMultiplication(&deltaS, &s) @@ -413,9 +485,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // free device/GPU memory that is not needed for future proofs (scalars/hpoly) go func() { - iciclegnark.FreeDevicePointer(wireValuesADevice.P) - iciclegnark.FreeDevicePointer(wireValuesBDevice.P) - iciclegnark.FreeDevicePointer(h) + wireValuesADevice.Free() + wireValuesBDevice.Free() + h.Free() }() return proof, nil @@ -450,7 +522,7 @@ func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr return } -func computeH(a, b, c []fr.Element, pk *ProvingKey) unsafe.Pointer { +func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { // H part of Krs // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) @@ -466,48 +538,44 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey) unsafe.Pointer { c = append(c, padding...) n = len(a) - sizeBytes := n * fr.Bytes - - /*********** Copy a,b,c to Device Start ************/ - // Individual channels are necessary to know which device pointers - // point to which vector - copyADone := make(chan unsafe.Pointer, 1) - copyBDone := make(chan unsafe.Pointer, 1) - copyCDone := make(chan unsafe.Pointer, 1) - - go iciclegnark.CopyToDevice(a, sizeBytes, copyADone) - go iciclegnark.CopyToDevice(b, sizeBytes, copyBDone) - go iciclegnark.CopyToDevice(c, sizeBytes, copyCDone) - - a_device := <-copyADone - b_device := <-copyBDone - c_device := <-copyCDone - /*********** Copy a,b,c to Device End ************/ - - computeInttNttDone := make(chan error, 1) - computeInttNttOnDevice := func(devicePointer unsafe.Pointer) { - a_intt_d := iciclegnark.INttOnDevice(devicePointer, pk.DomainDevice.TwiddlesInv, nil, n, sizeBytes, false) - iciclegnark.NttOnDevice(devicePointer, a_intt_d, pk.DomainDevice.Twiddles, pk.DomainDevice.CosetTable, n, n, sizeBytes, true) - computeInttNttDone <- nil - iciclegnark.FreeDevicePointer(a_intt_d) + computeADone := make(chan icicle_core.DeviceSlice, 1) + computeBDone := make(chan icicle_core.DeviceSlice, 1) + computeCDone := make(chan icicle_core.DeviceSlice, 1) + + computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { + cfg := icicle_ntt.GetDefaultNttConfig() + scalarsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &scalarsStream + cfg.Ordering = icicle_core.KNR + cfg.IsAsync = true + scalarsHost := icicle_core.HostSliceFromElements(scalars) + var scalarsDevice icicle_core.DeviceSlice + scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) + icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) + cfg.Ordering = icicle_core.KRN + cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGen[:])) + icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) + icicle_cr.SynchronizeStream(&scalarsStream) + channel <-scalarsDevice } - - go computeInttNttOnDevice(a_device) - go computeInttNttOnDevice(b_device) - go computeInttNttOnDevice(c_device) - _, _, _ = <-computeInttNttDone, <-computeInttNttDone, <-computeInttNttDone - - iciclegnark.PolyOps(a_device, b_device, c_device, pk.DenDevice, n) - - h := iciclegnark.INttOnDevice(a_device, pk.DomainDevice.TwiddlesInv, pk.DomainDevice.CosetTableInv, n, sizeBytes, true) - - go func() { - iciclegnark.FreeDevicePointer(a_device) - iciclegnark.FreeDevicePointer(b_device) - iciclegnark.FreeDevicePointer(c_device) - }() - - iciclegnark.ReverseScalars(h, n) - - return h + + go computeInttNttOnDevice(a, computeADone) + go computeInttNttOnDevice(b, computeBDone) + go computeInttNttOnDevice(c, computeCDone) + + aDevice := <-computeADone + bDevice := <-computeBDone + cDevice := <-computeCDone + + vecCfg := icicle_core.DefaultVecOpsConfig() + icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) + icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) + icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + + cfg := icicle_ntt.GetDefaultNttConfig() + cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGenInv[:])) + cfg.Ordering = icicle_core.KNR + icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + + return aDevice } diff --git a/backend/groth16/bn254/icicle/marshal_test.go b/backend/groth16/bn254/icicle/marshal_test.go index 75c5a2b57e..39cc66a85d 100644 --- a/backend/groth16/bn254/icicle/marshal_test.go +++ b/backend/groth16/bn254/icicle/marshal_test.go @@ -1,4 +1,4 @@ -package icicle_bn254_test +package icicle_test import ( "bytes" diff --git a/backend/groth16/bn254/icicle/noicicle.go b/backend/groth16/bn254/icicle/noicicle.go index 87703339ce..5925dd0b11 100644 --- a/backend/groth16/bn254/icicle/noicicle.go +++ b/backend/groth16/bn254/icicle/noicicle.go @@ -1,6 +1,6 @@ //go:build !icicle -package icicle_bn254 +package icicle import ( "fmt" diff --git a/backend/groth16/bn254/icicle/provingkey.go b/backend/groth16/bn254/icicle/provingkey.go index 146a794255..758930436d 100644 --- a/backend/groth16/bn254/icicle/provingkey.go +++ b/backend/groth16/bn254/icicle/provingkey.go @@ -1,25 +1,23 @@ -package icicle_bn254 +package icicle import ( - "unsafe" groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" cs "github.com/consensys/gnark/constraint/bn254" + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" ) type deviceInfo struct { G1Device struct { - A, B, K, Z unsafe.Pointer + A, B, K, Z icicle_core.DeviceSlice } DomainDevice struct { - Twiddles, TwiddlesInv unsafe.Pointer - CosetTable, CosetTableInv unsafe.Pointer + CosetTable, CosetTableInv icicle_core.DeviceSlice } G2Device struct { - B unsafe.Pointer + B icicle_core.DeviceSlice } - DenDevice unsafe.Pointer - InfinityPointIndicesK []int + DenDevice icicle_core.DeviceSlice } type ProvingKey struct { diff --git a/go.mod b/go.mod index 0a1fe8218d..282b9fbdc1 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b github.com/icza/bitio v1.1.0 - github.com/ingonyama-zk/iciclegnark v0.1.0 + github.com/ingonyama-zk/icicle/v2 v2.0.1 github.com/leanovate/gopter v0.2.9 github.com/rs/zerolog v1.30.0 github.com/stretchr/testify v1.8.4 @@ -25,14 +25,14 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/ingonyama-zk/icicle v0.0.0-20230928131117-97f0079e5c71 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/x448/float16 v0.8.4 // indirect golang.org/x/sys v0.15.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/ingonyama-zk/icicle/v2 v2.0.1 => github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b diff --git a/go.sum b/go.sum index ef932c9016..bf31e03076 100644 --- a/go.sum +++ b/go.sum @@ -23,10 +23,8 @@ github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= -github.com/ingonyama-zk/icicle v0.0.0-20230928131117-97f0079e5c71 h1:YxI1RTPzpFJ3MBmxPl3Bo0F7ume7CmQEC1M9jL6CT94= -github.com/ingonyama-zk/icicle v0.0.0-20230928131117-97f0079e5c71/go.mod h1:kAK8/EoN7fUEmakzgZIYdWy1a2rBnpCaZLqSHwZWxEk= -github.com/ingonyama-zk/iciclegnark v0.1.0 h1:88MkEghzjQBMjrYRJFxZ9oR9CTIpB8NG2zLeCJSvXKQ= -github.com/ingonyama-zk/iciclegnark v0.1.0/go.mod h1:wz6+IpyHKs6UhMMoQpNqz1VY+ddfKqC/gRwR/64W6WU= +github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b h1:Xmssae9C6ZPmct4uJeII0Mk2ZZDPOQX8Opkhubb9YoI= +github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b/go.mod h1:rr3B+xKQKW1U40A+vEzA4hI2ilTrPSJBtxedfnaUYHw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -46,8 +44,8 @@ github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFV github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= diff --git a/std/math/emulated/element.go b/std/math/emulated/element.go index f3da9d3c7c..c7233e51ed 100644 --- a/std/math/emulated/element.go +++ b/std/math/emulated/element.go @@ -89,6 +89,12 @@ func (f *Field[T]) newInternalElement(limbs []frontend.Variable, overflow uint) return &Element[T]{Limbs: limbs, overflow: overflow, internal: true} } +// newInternalElement sets the limbs and overflow. Given as a function for later +// possible refactor. +func (f *Field[T]) NewInternalElement(limbs []frontend.Variable, overflow uint) *Element[T] { + return &Element[T]{Limbs: limbs, overflow: overflow, internal: true} +} + // GnarkInitHook describes how to initialise the element. func (e *Element[T]) GnarkInitHook() { if e.Limbs == nil { From aa206644997304a047a1b7677ce77ea56cefd93b Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Thu, 25 Apr 2024 11:45:43 +0300 Subject: [PATCH 02/11] Update mod to use released v2.0.2 of icicle --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 282b9fbdc1..2224d72231 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b github.com/icza/bitio v1.1.0 - github.com/ingonyama-zk/icicle/v2 v2.0.1 + github.com/ingonyama-zk/icicle/v2 v2.0.2 github.com/leanovate/gopter v0.2.9 github.com/rs/zerolog v1.30.0 github.com/stretchr/testify v1.8.4 @@ -34,5 +34,3 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) - -replace github.com/ingonyama-zk/icicle/v2 v2.0.1 => github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b diff --git a/go.sum b/go.sum index bf31e03076..6f42ac4348 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= -github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b h1:Xmssae9C6ZPmct4uJeII0Mk2ZZDPOQX8Opkhubb9YoI= -github.com/ingonyama-zk/icicle/v2 v2.0.0-20240424145405-78befff3d61b/go.mod h1:rr3B+xKQKW1U40A+vEzA4hI2ilTrPSJBtxedfnaUYHw= +github.com/ingonyama-zk/icicle/v2 v2.0.2 h1:9eNDqgS/4tmxGk5owbJ7+eTwLY35f1T3aHp6aQDcgJU= +github.com/ingonyama-zk/icicle/v2 v2.0.2/go.mod h1:rr3B+xKQKW1U40A+vEzA4hI2ilTrPSJBtxedfnaUYHw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= From d9713f8ebf29024ff5b3460e77b29b3c9335985c Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Thu, 25 Apr 2024 17:20:53 +0300 Subject: [PATCH 03/11] MSMs and non-coset ntts working. Coset ntt fails in computeH --- backend/groth16/bn254/icicle/icicle.go | 288 ++++++++++++--------- backend/groth16/bn254/icicle/provingkey.go | 4 +- go.mod | 2 + 3 files changed, 175 insertions(+), 119 deletions(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 6be02c87cd..322bf0f7e9 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -6,11 +6,14 @@ import ( "fmt" "math/big" "math/bits" + "runtime" "time" + "github.com/consensys/gnark-crypto/ecc" curve "github.com/consensys/gnark-crypto/ecc/bn254" - "github.com/consensys/gnark-crypto/ecc/bn254/fr" "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/consensys/gnark-crypto/ecc/bn254/fr/fft" "github.com/consensys/gnark-crypto/ecc/bn254/fr/hash_to_field" "github.com/consensys/gnark-crypto/ecc/bn254/fr/pedersen" "github.com/consensys/gnark/backend" @@ -29,7 +32,7 @@ import ( icicle_g2 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/g2" icicle_msm "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/msm" icicle_ntt "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/ntt" - icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" + // icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" fcs "github.com/consensys/gnark/frontend/cs" ) @@ -41,38 +44,13 @@ func (pk *ProvingKey) setupDevicePointers() error { return nil } pk.deviceInfo = &deviceInfo{} - /************************* Start Domain Device Setup ***************************/ - /************************* CosetTableInv ***************************/ - copyCosetInvDone := make(chan bool, 1) - go func() { - cosetTableInv, err := pk.Domain.CosetTableInv() - if err != nil { - panic("Something went wrong") // TODO - } - cosetTableInvHost := (icicle_core.HostSlice[fr.Element])(cosetTableInv) - cosetTableInvHost.CopyToDevice(&pk.DomainDevice.CosetTableInv, true) - copyCosetInvDone <- true - }() - - /************************* CosetTable ***************************/ - copyCosetDone := make(chan bool, 1) - go func() { - coestTable, err := pk.Domain.CosetTable() - if err != nil { - panic("SOmething went wrong") // TODO - } - cosetTableHost := (icicle_core.HostSlice[fr.Element])(coestTable) - cosetTableHost.CopyToDevice(&pk.DomainDevice.CosetTable, true) - copyCosetDone <- true - }() - /************************* Den ***************************/ n := int(pk.Domain.Cardinality) var denI, oneI fr.Element oneI.SetOne() denI.Exp(pk.Domain.FrMultiplicativeGen, big.NewInt(int64(pk.Domain.Cardinality))) denI.Sub(&denI, &oneI).Inverse(&denI) - + log2SizeFloor := bits.Len(uint(n)) - 1 denIcicleArr := []fr.Element{denI} for i := 0; i < log2SizeFloor; i++ { @@ -82,34 +60,32 @@ func (pk *ProvingKey) setupDevicePointers() error { for i := 0; i < pow2Remainder; i++ { denIcicleArr = append(denIcicleArr, denI) } - + copyDenDone := make(chan bool, 1) go func() { denIcicleArrHost := (icicle_core.HostSlice[fr.Element])(denIcicleArr) denIcicleArrHost.CopyToDevice(&pk.DenDevice, true) copyDenDone <- true - }() - - /************************* Twiddles and Twiddles Inv ***************************/ + }() + + /************************* Init Domain Device ***************************/ ctx, err := icicle_cr.GetDefaultDeviceContext() if err != icicle_cr.CudaSuccess { panic("Couldn't create device context") // TODO } - rou := pk.Domain.Generator.Bits() - rouIcicle := icicle_bn254.ScalarField{} - limbs := icicle_core.ConvertUint64ArrToUint32Arr(rou[:]) + gen, _ := fft.Generator(2 * pk.Domain.Cardinality) + genBits := gen.Bits() + limbs := icicle_core.ConvertUint64ArrToUint32Arr(genBits[:]) + pk.CosetGenerator = limbs + var rouIcicle icicle_bn254.ScalarField rouIcicle.FromLimbs(limbs) e := icicle_ntt.InitDomain(rouIcicle, ctx, false) if e.IcicleErrorCode != icicle_core.IcicleSuccess { panic("Couldn't initialize domain") // TODO } - /************************* End Domain Device Setup ***************************/ - - <-copyCosetInvDone - <-copyCosetDone - <-copyDenDone + /************************* End Init Domain Device ***************************/ /************************* Start G1 Device Setup ***************************/ /************************* A ***************************/ copyADone := make(chan bool, 1) @@ -125,7 +101,7 @@ func (pk *ProvingKey) setupDevicePointers() error { g1BHost.CopyToDevice(&pk.G1Device.B, true) copyBDone <- true }() - /************************* K ***************************/ + /************************* K ***************************/ copyKDone := make(chan bool, 1) go func() { g1KHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.K) @@ -140,6 +116,7 @@ func (pk *ProvingKey) setupDevicePointers() error { copyZDone <- true }() /************************* End G1 Device Setup ***************************/ + <-copyDenDone <-copyADone <-copyBDone <-copyKDone @@ -174,21 +151,21 @@ func projectiveToGnarkAffine(p icicle_bn254.Projective) *curve.G1Affine { func g1ProjectiveToG1Jac(p icicle_bn254.Projective) curve.G1Jac { var p1 curve.G1Jac p1.FromAffine(projectiveToGnarkAffine(p)) - + return p1 } func toGnarkE2(f icicle_g2.G2BaseField) curve.E2 { bytes := f.ToBytesLittleEndian() - a0, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[:len(bytes)/2])) - a1, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[len(bytes)/2:])) + a0, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[:fp.Bytes])) + a1, _ := fp.LittleEndian.Element((*[fp.Bytes]byte)(bytes[fp.Bytes:])) return curve.E2{ A0: a0, A1: a1, } } -func g2ProjectiveToGnarkAffine(p icicle_g2.G2Projective) *curve.G2Affine { +func g2ProjectiveToG2Jac(p *icicle_g2.G2Projective) curve.G2Jac { x := toGnarkE2(p.X) y := toGnarkE2(p.Y) z := toGnarkE2(p.Z) @@ -201,19 +178,13 @@ func g2ProjectiveToGnarkAffine(p icicle_g2.G2Projective) *curve.G2Affine { var Y curve.E2 Y.Mul(&y, &zSquared) - return &curve.G2Affine{ - X: x, - Y: y, + return curve.G2Jac{ + X: X, + Y: Y, + Z: z, } } -func g2ProjectiveToG2Jac(p icicle_g2.G2Projective) curve.G2Jac{ - var p1 curve.G2Jac - p1.FromAffine(g2ProjectiveToGnarkAffine(p)) - - return p1 -} - // Prove generates the proof of knowledge of a r1cs with full witness (secret + public part). func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...backend.ProverOption) (*groth16_bn254.Proof, error) { opt, err := backend.NewProverConfig(opts...) @@ -292,10 +263,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b } // H (witness reduction / FFT part) - var h icicle_core.DeviceSlice + // var h icicle_core.DeviceSlice + var hCPU []fr.Element chHDone := make(chan struct{}, 1) go func() { - h = computeH(solution.A, solution.B, solution.C, pk) + hCPU = computeH(solution.A, solution.B, solution.C, &pk.Domain) + solution.A = nil solution.B = nil solution.C = nil @@ -356,6 +329,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // computes r[δ], s[δ], kr[δ] deltas := curve.BatchScalarMultiplicationG1(&pk.G1.Delta, []fr.Element{_r, _s, _kr}) + n := runtime.NumCPU() var bs1, ar curve.G1Jac @@ -365,12 +339,8 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg := icicle_msm.GetDefaultMSMConfig() cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true - bs1Stream, _ := icicle_cr.CreateStream() - cfg.Ctx.Stream = &bs1Stream res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) icicle_msm.Msm(wireValuesBDevice, pk.G1Device.B, &cfg, res) - - // TODO - convert res[0] back to G1Jac bs1 = g1ProjectiveToG1Jac(res[0]) bs1.AddMixed(&pk.G1.Beta) @@ -385,12 +355,8 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg := icicle_msm.GetDefaultMSMConfig() cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true - arStream, _ := icicle_cr.CreateStream() - cfg.Ctx.Stream = &arStream res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) icicle_msm.Msm(wireValuesADevice, pk.G1Device.A, &cfg, res) - - // TODO - convert res[0] back to G1Jac ar = g1ProjectiveToG1Jac(res[0]) ar.AddMixed(&pk.G1.Alpha) @@ -402,17 +368,31 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b computeKRS := func() error { var krs, krs2, p1 curve.G1Jac + var krs2CPU curve.G1Jac + sizeH := int(pk.Domain.Cardinality - 1) + + // CPU START + + if _, err := krs2CPU.MultiExp(pk.G1.Z, hCPU[:sizeH], ecc.MultiExpConfig{NbTasks: n / 2}); err != nil { + panic("krs2CPU didn't complete") + } + + // CPU END cfg := icicle_msm.GetDefaultMSMConfig() cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true - krsStream, _ := icicle_cr.CreateStream() - cfg.Ctx.Stream = &krsStream resKrs2 := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) - icicle_msm.Msm(h, pk.G1Device.Z, &cfg, resKrs2) - // TODO - convert res[0] back to G1Jac + // icicle_msm.Msm(h.RangeTo(sizeH, false), pk.G1Device.Z, &cfg, resKrs2) + icicle_msm.Msm(icicle_core.HostSliceFromElements(hCPU[:sizeH]), pk.G1Device.Z, &cfg, resKrs2) + krs2 = g1ProjectiveToG1Jac(resKrs2[0]) + if krs2.Equal(&krs2CPU) { + fmt.Println("krs2 succeeded") + } else { + fmt.Println("krs2 failed correctness") + } // filter the wire values if needed // TODO Perf @Tabaie worst memory allocation offender toRemove := commitmentInfo.GetPrivateCommitted() @@ -421,8 +401,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b _wireValuesHost := (icicle_core.HostSlice[fr.Element])(_wireValues) resKrs := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) icicle_msm.Msm(_wireValuesHost, pk.G1Device.K, &cfg, resKrs) - // TODO - convert res[0] back to G1Jac - krs2 = g1ProjectiveToG1Jac(resKrs[0]) + krs = g1ProjectiveToG1Jac(resKrs[0]) krs.AddMixed(&deltas[2]) @@ -445,15 +424,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b <-chWireValuesB - cfg := icicle_msm.GetDefaultMSMConfig() + cfg := icicle_g2.G2GetDefaultMSMConfig() cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true - krsStream, _ := icicle_cr.CreateStream() - cfg.Ctx.Stream = &krsStream res := make(icicle_core.HostSlice[icicle_g2.G2Projective], 1) - icicle_msm.Msm(wireValuesBDevice, pk.G2Device.B, &cfg, res) - // TODO - convert res[0] back to G1Jac - Bs = g2ProjectiveToG2Jac(res[0]) + icicle_g2.G2Msm(wireValuesBDevice, pk.G2Device.B, &cfg, res) + Bs = g2ProjectiveToG2Jac(&res[0]) deltaS.FromAffine(&pk.G2.Delta) deltaS.ScalarMultiplication(&deltaS, &s) @@ -487,7 +463,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b go func() { wireValuesADevice.Free() wireValuesBDevice.Free() - h.Free() + // h.Free() }() return proof, nil @@ -522,7 +498,68 @@ func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr return } -func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { +// func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { +// // H part of Krs +// // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) +// // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) +// // 2 - ca = fft_coset(_a), ba = fft_coset(_b), cc = fft_coset(_c) +// // 3 - h = ifft_coset(ca o cb - cc) + +// n := len(a) + +// // add padding to ensure input length is domain cardinality +// padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) +// a = append(a, padding...) +// b = append(b, padding...) +// c = append(c, padding...) +// n = len(a) + +// computeADone := make(chan icicle_core.DeviceSlice, 1) +// computeBDone := make(chan icicle_core.DeviceSlice, 1) +// computeCDone := make(chan icicle_core.DeviceSlice, 1) + +// computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { +// cfg := icicle_ntt.GetDefaultNttConfig() +// scalarsStream, _ := icicle_cr.CreateStream() +// cfg.Ctx.Stream = &scalarsStream +// cfg.Ordering = icicle_core.KNR +// cfg.IsAsync = true +// scalarsHost := icicle_core.HostSliceFromElements(scalars) +// var scalarsDevice icicle_core.DeviceSlice +// scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) +// icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) +// cfg.Ordering = icicle_core.KRN +// cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGen[:])) +// icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) +// icicle_cr.SynchronizeStream(&scalarsStream) +// channel <-scalarsDevice +// } + +// go computeInttNttOnDevice(a, computeADone) +// go computeInttNttOnDevice(b, computeBDone) +// go computeInttNttOnDevice(c, computeCDone) + +// aDevice := <-computeADone +// bDevice := <-computeBDone +// cDevice := <-computeCDone + +// vecCfg := icicle_core.DefaultVecOpsConfig() +// icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) +// icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) +// icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + +// cfg := icicle_ntt.GetDefaultNttConfig() +// cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGenInv[:])) +// cfg.Ordering = icicle_core.KNR +// icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + +// resHost := make(icicle_core.HostSlice[fr.Element], n) +// resHost.CopyFromDevice(&aDevice) + +// return aDevice +// } + +func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { // H part of Krs // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) @@ -532,50 +569,69 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { n := len(a) // add padding to ensure input length is domain cardinality - padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) + padding := make([]fr.Element, int(domain.Cardinality)-n) a = append(a, padding...) b = append(b, padding...) c = append(c, padding...) n = len(a) - computeADone := make(chan icicle_core.DeviceSlice, 1) - computeBDone := make(chan icicle_core.DeviceSlice, 1) - computeCDone := make(chan icicle_core.DeviceSlice, 1) - - computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { - cfg := icicle_ntt.GetDefaultNttConfig() - scalarsStream, _ := icicle_cr.CreateStream() - cfg.Ctx.Stream = &scalarsStream - cfg.Ordering = icicle_core.KNR - cfg.IsAsync = true - scalarsHost := icicle_core.HostSliceFromElements(scalars) - var scalarsDevice icicle_core.DeviceSlice - scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) - icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) - cfg.Ordering = icicle_core.KRN - cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGen[:])) - icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) - icicle_cr.SynchronizeStream(&scalarsStream) - channel <-scalarsDevice - } - - go computeInttNttOnDevice(a, computeADone) - go computeInttNttOnDevice(b, computeBDone) - go computeInttNttOnDevice(c, computeCDone) - - aDevice := <-computeADone - bDevice := <-computeBDone - cDevice := <-computeCDone - - vecCfg := icicle_core.DefaultVecOpsConfig() - icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) - icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) - icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + aCopy := make([]fr.Element, n) + copy(aCopy, a) cfg := icicle_ntt.GetDefaultNttConfig() - cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGenInv[:])) cfg.Ordering = icicle_core.KNR - icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + scalarsHost := icicle_core.HostSliceFromElements(aCopy) + scalarsHostOut := make(icicle_core.HostSlice[fr.Element], len(aCopy)) + icicle_ntt.Ntt(scalarsHost, icicle_core.KInverse, &cfg, scalarsHostOut) + + domain.FFTInverse(a, fft.DIF) + + for i, elem := range a { + if !elem.Equal(&scalarsHostOut[i]) { + fmt.Println("computeH: A failed") + } + } + + domain.FFTInverse(b, fft.DIF) + domain.FFTInverse(c, fft.DIF) + + + gen, _ := fft.Generator(2 * domain.Cardinality) + // genBits := gen.Bits() + // limbs := icicle_core.ConvertUint64ArrToUint32Arr(genBits[:]) + // var rouIcicle icicle_bn254.ScalarField + // rouIcicle.FromLimbs(limbs) + cfgCustom := icicle_ntt.GetDefaultNttConfig() + cfg.CosetGen = ([8]uint32)(icicle_core.ConvertUint64ArrToUint32Arr(gen[:])) + cfgCustom.Ordering = icicle_core.KRN + icicle_ntt.Ntt(scalarsHostOut, icicle_core.KForward, &cfgCustom, scalarsHost) + + domain.FFT(a, fft.DIT, fft.OnCoset()) + + if !scalarsHost[0].Equal(&a[0]) { + fmt.Println("computeH: A Forward failed") + } + + domain.FFT(b, fft.DIT, fft.OnCoset()) + domain.FFT(c, fft.DIT, fft.OnCoset()) + + var den, one fr.Element + one.SetOne() + den.Exp(domain.FrMultiplicativeGen, big.NewInt(int64(domain.Cardinality))) + den.Sub(&den, &one).Inverse(&den) + + // h = ifft_coset(ca o cb - cc) + // reusing a to avoid unnecessary memory allocation + utils.Parallelize(n, func(start, end int) { + for i := start; i < end; i++ { + a[i].Mul(&a[i], &b[i]). + Sub(&a[i], &c[i]). + Mul(&a[i], &den) + } + }) + + // ifft_coset + domain.FFTInverse(a, fft.DIF, fft.OnCoset()) - return aDevice + return a } diff --git a/backend/groth16/bn254/icicle/provingkey.go b/backend/groth16/bn254/icicle/provingkey.go index 758930436d..e0fa5a210a 100644 --- a/backend/groth16/bn254/icicle/provingkey.go +++ b/backend/groth16/bn254/icicle/provingkey.go @@ -8,12 +8,10 @@ import ( ) type deviceInfo struct { + CosetGenerator []uint32 G1Device struct { A, B, K, Z icicle_core.DeviceSlice } - DomainDevice struct { - CosetTable, CosetTableInv icicle_core.DeviceSlice - } G2Device struct { B icicle_core.DeviceSlice } diff --git a/go.mod b/go.mod index 2224d72231..f804479930 100644 --- a/go.mod +++ b/go.mod @@ -34,3 +34,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +// replace github.com/ingonyama-zk/icicle/v2 v2.0.2 => ../icicle From ec3c98052a002109758f4607ff017ff5c3e76eef Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Fri, 26 Apr 2024 08:17:13 +0300 Subject: [PATCH 04/11] Fix coset ntt and have correctness --- backend/groth16/bn254/icicle/icicle.go | 187 +++++++------------------ 1 file changed, 47 insertions(+), 140 deletions(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 322bf0f7e9..8f4d914447 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -6,10 +6,8 @@ import ( "fmt" "math/big" "math/bits" - "runtime" "time" - "github.com/consensys/gnark-crypto/ecc" curve "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fp" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -32,7 +30,7 @@ import ( icicle_g2 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/g2" icicle_msm "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/msm" icicle_ntt "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/ntt" - // icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" + icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bn254/vecOps" fcs "github.com/consensys/gnark/frontend/cs" ) @@ -263,11 +261,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b } // H (witness reduction / FFT part) - // var h icicle_core.DeviceSlice - var hCPU []fr.Element + var h icicle_core.DeviceSlice chHDone := make(chan struct{}, 1) go func() { - hCPU = computeH(solution.A, solution.B, solution.C, &pk.Domain) + h = computeH(solution.A, solution.B, solution.C, pk) solution.A = nil solution.B = nil @@ -329,7 +326,6 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // computes r[δ], s[δ], kr[δ] deltas := curve.BatchScalarMultiplicationG1(&pk.G1.Delta, []fr.Element{_r, _s, _kr}) - n := runtime.NumCPU() var bs1, ar curve.G1Jac @@ -368,31 +364,15 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b computeKRS := func() error { var krs, krs2, p1 curve.G1Jac - var krs2CPU curve.G1Jac sizeH := int(pk.Domain.Cardinality - 1) - // CPU START - - if _, err := krs2CPU.MultiExp(pk.G1.Z, hCPU[:sizeH], ecc.MultiExpConfig{NbTasks: n / 2}); err != nil { - panic("krs2CPU didn't complete") - } - - // CPU END - cfg := icicle_msm.GetDefaultMSMConfig() cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true resKrs2 := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) - // icicle_msm.Msm(h.RangeTo(sizeH, false), pk.G1Device.Z, &cfg, resKrs2) - icicle_msm.Msm(icicle_core.HostSliceFromElements(hCPU[:sizeH]), pk.G1Device.Z, &cfg, resKrs2) - + icicle_msm.Msm(h.RangeTo(sizeH, false), pk.G1Device.Z, &cfg, resKrs2) krs2 = g1ProjectiveToG1Jac(resKrs2[0]) - if krs2.Equal(&krs2CPU) { - fmt.Println("krs2 succeeded") - } else { - fmt.Println("krs2 failed correctness") - } // filter the wire values if needed // TODO Perf @Tabaie worst memory allocation offender toRemove := commitmentInfo.GetPrivateCommitted() @@ -498,68 +478,7 @@ func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr return } -// func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { -// // H part of Krs -// // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) -// // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) -// // 2 - ca = fft_coset(_a), ba = fft_coset(_b), cc = fft_coset(_c) -// // 3 - h = ifft_coset(ca o cb - cc) - -// n := len(a) - -// // add padding to ensure input length is domain cardinality -// padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) -// a = append(a, padding...) -// b = append(b, padding...) -// c = append(c, padding...) -// n = len(a) - -// computeADone := make(chan icicle_core.DeviceSlice, 1) -// computeBDone := make(chan icicle_core.DeviceSlice, 1) -// computeCDone := make(chan icicle_core.DeviceSlice, 1) - -// computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { -// cfg := icicle_ntt.GetDefaultNttConfig() -// scalarsStream, _ := icicle_cr.CreateStream() -// cfg.Ctx.Stream = &scalarsStream -// cfg.Ordering = icicle_core.KNR -// cfg.IsAsync = true -// scalarsHost := icicle_core.HostSliceFromElements(scalars) -// var scalarsDevice icicle_core.DeviceSlice -// scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) -// icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) -// cfg.Ordering = icicle_core.KRN -// cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGen[:])) -// icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) -// icicle_cr.SynchronizeStream(&scalarsStream) -// channel <-scalarsDevice -// } - -// go computeInttNttOnDevice(a, computeADone) -// go computeInttNttOnDevice(b, computeBDone) -// go computeInttNttOnDevice(c, computeCDone) - -// aDevice := <-computeADone -// bDevice := <-computeBDone -// cDevice := <-computeCDone - -// vecCfg := icicle_core.DefaultVecOpsConfig() -// icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) -// icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) -// icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) - -// cfg := icicle_ntt.GetDefaultNttConfig() -// cfg.CosetGen = [8]uint32(icicle_core.ConvertUint64ArrToUint32Arr(pk.Domain.FrMultiplicativeGenInv[:])) -// cfg.Ordering = icicle_core.KNR -// icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) - -// resHost := make(icicle_core.HostSlice[fr.Element], n) -// resHost.CopyFromDevice(&aDevice) - -// return aDevice -// } - -func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { +func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { // H part of Krs // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) @@ -569,69 +488,57 @@ func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { n := len(a) // add padding to ensure input length is domain cardinality - padding := make([]fr.Element, int(domain.Cardinality)-n) + padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) a = append(a, padding...) b = append(b, padding...) c = append(c, padding...) n = len(a) - aCopy := make([]fr.Element, n) - copy(aCopy, a) - - cfg := icicle_ntt.GetDefaultNttConfig() - cfg.Ordering = icicle_core.KNR - scalarsHost := icicle_core.HostSliceFromElements(aCopy) - scalarsHostOut := make(icicle_core.HostSlice[fr.Element], len(aCopy)) - icicle_ntt.Ntt(scalarsHost, icicle_core.KInverse, &cfg, scalarsHostOut) - - domain.FFTInverse(a, fft.DIF) - - for i, elem := range a { - if !elem.Equal(&scalarsHostOut[i]) { - fmt.Println("computeH: A failed") - } + computeADone := make(chan icicle_core.DeviceSlice, 1) + computeBDone := make(chan icicle_core.DeviceSlice, 1) + computeCDone := make(chan icicle_core.DeviceSlice, 1) + + cosetGenBits := pk.Domain.FrMultiplicativeGen.Bits() + cosetGen := icicle_core.ConvertUint64ArrToUint32Arr(cosetGenBits[:]) + var configCosetGen [8]uint32 + copy(configCosetGen[:], cosetGen[:8]) + + computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { + cfg := icicle_ntt.GetDefaultNttConfig() + scalarsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &scalarsStream + cfg.Ordering = icicle_core.KNR + cfg.IsAsync = true + scalarsHost := icicle_core.HostSliceFromElements(scalars) + var scalarsDevice icicle_core.DeviceSlice + scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) + icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) + cfg.Ordering = icicle_core.KRN + cfg.CosetGen = configCosetGen + icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) + icicle_cr.SynchronizeStream(&scalarsStream) + channel <-scalarsDevice } - domain.FFTInverse(b, fft.DIF) - domain.FFTInverse(c, fft.DIF) - + go computeInttNttOnDevice(a, computeADone) + go computeInttNttOnDevice(b, computeBDone) + go computeInttNttOnDevice(c, computeCDone) - gen, _ := fft.Generator(2 * domain.Cardinality) - // genBits := gen.Bits() - // limbs := icicle_core.ConvertUint64ArrToUint32Arr(genBits[:]) - // var rouIcicle icicle_bn254.ScalarField - // rouIcicle.FromLimbs(limbs) - cfgCustom := icicle_ntt.GetDefaultNttConfig() - cfg.CosetGen = ([8]uint32)(icicle_core.ConvertUint64ArrToUint32Arr(gen[:])) - cfgCustom.Ordering = icicle_core.KRN - icicle_ntt.Ntt(scalarsHostOut, icicle_core.KForward, &cfgCustom, scalarsHost) + aDevice := <-computeADone + bDevice := <-computeBDone + cDevice := <-computeCDone - domain.FFT(a, fft.DIT, fft.OnCoset()) - - if !scalarsHost[0].Equal(&a[0]) { - fmt.Println("computeH: A Forward failed") - } + vecCfg := icicle_core.DefaultVecOpsConfig() + icicle_bn254.FromMontgomery(&aDevice) + icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) + icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) + icicle_bn254.FromMontgomery(&aDevice) + icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) - domain.FFT(b, fft.DIT, fft.OnCoset()) - domain.FFT(c, fft.DIT, fft.OnCoset()) - - var den, one fr.Element - one.SetOne() - den.Exp(domain.FrMultiplicativeGen, big.NewInt(int64(domain.Cardinality))) - den.Sub(&den, &one).Inverse(&den) - - // h = ifft_coset(ca o cb - cc) - // reusing a to avoid unnecessary memory allocation - utils.Parallelize(n, func(start, end int) { - for i := start; i < end; i++ { - a[i].Mul(&a[i], &b[i]). - Sub(&a[i], &c[i]). - Mul(&a[i], &den) - } - }) - - // ifft_coset - domain.FFTInverse(a, fft.DIF, fft.OnCoset()) + cfg := icicle_ntt.GetDefaultNttConfig() + cfg.CosetGen = configCosetGen + cfg.Ordering = icicle_core.KNR + icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) - return a + return aDevice } From 9710f58c5d2a66ee5a79d319ae74bb00ba84b0f5 Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Fri, 26 Apr 2024 09:18:11 +0300 Subject: [PATCH 05/11] bump icicle to v2.0.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f804479930..eb81dccb07 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.5.9 github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b github.com/icza/bitio v1.1.0 - github.com/ingonyama-zk/icicle/v2 v2.0.2 + github.com/ingonyama-zk/icicle/v2 v2.0.3 github.com/leanovate/gopter v0.2.9 github.com/rs/zerolog v1.30.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 6f42ac4348..29bedfde79 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= -github.com/ingonyama-zk/icicle/v2 v2.0.2 h1:9eNDqgS/4tmxGk5owbJ7+eTwLY35f1T3aHp6aQDcgJU= -github.com/ingonyama-zk/icicle/v2 v2.0.2/go.mod h1:rr3B+xKQKW1U40A+vEzA4hI2ilTrPSJBtxedfnaUYHw= +github.com/ingonyama-zk/icicle/v2 v2.0.3 h1:qNFXWQqUuOdJXh+25lIdCRJLqLrUwPkAfcK4wJXBap0= +github.com/ingonyama-zk/icicle/v2 v2.0.3/go.mod h1:rr3B+xKQKW1U40A+vEzA4hI2ilTrPSJBtxedfnaUYHw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= From 5b8506fb845e9da27beb5d3a7c38ea7b081e3679 Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Sun, 28 Apr 2024 07:35:59 +0300 Subject: [PATCH 06/11] Use double size domain instead of FrMultiplicativeGen, Add timings --- backend/groth16/bn254/icicle/icicle.go | 46 ++++++++++++------- backend/groth16/bn254/icicle/provingkey.go | 4 +- backend/groth16/bn254/prove.go | 52 +++++++++++++++++----- 3 files changed, 72 insertions(+), 30 deletions(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 8f4d914447..02e887c9ee 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -23,6 +23,7 @@ import ( "github.com/consensys/gnark/constraint/solver" "github.com/consensys/gnark/internal/utils" "github.com/consensys/gnark/logger" + "github.com/rs/zerolog" icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" icicle_cr "github.com/ingonyama-zk/icicle/v2/wrappers/golang/cuda_runtime" @@ -42,11 +43,12 @@ func (pk *ProvingKey) setupDevicePointers() error { return nil } pk.deviceInfo = &deviceInfo{} + gen, _ := fft.Generator(2 * pk.Domain.Cardinality) /************************* Den ***************************/ n := int(pk.Domain.Cardinality) var denI, oneI fr.Element oneI.SetOne() - denI.Exp(pk.Domain.FrMultiplicativeGen, big.NewInt(int64(pk.Domain.Cardinality))) + denI.Exp(gen, big.NewInt(int64(pk.Domain.Cardinality))) denI.Sub(&denI, &oneI).Inverse(&denI) log2SizeFloor := bits.Len(uint(n)) - 1 @@ -63,6 +65,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { denIcicleArrHost := (icicle_core.HostSlice[fr.Element])(denIcicleArr) denIcicleArrHost.CopyToDevice(&pk.DenDevice, true) + icicle_bn254.FromMontgomery(&pk.DenDevice) copyDenDone <- true }() @@ -72,10 +75,9 @@ func (pk *ProvingKey) setupDevicePointers() error { panic("Couldn't create device context") // TODO } - gen, _ := fft.Generator(2 * pk.Domain.Cardinality) genBits := gen.Bits() limbs := icicle_core.ConvertUint64ArrToUint32Arr(genBits[:]) - pk.CosetGenerator = limbs + copy(pk.CosetGenerator[:], limbs[:fr.Limbs*2]) var rouIcicle icicle_bn254.ScalarField rouIcicle.FromLimbs(limbs) e := icicle_ntt.InitDomain(rouIcicle, ctx, false) @@ -264,7 +266,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b var h icicle_core.DeviceSlice chHDone := make(chan struct{}, 1) go func() { - h = computeH(solution.A, solution.B, solution.C, pk) + h = computeH(solution.A, solution.B, solution.C, pk, log) solution.A = nil solution.B = nil @@ -336,7 +338,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + start := time.Now() icicle_msm.Msm(wireValuesBDevice, pk.G1Device.B, &cfg, res) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs1") bs1 = g1ProjectiveToG1Jac(res[0]) bs1.AddMixed(&pk.G1.Beta) @@ -352,7 +356,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + start := time.Now() icicle_msm.Msm(wireValuesADevice, pk.G1Device.A, &cfg, res) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Ar1") ar = g1ProjectiveToG1Jac(res[0]) ar.AddMixed(&pk.G1.Alpha) @@ -370,7 +376,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true resKrs2 := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + start := time.Now() icicle_msm.Msm(h.RangeTo(sizeH, false), pk.G1Device.Z, &cfg, resKrs2) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs2") krs2 = g1ProjectiveToG1Jac(resKrs2[0]) // filter the wire values if needed @@ -380,7 +388,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b _wireValues := filterHeap(wireValues[r1cs.GetNbPublicVariables():], r1cs.GetNbPublicVariables(), internal.ConcatAll(toRemove...)) _wireValuesHost := (icicle_core.HostSlice[fr.Element])(_wireValues) resKrs := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + start = time.Now() icicle_msm.Msm(_wireValuesHost, pk.G1Device.K, &cfg, resKrs) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs") krs = g1ProjectiveToG1Jac(resKrs[0]) krs.AddMixed(&deltas[2]) @@ -408,7 +418,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b cfg.ArePointsMontgomeryForm = true cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_g2.G2Projective], 1) + start := time.Now() icicle_g2.G2Msm(wireValuesBDevice, pk.G2Device.B, &cfg, res) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs2 G2") Bs = g2ProjectiveToG2Jac(&res[0]) deltaS.FromAffine(&pk.G2.Delta) @@ -478,7 +490,7 @@ func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr return } -func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { +func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_core.DeviceSlice { // H part of Krs // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) @@ -498,25 +510,22 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { computeBDone := make(chan icicle_core.DeviceSlice, 1) computeCDone := make(chan icicle_core.DeviceSlice, 1) - cosetGenBits := pk.Domain.FrMultiplicativeGen.Bits() - cosetGen := icicle_core.ConvertUint64ArrToUint32Arr(cosetGenBits[:]) - var configCosetGen [8]uint32 - copy(configCosetGen[:], cosetGen[:8]) - computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { cfg := icicle_ntt.GetDefaultNttConfig() scalarsStream, _ := icicle_cr.CreateStream() cfg.Ctx.Stream = &scalarsStream - cfg.Ordering = icicle_core.KNR + cfg.Ordering = icicle_core.KNM cfg.IsAsync = true scalarsHost := icicle_core.HostSliceFromElements(scalars) var scalarsDevice icicle_core.DeviceSlice scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) + start := time.Now() icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) - cfg.Ordering = icicle_core.KRN - cfg.CosetGen = configCosetGen + cfg.Ordering = icicle_core.KMN + cfg.CosetGen = pk.CosetGenerator icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) icicle_cr.SynchronizeStream(&scalarsStream) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") channel <-scalarsDevice } @@ -529,16 +538,21 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey) icicle_core.DeviceSlice { cDevice := <-computeCDone vecCfg := icicle_core.DefaultVecOpsConfig() + start := time.Now() icicle_bn254.FromMontgomery(&aDevice) icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) - icicle_bn254.FromMontgomery(&aDevice) icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") + defer bDevice.Free() + defer cDevice.Free() cfg := icicle_ntt.GetDefaultNttConfig() - cfg.CosetGen = configCosetGen - cfg.Ordering = icicle_core.KNR + cfg.CosetGen = pk.CosetGenerator + cfg.Ordering = icicle_core.KNM + start = time.Now() icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") return aDevice } diff --git a/backend/groth16/bn254/icicle/provingkey.go b/backend/groth16/bn254/icicle/provingkey.go index e0fa5a210a..501c2d22da 100644 --- a/backend/groth16/bn254/icicle/provingkey.go +++ b/backend/groth16/bn254/icicle/provingkey.go @@ -1,14 +1,14 @@ package icicle import ( - + "github.com/consensys/gnark-crypto/ecc/bn254/fr" groth16_bn254 "github.com/consensys/gnark/backend/groth16/bn254" cs "github.com/consensys/gnark/constraint/bn254" icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" ) type deviceInfo struct { - CosetGenerator []uint32 + CosetGenerator [fr.Limbs*2]uint32 G1Device struct { A, B, K, Z icicle_core.DeviceSlice } diff --git a/backend/groth16/bn254/prove.go b/backend/groth16/bn254/prove.go index 100f30e85a..8674a50651 100644 --- a/backend/groth16/bn254/prove.go +++ b/backend/groth16/bn254/prove.go @@ -18,6 +18,10 @@ package groth16 import ( "fmt" + "math/big" + "runtime" + "time" + "github.com/consensys/gnark-crypto/ecc" curve "github.com/consensys/gnark-crypto/ecc/bn254" "github.com/consensys/gnark-crypto/ecc/bn254/fr" @@ -32,9 +36,7 @@ import ( "github.com/consensys/gnark/constraint/solver" "github.com/consensys/gnark/internal/utils" "github.com/consensys/gnark/logger" - "math/big" - "runtime" - "time" + "github.com/rs/zerolog" fcs "github.com/consensys/gnark/frontend/cs" ) @@ -132,7 +134,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b var h []fr.Element chHDone := make(chan struct{}, 1) go func() { - h = computeH(solution.A, solution.B, solution.C, &pk.Domain) + h = computeH(solution.A, solution.B, solution.C, &pk.Domain, log) solution.A = nil solution.B = nil solution.C = nil @@ -191,7 +193,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b chBs1Done := make(chan error, 1) computeBS1 := func() { <-chWireValuesB - if _, err := bs1.MultiExp(pk.G1.B, wireValuesB, ecc.MultiExpConfig{NbTasks: n / 2}); err != nil { + start := time.Now() + _, err := bs1.MultiExp(pk.G1.B, wireValuesB, ecc.MultiExpConfig{NbTasks: n / 2}) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs1") + if err != nil { chBs1Done <- err close(chBs1Done) return @@ -204,7 +209,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b chArDone := make(chan error, 1) computeAR1 := func() { <-chWireValuesA - if _, err := ar.MultiExp(pk.G1.A, wireValuesA, ecc.MultiExpConfig{NbTasks: n / 2}); err != nil { + start := time.Now() + _, err := ar.MultiExp(pk.G1.A, wireValuesA, ecc.MultiExpConfig{NbTasks: n / 2}) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Ar1") + if err != nil { chArDone <- err close(chArDone) return @@ -224,7 +232,9 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b chKrs2Done := make(chan error, 1) sizeH := int(pk.Domain.Cardinality - 1) // comes from the fact the deg(H)=(n-1)+(n-1)-n=n-2 go func() { + start := time.Now() _, err := krs2.MultiExp(pk.G1.Z, h[:sizeH], ecc.MultiExpConfig{NbTasks: n / 2}) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs2") chKrs2Done <- err }() @@ -234,7 +244,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b toRemove = append(toRemove, commitmentInfo.CommitmentIndexes()) _wireValues := filterHeap(wireValues[r1cs.GetNbPublicVariables():], r1cs.GetNbPublicVariables(), internal.ConcatAll(toRemove...)) - if _, err := krs.MultiExp(pk.G1.K, _wireValues, ecc.MultiExpConfig{NbTasks: n / 2}); err != nil { + start := time.Now() + _, err := krs.MultiExp(pk.G1.K, _wireValues, ecc.MultiExpConfig{NbTasks: n / 2}) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs") + if err != nil { chKrsDone <- err return } @@ -280,7 +293,10 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b nbTasks *= 2 } <-chWireValuesB - if _, err := Bs.MultiExp(pk.G2.B, wireValuesB, ecc.MultiExpConfig{NbTasks: nbTasks}); err != nil { + start := time.Now() + _, err := Bs.MultiExp(pk.G2.B, wireValuesB, ecc.MultiExpConfig{NbTasks: nbTasks}) + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs2 G2") + if err != nil { return err } @@ -343,7 +359,7 @@ func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr return } -func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { +func computeH(a, b, c []fr.Element, domain *fft.Domain, log zerolog.Logger) []fr.Element { // H part of Krs // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) @@ -359,13 +375,21 @@ func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { c = append(c, padding...) n = len(a) + start := time.Now() domain.FFTInverse(a, fft.DIF) - domain.FFTInverse(b, fft.DIF) - domain.FFTInverse(c, fft.DIF) - domain.FFT(a, fft.DIT, fft.OnCoset()) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + + start = time.Now() + domain.FFTInverse(b, fft.DIF) domain.FFT(b, fft.DIT, fft.OnCoset()) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + + start = time.Now() + domain.FFTInverse(c, fft.DIF) domain.FFT(c, fft.DIT, fft.OnCoset()) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + var den, one fr.Element one.SetOne() @@ -374,6 +398,7 @@ func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { // h = ifft_coset(ca o cb - cc) // reusing a to avoid unnecessary memory allocation + start = time.Now() utils.Parallelize(n, func(start, end int) { for i := start; i < end; i++ { a[i].Mul(&a[i], &b[i]). @@ -381,9 +406,12 @@ func computeH(a, b, c []fr.Element, domain *fft.Domain) []fr.Element { Mul(&a[i], &den) } }) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") // ifft_coset + start = time.Now() domain.FFTInverse(a, fft.DIF, fft.OnCoset()) + log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") return a } From 89c49e44f6feb8e905f89dba2095bf2bb8a26a9f Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Sun, 28 Apr 2024 09:10:35 +0300 Subject: [PATCH 07/11] Fix invalid proofs for > 2^15 --- backend/groth16/bn254/icicle/icicle.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 02e887c9ee..5e4ef0829d 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -80,7 +80,7 @@ func (pk *ProvingKey) setupDevicePointers() error { copy(pk.CosetGenerator[:], limbs[:fr.Limbs*2]) var rouIcicle icicle_bn254.ScalarField rouIcicle.FromLimbs(limbs) - e := icicle_ntt.InitDomain(rouIcicle, ctx, false) + e := icicle_ntt.InitDomain(rouIcicle, ctx, true) if e.IcicleErrorCode != icicle_core.IcicleSuccess { panic("Couldn't initialize domain") // TODO } @@ -520,6 +520,7 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c var scalarsDevice icicle_core.DeviceSlice scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) start := time.Now() + cfg.NttAlgorithm = icicle_core.Radix2 icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) cfg.Ordering = icicle_core.KMN cfg.CosetGen = pk.CosetGenerator @@ -550,6 +551,7 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c cfg := icicle_ntt.GetDefaultNttConfig() cfg.CosetGen = pk.CosetGenerator cfg.Ordering = icicle_core.KNM + cfg.NttAlgorithm = icicle_core.Radix2 start = time.Now() icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") From a6ea30d3563462e71d1fb6ef1942533833a1d97c Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Tue, 30 Apr 2024 11:17:59 +0300 Subject: [PATCH 08/11] Fix ordering issue for computeH --- backend/groth16/bn254/icicle/icicle.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index 5e4ef0829d..a31de6cbc9 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -520,7 +520,6 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c var scalarsDevice icicle_core.DeviceSlice scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) start := time.Now() - cfg.NttAlgorithm = icicle_core.Radix2 icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) cfg.Ordering = icicle_core.KMN cfg.CosetGen = pk.CosetGenerator @@ -550,8 +549,7 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c cfg := icicle_ntt.GetDefaultNttConfig() cfg.CosetGen = pk.CosetGenerator - cfg.Ordering = icicle_core.KNM - cfg.NttAlgorithm = icicle_core.Radix2 + cfg.Ordering = icicle_core.KNR start = time.Now() icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") From a7780a8841f46a507b20b043aa8ff5b5bd384b31 Mon Sep 17 00:00:00 2001 From: Jeremy Felder Date: Tue, 30 Apr 2024 13:38:41 +0300 Subject: [PATCH 09/11] Convert from mont at setup --- backend/groth16/bn254/icicle/icicle.go | 63 ++++++++++++++++++-------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/backend/groth16/bn254/icicle/icicle.go b/backend/groth16/bn254/icicle/icicle.go index a31de6cbc9..7af3332c65 100644 --- a/backend/groth16/bn254/icicle/icicle.go +++ b/backend/groth16/bn254/icicle/icicle.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" "math/bits" + "os" "time" curve "github.com/consensys/gnark-crypto/ecc/bn254" @@ -92,6 +93,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { g1AHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.A) g1AHost.CopyToDevice(&pk.G1Device.A, true) + icicle_bn254.AffineFromMontgomery(&pk.G1Device.A) copyADone <- true }() /************************* B ***************************/ @@ -99,6 +101,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { g1BHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.B) g1BHost.CopyToDevice(&pk.G1Device.B, true) + icicle_bn254.AffineFromMontgomery(&pk.G1Device.B) copyBDone <- true }() /************************* K ***************************/ @@ -106,6 +109,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { g1KHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.K) g1KHost.CopyToDevice(&pk.G1Device.K, true) + icicle_bn254.AffineFromMontgomery(&pk.G1Device.K) copyKDone <- true }() /************************* Z ***************************/ @@ -113,6 +117,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { g1ZHost := (icicle_core.HostSlice[curve.G1Affine])(pk.G1.Z) g1ZHost.CopyToDevice(&pk.G1Device.Z, true) + icicle_bn254.AffineFromMontgomery(&pk.G1Device.Z) copyZDone <- true }() /************************* End G1 Device Setup ***************************/ @@ -126,6 +131,7 @@ func (pk *ProvingKey) setupDevicePointers() error { go func() { g2BHost := (icicle_core.HostSlice[curve.G2Affine])(pk.G2.B) g2BHost.CopyToDevice(&pk.G2Device.B, true) + icicle_g2.G2AffineFromMontgomery(&pk.G2Device.B) copyG2BDone <- true }() @@ -205,6 +211,8 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b } } + _, isProfile := os.LookupEnv("profile") + commitmentInfo := r1cs.CommitmentInfo.(constraint.Groth16Commitments) proof := &groth16_bn254.Proof{Commitments: make([]curve.G1Affine, len(commitmentInfo))} @@ -292,6 +300,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // Copy scalars to the device and retain ptr to them wireValuesAHost := (icicle_core.HostSlice[fr.Element])(wireValuesA) wireValuesAHost.CopyToDevice(&wireValuesADevice, true) + icicle_bn254.FromMontgomery(&wireValuesADevice) close(chWireValuesA) }() @@ -308,6 +317,7 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b // Copy scalars to the device and retain ptr to them wireValuesBHost := (icicle_core.HostSlice[fr.Element])(wireValuesB) wireValuesBHost.CopyToDevice(&wireValuesBDevice, true) + icicle_bn254.FromMontgomery(&wireValuesBDevice) close(chWireValuesB) }() @@ -335,12 +345,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b <-chWireValuesB cfg := icicle_msm.GetDefaultMSMConfig() - cfg.ArePointsMontgomeryForm = true - cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) start := time.Now() icicle_msm.Msm(wireValuesBDevice, pk.G1Device.B, &cfg, res) - log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs1") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs1") + } bs1 = g1ProjectiveToG1Jac(res[0]) bs1.AddMixed(&pk.G1.Beta) @@ -353,12 +363,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b <-chWireValuesA cfg := icicle_msm.GetDefaultMSMConfig() - cfg.ArePointsMontgomeryForm = true - cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) start := time.Now() icicle_msm.Msm(wireValuesADevice, pk.G1Device.A, &cfg, res) - log.Debug().Dur("took", time.Since(start)).Msg("MSM Ar1") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("MSM Ar1") + } ar = g1ProjectiveToG1Jac(res[0]) ar.AddMixed(&pk.G1.Alpha) @@ -373,14 +383,14 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b sizeH := int(pk.Domain.Cardinality - 1) cfg := icicle_msm.GetDefaultMSMConfig() - cfg.ArePointsMontgomeryForm = true - cfg.AreScalarsMontgomeryForm = true resKrs2 := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) start := time.Now() icicle_msm.Msm(h.RangeTo(sizeH, false), pk.G1Device.Z, &cfg, resKrs2) - log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs2") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs2") + } krs2 = g1ProjectiveToG1Jac(resKrs2[0]) - + // filter the wire values if needed // TODO Perf @Tabaie worst memory allocation offender toRemove := commitmentInfo.GetPrivateCommitted() @@ -388,9 +398,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b _wireValues := filterHeap(wireValues[r1cs.GetNbPublicVariables():], r1cs.GetNbPublicVariables(), internal.ConcatAll(toRemove...)) _wireValuesHost := (icicle_core.HostSlice[fr.Element])(_wireValues) resKrs := make(icicle_core.HostSlice[icicle_bn254.Projective], 1) + cfg.AreScalarsMontgomeryForm = true start = time.Now() icicle_msm.Msm(_wireValuesHost, pk.G1Device.K, &cfg, resKrs) - log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("MSM Krs") + } krs = g1ProjectiveToG1Jac(resKrs[0]) krs.AddMixed(&deltas[2]) @@ -415,12 +428,12 @@ func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...b <-chWireValuesB cfg := icicle_g2.G2GetDefaultMSMConfig() - cfg.ArePointsMontgomeryForm = true - cfg.AreScalarsMontgomeryForm = true res := make(icicle_core.HostSlice[icicle_g2.G2Projective], 1) start := time.Now() icicle_g2.G2Msm(wireValuesBDevice, pk.G2Device.B, &cfg, res) - log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs2 G2") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("MSM Bs2 G2") + } Bs = g2ProjectiveToG2Jac(&res[0]) deltaS.FromAffine(&pk.G2.Delta) @@ -496,9 +509,11 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) // 2 - ca = fft_coset(_a), ba = fft_coset(_b), cc = fft_coset(_c) // 3 - h = ifft_coset(ca o cb - cc) - + _, isProfile := os.LookupEnv("profile") + startTotal := time.Now() n := len(a) + // add padding to ensure input length is domain cardinality padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) a = append(a, padding...) @@ -525,7 +540,9 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c cfg.CosetGen = pk.CosetGenerator icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) icicle_cr.SynchronizeStream(&scalarsStream) - log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + } channel <-scalarsDevice } @@ -543,7 +560,9 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) - log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") + } defer bDevice.Free() defer cDevice.Free() @@ -552,7 +571,13 @@ func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_c cfg.Ordering = icicle_core.KNR start = time.Now() icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) - log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") - + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") + } + icicle_bn254.FromMontgomery(&aDevice) + + if isProfile { + log.Debug().Dur("took", time.Since(startTotal)).Msg("computeH: Total") + } return aDevice } From f71ae40b66dbbb19fab1dd13aab19c345c81d2ff Mon Sep 17 00:00:00 2001 From: liuxiaobleach <1241368737@qq.com> Date: Tue, 7 May 2024 14:44:43 +0800 Subject: [PATCH 10/11] add bls12377 icicle --- backend/groth16/bls12-377/icicle/icicle.go | 582 ++++++++++++++++++ backend/groth16/bls12-377/icicle/noicicle.go | 18 + .../groth16/bls12-377/icicle/provingkey.go | 32 + backend/groth16/groth16.go | 23 +- 4 files changed, 654 insertions(+), 1 deletion(-) create mode 100644 backend/groth16/bls12-377/icicle/icicle.go create mode 100644 backend/groth16/bls12-377/icicle/noicicle.go create mode 100644 backend/groth16/bls12-377/icicle/provingkey.go diff --git a/backend/groth16/bls12-377/icicle/icicle.go b/backend/groth16/bls12-377/icicle/icicle.go new file mode 100644 index 0000000000..411568c31a --- /dev/null +++ b/backend/groth16/bls12-377/icicle/icicle.go @@ -0,0 +1,582 @@ +//go:build icicle + +package icicle + +import ( + "fmt" + "math/big" + "math/bits" + "os" + "time" + + curve "github.com/consensys/gnark-crypto/ecc/bls12-377" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fp" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fr" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fr/fft" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fr/hash_to_field" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fr/pedersen" + "github.com/consensys/gnark/backend" + groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" + "github.com/consensys/gnark/backend/groth16/internal" + "github.com/consensys/gnark/backend/witness" + "github.com/consensys/gnark/constraint" + cs "github.com/consensys/gnark/constraint/bls12-377" + "github.com/consensys/gnark/constraint/solver" + "github.com/consensys/gnark/internal/utils" + "github.com/consensys/gnark/logger" + "github.com/rs/zerolog" + + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" + icicle_cr "github.com/ingonyama-zk/icicle/v2/wrappers/golang/cuda_runtime" + icicle_bls12377 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bls12377" + icicle_g2 "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bls12377/g2" + icicle_msm "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bls12377/msm" + icicle_ntt "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bls12377/ntt" + icicle_vecops "github.com/ingonyama-zk/icicle/v2/wrappers/golang/curves/bls12377/vecOps" + + fcs "github.com/consensys/gnark/frontend/cs" +) + +const HasIcicle = true + +func (pk *ProvingKey) setupDevicePointers() error { + if pk.deviceInfo != nil { + return nil + } + pk.deviceInfo = &deviceInfo{} + gen, _ := fft.Generator(2 * pk.Domain.Cardinality) + /************************* Den ***************************/ + n := int(pk.Domain.Cardinality) + var denI, oneI fr.Element + oneI.SetOne() + denI.Exp(gen, big.NewInt(int64(pk.Domain.Cardinality))) + denI.Sub(&denI, &oneI).Inverse(&denI) + + log2SizeFloor := bits.Len(uint(n)) - 1 + denIcicleArr := []fr.Element{denI} + for i := 0; i < log2SizeFloor; i++ { + denIcicleArr = append(denIcicleArr, denIcicleArr...) + } + pow2Remainder := n - 1< len(toRemove) +// filterHeap modifies toRemove +func filterHeap(slice []fr.Element, sliceFirstIndex int, toRemove []int) (r []fr.Element) { + + if len(toRemove) == 0 { + return slice + } + + heap := utils.IntHeap(toRemove) + heap.Heapify() + + r = make([]fr.Element, 0, len(slice)) + + // note: we can optimize that for the likely case where len(slice) >>> len(toRemove) + for i := 0; i < len(slice); i++ { + if len(heap) > 0 && i+sliceFirstIndex == heap[0] { + for len(heap) > 0 && i+sliceFirstIndex == heap[0] { + heap.Pop() + } + continue + } + r = append(r, slice[i]) + } + + return +} + +func computeH(a, b, c []fr.Element, pk *ProvingKey, log zerolog.Logger) icicle_core.DeviceSlice { + // H part of Krs + // Compute H (hz=ab-c, where z=-2 on ker X^n+1 (z(x)=x^n-1)) + // 1 - _a = ifft(a), _b = ifft(b), _c = ifft(c) + // 2 - ca = fft_coset(_a), ba = fft_coset(_b), cc = fft_coset(_c) + // 3 - h = ifft_coset(ca o cb - cc) + _, isProfile := os.LookupEnv("profile") + startTotal := time.Now() + n := len(a) + + // add padding to ensure input length is domain cardinality + padding := make([]fr.Element, int(pk.Domain.Cardinality)-n) + a = append(a, padding...) + b = append(b, padding...) + c = append(c, padding...) + n = len(a) + + computeADone := make(chan icicle_core.DeviceSlice, 1) + computeBDone := make(chan icicle_core.DeviceSlice, 1) + computeCDone := make(chan icicle_core.DeviceSlice, 1) + + computeInttNttOnDevice := func(scalars []fr.Element, channel chan icicle_core.DeviceSlice) { + cfg := icicle_ntt.GetDefaultNttConfig() + scalarsStream, _ := icicle_cr.CreateStream() + cfg.Ctx.Stream = &scalarsStream + cfg.Ordering = icicle_core.KNM + cfg.IsAsync = true + scalarsHost := icicle_core.HostSliceFromElements(scalars) + var scalarsDevice icicle_core.DeviceSlice + scalarsHost.CopyToDeviceAsync(&scalarsDevice, scalarsStream, true) + start := time.Now() + icicle_ntt.Ntt(scalarsDevice, icicle_core.KInverse, &cfg, scalarsDevice) + cfg.Ordering = icicle_core.KMN + cfg.CosetGen = pk.CosetGenerator + icicle_ntt.Ntt(scalarsDevice, icicle_core.KForward, &cfg, scalarsDevice) + icicle_cr.SynchronizeStream(&scalarsStream) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: NTT + INTT") + } + channel <- scalarsDevice + } + + go computeInttNttOnDevice(a, computeADone) + go computeInttNttOnDevice(b, computeBDone) + go computeInttNttOnDevice(c, computeCDone) + + aDevice := <-computeADone + bDevice := <-computeBDone + cDevice := <-computeCDone + + vecCfg := icicle_core.DefaultVecOpsConfig() + start := time.Now() + icicle_bls12377.FromMontgomery(&aDevice) + icicle_vecops.VecOp(aDevice, bDevice, aDevice, vecCfg, icicle_core.Mul) + icicle_vecops.VecOp(aDevice, cDevice, aDevice, vecCfg, icicle_core.Sub) + icicle_vecops.VecOp(aDevice, pk.DenDevice, aDevice, vecCfg, icicle_core.Mul) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: vecOps") + } + defer bDevice.Free() + defer cDevice.Free() + + cfg := icicle_ntt.GetDefaultNttConfig() + cfg.CosetGen = pk.CosetGenerator + cfg.Ordering = icicle_core.KNR + start = time.Now() + icicle_ntt.Ntt(aDevice, icicle_core.KInverse, &cfg, aDevice) + if isProfile { + log.Debug().Dur("took", time.Since(start)).Msg("computeH: INTT final") + } + icicle_bls12377.FromMontgomery(&aDevice) + + if isProfile { + log.Debug().Dur("took", time.Since(startTotal)).Msg("computeH: Total") + } + return aDevice +} diff --git a/backend/groth16/bls12-377/icicle/noicicle.go b/backend/groth16/bls12-377/icicle/noicicle.go new file mode 100644 index 0000000000..f34c125309 --- /dev/null +++ b/backend/groth16/bls12-377/icicle/noicicle.go @@ -0,0 +1,18 @@ +//go:build !icicle + +package icicle + +import ( + "fmt" + + "github.com/consensys/gnark/backend" + groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" + "github.com/consensys/gnark/backend/witness" + cs "github.com/consensys/gnark/constraint/bls12-377" +) + +const HasIcicle = false + +func Prove(r1cs *cs.R1CS, pk *ProvingKey, fullWitness witness.Witness, opts ...backend.ProverOption) (*groth16_bls12377.Proof, error) { + return nil, fmt.Errorf("icicle backend requested but program compiled without 'icicle' build tag") +} diff --git a/backend/groth16/bls12-377/icicle/provingkey.go b/backend/groth16/bls12-377/icicle/provingkey.go new file mode 100644 index 0000000000..289b4e7cf9 --- /dev/null +++ b/backend/groth16/bls12-377/icicle/provingkey.go @@ -0,0 +1,32 @@ +package icicle + +import ( + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" + cs "github.com/consensys/gnark/constraint/bls12-377" + icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core" +) + +type deviceInfo struct { + CosetGenerator [fr.Limbs * 2]uint32 + G1Device struct { + A, B, K, Z icicle_core.DeviceSlice + } + G2Device struct { + B icicle_core.DeviceSlice + } + DenDevice icicle_core.DeviceSlice +} + +type ProvingKey struct { + groth16_bls12377.ProvingKey + *deviceInfo +} + +func Setup(r1cs *cs.R1CS, pk *ProvingKey, vk *groth16_bls12377.VerifyingKey) error { + return groth16_bls12377.Setup(r1cs, &pk.ProvingKey, vk) +} + +func DummySetup(r1cs *cs.R1CS, pk *ProvingKey) error { + return groth16_bls12377.DummySetup(r1cs, &pk.ProvingKey) +} diff --git a/backend/groth16/groth16.go b/backend/groth16/groth16.go index ca5b8bdc61..bc05b9726c 100644 --- a/backend/groth16/groth16.go +++ b/backend/groth16/groth16.go @@ -45,6 +45,7 @@ import ( gnarkio "github.com/consensys/gnark/io" groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" + icicle_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377/icicle" groth16_bls12381 "github.com/consensys/gnark/backend/groth16/bls12-381" groth16_bls24315 "github.com/consensys/gnark/backend/groth16/bls24-315" groth16_bls24317 "github.com/consensys/gnark/backend/groth16/bls24-317" @@ -170,6 +171,9 @@ func Verify(proof Proof, vk VerifyingKey, publicWitness witness.Witness, opts .. func Prove(r1cs constraint.ConstraintSystem, pk ProvingKey, fullWitness witness.Witness, opts ...backend.ProverOption) (Proof, error) { switch _r1cs := r1cs.(type) { case *cs_bls12377.R1CS: + if icicle_bls12377.HasIcicle { + return icicle_bls12377.Prove(_r1cs, pk.(*icicle_bls12377.ProvingKey), fullWitness, opts...) + } return groth16_bls12377.Prove(_r1cs, pk.(*groth16_bls12377.ProvingKey), fullWitness, opts...) case *cs_bls12381.R1CS: @@ -210,8 +214,15 @@ func Setup(r1cs constraint.ConstraintSystem) (ProvingKey, VerifyingKey, error) { switch _r1cs := r1cs.(type) { case *cs_bls12377.R1CS: - var pk groth16_bls12377.ProvingKey var vk groth16_bls12377.VerifyingKey + if icicle_bls12377.HasIcicle { + var pk groth16_bls12377.ProvingKey + if err := groth16_bls12377.Setup(_r1cs, &pk, &vk); err != nil { + return nil, nil, err + } + return &pk, &vk, nil + } + var pk groth16_bls12377.ProvingKey if err := groth16_bls12377.Setup(_r1cs, &pk, &vk); err != nil { return nil, nil, err } @@ -275,6 +286,13 @@ func Setup(r1cs constraint.ConstraintSystem) (ProvingKey, VerifyingKey, error) { func DummySetup(r1cs constraint.ConstraintSystem) (ProvingKey, error) { switch _r1cs := r1cs.(type) { case *cs_bls12377.R1CS: + if icicle_bls12377.HasIcicle { + var pk icicle_bls12377.ProvingKey + if err := icicle_bls12377.DummySetup(_r1cs, &pk); err != nil { + return nil, err + } + return &pk, nil + } var pk groth16_bls12377.ProvingKey if err := groth16_bls12377.DummySetup(_r1cs, &pk); err != nil { return nil, err @@ -340,6 +358,9 @@ func NewProvingKey(curveID ecc.ID) ProvingKey { } case ecc.BLS12_377: pk = &groth16_bls12377.ProvingKey{} + if icicle_bls12377.HasIcicle { + pk = &icicle_bls12377.ProvingKey{} + } case ecc.BLS12_381: pk = &groth16_bls12381.ProvingKey{} case ecc.BW6_761: From ed607de0156cbcb9e7ba6e225b717146acf301bb Mon Sep 17 00:00:00 2001 From: liuxiaobleach <1241368737@qq.com> Date: Tue, 7 May 2024 14:45:54 +0800 Subject: [PATCH 11/11] fix --- backend/groth16/bls12-377/icicle/provingkey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/groth16/bls12-377/icicle/provingkey.go b/backend/groth16/bls12-377/icicle/provingkey.go index 289b4e7cf9..5151800743 100644 --- a/backend/groth16/bls12-377/icicle/provingkey.go +++ b/backend/groth16/bls12-377/icicle/provingkey.go @@ -1,7 +1,7 @@ package icicle import ( - "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/consensys/gnark-crypto/ecc/bls12-377/fr" groth16_bls12377 "github.com/consensys/gnark/backend/groth16/bls12-377" cs "github.com/consensys/gnark/constraint/bls12-377" icicle_core "github.com/ingonyama-zk/icicle/v2/wrappers/golang/core"