-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix OOM hugeCapacity in large scale, use kyro serializer, merge perso…
…n and company activities (#94)
- Loading branch information
1 parent
d05e55d
commit 15ab7ac
Showing
16 changed files
with
321 additions
and
200 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
src/main/java/ldbc/finbench/datagen/generation/events/CompanyActivitiesEvent.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| package ldbc.finbench.datagen.generation.events; | ||
|
|
||
| import java.io.Serializable; | ||
| import java.util.List; | ||
| import java.util.Random; | ||
| import ldbc.finbench.datagen.entities.edges.CompanyApplyLoan; | ||
| import ldbc.finbench.datagen.entities.edges.CompanyGuaranteeCompany; | ||
| import ldbc.finbench.datagen.entities.edges.CompanyOwnAccount; | ||
| import ldbc.finbench.datagen.entities.nodes.Account; | ||
| import ldbc.finbench.datagen.entities.nodes.Company; | ||
| import ldbc.finbench.datagen.entities.nodes.Loan; | ||
| import ldbc.finbench.datagen.generation.DatagenParams; | ||
| import ldbc.finbench.datagen.generation.dictionary.Dictionaries; | ||
| import ldbc.finbench.datagen.generation.generators.AccountGenerator; | ||
| import ldbc.finbench.datagen.generation.generators.LoanGenerator; | ||
| import ldbc.finbench.datagen.util.RandomGeneratorFarm; | ||
|
|
||
| public class CompanyActivitiesEvent implements Serializable { | ||
| private final RandomGeneratorFarm randomFarm; | ||
| private final Random randIndex; | ||
|
|
||
| public CompanyActivitiesEvent() { | ||
| randomFarm = new RandomGeneratorFarm(); | ||
| randIndex = new Random(DatagenParams.defaultSeed); | ||
| } | ||
|
|
||
| private void resetState(int seed) { | ||
| randomFarm.resetRandomGenerators(seed); | ||
| randIndex.setSeed(seed); | ||
| } | ||
|
|
||
| public List<Company> companyActivities(List<Company> companies, AccountGenerator accountGenerator, | ||
| LoanGenerator loanGenerator, int blockId) { | ||
| resetState(blockId); | ||
| accountGenerator.resetState(blockId); | ||
|
|
||
| Random numAccRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_ACCOUNTS_PER_COMPANY); | ||
|
|
||
| Random pickCompanyGuaRand = randomFarm.get(RandomGeneratorFarm.Aspect.PICK_COMPANY_GUARANTEE); | ||
| Random numGuaranteesRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_GUARANTEES_PER_COMPANY); | ||
|
|
||
| Random pickCompanyLoanRand = randomFarm.get(RandomGeneratorFarm.Aspect.PICK_COMPANY_FOR_LOAN); | ||
| Random numLoansRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LOANS_PER_COMPANY); | ||
| Random dateRand = randomFarm.get(RandomGeneratorFarm.Aspect.COMPANY_APPLY_LOAN_DATE); | ||
|
|
||
| for (Company from : companies) { | ||
| // register accounts | ||
| int numAccounts = numAccRand.nextInt(DatagenParams.maxAccountsPerOwner); | ||
| for (int i = 0; i < Math.max(1, numAccounts); i++) { | ||
| Account account = accountGenerator.generateAccount(from.getCreationDate(), "company", blockId); | ||
| CompanyOwnAccount.createCompanyOwnAccount(randomFarm, from, account, account.getCreationDate()); | ||
| } | ||
| // guarantee other companies | ||
| if (pickCompanyGuaRand.nextDouble() < DatagenParams.companyGuaranteeFraction) { | ||
| int numGuarantees = numGuaranteesRand.nextInt(DatagenParams.maxTargetsToGuarantee); | ||
| for (int i = 0; i < Math.max(1, numGuarantees); i++) { | ||
| Company to = companies.get(randIndex.nextInt(companies.size())); | ||
| if (from.canGuarantee(to)) { | ||
| CompanyGuaranteeCompany.createCompanyGuaranteeCompany(randomFarm, from, to); | ||
| } | ||
| } | ||
| } | ||
| // apply loans | ||
| if (pickCompanyLoanRand.nextDouble() < DatagenParams.companyLoanFraction) { | ||
| int numLoans = numLoansRand.nextInt(DatagenParams.maxLoans); | ||
| for (int i = 0; i < Math.max(1, numLoans); i++) { | ||
| long applyDate = Dictionaries.dates.randomCompanyToLoanDate(dateRand, from); | ||
| Loan to = loanGenerator.generateLoan(applyDate, "company", blockId); | ||
| CompanyApplyLoan.createCompanyApplyLoan(randomFarm, applyDate, from, to); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return companies; | ||
| } | ||
| } |
77 changes: 77 additions & 0 deletions
77
src/main/java/ldbc/finbench/datagen/generation/events/PersonActivitiesEvent.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| package ldbc.finbench.datagen.generation.events; | ||
|
|
||
| import java.io.Serializable; | ||
| import java.util.List; | ||
| import java.util.Random; | ||
| import ldbc.finbench.datagen.entities.edges.PersonApplyLoan; | ||
| import ldbc.finbench.datagen.entities.edges.PersonGuaranteePerson; | ||
| import ldbc.finbench.datagen.entities.edges.PersonOwnAccount; | ||
| import ldbc.finbench.datagen.entities.nodes.Account; | ||
| import ldbc.finbench.datagen.entities.nodes.Loan; | ||
| import ldbc.finbench.datagen.entities.nodes.Person; | ||
| import ldbc.finbench.datagen.generation.DatagenParams; | ||
| import ldbc.finbench.datagen.generation.dictionary.Dictionaries; | ||
| import ldbc.finbench.datagen.generation.generators.AccountGenerator; | ||
| import ldbc.finbench.datagen.generation.generators.LoanGenerator; | ||
| import ldbc.finbench.datagen.util.RandomGeneratorFarm; | ||
|
|
||
| public class PersonActivitiesEvent implements Serializable { | ||
| private final RandomGeneratorFarm randomFarm; | ||
| private final Random randIndex; | ||
|
|
||
| public PersonActivitiesEvent() { | ||
| randomFarm = new RandomGeneratorFarm(); | ||
| randIndex = new Random(DatagenParams.defaultSeed); | ||
| } | ||
|
|
||
| private void resetState(int seed) { | ||
| randomFarm.resetRandomGenerators(seed); | ||
| randIndex.setSeed(seed); | ||
| } | ||
|
|
||
| // Generate accounts, guarantees, and loans for persons | ||
| public List<Person> personActivities(List<Person> persons, AccountGenerator accountGenerator, | ||
| LoanGenerator loanGenerator, int blockId) { | ||
| resetState(blockId); | ||
| accountGenerator.resetState(blockId); | ||
|
|
||
| Random numAccRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_ACCOUNTS_PER_PERSON); | ||
|
|
||
| Random pickPersonGuaRand = randomFarm.get(RandomGeneratorFarm.Aspect.PICK_PERSON_GUARANTEE); | ||
| Random numGuaranteesRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_GUARANTEES_PER_PERSON); | ||
|
|
||
| Random pickPersonLoanRand = randomFarm.get(RandomGeneratorFarm.Aspect.PICK_PERSON_LOAN); | ||
| Random numLoansRand = randomFarm.get(RandomGeneratorFarm.Aspect.NUM_LOANS_PER_PERSON); | ||
| Random dateRand = randomFarm.get(RandomGeneratorFarm.Aspect.PERSON_APPLY_LOAN_DATE); | ||
|
|
||
| for (Person from : persons) { | ||
| // register accounts | ||
| int numAccounts = numAccRand.nextInt(DatagenParams.maxAccountsPerOwner); | ||
| for (int i = 0; i < Math.max(1, numAccounts); i++) { | ||
| Account to = accountGenerator.generateAccount(from.getCreationDate(), "person", blockId); | ||
| PersonOwnAccount.createPersonOwnAccount(randomFarm, from, to, to.getCreationDate()); | ||
| } | ||
| // guarantee other persons | ||
| if (pickPersonGuaRand.nextDouble() < DatagenParams.personGuaranteeFraction) { | ||
| int numGuarantees = numGuaranteesRand.nextInt(DatagenParams.maxTargetsToGuarantee); | ||
| for (int i = 0; i < Math.max(1, numGuarantees); i++) { | ||
| Person to = persons.get(randIndex.nextInt(persons.size())); | ||
| if (from.canGuarantee(to)) { | ||
| PersonGuaranteePerson.createPersonGuaranteePerson(randomFarm, from, to); | ||
| } | ||
| } | ||
| } | ||
| // apply loans | ||
| if (pickPersonLoanRand.nextDouble() < DatagenParams.personLoanFraction) { | ||
| int numLoans = numLoansRand.nextInt(DatagenParams.maxLoans); | ||
| for (int i = 0; i < Math.max(1, numLoans); i++) { | ||
| long applyDate = Dictionaries.dates.randomPersonToLoanDate(dateRand, from); | ||
| Loan to = loanGenerator.generateLoan(applyDate, "person", blockId); | ||
| PersonApplyLoan.createPersonApplyLoan(randomFarm, applyDate, from, to); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return persons; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
24 changes: 2 additions & 22 deletions
24
src/main/scala/ldbc/finbench/datagen/factors/FactorGenerationStage.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.