Merge branch 'main' into dev

ldbc · Sep 19, 2024 · 794f4ea · 794f4ea
2 parents 4c95a33 + cfa327a
commit 794f4ea
Show file tree

Hide file tree

Showing 37 changed files with 402 additions and 941 deletions.
diff --git a/logs/.gitignore b/logs/.gitignore
@@ -1,2 +1,3 @@
 *.log
-*/*.log
+*/*.log
+*.png
diff --git a/scripts/run_local.sh b/scripts/run_local.sh
@@ -9,18 +9,18 @@ OUTPUT_DIR=out
 # run locally with spark-submit command
 # **({'spark.driver.extraJavaOptions': '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005'}), # Debug
 # **({'spark.executor.extraJavaOptions': '-verbose:gc -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps'}),
-#    --conf "spark.memory.offHeap.enabled=true" \
-#    --conf "spark.memory.offHeap.size=100g" \
-#    --conf "spark.storage.memoryFraction=0" \
 time spark-submit --master local[*] \
     --class ldbc.finbench.datagen.LdbcDatagen \
-    --driver-memory 500g \
+    --driver-memory 480g \
     --conf "spark.default.parallelism=500" \
     --conf "spark.shuffle.compress=true" \
     --conf "spark.shuffle.spill.compress=true" \
     --conf "spark.serializer=org.apache.spark.serializer.KryoSerializer" \
+    --conf "spark.memory.offHeap.enabled=true" \
+    --conf "spark.memory.offHeap.size=100g" \
+    --conf "spark.storage.memoryFraction=0" \
     --conf "spark.driver.maxResultSize=0" \
     --conf "spark.executor.extraJavaOptions=-XX:+UseG1GC" \
     ${LDBC_FINBENCH_DATAGEN_JAR} \
-    --scale-factor 1 \
+    --scale-factor 10 \
     --output-dir ${OUTPUT_DIR}
diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyApplyLoan.java b/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyApplyLoan.java
@@ -9,23 +9,27 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class CompanyApplyLoan implements DynamicActivity, Serializable {
-    private final Company company;
-    private final Loan loan;
+    private final long companyId;
+    private final long loanId;
+    private final Loan loan; // TODO: can be removed
     private final long creationDate;
     private final long deletionDate;
     private final boolean isExplicitlyDeleted;
     private final String organization;
     private final String comment;
+    private final double loanAmount;
 
     public CompanyApplyLoan(Company company, Loan loan, long creationDate, long deletionDate,
                             boolean isExplicitlyDeleted, String organization, String comment) {
-        this.company = company;
+        this.companyId = company.getCompanyId();
+        this.loanId = loan.getLoanId();
         this.loan = loan;
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
         this.organization = organization;
         this.comment = comment;
+        this.loanAmount = loan.getLoanAmount();
     }
 
     public static void createCompanyApplyLoan(RandomGeneratorFarm farm, long creationDate, Company company, Loan loan) {
@@ -41,8 +45,12 @@ public static void createCompanyApplyLoan(RandomGeneratorFarm farm, long creatio
         company.getCompanyApplyLoans().add(companyApplyLoan);
     }
 
-    public Company getCompany() {
-        return company;
+    public long getCompanyId() {
+        return companyId;
+    }
+
+    public long getLoanId() {
+        return loanId;
     }
 
     public Loan getLoan() {
@@ -71,4 +79,8 @@ public String getOrganization() {
     public String getComment() {
         return comment;
     }
+
+    public double getLoanAmount() {
+        return loanAmount;
+    }
 }
diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyGuaranteeCompany.java b/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyGuaranteeCompany.java
@@ -8,8 +8,8 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class CompanyGuaranteeCompany implements DynamicActivity, Serializable {
-    private final Company fromCompany;
-    private final Company toCompany;
+    private final long fromCompanyId;
+    private final long toCompanyId;
     private final long creationDate;
     private final long deletionDate;
     private final boolean isExplicitlyDeleted;
@@ -19,8 +19,8 @@ public class CompanyGuaranteeCompany implements DynamicActivity, Serializable {
     public CompanyGuaranteeCompany(Company fromCompany, Company toCompany,
                                    long creationDate, long deletionDate, boolean isExplicitlyDeleted, String relation,
                                    String comment) {
-        this.fromCompany = fromCompany;
-        this.toCompany = toCompany;
+        this.fromCompanyId = fromCompany.getCompanyId();
+        this.toCompanyId = toCompany.getCompanyId();
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
@@ -40,12 +40,12 @@ public static void createCompanyGuaranteeCompany(RandomGeneratorFarm farm, Compa
         fromCompany.getGuaranteeSrc().add(companyGuaranteeCompany);
     }
 
-    public Company getFromCompany() {
-        return fromCompany;
+    public long getFromCompanyId() {
+        return fromCompanyId;
     }
 
-    public Company getToCompany() {
-        return toCompany;
+    public long getToCompanyId() {
+        return toCompanyId;
     }
 
     @Override

diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyInvestCompany.java b/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyInvestCompany.java
@@ -8,8 +8,8 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class CompanyInvestCompany implements DynamicActivity, Serializable {
-    private final Company fromCompany;
-    private final Company toCompany;
+    private final long fromCompanyId;
+    private final long toCompanyId;
     private double ratio;
     private final long creationDate;
     private final long deletionDate;
@@ -19,8 +19,8 @@ public class CompanyInvestCompany implements DynamicActivity, Serializable {
     public CompanyInvestCompany(Company fromCompany, Company toCompany,
                                 long creationDate, long deletionDate, double ratio, boolean isExplicitlyDeleted,
                                 String comment) {
-        this.fromCompany = fromCompany;
-        this.toCompany = toCompany;
+        this.fromCompanyId = fromCompany.getCompanyId();
+        this.toCompanyId = toCompany.getCompanyId();
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.ratio = ratio;
@@ -49,12 +49,12 @@ public double getRatio() {
         return ratio;
     }
 
-    public Company getFromCompany() {
-        return fromCompany;
+    public long getFromCompanyId() {
+        return fromCompanyId;
     }
 
-    public Company getToCompany() {
-        return toCompany;
+    public long getToCompanyId() {
+        return toCompanyId;
     }
 
     @Override

diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyOwnAccount.java b/src/main/java/ldbc/finbench/datagen/entities/edges/CompanyOwnAccount.java
@@ -9,17 +9,19 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class CompanyOwnAccount implements DynamicActivity, Serializable {
-    private final Company company;
-    private final Account account;
+    private final long companyId;
+    private final long accountId;
     private final long creationDate;
     private final long deletionDate;
     private final boolean isExplicitlyDeleted;
     private final String comment;
+    private final Account account; // TODO: can be removed
 
     public CompanyOwnAccount(Company company, Account account, long creationDate, long deletionDate,
                              boolean isExplicitlyDeleted, String comment) {
-        this.company = company;
-        this.account = account;
+        this.companyId = company.getCompanyId();
+        this.accountId = account.getAccountId();
+        this.account = account; // TODO: can be removed
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
@@ -39,12 +41,12 @@ public static void createCompanyOwnAccount(RandomGeneratorFarm farm, Company com
         company.getCompanyOwnAccounts().add(companyOwnAccount);
     }
 
-    public Company getCompany() {
-        return company;
+    public long getCompanyId() {
+        return companyId;
     }
 
-    public Account getAccount() {
-        return account;
+    public long getAccountId() {
+        return accountId;
     }
 
     @Override
@@ -65,4 +67,8 @@ public boolean isExplicitlyDeleted() {
     public String getComment() {
         return comment;
     }
+
+    public Account getAccount() {
+        return account;
+    }
 }
diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/Deposit.java b/src/main/java/ldbc/finbench/datagen/entities/edges/Deposit.java
@@ -8,8 +8,8 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class Deposit implements DynamicActivity, Serializable {
-    private final Loan loan;
-    private final Account account;
+    private final long loanId;
+    private final long accountId;
     private final double amount;
     private final long creationDate;
     private final long deletionDate;
@@ -18,16 +18,16 @@ public class Deposit implements DynamicActivity, Serializable {
 
     public Deposit(Loan loan, Account account, double amount, long creationDate, long deletionDate,
                    boolean isExplicitlyDeleted, String comment) {
-        this.loan = loan;
-        this.account = account;
+        this.loanId = loan.getLoanId();
+        this.accountId = account.getAccountId();
         this.amount = amount;
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
         this.comment = comment;
     }
 
-    public static Deposit createDeposit(RandomGeneratorFarm farm, Loan loan, Account account, double amount) {
+    public static void createDeposit(RandomGeneratorFarm farm, Loan loan, Account account, double amount) {
         long creationDate =
             Dictionaries.dates.randomLoanToAccountDate(farm.get(RandomGeneratorFarm.Aspect.LOAN_SUBEVENTS_DATE), loan,
                                                        account, account.getDeletionDate());
@@ -38,21 +38,19 @@ public static Deposit createDeposit(RandomGeneratorFarm farm, Loan loan, Account
             new Deposit(loan, account, amount, creationDate, account.getDeletionDate(), account.isExplicitlyDeleted(),
                         comment);
         loan.addDeposit(deposit);
-        account.getDeposits().add(deposit);
-
-        return deposit;
+        //account.getDeposits().add(deposit);
     }
 
     public double getAmount() {
         return amount;
     }
 
-    public Loan getLoan() {
-        return loan;
+    public long getLoanId() {
+        return loanId;
     }
 
-    public Account getAccount() {
-        return account;
+    public long getAccountId() {
+        return accountId;
     }
 
     @Override

diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/PersonApplyLoan.java b/src/main/java/ldbc/finbench/datagen/entities/edges/PersonApplyLoan.java
@@ -9,23 +9,27 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class PersonApplyLoan implements DynamicActivity, Serializable {
-    private final Person person;
-    private final Loan loan;
+    private final long personId;
+    private final long loanId;
+    private final Loan loan; // TODO: can be removed
     private final long creationDate;
     private final long deletionDate;
     private final boolean isExplicitlyDeleted;
     private final String organization;
     private final String comment;
+    private final double loanAmount;
 
     public PersonApplyLoan(Person person, Loan loan, long creationDate, long deletionDate,
                            boolean isExplicitlyDeleted, String organization, String comment) {
-        this.person = person;
+        this.personId = person.getPersonId();
+        this.loanId = loan.getLoanId();
         this.loan = loan;
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
         this.organization = organization;
         this.comment = comment;
+        this.loanAmount = loan.getLoanAmount();
     }
 
     public static void createPersonApplyLoan(RandomGeneratorFarm farm, long creationDate, Person person, Loan loan) {
@@ -41,8 +45,12 @@ public static void createPersonApplyLoan(RandomGeneratorFarm farm, long creation
         person.getPersonApplyLoans().add(personApplyLoan);
     }
 
-    public Person getPerson() {
-        return person;
+    public long getPersonId() {
+        return personId;
+    }
+
+    public long getLoanId() {
+        return loanId;
     }
 
     public Loan getLoan() {
@@ -72,4 +80,7 @@ public String getComment() {
         return comment;
     }
 
+    public double getLoanAmount() {
+        return loanAmount;
+    }
 }
diff --git a/src/main/java/ldbc/finbench/datagen/entities/edges/PersonGuaranteePerson.java b/src/main/java/ldbc/finbench/datagen/entities/edges/PersonGuaranteePerson.java
@@ -7,8 +7,8 @@
 import ldbc.finbench.datagen.util.RandomGeneratorFarm;
 
 public class PersonGuaranteePerson implements DynamicActivity, Serializable {
-    private final Person fromPerson;
-    private final Person toPerson;
+    private final long fromPersonId;
+    private final long toPersonId;
     private final long creationDate;
     private final long deletionDate;
     private final boolean isExplicitlyDeleted;
@@ -18,8 +18,8 @@ public class PersonGuaranteePerson implements DynamicActivity, Serializable {
     public PersonGuaranteePerson(Person fromPerson, Person toPerson,
                                  long creationDate, long deletionDate, boolean isExplicitlyDeleted, String relation,
                                  String comment) {
-        this.fromPerson = fromPerson;
-        this.toPerson = toPerson;
+        this.fromPersonId = fromPerson.getPersonId();
+        this.toPersonId = toPerson.getPersonId();
         this.creationDate = creationDate;
         this.deletionDate = deletionDate;
         this.isExplicitlyDeleted = isExplicitlyDeleted;
@@ -40,12 +40,12 @@ public static void createPersonGuaranteePerson(RandomGeneratorFarm farm, Person
         fromPerson.getGuaranteeSrc().add(personGuaranteePerson);
     }
 
-    public Person getFromPerson() {
-        return fromPerson;
+    public long getFromPersonId() {
+        return fromPersonId;
     }
 
-    public Person getToPerson() {
-        return toPerson;
+    public long getToPersonId() {
+        return toPersonId;
     }
 
     @Override
-Original file line number
+Diff line change
@@ -1,2 +1,3 @@
     *.log
-    */*.log
+    */*.log
+    *.png