Skip to content

Commit

Permalink
Merge pull request #127 from navinrathore/zNewTrainingSamples
Browse files Browse the repository at this point in the history 
unionByName() with allowMissingColumns=true for training data
  • Loading branch information
@sonalgoyal
sonalgoyal authored Jan 12, 2022
2 parents 46509e7 + eeed8a4 commit 21dde93
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 72 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/zingg/util/DSUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ private static Dataset<Row> getTraining(SparkSession spark, Arguments args, Pipe
Dataset<Row> trSamples = PipeUtil.read(spark,
true, false, args.getTrainingSamples());
LOG.warn("Read all training samples ");
trFile = (trFile == null) ? trSamples : trFile.unionByName(trSamples);
trFile = (trFile == null) ? trSamples : trFile.unionByName(trSamples, true);
}
else {
LOG.warn("No configured training samples");
Expand Down
1 change: 0 additions & 1 deletion examples/febrl/configWithTrainingSamples.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

{\"name\" : \"z_cluster\",\"type\" : \"string\",\"nullable\" : false, \"metadata\" : {}},
{\"name\":\"z_ismatch\",\"type\":\"integer\",\"nullable\":true,\"metadata\" : {}},
{\"name\":\"id\", \"type\":\"string\", \"nullable\":false,\"metadata\" : {}},
{\"name\":\"fname\", \"type\":\"string\", \"nullable\":true,\"metadata\" : {}},
{\"name\":\"lname\",\"type\":\"string\",\"nullable\":true,\"metadata\" : {}},
{\"name\":\"stNo\", \"type\":\"string\", \"nullable\":true,\"metadata\" : {}},
Expand Down
140 changes: 70 additions & 70 deletions examples/febrl/training.csv
View file
Original file line number Diff line number Diff line change
@@ -1,70 +1,70 @@
10,1,rec-10-dup-0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769
10,1,rec-10-dup-1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1,rec-10-dup-2, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769
10,1,rec-10-dup-3, mikhayla, hannagan,, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1,rec-10-org, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1,rec-10-org, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,,1030769
100,1,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
100,1,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,, q,19161017,4975843
100,1,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
100,1,rec-100-dup-2, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161027,4975843
100,1,rec-100-dup-3, domenique, paterson,15, lubum a place, merewether pub lic school, greensborough,3138, qld,19161017,4975843
100,1,rec-100-dup-4, domenique, paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843
100,1,rec-100-dup-4, , paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843
100,1,rec-100-org, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161017,4975843
100,1,rec-100-org, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,,4975843
1000,1,rec-1000-dup-0, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902
1000,1,rec-1000-dup-1, sophie, waller,24, suunybrae, goldfinch circuit, mona vale,2285, vic,19380705,2972902
1000,1,rec-1000-dup-2, sophie, waller,24, suunybrae, , mona vale,2285, vic,19380705,2972902
1000,1,rec-1000-org, sophie, waller,24, goldfinch circuit, suunybrae, mona vale,2285, vic,19380705,2972902
1001,1,rec-1001-dup-0, , pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19991009,1859104
1001,1,rec-1001-org, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
1001,1,rec-1001-org, casey, ,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
1002,1,rec-1002-dup-0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819
1002,1,rec-1002-org, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
1002,1,rec-1002-org, , matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
1002,1,rec-1002-dup-0, xavier, ,10, , rank in park, kellyville,3039, vdc,19620202,6169819
1004,1,rec-1004-dup-0, tayla, green,41, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1,rec-1004-dup-1, tayla, greecn,38, kambalda crescent, ambevrale, bonville,2905, vic,19310175,2230673
1004,1,rec-1004-dup-2, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1,rec-1004-org, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1,rec-1004-org, tayla, gren,38, kambaldcrescent, ambervale, bonville,2905, ,19310715,2230673
1006,1,rec-1006-dup-0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819
1006,1,rec-1006-dup-1, sam, bentham,1, narran street, lake chatsworth, westlake,2306, vic, ,8928819
1006,1,rec-1006-dup-2, sam, bentham,1, narranstreet, lake chagsworth, westlake,2036, vic, ,8928819
1006,1,rec-1006-dup-3, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic,19330165,8928819
1006,1,rec-1006-dup-4, sam, bentham,1, narranystreet, , westl ake,2036, vic,19330165,8928819
1006,1,rec-1006-org, , bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819
1007,1,rec-1007-dup-0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023
1007,1,rec-1007-dup-1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1,rec-1007-dup-2, rebekah, modystach,34, tausx place, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1,rec-1007-dup-3, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1,rec-1007-dup-4, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, np,19341113,8796023
1007,1,rec-1007-org, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n0,0,rec-0-org, jinni, dreyer,11, were street, marriott downs, south melbourne,3172, nsw,19420127,3787407
n0,0,rec-1-org, jonah, browne,6, bingley crescent, , pindimar,3182, nsw,19250413,8328406
n10,0,rec-10-dup-0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769
n10,0,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
n11,0,rec-10-dup-1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769
n11,0,rec-100-dup-0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
n12,0,rec-10-dup-2, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769
n12,0,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
n13,0,rec-100-dup-1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
n13,0,rec-13-org, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902
n1001,0,rec-1001-org, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
n1001,0,rec-1002-dup-0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819
n1002,0,rec-1002-org, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
n1002,0,rec-1003-org, connor, clarke,39, de chair street, wilding, peakhurst,5204, vic,19450907,1230316
n1004,0,rec-1004-dup-2, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673
n1004,0,rec-1005-org, jamie, kranz,8, , rmbh 884, port macquarie,5085, vic,19921217,4602627
n1005,0,rec-1004-org, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
n1005,0,rec-1006-dup-0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819
n1006,0,rec-1006-org, sam, bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819
n1006,0,rec-1007-dup-0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023
n1007,0,rec-1007-dup-1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n1007,0,rec-1008-org, alex, ballantyne,212, aronson crescent, logan central plaza, semaphore park,2153, wa,19450621,4145113
n1008,0,rec-1007-org, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n1008,0,rec-1085-org, schkirra, walpole,4, namadgi circuit, rydellan, bibra lake,2234, nsw,19560330,1950180
n511,0,rec-511-org-0, mason, ashlie,6, sinclair street, young retirement village, ,3201, nsw,19540731,9955736
n511,0,rec-1093-org, dylan, wilkins,18, denovan circuit, two tree hill, brighton-le-sands,2320, nsw, ,8735875
n511,0,rec-74-org, jordan, gold,18, sharwood crescent, sunforest therapy centre, belmont,2155, nsw,19941117,1139906
10,1, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769
10,1, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769
10,1, mikhayla, hannagan,, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,19770501,1030769
10,1, mikhayla, hannagan,20, rupp place, brentwood vlge, penshurst,2257, vic,,1030769
100,1, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
100,1, domenique, paterson,15, , merewether pufblic school, greensborough,, q,19161017,4975843
100,1, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161027,4975843
100,1, domenique, paterson,15, lubum a place, merewether pub lic school, greensborough,3138, qld,19161017,4975843
100,1, domenique, paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843
100,1, , paterson,15, merewether pu blic school, lubum aplace, greensborough,3138, qld,19160017,4957843
100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,19161017,4975843
100,1, domenique, paterson,15, lubuma place, merewether public school, greensborough,3138, qld,,4975843
1000,1, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902
1000,1, sophie, waller,24, suunybrae, goldfinch circuit, mona vale,2285, vic,19380705,2972902
1000,1, sophie, waller,24, suunybrae, , mona vale,2285, vic,19380705,2972902
1000,1, sophie, waller,24, goldfinch circuit, suunybrae, mona vale,2285, vic,19380705,2972902
1001,1, , pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19991009,1859104
1001,1, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
1001,1, casey, ,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
1002,1, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819
1002,1, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
1002,1, , matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
1002,1, xavier, ,10, , rank in park, kellyville,3039, vdc,19620202,6169819
1004,1, tayla, green,41, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1, tayla, greecn,38, kambalda crescent, ambevrale, bonville,2905, vic,19310175,2230673
1004,1, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
1004,1, tayla, gren,38, kambaldcrescent, ambervale, bonville,2905, ,19310715,2230673
1006,1, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819
1006,1, sam, bentham,1, narran street, lake chatsworth, westlake,2306, vic, ,8928819
1006,1, sam, bentham,1, narranstreet, lake chagsworth, westlake,2036, vic, ,8928819
1006,1, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic,19330165,8928819
1006,1, sam, bentham,1, narranystreet, , westl ake,2036, vic,19330165,8928819
1006,1, , bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819
1007,1, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023
1007,1, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1, rebekah, modystach,34, tausx place, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, nsp,19341113,8796023
1007,1, rebekah, modystach,34, hemmant street, pyramid cara park, iluka,2216, np,19341113,8796023
1007,1, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n0,0, jinni, dreyer,11, were street, marriott downs, south melbourne,3172, nsw,19420127,3787407
n0,0, jonah, browne,6, bingley crescent, , pindimar,3182, nsw,19250413,8328406
n10,0, hannagan, mikhayla,20, rupp lace, brentwoo dvlge, penshurst,2283, vic,19770501,1030769
n10,0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
n11,0, mikhvyla, hannagan,20, windradyen street, brentwood vlge, penshurst,2257, vic,19770501,1030769
n11,0, domenique, paterson,15, , merewether pufblic school, greensborough,3138, qld,19161017,4975843
n12,0, mikhayla, hannaan,20, , brentwoodvlge, penshurst,2257, vic,19770501,1030769
n12,0, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
n13,0, domenique, paterson,15, lubumaplace, merewether public school, greensborough,3138, qld,19161017,4975843
n13,0, sophie, waller,24, copland drive, suunybrae, mona vale,2285, vic,19380705,2972902
n1001,0, casey, pacey,22, officer crescent, ken hunter lodge, carlingford,6105, wa,19220930,1859104
n1001,0, xavier, matosn,10, , rank in park, kellyville,3039, vdc,19620202,6169819
n1002,0, xavier, matson,10, marchant circuit, rankin park, kellyville,2085, vdc,19620202,6169819
n1002,0, connor, clarke,39, de chair street, wilding, peakhurst,5204, vic,19450907,1230316
n1004,0, tayla, white,38, kambalda ctescent, ambervale, bonville,2905, vic,19310715,2230673
n1004,0, jamie, kranz,8, , rmbh 884, port macquarie,5085, vic,19921217,4602627
n1005,0, tayla, green,38, kambalda crescent, ambervale, bonville,2905, vic,19310715,2230673
n1005,0, sam, bentham,1, narranystreet, lake chat sworth, westl ake,2036, vic, ,8928819
n1006,0, sam, bentham,1, narran street, lake chatsworth, westlake,2036, vic, ,8928819
n1006,0, reekah, modystach,34, tauss place, pyramid c ara park, revesyb,2216, nsp,19341113,8796023
n1007,0, rebekah, modystach,34, taussw place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n1007,0, alex, ballantyne,212, aronson crescent, logan central plaza, semaphore park,2153, wa,19450621,4145113
n1008,0, rebekah, modystach,34, tauss place, pyramid cara park, iluka,2216, nsp,19341113,8796023
n1008,0, schkirra, walpole,4, namadgi circuit, rydellan, bibra lake,2234, nsw,19560330,1950180
n511,0, mason, ashlie,6, sinclair street, young retirement village, ,3201, nsw,19540731,9955736
n511,0, dylan, wilkins,18, denovan circuit, two tree hill, brighton-le-sands,2320, nsw, ,8735875
n511,0, jordan, gold,18, sharwood crescent, sunforest therapy centre, belmont,2155, nsw,19941117,1139906

0 comments on commit 21dde93

Please sign in to comment.