Skip to content

Commit

Permalink
ofac: return maps for Addresses, Alts, Comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamdecaf
adamdecaf committed Jan 15, 2025
1 parent 7d2112c commit 6b5e58d
Show file tree
Hide file tree
Showing 3 changed files with 86 additions and 62 deletions.
28 changes: 3 additions & 25 deletions pkg/ofac/mapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,34 +145,12 @@ func extractCountry(remark string) string {
return ""
}

func GroupIntoEntities(sdns []SDN, addresses []Address, comments []SDNComments, altIds []AlternateIdentity) []search.Entity[search.Value] {
func GroupIntoEntities(sdns []SDN, addresses map[string][]Address, comments map[string][]SDNComments, altIds map[string][]AlternateIdentity) []search.Entity[search.Value] {
fn := func(sdn SDN) search.Entity[search.Value] {
var addrs []Address
for _, addr := range addresses {
if sdn.EntityID == addr.EntityID {
addrs = append(addrs, addr)
}
}

var cmts []SDNComments
for _, comment := range comments {
if sdn.EntityID == comment.EntityID {
cmts = append(cmts, comment)
}
}

var alts []AlternateIdentity
for _, alt := range altIds {
if sdn.EntityID == alt.EntityID {
alts = append(alts, alt)
}
}

return ToEntity(sdn, addrs, cmts, alts)
return ToEntity(sdn, addresses[sdn.EntityID], comments[sdn.EntityID], altIds[sdn.EntityID])
}

groups := runtime.NumCPU() // arbitrary group size // TODO(adam):

groups := runtime.NumCPU()
return indices.ProcessSlice(sdns, groups, fn)
}

Expand Down
83 changes: 61 additions & 22 deletions pkg/ofac/reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@ import (
//
// For more details on the raw OFAC files see https://moov-io.github.io/watchman/file-structure.html
func Read(files map[string]io.ReadCloser) (*Results, error) {
res := new(Results)
res := &Results{
SDNs: []SDN{},
Addresses: make(map[string][]Address, 0),
AlternateIdentities: make(map[string][]AlternateIdentity, 0),
SDNComments: make(map[string][]SDNComments, 0),
}

for filename, file := range files {
switch strings.ToLower(filepath.Base(filename)) {
case "add.csv":
Expand Down Expand Up @@ -57,33 +63,56 @@ func Read(files map[string]io.ReadCloser) (*Results, error) {
}

type Results struct {
// SDNs returns an array of OFAC Specially Designated Nationals
SDNs []SDN `json:"sdn"`

// Addresses returns an array of OFAC Specially Designated National Addresses
Addresses []Address `json:"address"`
Addresses map[string][]Address `json:"address"`

// AlternateIdentities returns an array of OFAC Specially Designated National Alternate Identity
AlternateIdentities []AlternateIdentity `json:"alternateIdentity"`

// SDNs returns an array of OFAC Specially Designated Nationals
SDNs []SDN `json:"sdn"`
AlternateIdentities map[string][]AlternateIdentity `json:"alternateIdentity"`

// SDNComments returns an array of OFAC Specially Designated National Comments
SDNComments []SDNComments `json:"sdnComments"`
SDNComments map[string][]SDNComments `json:"sdnComments"`
}

func (r *Results) append(rr *Results, err error) error {
if err != nil {
return err
}
r.Addresses = append(r.Addresses, rr.Addresses...)
r.AlternateIdentities = append(r.AlternateIdentities, rr.AlternateIdentities...)
r.SDNs = append(r.SDNs, rr.SDNs...)
r.SDNComments = append(r.SDNComments, rr.SDNComments...)

if rr.Addresses != nil {
for _, addresses := range rr.Addresses {
for _, addr := range addresses {
r.Addresses[addr.EntityID] = append(r.Addresses[addr.EntityID], addr)
}
}
}

if rr.AlternateIdentities != nil {
for _, alts := range rr.AlternateIdentities {
for _, alt := range alts {
r.AlternateIdentities[alt.EntityID] = append(r.AlternateIdentities[alt.EntityID], alt)
}
}
}

if rr.SDNComments != nil {
for _, comments := range rr.SDNComments {
for _, comment := range comments {
r.SDNComments[comment.EntityID] = append(r.SDNComments[comment.EntityID], comment)
}
}
}

return nil
}

func csvAddressFile(f io.ReadCloser) (*Results, error) {
defer f.Close()
var out []Address

out := make(map[string][]Address, 0)

// Read File into a Variable
reader := csv.NewReader(f)
Expand All @@ -107,8 +136,10 @@ func csvAddressFile(f io.ReadCloser) (*Results, error) {
}

record = replaceNull(record)
out = append(out, Address{
EntityID: record[0],

entityID := record[0]
out[entityID] = append(out[entityID], Address{
EntityID: entityID,
AddressID: record[1],
Address: record[2],
CityStateProvincePostalCode: record[3],
Expand All @@ -121,7 +152,8 @@ func csvAddressFile(f io.ReadCloser) (*Results, error) {

func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) {
defer f.Close()
var out []AlternateIdentity

out := make(map[string][]AlternateIdentity, 0)

// Read File into a Variable
reader := csv.NewReader(f)
Expand All @@ -144,7 +176,9 @@ func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) {
continue
}
record = replaceNull(record)
out = append(out, AlternateIdentity{

entityID := record[0]
out[entityID] = append(out[entityID], AlternateIdentity{
EntityID: record[0],
AlternateID: record[1],
AlternateType: record[2],
Expand All @@ -157,6 +191,7 @@ func csvAlternateIdentityFile(f io.ReadCloser) (*Results, error) {

func csvSDNFile(f io.ReadCloser) (*Results, error) {
defer f.Close()

var out []SDN

// Read File into a Variable
Expand Down Expand Up @@ -205,7 +240,7 @@ func csvSDNCommentsFile(f io.ReadCloser) (*Results, error) {
r.LazyQuotes = true

// Loop through lines & turn into object
var out []SDNComments
out := make(map[string][]SDNComments, 0)
for {
line, err := r.Read()
if err != nil {
Expand All @@ -225,8 +260,10 @@ func csvSDNCommentsFile(f io.ReadCloser) (*Results, error) {
continue
}
line = replaceNull(line)
out = append(out, SDNComments{
EntityID: line[0],

entityID := line[0]
out[entityID] = append(out[entityID], SDNComments{
EntityID: entityID,
RemarksExtended: line[1],
DigitalCurrencyAddresses: readDigitalCurrencyAddresses(line[1]),
})
Expand Down Expand Up @@ -379,11 +416,13 @@ func readDigitalCurrencyAddresses(remarks string) []DigitalCurrencyAddress {
return out
}

func mergeSpilloverRecords(sdns []SDN, comments []SDNComments) []SDN {
func mergeSpilloverRecords(sdns []SDN, allComments map[string][]SDNComments) []SDN {
for i := range sdns {
for j := range comments {
if sdns[i].EntityID == comments[j].EntityID {
sdns[i].Remarks += comments[j].RemarksExtended
comments := allComments[sdns[i].EntityID]

for _, comment := range comments {
if sdns[i].EntityID == comment.EntityID {
sdns[i].Remarks += comment.RemarksExtended // has to be index to update
}
}
}
Expand Down
37 changes: 22 additions & 15 deletions pkg/ofac/reader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ import (
"github.com/stretchr/testify/require"
)

// TestOFAC__read validates reading an OFAC Address CSV File
func TestOFAC__read(t *testing.T) {
func TestRead(t *testing.T) {
testdata := func(fn string) map[string]io.ReadCloser {
fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", fn))
if err != nil {
Expand All @@ -27,15 +26,16 @@ func TestOFAC__read(t *testing.T) {
}
res, err := Read(testdata("add.csv"))
require.NoError(t, err)
require.Len(t, res.Addresses, 11696)

require.Len(t, res.Addresses, 7379)
require.Len(t, res.AlternateIdentities, 0)
require.Len(t, res.SDNs, 0)
require.Len(t, res.SDNComments, 0)

res, err = Read(testdata("alt.csv"))
require.NoError(t, err)
require.Len(t, res.Addresses, 0)
require.Len(t, res.AlternateIdentities, 9682)
require.Len(t, res.AlternateIdentities, 3196)
require.Len(t, res.SDNs, 0)
require.Len(t, res.SDNComments, 0)

Expand Down Expand Up @@ -108,18 +108,20 @@ func TestSDNComments(t *testing.T) {
if _, err := fd.WriteString(`28264,"hone Number 8613314257947; alt. Phone Number 8618004121000; Identification Number 210302198701102136 (China); a.k.a. "blackjack1987"; a.k.a. "khaleesi"; Linked To: LAZARUS GROUP."`); err != nil {
t.Fatal(err)
}

fd.Seek(0, 0)

// read with lazy quotes enabled
if res, err := csvSDNCommentsFile(fd); err != nil {
t.Errorf("unexpected error: %v", err)
} else {
if len(res.SDNComments) != 1 {
t.Errorf("SDNComments=%#v", res.SDNComments)
}
for i := range res.SDNComments {
t.Logf("\ncomment #%d\n entity=%s\n remarks=%v", i, res.SDNComments[i].EntityID, res.SDNComments[i].RemarksExtended)
}
}
res, err := csvSDNCommentsFile(fd)
require.NoError(t, err)
require.Len(t, res.SDNComments, 1)

comments, found := res.SDNComments["28264"]
require.True(t, found)
require.Len(t, comments, 1)

comment := comments[0]
require.NotEmpty(t, comment.RemarksExtended)
}

func TestSDN__remarks(t *testing.T) {
Expand Down Expand Up @@ -174,11 +176,16 @@ func TestSDNComments_CryptoCurrencies(t *testing.T) {
_, err = fd.WriteString(`42496," alt. Digital Currency Address - XBT 12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax; alt. Digital Currency Address - XBT 1J378PbmTKn2sEw6NBrSWVfjZLBZW3DZem; alt. Digital Currency Address - XBT 18aqbRhHupgvC9K8qEqD78phmTQQWs7B5d; alt. Digital Currency Address - XBT 16ti2EXaae5izfkUZ1Zc59HMcsdnHpP5QJ; Secondary sanctions risk: North Korea Sanctions Regulations, sections 510.201 and 510.210; Transactions Prohibited For Persons Owned or Controlled By U.S. Financial Institutions: North Korea Sanctions Regulations section 510.214; Passport E59165201 (China) expires 01 Sep 2025; Identification Number 371326198812157611 (China); a.k.a. 'WAKEMEUPUPUP'; a.k.a. 'FAST4RELEASE'; Linked To: LAZARUS GROUP."`)
require.NoError(t, err)
fd.Seek(0, 0)

sdn, err := csvSDNCommentsFile(fd)
require.NoError(t, err)
require.Len(t, sdn.SDNComments, 1)

addresses := sdn.SDNComments[0].DigitalCurrencyAddresses
comments, found := sdn.SDNComments["42496"]
require.True(t, found)
require.Len(t, comments, 1)

addresses := comments[0].DigitalCurrencyAddresses
require.Len(t, addresses, 4)

expected := []DigitalCurrencyAddress{
Expand Down

0 comments on commit 6b5e58d

Please sign in to comment.