From 9cfa5c396b3cb58b9eda029482428e337e38fd9e Mon Sep 17 00:00:00 2001 From: Louis Chanouha Date: Sun, 4 Jun 2017 15:30:23 +0200 Subject: [PATCH] Adding buffer & channel for Marshall & UnMarshall, fixing performance issues & memory usage which large files --- ldif.go | 68 +++++++++++++++++++++++++++++++---------------- marshal.go | 78 ++++++++++++++++++++++++++++++++---------------------- 2 files changed, 91 insertions(+), 55 deletions(-) diff --git a/ldif.go b/ldif.go index fd8fc6b..594e6b4 100644 --- a/ldif.go +++ b/ldif.go @@ -12,7 +12,6 @@ import ( "net/url" "strconv" "strings" - "gopkg.in/ldap.v2" ) @@ -22,6 +21,7 @@ type Entry struct { Add *ldap.AddRequest Del *ldap.DelRequest Modify *ldap.ModifyRequest + Err *ParseError } // The LDIF struct is used for parsing an LDIF. The Controls @@ -72,25 +72,13 @@ func ParseWithControls(str string) (l *LDIF, err error) { err = Unmarshal(buf, l) return } - -// Unmarshal parses the LDIF from the given io.Reader into the LDIF struct. -// The caller is responsible for closing the io.Reader if that is -// needed. -func Unmarshal(r io.Reader, l *LDIF) (err error) { - if r == nil { - return &ParseError{Line: 0, Message: "No reader present"} - } - curLine := 0 - l.Version = 0 - l.changeType = "" - isComment := false - - reader := bufio.NewReader(r) - +func iterLines(r io.Reader, ch chan *Entry, l *LDIF){ + var err error var lines []string var line, nextLine string - l.firstEntry = true - + isComment := false + curLine := 0 + reader := bufio.NewReader(r) for { curLine++ nextLine, err = reader.ReadString(lf) @@ -101,7 +89,8 @@ func Unmarshal(r io.Reader, l *LDIF) (err error) { switch len(nextLine) { case 0: if len(line) == 0 && err == io.EOF { - return nil + close(ch) + return } if len(line) == 0 && len(lines) == 0 { continue @@ -109,13 +98,16 @@ func Unmarshal(r io.Reader, l *LDIF) (err error) { lines = append(lines, line) entry, perr := l.parseEntry(lines) if perr != nil { - return &ParseError{Line: curLine, Message: perr.Error()} + ch <- &Entry{Err: &ParseError{Line: curLine, Message: perr.Error()} } + }else{ + ch <- entry } - l.Entries = append(l.Entries, entry) + //l.Entries = append(l.Entries, entry) line = "" lines = []string{} if err == io.EOF { - return nil + close(ch) + return } default: switch nextLine[0] { @@ -140,9 +132,39 @@ func Unmarshal(r io.Reader, l *LDIF) (err error) { } } default: - return &ParseError{Line: curLine, Message: err.Error()} + ch <- &Entry{Err: &ParseError{Line: curLine, Message: err.Error()} } + close(ch) + return + } + } +} +func UnmarshalBuffer(r io.Reader, l *LDIF) (error, chan *Entry) { + if r == nil { + return &ParseError{Line: 0, Message: "No reader present"}, nil + } + ch := make (chan *Entry) + l.Version = 0 + l.changeType = "" + l.firstEntry = true + go iterLines(r, ch, l); + return nil, ch +} +// Unmarshal parses the LDIF from the given io.Reader into the LDIF struct. +// The caller is responsible for closing the io.Reader if that is +// needed. +func Unmarshal(r io.Reader, l *LDIF) (error) { + err, channel := UnmarshalBuffer(r, l) + if err != nil { + return err + } + for entry := range channel { + if entry.Err != nil { + return entry.Err + }else{ + l.Entries = append(l.Entries, entry) } } + return nil } func (l *LDIF) parseEntry(lines []string) (entry *Entry, err error) { diff --git a/marshal.go b/marshal.go index 51c3c48..71aff3e 100644 --- a/marshal.go +++ b/marshal.go @@ -4,6 +4,8 @@ import ( "encoding/base64" "errors" "fmt" + "bufio" + "bytes" "gopkg.in/ldap.v2" "io" ) @@ -14,36 +16,36 @@ var foldWidth = 76 // records in one LDIF var ErrMixed = errors.New("cannot mix change records and content records") -// Marshal returns an LDIF string from the given LDIF. +// Marshal writes LDIF string to the specified Buffer. // // The default line lenght is 76 characters. This can be changed by setting // the fw parameter to something else than 0. // For a fold width < 0, no folding will be done, with 0, the default is used. -func Marshal(l *LDIF) (data string, err error) { +func MarshalBuffer(l *LDIF, output io.Writer) (err error) { hasEntry := false hasChange := false - + buffer := bufio.NewWriter(output) if l.Version > 0 { - data = "version: 1\n" + buffer.WriteString("version: 1\n") } + fw := l.FoldWidth if fw == 0 { fw = foldWidth } - for _, e := range l.Entries { switch { case e.Add != nil: hasChange = true if hasEntry { - return "", ErrMixed + return ErrMixed } - data += foldLine("dn: "+e.Add.DN, fw) + "\n" - data += "changetype: add\n" + buffer.WriteString(foldLine("dn: "+e.Add.DN, fw) + "\n") + buffer.WriteString("changetype: add\n") for _, add := range e.Add.Attributes { if len(add.Vals) == 0 { - return "", errors.New("changetype 'add' requires non empty value list") + return errors.New("changetype 'add' requires non empty value list") } for _, v := range add.Vals { ev, t := encodeValue(v) @@ -51,75 +53,75 @@ func Marshal(l *LDIF) (data string, err error) { if t { col = ":: " } - data += foldLine(add.Type+col+ev, fw) + "\n" + buffer.WriteString(foldLine(add.Type+col+ev, fw) + "\n") } } case e.Del != nil: hasChange = true if hasEntry { - return "", ErrMixed + return ErrMixed } - data += foldLine("dn: "+e.Del.DN, fw) + "\n" - data += "changetype: delete\n" + buffer.WriteString(foldLine("dn: "+e.Del.DN, fw) + "\n") + buffer.WriteString("changetype: delete\n") case e.Modify != nil: hasChange = true if hasEntry { - return "", ErrMixed + return ErrMixed } - data += foldLine("dn: "+e.Modify.DN, fw) + "\n" - data += "changetype: modify\n" + buffer.WriteString(foldLine("dn: "+e.Modify.DN, fw) + "\n") + buffer.WriteString("changetype: modify\n") for _, mod := range e.Modify.AddAttributes { if len(mod.Vals) == 0 { - return "", errors.New("changetype 'modify', op 'add' requires non empty value list") + return errors.New("changetype 'modify', op 'add' requires non empty value list") } - data += "add: " + mod.Type + "\n" + buffer.WriteString("add: " + mod.Type + "\n") for _, v := range mod.Vals { ev, t := encodeValue(v) col := ": " if t { col = ":: " } - data += foldLine(mod.Type+col+ev, fw) + "\n" + buffer.WriteString(foldLine(mod.Type+col+ev, fw) + "\n") } - data += "-\n" + buffer.WriteString("-\n") } for _, mod := range e.Modify.DeleteAttributes { - data += "delete: " + mod.Type + "\n" + buffer.WriteString("delete: " + mod.Type + "\n") for _, v := range mod.Vals { ev, t := encodeValue(v) col := ": " if t { col = ":: " } - data += foldLine(mod.Type+col+ev, fw) + "\n" + buffer.WriteString(foldLine(mod.Type+col+ev, fw) + "\n") } - data += "-\n" + buffer.WriteString("-\n") } for _, mod := range e.Modify.ReplaceAttributes { if len(mod.Vals) == 0 { - return "", errors.New("changetype 'modify', op 'replace' requires non empty value list") + return errors.New("changetype 'modify', op 'replace' requires non empty value list") } - data += "replace: " + mod.Type + "\n" + buffer.WriteString("replace: " + mod.Type + "\n") for _, v := range mod.Vals { ev, t := encodeValue(v) col := ": " if t { col = ":: " } - data += foldLine(mod.Type+col+ev, fw) + "\n" + buffer.WriteString(foldLine(mod.Type+col+ev, fw) + "\n") } - data += "-\n" + buffer.WriteString("-\n") } default: hasEntry = true if hasChange { - return "", ErrMixed + return ErrMixed } - data += foldLine("dn: "+e.Entry.DN, fw) + "\n" + buffer.WriteString(foldLine("dn: "+e.Entry.DN, fw) + "\n") for _, av := range e.Entry.Attributes { for _, v := range av.Values { ev, t := encodeValue(v) @@ -127,13 +129,25 @@ func Marshal(l *LDIF) (data string, err error) { if t { col = ":: " } - data += foldLine(av.Name+col+ev, fw) + "\n" + buffer.WriteString(foldLine(av.Name+col+ev, fw) + "\n") } } } - data += "\n" + buffer.WriteString("\n") + } + buffer.Flush() + return nil +} + +// Marshal returns an LDIF string from the given LDIF. +func Marshal(l *LDIF) (data string, err error) { + temp := new(bytes.Buffer) + err2 := MarshalBuffer(l, temp) + if err2 == nil { + return temp.String(), nil + }else { + return "", err2 } - return data, nil } func encodeValue(value string) (string, bool) {