diff --git a/artifacts/definitions/Windows/Applications/outlook/emailheader.yaml b/artifacts/definitions/Windows/Applications/outlook/emailheader.yaml new file mode 100644 index 00000000000..004c67faa8b --- /dev/null +++ b/artifacts/definitions/Windows/Applications/outlook/emailheader.yaml @@ -0,0 +1,37 @@ +name: Windows.Applications.Outlook.PST.Header +author: "Sikha Puthanveedu @SikhaMohan" +description: | + This artifact fetch emails and header details such as SPF, DMARC and DKIM from outlook PST file. +parameters: + - name: outlookPSTfile + type: .pst + description: Full path to the outlook .pst file (For example - D:/MyPST/MyOutlookDataFile.pst) + +sources: + - precondition: + SELECT OS FROM info() WHERE OS = 'windows' + query: | + + LET PSTInfo = SELECT Sender as Sender, + Receiver as Receiver, + Subject as Subject, + Message as Message, + DateandTime as DeliveryTime, + Attachments as AttachmentNames, + Body as Body + from parse_pst(filename=outlookPSTfile) + + SELECT + Sender, + Receiver, + parse_string_with_regex(regex='spf=(\\w+)', string=Body).g1 AS SPF, + parse_string_with_regex(regex='dkim=(\\w+)', string=Body).g1 AS DKIM, + parse_string_with_regex(regex='dmarc=(\\w+)', string=Body).g1 AS DMARC, + parse_string_with_regex(regex='Return-Path: <(.*?)>', string=Body).g1 AS ReturnPath, + Subject, + DeliveryTime, + AttachmentNames, + Message, + parse_string_with_regex(regex='internet_message_id:"<(.*?)>"', string=Body).g1 AS msgId, + parse_string_with_regex(regex='Content-Type:(.*?);', string=Body).g1 AS ContentType + FROM PSTInfo \ No newline at end of file diff --git a/artifacts/definitions/Windows/Applications/outlook/pstparser.yaml b/artifacts/definitions/Windows/Applications/outlook/pstparser.yaml new file mode 100644 index 00000000000..4ad9a1d7a56 --- /dev/null +++ b/artifacts/definitions/Windows/Applications/outlook/pstparser.yaml @@ -0,0 +1,36 @@ +name: Windows.Applications.Outlook.PST +author: "Sikha Puthanveedu @SikhaMohan" +description: | + This artifact fetch emails and attachments from outlook PST file. + This artifact parse for outlook pst files and display the details and save + all attachments to user specified directory. +parameters: + - name: outlookPSTfile + type: .pst + description: Full path to the outlook .pst file (For example - D:/MyPST/MyOutlookDataFile.pst) + - name: AttachmentFolder + type: directory path + description: If selected it will save all the attachments from emails to the specified directory. + +sources: + - precondition: + SELECT OS FROM info() WHERE OS = 'windows' + query: | + + LET PSTInfo = SELECT Sender as Sender, + Receiver as Receiver, + Subject as Subject, + Message as Message, + DateandTime as DeliveryTime, + Attachments as AttachmentNames, + Body as Body + from parse_pst(filename=outlookPSTfile, FolderPath=AttachmentFolder) + + SELECT + Sender, + Receiver, + Subject, + DeliveryTime, + AttachmentNames, + Message + FROM PSTInfo \ No newline at end of file diff --git a/artifacts/testdata/files/pst/Outlook.pst b/artifacts/testdata/files/pst/Outlook.pst new file mode 100644 index 00000000000..d3621be9b59 Binary files /dev/null and b/artifacts/testdata/files/pst/Outlook.pst differ diff --git a/artifacts/testdata/server/testcases/pstparser.yaml b/artifacts/testdata/server/testcases/pstparser.yaml new file mode 100644 index 00000000000..5d45d8c11c2 --- /dev/null +++ b/artifacts/testdata/server/testcases/pstparser.yaml @@ -0,0 +1,37 @@ +name: Windows.Applications.Outlook.PST +author: "Sikha Puthanveedu" +description: | + This artifact fetch emails and attachments from outlook PST file. + This artifact parse for outlook pst files and display the details and save + all attachments to user specified directory. +parameters: + - name: outlookPSTfile + type: .pst + description: Full path to the outlook .pst file (For example - D:/MyPST/MyOutlookDataFile.pst) + - name: AttachmentFolder + type: directory path + description: If selected it will save all the attachments from emails to the specified directory. + +sources: + - precondition: + SELECT OS FROM info() WHERE OS = 'windows' + query: | + + LET PSTInfo = SELECT Sender as Sender, + Receiver as Receiver, + Subject as Subject, + Message as Message, + DateandTime as DeliveryTime, + Attachments as AttachmentNames, + AttachmentId as AttachmentId, + Body as Body + from parse_pst(filename=outlookPSTfile, FolderPath=AttachmentFolder) + + SELECT + Sender, + Receiver, + Subject, + DeliveryTime, + AttachmentNames, + Message + FROM PSTInfo \ No newline at end of file diff --git a/go.mod b/go.mod index 96041f8927d..5a7f5a85ed6 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( golang.org/x/mod v0.10.0 golang.org/x/net v0.10.0 golang.org/x/sys v0.8.0 - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/time v0.3.0 google.golang.org/api v0.126.0 google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect @@ -158,7 +158,10 @@ require ( github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e // indirect + github.com/emersion/go-message v0.16.0 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/geoffgarside/ber v1.1.0 // indirect + github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5 // indirect github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f // indirect github.com/golang/glog v1.1.0 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -176,6 +179,7 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/lestrrat-go/strftime v1.0.5 // indirect + github.com/libp2p/go-sockaddr v0.1.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -183,15 +187,20 @@ require ( github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/mooijtech/go-pst/v6 v6.0.2 // indirect github.com/paulmach/orb v0.1.5 // indirect + github.com/philhofer/fwd v1.1.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/rivo/uniseg v0.4.2 // indirect + github.com/rotisserie/eris v0.5.4 // indirect github.com/russellhaering/goxmldsig v1.3.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect + github.com/tidwall/btree v1.6.0 // indirect + github.com/tinylib/msgp v1.1.8 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect diff --git a/go.sum b/go.sum index 83ec6ef6fd5..28f6161497d 100644 --- a/go.sum +++ b/go.sum @@ -249,6 +249,10 @@ github.com/elastic/go-elasticsearch/v7 v7.3.0 h1:H29Nqf9cB9dVxX6LwS+zTDC2D4t9s+8 github.com/elastic/go-elasticsearch/v7 v7.3.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= github.com/elastic/go-libaudit v0.4.0 h1:pxLCycMJKW91W8ZmZT74DQmryTZuXryKESo6sXdu1XY= github.com/elastic/go-libaudit v0.4.0/go.mod h1:lNJ7gX+arohEQTwqinAc8xycVuFNqsaunba1mwcBdvE= +github.com/emersion/go-message v0.16.0 h1:uZLz8ClLv3V5fSFF/fFdW9jXjrZkXIpE1Fn8fKx7pO4= +github.com/emersion/go-message v0.16.0/go.mod h1:pDJDgf/xeUIF+eicT6B/hPX/ZbEorKkUMPOxrPVG2eQ= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -272,11 +276,14 @@ github.com/glaslos/tlsh v0.2.0/go.mod h1:S/OBGINihiGogV6WoaLeMY2UrS5Rl1iqMnplLon github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.6.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5 h1:5zELAgnSz0gqmr4Q5DWCoOzNHoeBAxVUXB7LS1eG+sw= +github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5/go.mod h1:ermjEDUoT/fS+3Ona5Vd6t6mZkw1eHp99ILO5jGRBkM= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= @@ -446,10 +453,13 @@ github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXL github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQymUhg= github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -478,6 +488,8 @@ github.com/lestrrat-go/strftime v1.0.5 h1:A7H3tT8DhTz8u65w+JRpiBxM4dINQhUXAZnhBa github.com/lestrrat-go/strftime v1.0.5/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR76fd03sz+Qz4g= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/libp2p/go-sockaddr v0.1.1 h1:yD80l2ZOdGksnOyHrhxDdTDFrf7Oy+v3FMVArIRgZxQ= +github.com/libp2p/go-sockaddr v0.1.1/go.mod h1:syPvOmNs24S3dFVGJA1/mrqdeijPxLV2Le3BRLKd68k= github.com/lpar/gzipped v1.1.0 h1:FEQnBzF06KTMh8Wnse6wNJvGwe7+vILQIFzuTq6ipGs= github.com/lpar/gzipped v1.1.0/go.mod h1:JBo67wiCld7AmFYfSNA75NmFG65roJiGwrVohF8uYGE= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -523,8 +535,13 @@ github.com/mitchellh/panicwrap v1.0.0/go.mod h1:pKvZHwWrZowLUzftuFq7coarnxbBXU4a github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mooijtech/go-pst/v6 v6.0.2 h1:mXzOgcSZ30UPuCWpz4DAQCTm0ttOmiejOuF/CN32C2Q= +github.com/mooijtech/go-pst/v6 v6.0.2/go.mod h1:pF4o5rQwF33uLJQ0c+CZICeK4GwcKTpGVq6yVOHrvkY= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/octago/sflags v0.2.0 h1:XceYzkRXGAHa/lSFmKLcaxSrsh4MTuOMQdIGsUD0wlk= github.com/octago/sflags v0.2.0/go.mod h1:G0bjdxh4qPRycF74a2B8pU36iTp9QHGx0w0dFZXPt80= @@ -540,6 +557,8 @@ github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19 github.com/paulmach/orb v0.1.5 h1:GUcATabvxciqEzGd+c01/9ek3B6pUp9OdcIHFSDDSSg= github.com/paulmach/orb v0.1.5/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -578,6 +597,8 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rotisserie/eris v0.5.4 h1:Il6IvLdAapsMhvuOahHWiBnl1G++Q0/L5UIkI5mARSk= +github.com/rotisserie/eris v0.5.4/go.mod h1:Z/kgYTJiJtocxCbFfvRmO+QejApzG6zpyky9G1A4g9s= github.com/russellhaering/goxmldsig v1.2.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= github.com/russellhaering/goxmldsig v1.3.0 h1:DllIWUgMy0cRUMfGiASiYEa35nsieyD3cigIwLonTPM= github.com/russellhaering/goxmldsig v1.3.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= @@ -627,8 +648,12 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tink-ab/tempfile v0.0.0-20180226111222-33beb0518f1a h1:Qhm/9UKGO1+AjEKIsq8G72uCq4SrYxSxS5wiD0F3IC4= github.com/tink-ab/tempfile v0.0.0-20180226111222-33beb0518f1a/go.mod h1:Wt5qSdcHgX6XkqZKAZTxnN+93jnqtx0jEgTQakpZ1CE= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= @@ -718,6 +743,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -769,6 +795,7 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -809,12 +836,14 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -889,12 +918,14 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -907,9 +938,12 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -971,6 +1005,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vql/parsers/pst_parser/pst_parser.go b/vql/parsers/pst_parser/pst_parser.go new file mode 100644 index 00000000000..267fcc1b9f8 --- /dev/null +++ b/vql/parsers/pst_parser/pst_parser.go @@ -0,0 +1,214 @@ +package pst_parser + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/Velocidex/ordereddict" + pst "github.com/mooijtech/go-pst/v6/pkg" + "github.com/mooijtech/go-pst/v6/pkg/properties" + "github.com/rotisserie/eris" + "www.velocidex.com/golang/velociraptor/acls" + "www.velocidex.com/golang/velociraptor/vql" + vql_subsystem "www.velocidex.com/golang/velociraptor/vql" + vfilter "www.velocidex.com/golang/vfilter" +) + +type PSTParserArgs struct { + Filename string `vfilter:"required,field=filename,doc=The PST file to parse."` + FolderPath string `vfilter:"field=FolderPath,doc=The folder path to save the attachments from emails."` + Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use"` +} + +type PSTParser struct{} + +func (self *PSTParser) Call(ctx context.Context, scope vfilter.Scope, args *ordereddict.Dict) <-chan vfilter.Row { + outputChan := make(chan vfilter.Row) + + go func() { + defer close(outputChan) + + arg := &PSTParserArgs{} + err := vfilter.ExtractArgs(scope, args, arg) + if err != nil { + scope.Log("pst_parser: ExtractArgs %v", err) + return + } + + reader, err := os.Open(arg.Filename) + if err != nil { + scope.Log(arg.Filename) + scope.Log("pst_parser: os.Open %s", arg.Filename) + scope.Log("pst_parser: %v", err) + return + } + + pstFile, err := pst.New(reader) + if err != nil { + scope.Log("pst_parser:pst.New(reader) %v", err) + return + } + + defer func() { + pstFile.Cleanup() + + if errClosing := reader.Close(); errClosing != nil { + scope.Log("pst_parser:reader.Close() %v", errClosing) + } + }() + + // For writing attachment + switch arg.Accessor { + case "", "auto", "file": + err := vql_subsystem.CheckAccess(scope, acls.FILESYSTEM_WRITE) + if err != nil { + scope.Log("write_parser: %s", err) + } + + // Create attachments directory + if len(arg.FolderPath) != 0 { + if _, err := os.Stat(arg.FolderPath); err != nil { + if err := os.Mkdir(arg.FolderPath, 0755); err != nil { + scope.Log("Failed to create attachments directory: %+v", err) + } + } + } + + default: + scope.Log("write_parser: Unsupported accessor for writing %v", arg.Accessor) + } + + // Walk through folders and process messages + if err := pstFile.WalkFolders(func(folder *pst.Folder) error { + + messageIterator, err := folder.GetMessageIterator() + + if eris.Is(err, pst.ErrMessagesNotFound) { + // Folder has no messages. + return nil + } else if err != nil { + scope.Log("Walking folder error: %s\n", folder.Name) + } + + for messageIterator.Next() { + message := messageIterator.Value() + + // Process the message and send it to the output channel + output := ordereddict.NewDict() + message_props, ok := message.Properties.(*properties.Message) + if !ok { + continue + } + + output.Set("Sender", message_props.GetSenderEmailAddress()) + output.Set("Receiver", message_props.GetReceivedByEmailAddress()) + output.Set("Subject", message_props.GetSubject()) + output.Set("Message", message_props.GetBody()) + output.Set("Body", message_props.String()) + + // Convert the int64 timestamp (in milliseconds) to a time.Time value + deliveryTime := message_props.GetMessageDeliveryTime() / 1e9 + deliveryTimeValue := time.Unix(deliveryTime, 0).UTC() + + // Format the date and time in UTC + formattedTime := deliveryTimeValue.Format("2006-01-02 15:04:05 MST") + output.Set("DateandTime", formattedTime) + + attachmentIterator, err := message.GetAttachmentIterator() + attachmentName := make([]string, 0) + attachmentId := make([]string, 0) + + if eris.Is(err, pst.ErrAttachmentsNotFound) { + // This message has no attachments. + output.Set("AttachmentId", "NIL") + output.Set("Attachments", "NIL") + + outputChan <- output + continue + } else if err != nil { + scope.Log("ErrAttachmentsNotFound") + } + + // Iterate through attachments. + for attachmentIterator.Next() { + attachment := attachmentIterator.Value() + + var attachmentNameId string + var attachtId string + + if attachment.GetAttachLongFilename() != "" { + attachmentNameId = fmt.Sprintf("%d-%s", attachment.Identifier, attachment.GetAttachLongFilename()) + attachmentName = append(attachmentName, attachmentNameId) + attachtId = fmt.Sprintf("%d", attachment.Identifier) + attachmentId = append(attachmentId, attachtId) + + } else { + attachmentNameId = fmt.Sprintf("%d-%s", attachment.Identifier, "") + attachmentName = append(attachmentName, attachmentNameId) + attachtId = fmt.Sprintf("%d", attachment.Identifier) + attachmentId = append(attachmentId, attachtId) + scope.Log("attachments/UNKNOWN_%d", attachment.Identifier) + } + // Set attachment name and Id to the output channel + output.Set("AttachmentId", attachmentId) + output.Set("Attachments", attachmentName) + + // Save to attachments folder only if FolderPath is specified + if arg.FolderPath != "" { + var attachmentOutputPath string + + if attachment.GetAttachLongFilename() != "" { + attachmentOutputPath = fmt.Sprintf(arg.FolderPath+"/%d-%s", attachment.Identifier, attachment.GetAttachLongFilename()) + } else { + attachmentOutputPath = fmt.Sprintf("attachments/UNKNOWN_%d", attachment.Identifier) + } + + attachmentOutput, err := os.Create(attachmentOutputPath) + if err != nil { + scope.Log("Error while creating Attchment_%d\n", attachment.Identifier) + continue + } + + if _, err := attachment.WriteTo(attachmentOutput); err != nil { + scope.Log("Error while writing Attchment_%d\n", attachment.Identifier) + } + + if err := attachmentOutput.Close(); err != nil { + scope.Log("Error while closing attachmentOutput_%d\n", attachment.Identifier) + } + if attachmentIterator.Err() != nil { + scope.Log("Error while iterating atachments\n") + continue + } + } + } + outputChan <- output + } + + if messageIterator.Err() != nil { + scope.Log("Error while iterating messages\n") + } + + return nil + }); err != nil { + scope.Log("pst_parser_WalkFolders error: %v", err) + } + }() + + return outputChan +} + +func (self *PSTParser) Info(scope vfilter.Scope, typeMap *vfilter.TypeMap) *vfilter.PluginInfo { + return &vfilter.PluginInfo{ + Name: "parse_pst", + Doc: "Parse a PST file and extract email data.", + ArgType: typeMap.AddType(scope, &PSTParserArgs{}), + Metadata: vql.VQLMetadata().Permissions(acls.FILESYSTEM_READ, acls.FILESYSTEM_WRITE).Build(), + } +} + +func init() { + vql_subsystem.RegisterPlugin(&PSTParser{}) +} diff --git a/vql_plugins/plugins.go b/vql_plugins/plugins.go index 9289b5c81ec..6297febd793 100644 --- a/vql_plugins/plugins.go +++ b/vql_plugins/plugins.go @@ -41,4 +41,5 @@ import ( _ "www.velocidex.com/golang/velociraptor/vql/tools" _ "www.velocidex.com/golang/velociraptor/vql/tools/collector" _ "www.velocidex.com/golang/velociraptor/vql/tools/process" + _ "www.velocidex.com/golang/velociraptor/vql/parsers/pst_parser" )