From df1f92502a5a6a49959cd79530037ddbe549fd90 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Sat, 6 Oct 2018 11:23:42 +0800 Subject: [PATCH 01/13] add mongodb op_msg 2013 --- packetbeat/protos/mongodb/mongodb.go | 1 + packetbeat/protos/mongodb/mongodb_parser.go | 77 ++++++++++++++++++-- packetbeat/protos/mongodb/mongodb_structs.go | 11 ++- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/packetbeat/protos/mongodb/mongodb.go b/packetbeat/protos/mongodb/mongodb.go index 5b7eb9d3dea..559902da02c 100644 --- a/packetbeat/protos/mongodb/mongodb.go +++ b/packetbeat/protos/mongodb/mongodb.go @@ -293,6 +293,7 @@ func newTransaction(requ, resp *mongodbMessage) *transaction { trans.params = requ.params trans.resource = requ.resource trans.bytesIn = requ.messageLength + trans.documents = requ.documents } // fill response diff --git a/packetbeat/protos/mongodb/mongodb_parser.go b/packetbeat/protos/mongodb/mongodb_parser.go index 58d6aa145da..d65f22dbca3 100644 --- a/packetbeat/protos/mongodb/mongodb_parser.go +++ b/packetbeat/protos/mongodb/mongodb_parser.go @@ -56,14 +56,14 @@ func mongodbMessageParser(s *stream) (bool, bool) { opCode := opCode(code) if !validOpcode(opCode) { - logp.Err("Unknown operation code: %v", opCode) + logp.Err("Unknown operation code: %d (%v)", opCode, opCode) return false, false } s.message.opCode = opCode s.message.isResponse = false // default is that the message is a request. If not opReplyParse will set this to false s.message.expectsResponse = false - debugf("opCode = %v", s.message.opCode) + debugf("opCode = %d (%v)", s.message.opCode, s.message.opCode) // then split depending on operation type s.message.event = common.MapStr{} @@ -72,9 +72,9 @@ func mongodbMessageParser(s *stream) (bool, bool) { case opReply: s.message.isResponse = true return opReplyParse(d, s.message) - case opMsg: + case opMsgLegacy: s.message.method = "msg" - return opMsgParse(d, s.message) + return opMsgLegacyParse(d, s.message) case opUpdate: s.message.method = "update" return opUpdateParse(d, s.message) @@ -94,6 +94,9 @@ func mongodbMessageParser(s *stream) (bool, bool) { case opKillCursor: s.message.method = "killCursors" return opKillCursorsParse(d, s.message) + case opMsg: + s.message.method = "msg" + return opMsgParse(d, s.message) } return false, false @@ -137,7 +140,7 @@ func opReplyParse(d *decoder, m *mongodbMessage) (bool, bool) { return true, true } -func opMsgParse(d *decoder, m *mongodbMessage) (bool, bool) { +func opMsgLegacyParse(d *decoder, m *mongodbMessage) (bool, bool) { var err error m.event["message"], err = d.readCStr() if err != nil { @@ -292,6 +295,61 @@ func opKillCursorsParse(d *decoder, m *mongodbMessage) (bool, bool) { return true, true } +func opMsgParse(d *decoder, m *mongodbMessage) (bool, bool) { + // ignore flagbits + _, err := d.readInt32() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + return false, false + } + + // read sections + kind, err := d.readByte() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + return false, false + } + + switch msgKind(kind) { + case msgKindBody: + document, err := d.readDocument() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + return false, false + } + m.documents = []interface{}{document} + + case msgKindDocumentSequence: + start := d.i + size, err := d.readInt32() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + return false, false + } + cstring, err := d.readCStr() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + return false, false + } + m.event["message"] = cstring + var documents []interface{} + for d.i < start+size { + document, err := d.readDocument() + if err != nil { + logp.Err("An error occurred while parsing OP_MSG message: %s", err) + } + documents = append(documents, document) + } + m.documents = documents + + default: + logp.Err("Unknown message kind: %v", kind) + return false, false + } + + return true, true +} + // NOTE: The following functions are inspired by the source of the go-mgo/mgo project // https://github.com/go-mgo/mgo/blob/v2/bson/decode.go @@ -324,6 +382,15 @@ func (d *decoder) readCStr() (string, error) { return string(d.in[start:end]), nil } +func (d *decoder) readByte() (byte, error) { + i := d.i + d.i++ + if d.i > len(d.in) { + return 0, errors.New("Read byte failed") + } + return d.in[i], nil +} + func (d *decoder) readInt32() (int, error) { b, err := d.readBytes(4) diff --git a/packetbeat/protos/mongodb/mongodb_structs.go b/packetbeat/protos/mongodb/mongodb_structs.go index 759117c33e6..9bb2abc1493 100644 --- a/packetbeat/protos/mongodb/mongodb_structs.go +++ b/packetbeat/protos/mongodb/mongodb_structs.go @@ -97,11 +97,18 @@ type transaction struct { documents []interface{} } +type msgKind byte + +const ( + msgKindBody msgKind = 0 + msgKindDocumentSequence msgKind = 1 +) + type opCode int32 const ( opReply opCode = 1 - opMsg opCode = 1000 + opMsgLegacy opCode = 1000 opUpdate opCode = 2001 opInsert opCode = 2002 opReserved opCode = 2003 @@ -109,6 +116,7 @@ const ( opGetMore opCode = 2005 opDelete opCode = 2006 opKillCursor opCode = 2007 + opMsg opCode = 2013 ) // List of valid mongodb wire protocol operation codes @@ -123,6 +131,7 @@ var opCodeNames = map[opCode]string{ 2005: "OP_GET_MORE", 2006: "OP_DELETE", 2007: "OP_KILL_CURSORS", + 2013: "OP_MSG", } func validOpcode(o opCode) bool { From 40c5ea7dbfead847a1d252d5a3d7cfafe2c6a00e Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Sat, 6 Oct 2018 11:31:33 +0800 Subject: [PATCH 02/13] add changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6cee98d441a..761e0c12095 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -156,6 +156,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Added DHCP protocol support. {pull}7647[7647] - Add support to decode HTTP bodies compressed with `gzip` and `deflate`. {pull}7915[7915] - Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180] +- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] *Winlogbeat* From 5a3e804e2db42d8b67f656e6d6b2ca8776ab85fb Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Sat, 6 Oct 2018 11:32:31 +0800 Subject: [PATCH 03/13] add changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6cee98d441a..761e0c12095 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -156,6 +156,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Added DHCP protocol support. {pull}7647[7647] - Add support to decode HTTP bodies compressed with `gzip` and `deflate`. {pull}7915[7915] - Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180] +- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] *Winlogbeat* From 63298d30d4b88e9e070980607f25e3e01fd6c546 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Sat, 5 Jan 2019 16:52:01 +0800 Subject: [PATCH 04/13] update doc --- CHANGELOG.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 79f4a0d0691..c2ebcf73dc1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -101,6 +101,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Packetbeat* +- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] + *Functionbeat* ==== Deprecated @@ -580,7 +582,6 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Added DHCP protocol support. {pull}7647[7647] - Add support to decode HTTP bodies compressed with `gzip` and `deflate`. {pull}7915[7915] - Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180] -- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] *Functionbeat* From 9b3af92cf5f04689d15baf901551651e201a3cef Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Sat, 5 Jan 2019 18:31:10 +0800 Subject: [PATCH 05/13] fix merge error --- CHANGELOG.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c2ebcf73dc1..c21406fccbf 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -580,8 +580,6 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] *Packetbeat* - Added DHCP protocol support. {pull}7647[7647] -- Add support to decode HTTP bodies compressed with `gzip` and `deflate`. {pull}7915[7915] -- Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180] *Functionbeat* From 89fd46501dbd6466671182e1a9c726ad74d1b35c Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Wed, 20 Feb 2019 06:15:25 +0800 Subject: [PATCH 06/13] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a07d88a2a2a..9492ab3d634 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -143,6 +143,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - `http.response.body` moves to `http.response.body.content` - Changed Packetbeat fields to align with ECS. {issue}7968[7968] - Removed trailing dot from domain names reported by the DNS protocol. {pull}9941[9941] +- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] *Winlogbeat* From 63b4492a6400628782d0cb71ab6dbcf197bfda3a Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Wed, 20 Feb 2019 07:01:45 +0800 Subject: [PATCH 07/13] update git user config From c3965c34c6b5bc32863e42357c4d2ffe9b41d46b Mon Sep 17 00:00:00 2001 From: poh Date: Tue, 26 Mar 2019 07:19:57 +0800 Subject: [PATCH 08/13] remove test --- packetbeat/tests/system/x.txt | 162 ---------------------------------- 1 file changed, 162 deletions(-) delete mode 100644 packetbeat/tests/system/x.txt diff --git a/packetbeat/tests/system/x.txt b/packetbeat/tests/system/x.txt deleted file mode 100644 index 59740e61cfe..00000000000 --- a/packetbeat/tests/system/x.txt +++ /dev/null @@ -1,162 +0,0 @@ -22:09:43.610120 IP 127.0.0.1.57203 > 127.0.0.1.27017: tcp 50 -E..fHN@.@..A.........si....V"^.o.. ..Z..... -........2...................test.restaurants.............. -22:09:43.610408 IP 127.0.0.1.27017 > 127.0.0.1.57203: tcp 32768 -E..4%.@.@...........i..s"^.o........~)..... -........[...D..................+...`....e........_id.U..._.Ve...'.address.i....building.....1007..coord......0..v/..vR..1...J..lD@..street.....Morris Park Ave..zipcode.....10462...borough.....Bronx..cuisine.....Bakery..grades......0.+... date..p=.D....grade.....A..score.......1.+... date..DQ -A....grade.....A..score.......2.+... date....i<....grade.....A..score. -.....3.+... date.....3....grade.....A..score. .....4.+... date..........grade.....B..score........name.....Morris Park Bake Shop..restaurant_id. ...30075445......._id.U..._.Ve...(.address.h....building.....469..coord......0......}R..1....H.TD@..street.....Flatbush Avenue..zipcode.....11225...borough. ...Brooklyn..cuisine.....Hamburgers..grades......0.+... date...}.J....grade.....A..score.......1.+... date...8.F....grade.....B..score.......2.+... date...=X>....grade.....A..score.......3.+... date....)7....grade.....A..score........name.....Wendy'S..restaurant_id. ...30112340......._id.U..._.Ve...).address.i....building.....351..coord......0...0v..R..1...i.CbD@..street.....West 57 Street..zipcode.....10019...borough. -...Manhattan..cuisine.....Irish..grades......0.+... date...BHH....grade.....A..score.......1.+... date.....@....grade.....A..score.......2.+... date...U.8....grade.....A..score.......3.+... date.....4....grade.....A..score........name.....Dj Reynolds Pub And Restaurant..restaurant_id. ...30191841......._id.U..._.Ve...*.address.j....building.....2780..coord......0..."..~R..1.*oG8-JD@..street.....Stillwell Avenue..zipcode.....11224...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date.....F....grade.....A..score.......1.+... date.....?....grade.....A..score.......2.+... date.....6....grade.....A..score.......3.+... date...m.2....grade.....A..score........name.....Riviera Caterer..restaurant_id. ...40356018......._id.U..._.Ve...+.address.b....building.....97-22..coord......0.... .wR..1.*.:..]D@..street.....63 Road..zipcode.....11374...borough.....Queens..cuisine.....Jewish/Kosher..grades......0.+... date.....I....grade.....Z..score.......1.+... date....E<....grade.....A..score.......2.+... date.....8....grade.....A..score.......3.+... date....?4....grade.....B..score........name.....Tov Kosher Kitchen..restaurant_id. ...40356068......._id.U..._.Ve...,.address.k....building.....8825..coord......0....0XxR..1......aD@..street.....Astoria Boulevard..zipcode.....11369...borough.....Queens..cuisine. -...American ..grades......0.+... date.....I....grade.....Z..score.&.....1.+... date...;.E....grade.....A..score. -.....2.+... date...e(=....grade.....A..score.......3.+... date..0.d5....grade.....A..score........name.....Brunos On The Boulevard..restaurant_id. ...40356151......._id.U..._.Ve...-.address.k....building.....2206..coord......0.T.....R..1..F..TND@..street.....Victory Boulevard..zipcode.....10314...borough.....Staten Island..cuisine.....Jewish/Kosher..grades......0.+... date..l..H....grade.....A..score. .....1.+... date..x..F....grade.....A..score.......2.+... date...W.=....grade.....A..score.......3.+... date.....5....grade.....A..score. ......name.....Kosher Island..restaurant_id. ...40356442......._id.U..._.Ve.....address.b....building.....7114..coord......0..V.. zR..1.....XOD@..street. ...Avenue U..zipcode.....11234...borough. ...Brooklyn..cuisine.....Delicatessen..grades......0.+... date...FEF....grade.....A..score. -.....1.+... date..0..C....grade.....A..score. -.....2.+... date..@yA@....grade.....A..score.......3.+... date..Tb.8....grade.....A..score. -.....4.+... date..@..5....grade.....A..score.......5.+... date..l..2....grade.....A..score. ......name.....Wilken'S Fine Food..restaurant_id. ...40356483......._id.U..._.Ve.../.address.c....building.....6409..coord......0.....V.R..1..t!V.PD@..street. -...11 Avenue..zipcode.....11219...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date....FG....grade.....A..score.......1.+... date....,@....grade.....A..score.......2.+... date.....<....grade.....A..score.......3.+... date....,9....grade.....A..score.......4.+... date... .1....grade.....A..score........name.....Regina Caterers..restaurant_id. ...40356649......._id.U..._.Ve...0.address.i....building.....1839..coord......0..d|N.|R..1..qV..RD@..street.....Nostrand Avenue..zipcode.....11226...borough. ...Brooklyn..cuisine. ...Ice Cream, Gelato, Yogurt, Ices..grades......0.+... date..<+2G....grade.....A..score.......1.+... date.....?....grade.....A..score.......2.+... date...Us8....grade.....A..score.......3.+... date.....5....grade.....A..score........name.....Taste The Tropics Ice Cream..restaurant_id. ...40356731..|...._id.U..._.Ve...1.address.l....building.....2300..coord......0.Y..*;xR..1..U....grade.....A..score.......2.+... date.....6....grade.....A..score.......3.+... date...+D4....grade.....A..score........name.....C & C Catering Service..restaurant_id. ...40357437......._id.U..._.Ve...3.address.g....building.....1269..coord......0.,.z..wR..1.z...(VD@..street.....Sutter Avenue..zipcode.....11208...borough. ...Brooklyn..cuisine.....Chinese..grades......0.+... date..<.{H....grade.....B..score.......1.+... date..<8.@....grade.....A..score.......2.+... date.....=....grade.....C..score.8.....3.+... date..d.'9....grade.....B..score.......4.+... date....V6....grade.....B..score........name.....May May Kitchen..restaurant_id. ...40358429......._id.U..._.Ve...4.address.g....building.....1..coord......0.~aC..~R..1..Za.^bD@..street.....East 66 Street..zipcode.....10065...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date.....E....grade.....A..score.......1.+... date..0.g>....grade.....A..score.......2.+... date.....7....grade.....A..score.......3.+... date....|4....grade.....A..score........name.....1 East 66Th Street Kitchen..restaurant_id. ...40359480......._id.U..._.Ve...5.address.f....building.....705..coord......0...=..}R..1...C..MD@..street.....Kings Highway..zipcode.....11223...borough. ...Brooklyn..cuisine.....Jewish/Kosher..grades......0.+... date.....I....grade.....A..score.......1.+... date.....A....grade.....A..score.......2.+... date..\.):....grade.....A..score.......3.+... date..|.l7....grade.....A..score. .....4.+... date...D.4....grade.....B..score........name.....Seuda Foods..restaurant_id. ...40360045......._id.U..._.Ve...6.address.f....building.....203..coord......0....).~R..1...Q -_RD@..street.....Church Avenue..zipcode.....11218...borough. ...Brooklyn..cuisine. ...Ice Cream, Gelato, Yogurt, Ices..grades......0.+... date.....D....grade.....A..score.......1.+... date.....;....grade.....A..score.......2.+... date.....4....grade.....A..score.......3.+... date...S{3....grade.....P..score.......4.+... date....I1....grade.....A..score........name.....Carvel Ice Cream..restaurant_id. ...40360076......._id.U..._.Ve...7.address.k....building.....265-15..coord......0.:..6.mR..1..{...^D@..street.....Hillside Avenue..zipcode.....11004...borough.....Queens..cuisine. ...Ice Cream, Gelato, Yogurt, Ices..grades......0.+... date..T.TI....grade.....A..score. .....1.+... date...].A....grade.....A..score. -.....2.+... date..T..9....grade.....A..score........name.....Carvel Ice Cream..restaurant_id. ...40361322......._id.U..._.Ve...8.address.b....building.....6909..coord......0.,gJF..R..1.....SQD@..street. ...3 Avenue..zipcode.....11209...borough. ...Brooklyn..cuisine.....Delicatessen..grades......0.+... date.....G....grade.....A..score.......1.+... date..(..D....grade.....A..score.......2.+... date....!<....grade.....A..score. -......name.....Nordic Delicacies..restaurant_id. ...40361390......._id.U..._.Ve...9.address.i....building.....522..coord......0..Q...|R..1.L.{)....grade.....A..score. .....3.+... date..\.r;....grade.....A..score.......4.+... date..p.[6....grade.....A..score........name.....Glorious Food..restaurant_id. ...40361521......._id.U..._.Ve...:.address.k....building.....284..coord......0.<=.9.~R..1.....;TD@..street.....Prospect Park West..zipcode.....11215...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date...address.i....building.....502..coord......0..Nw.x~R..1...Z..dD@..street.....Amsterdam Avenue..zipcode.....10024...borough. -...Manhattan..cuisine.....Chicken..grades......0.+... date....vH....grade.....A..score. -.....1.+... date...c.D....grade.....A..score.......2.+... date.....?....grade.....A..score.......3.+... date..8..<....grade.....A..score.......4.+... date.....6....grade.....A..score. -.....5.+... date.....3....grade.....A..score........name.....Harriet'S Kitchen..restaurant_id. ...40362098......._id.U..._.Ve...?.address.h....building.....730..coord......0.../..}R..1..T@.reD@..street.....Columbus Avenue..zipcode.....10025...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...(gH....grade.....B..score.......1.+... date..<8.@....grade.....A..score. .....2.+... date..,..=....grade.....B..score.......3.+... date...(y5....grade.....A..score........name.....P & S Deli Grocery..restaurant_id. ...40362264......._id.U..._.Ve...@.address.k....building.....18..coord......0.......R..1.)?...\D@..street.....West Houston Street..zipcode.....10012...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date....$E....grade.....A..score. .....1.+... date.. ~.=....grade.....A..score.......2.+... date....26....grade.....A..score.......3.+... date..TA./....grade.....A..score........name.....Angelika Film Center..restaurant_id. ...40362274......._id.U..._.Ve...A.address.f....building.....531..coord......0......}R..1..u...XD@..street.....Myrtle Avenue..zipcode.....11205...borough. ...Brooklyn..cuisine.....Hamburgers..grades......0.+... date...|.D....grade.....A..score.......1.+... date....z=....grade.....A..score.......2.+... date....G:....grade.....A..score.......3.+... date...n.2....grade.....A..score........name.....White Castle..restaurant_id. ...40362344..b...._id.U..._.Ve...B.address.e....building.....103-05..coord......0.....OwR..1.q.q.t`D@..street. -...37 Avenue..zipcode.....11368...borough.....Queens..cuisine.....Chinese..grades.u....0.+... date.....E....grade.....A..score. -.....1.+... date....IB....grade.....A..score.......2.+... date...{.?....grade.....A..score.......3.+... date...7.:....grade.....A..score. -.....4.+... date....M:....grade.....P..score.......5.+... date...$|7....grade.....A..score.......6.+... date..0..4....grade.....A..score.......7.+... date....D1....grade.....A..score........name.....Ho Mei Restaurant..restaurant_id. ...40362432......._id.U..._.Ve...C.address.c....building.....60..coord......0.M.R...R..1...D.dZD@..street.....Wall Street..zipcode.....10005...borough. -...Manhattan..cuisine.....Turkish..grades......0.+... date...A.H....grade.....A..score. .....1.+... date...].A....grade.....A..score.......2.+... date.....9....grade.....A..score. .....3.+... date..,..7....grade.....A..score........name.....The Country Cafe..restaurant_id. ...40362715......._id.U..._.Ve...D.address.i....building.....195..coord......0...8.,{R..1...S.|SD@..street.....East 56 Street..zipcode.....11203...borough. ...Brooklyn..cuisine. -...Caribbean..grades......0.+... date.....E....grade.....A..score.......1.+... date...o.>....grade.....A..score.......2.+... date...F.9....grade.....A..score.......3.+... date...?b0....grade.....A..score........name.....Shashemene Int'L Restaura..restaurant_id. ...40362869......._id.U..._.Ve...E.address.f....building.....107..coord......0.M6....R..1...++M[D@..street.....Church Street..zipcode.....10007...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date....FG....grade.....A..score.......1.+... date...}kD....grade.....A..score. .....2.+... date.....@....grade.....A..score. .....3.+... date..|..<....grade.....A..score.......4.+... date.....4....grade.....A..score.......5.+... date...S.3....grade.....A..score........name.....Downtown Deli..restaurant_id. ...40363021......._id.U..._.Ve...F.address.i....building.....1006..coord......0.2...NvR..1..I...qD@..street.....East 233 Street..zipcode.....10466...borough.....Bronx..cuisine. ...Ice Cream, Gelato, Yogurt, Ices..grades......0.+... date.. ..E....grade.....A..score. -.....1.+... date...k.@....grade.....A..score. -.....2.+... date.....<....grade.....A..score. .....3.+... date..."J8....grade.....A..score.......4.+... date..H~.1....grade.....A..score........name.....Carvel Ice Cream..restaurant_id. ...40363093......._id.U..._.Ve...G.address.d....building.....56..coord......0.q.q.t.R..1..s.f.XD@..street.....Court Street..zipcode.....11201...borough. ...Brooklyn..cuisine.....Donuts..grades......0.+... date...}.J....grade.....A..score.......1.+... date...2.C....grade.....A..score. .....2.+... date...t.<....grade.....A..score.......3.+... date..HC.4....grade.....A..score. -......name.....Dunkin' Donuts..restaurant_id. ...40363098......._id.U..._.Ve...H.address.b....building.....7615..coord......0.X.tJv.R..1..9[@hPD@..street. ...5 Avenue..zipcode.....11209...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date.....J....grade.....A..score. -.....1.+... date.....A....grade.....A..score.......2.+... date...p.>....grade.....A..score.......3.+... date.....6....grade.....A..score........name.....Mejlander & Mulgannon..restaurant_id. ...40363117......._id.U..._.Ve...I.address.f....building.....120..coord......0..Y....R..1...h..\D@..street.....Prince Street..zipcode.....10012...borough. -...Manhattan..cuisine.....Bakery..grades......0.+... date..`g.I....grade.....A..score.......1.+... date...].A....grade.....A..score.......2.+... date...=X>....grade.....A..score.......3.+... date..X..6....grade.....A..score.......4.+... date..$.S4....grade.....A..score........name.....Olive'S..restaurant_id. ...40363151..{...._id.U..._.Ve...J.address.j....building.....1236..coord......0.j..\.xR..1....X)hD@..street.....238 Spofford Ave..zipcode.....10474...borough.....Bronx..cuisine.....Chinese..grades......0.+... date....@C....grade.....A..score.......1.+... date...t.<....grade.....A..score. -.....2.+... date..d..7....grade.....B..score........name.....Happy Garden..restaurant_id. ...40363289......._id.U..._.Ve...K.address.a....building.....625..coord......0..8.@d.R..1......`D@..street. ...8 Avenue..zipcode.....10018...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date....}F....grade.....A..score.......1.+... date...ryC....grade.....A..score. .....2.+... date..\.r;....grade.....A..score.......3.+... date....44....grade.....A..score. .....4.+... date...{K2....grade.....A..score........name.....Cafe Metro..restaurant_id. ...40363298......._id.U..._.Ve...L.address.h....building.....1069..coord......0.g.+..yR..1.N..E.XD@..street.....Wyckoff Avenue..zipcode.....11385...borough.....Queens..cuisine.....Delicatessen..grades......0.+... date..(!.E....grade.....A..score.......1.+... date..T..B....grade.....A..score.......2.+... date....d?....grade.....A..score.......3.+... date..x5.;....grade.....B..score.......4.+... date..8z.3....grade.....A..score.......5.+... date.....0....grade.....A..score. -......name.....Tony'S Deli..restaurant_id. ...40363333......._id.U..._.Ve...M.address.i....building.....405..coord......0.. h"l~R..1..%mO5`D@..street.....Lexington Avenue..zipcode.....10174...borough. -...Manhattan..cuisine.....Sandwiches/Salads/Mixed Buffet..grades......0.+... date....QD....grade.....A..score.......1.+... date.....A....grade.....A..score.......2.+... date....j9....grade.....A..score.......3.+... date..H.`2....grade.....A..score.......4.+... date..|'./....grade.....A..score........name.....Lexler Deli..restaurant_id. ...40363426......._id.U..._.Ve...N.address.k....building.....2491..coord......0..Y4.V.R..1...h. ND@..street.....Victory Boulevard..zipcode.....10314...borough.....Staten Island..cuisine.....Delicatessen..grades......0.+... date.....J....grade.....A..score.......1.+... date.....B....grade.....A..score. -.....2.+... date....Y?....grade.....A..score. -.....3.+... date...t.<....grade.....A..score........name.....Bagels N Buns..restaurant_id. ...40363427......._id.U..._.Ve...O.address.m....building.....7905..coord......0.._...wR..1...d.T[D@..street.....Metropolitan Avenue..zipcode.....11379...borough.....Queens..cuisine.....Bagels/Pretzels..grades......0.+... date.....H....grade.....A..score. -.....1.+... date...X.C....grade.....B..score.......2.+... date....V@....grade.....A..score. -.....3.+... date.....<....grade.....B..score.......4.+... date..D0.8....grade.....B..score.......5.+... date....15....grade.....B..score........name.....Hot Bagels..restaurant_id. ...40363565......._id.U..._.Ve...P.address.m....building.....87-69..coord......0.}.*J.uR..1.V..E.YD@..street.....Lefferts Boulevard..zipcode.....11418...borough.....Queens..cuisine. -...American ..grades......0.+... date..HWfD....grade.....A..score.......1.+... date..4.z@....grade.....A..score.......2.+... date...a.8....grade.....A..score.......3.+... date..\ML6....grade.....A..score. -.....4.+... date....k3....grade.....A..score.......5.+... date.....0....grade.....A..score........name.....Snack Time Grill..restaurant_id. ...40363590......._id.U..._.Ve...Q.address.f....building.....1418..coord......0.. -..=}R..1..c*X>cD@..street.....Third Avenue..zipcode.....10028...borough. -...Manhattan..cuisine.....Continental..grades......0.+... date..$.YF....grade.....A..score. .....1.+... date...Y1C....grade.....A..score.......2.+... date....z=....grade.....B..score.......3.+... date...565....grade.....A..score.......4.+... date..|..0....grade.....B..score........name.....Lorenzo & Maria'S..restaurant_id. ...40363630......._id.U..._.Ve...R.address.a....building.....464..coord......0..A%S.~R..1....#F_D@..street. ...3 Avenue..zipcode.....10016...borough. -...Manhattan..cuisine.....Pizza..grades......0.+... date..$w.G....grade.....A..score.......1.+... date.....D....grade.....A..score.......2.+... date..D..?....grade.....A..score.......3.+... date.....<....grade.....A..score.......4.+... date..@*.4....grade.....A..score.......5.+... date.....2....grade.....A..score........name.....Domino'S Pizza..restaurant_id. ...40363644......._id.U..._.Ve...S.address.g....building.....437..coord......0.....l~R..1.I..V.`D@..street.....Madison Avenue..zipcode.....10022...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...._F....grade.....A..score. .....1.+... date.....?....grade.....A..score.......2.+... date....58....grade.....A..score.......3.+... date....O5....grade.....A..score.......4.+... date.....0....grade.....A..score........name.....Berkely..restaurant_id. ...40363685......._id.U..._.Ve...T.address.j....building.....1031..coord......0.H7.\.zR..1..v.GjRD@..street.....East 92 Street..zipcode.....11236...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date...X.C....grade.....A..score.......1.+... date..h..<....grade.....A..score.......2.+... date..0..4....grade.....A..score. -......name.....Sonny'S Heros..restaurant_id. ...40363744......._id.U..._.Ve...U.address.i....building.....1111..coord......0.|Ux...R..1..18..LD@..street.....Hylan Boulevard..zipcode.....10305...borough.....Staten Island..cuisine. ...Ice Cream, Gelato, Yogurt, Ices..grades......0.+... date.. ..E....grade.....A..score.......1.+... date..x..=....grade.....A..score.......2.+... date..P\;5....grade.....A..score........name.....Carvel Ice Cream..restaurant_id. ...40363834......._id.U..._.Ve...V.address.e....building.....976..coord......0.{Y$7T{R..1.|....TD@..street.....Rutland Road..zipcode.....11212...borough. ...Brooklyn..cuisine.....Chinese..grades......0.+... date.....E....grade.....A..score.......1.+... date.....=....grade.....A..score. -.....2.+... date...Z 6....grade.....A..score.......3.+... date..H..3....grade.....A..score........name.....Golden Pavillion..restaurant_id. ...40363920......._id.U..._.Ve...W.address.i....building.....148..coord......0..*...~R..1..3d..cD@..street.....West 72 Street..zipcode.....10023...borough. -...Manhattan..cuisine.....Pizza..grades......0.+... date...2'J....grade.....A..score.......1.+... date.....E....grade.....B..score.......2.+... date.. ~.=....grade.....A..score.......3.+... date....`6....grade.....A..score. ......name.....Domino'S Pizza..restaurant_id. ...40363945......._id.U..._.Ve...X.address.i....building.....364..coord......0....l~}R..1.TA.G.fD@..street.....West 110 Street..zipcode.....10025...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date....=H....grade.....B..score.......1.+... date...}kD....grade.....B..score.......2.+... date..,..=....grade.....A..score.......3.+... date..$5.5....grade.....A..score........name.....Spoon Bread Catering..restaurant_id. ...40364179......._id.U..._.Ve...Y.address.b....building.....1423..coord......0..+.n.}R..1...e. -PD@..street. ...Avenue J..zipcode.....11230...borough. ...Brooklyn..cuisine.....Jewish/Kosher..grades......0.+... date...._J....grade.....A..score. -.....1.+... date.....B....grade.....A..score.......2.+... date....m;....grade.....A..score. ......name.....Kosher Bagel Hole..restaurant_id. ...40364220......._id.U..._.Ve...Z.address.n....building.....0..coord......0.iN....U..1....*P.#@..street.....Guardia Airport Parking..zipcode.....11371...borough.....Queens..cuisine. -...American ..grades......0.+... date...T.F....grade.....A..score. .....1.+... date.....>....grade.....A..score. -.....2.+... date..T.M7....grade.....A..score. .....3.+... date..@.a3....grade.....C..score. ......name.....Terminal Cafe/Yankee Clipper..restaurant_id. ...40364262......._id.U..._.Ve...[.address.f....building.....73..coord......0._.....R..1....#hID@..street.....New Dorp Plaza..zipcode.....10306...borough.....Staten Island..cuisine.....Delicatessen..grades......0.+... date...2.I....grade.....A..score.......1.+... date..../B....grade.....A..score.......2.+... date...V9>....grade.....A..score.......3.+... date..4g-6....grade.....A..score........name.....Plaza Bagels & Deli..restaurant_id. ...40364286......._id.U..._.Ve...\.address.n....building.....277..coord......0....e:yR..1....!.nD@..street.....East Kingsbridge Road..zipcode.....10458...borough.....Bronx..cuisine.....Chinese..grades......0.+... date..p=.D....grade.....A..score. -.....1.+... date....WA....grade.....A..score. -.....2.+... date.....=....grade.....A..score. -.....3.+... date..l.o9....grade.....A..score.......4.+... date... .1....grade.....A..score........name.....Happy Garden..restaurant_id. ...40364296......._id.U..._.Ve...].address.f....building.....203..coord......0..O.@..R..1.m%.P7GD@..street.....Giffords Lane..zipcode.....10308...borough.....Staten Island..cuisine.....Delicatessen..grades......0.+... date.. d.J....grade.....A..score.......1.+... date..p.bH....grade.....C..score.'.....2.+... date.....D....grade.....A..score.......3.+... date....i<....grade.....A..score.......4.+... date..4.v7....grade.....A..score. -......name.....B & M Hot Bagel & Grocery..restaurant_id. ...40364299......._id.U..._.Ve...^.address.e....building.....94..coord......0.s\.ye.R..1.v..0.ZD@..street.....Fulton Street..zipcode.....10038...borough. -...Manhattan..cuisine.....Chicken..grades......0.+... date..|..J....grade.....A..score.......1.+... date...Q7G....grade.....C..score.0.....2.+... date....b>....grade.....A..score.......3.+... date.....9....grade.....A..score.......4.+... date.....6....grade.....A..score........name.....Texas Rotisserie..restaurant_id. ...40364304......._id.U..._.Ve..._.address.c....building.....10004..coord......0...w"-.R..1....4mND@..street. ...4 Avenue..zipcode.....11209...borough. ...Brooklyn..cuisine.....Italian..grades......0.+... date..HWfD....grade.....A..score.......1.+... date.....?....grade.....A..score.......2.+... date....^;....grade.....A..score. -.....3.+... date.....3....grade.....A..score........name.....Philadelhia Grille Express..restaurant_id. ...40364305......._id.U..._.Ve...`.address.a....building.....178..coord......0..M...}R..1...Q..ZD@..street. ...Broadway..zipcode.....11211...borough. ...Brooklyn..cuisine.....Steak..grades......0.+... date..<..D....grade.....A..score.......1.+... date..`.aA....grade.....A..score. -.....2.+... date.....=....grade.....A..score.......3.+... date...y.9....grade.....A..score.......4.+... date.....1....grade.....A..score........name.....Peter Luger Steakhouse..restaurant_id. ...40364335......._id.U..._.Ve...a.address.g....building.....1..coord......0.^4../~R..1..w...aD@..street.....East 60 Street..zipcode.....10022...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...A.I....grade.....B..score.......1.+... date...;.E....grade.....A..score.......2.+... date.....=....grade.....A..score.......3.+... date...Qv:....grade.....A..score.......4.+... date.....6....grade.....B..score.......5.+... date.....3....grade.....A..score........name.....Metropolitan Club..restaurant_id. ...40364347......._id.U..._.Ve...b.address.a....building.....837..coord......0..x.$(~R..1.....7`D@..street. ...2 Avenue..zipcode.....10017...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...^[G....grade.....B..score.......1.+... date....WA....grade.....A..score.......2.+... date..x..=....grade.....A..score.......3.+... date.....7....grade.....A..score.......4.+... date..0a.2....grade.....A..score........name.....Palm Restaurant..restaurant_id. ...40364355..z...._id.U..._.Ve...c.address.h....building.....21..coord......0..4.].~R..1...h.VaD@..street.....West 52 Street..zipcode.....10019...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..P..E....grade.....A..score.......1.+... date....t@....grade.....A..score.......2.+... date....z6....grade.....A..score........name.....21 Club..restaurant_id. ...40364362..I...._id.U..._.Ve...d.address.e....building.....658..coord......0......tR..1....#*jD@..street.....Clarence Ave..zipcode.....10465...borough.....Bronx..cuisine. -...American ..grades.a....0.+... date.....F....grade.....A..score.......1.+... date...Us8....grade.....A..score. -......name.....Manhem Club..restaurant_id. ...40364363......._id.U..._.Ve...e.address.b....building.....1028..coord......0....w.}R..1..+.z.aD@..street. ...3 Avenue..zipcode.....10065...borough. -...Manhattan..cuisine.....Italian..grades......0.+... date..<.{H....grade.....A..score.......1.+... date...0aD....grade.....A..score. -.....2.+... date..0.g>....grade.....A..score. -.....3.+... date..0TA9....grade.....A..score.......4.+... date..D.t5....grade.....A..score. ......name.....Isle Of Capri Resturant..restaurant_id. ...40364373......._id.U..._.Ve...f.address.h....building.....45..coord......0...X.N.R..1.S..}h^D@..street.....East 18 Street..zipcode.....10003...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..$..H....grade.....A..score. -.....1.+... date.....A....grade.....A..score. .....2.+... date...V9>....grade.....C..score.$.....3.+... date.....4....grade.....A..score. ......name.....Old Town Bar & Restaurant..restaurant_id. ...40364389......._id.U..._.Ve...g.address.f....building.....261..coord......0....s.|R..1...Byz\D@..street.....Driggs Avenue..zipcode.....11222...borough. ...Brooklyn..cuisine.....Polish..grades......0.+... date..l.OF....grade.....A..score.......1.+... date.....>....grade.....A..score.......2.+... date.....5....grade.....A..score.......3.+... date..l..2....grade.....C..score.6......name.....Polish National Home..restaurant_id. ...40364404......._id.U..._.Ve...h.address.f....building.....62..coord......0...I.2.R..1.-....^D@..street.....Charles Street..zipcode.....10014...borough. -...Manhattan..cuisine.A...Latin (Cuban, Dominican, Puerto Rican, South & Central American)..grades......0.+... date...;.E....grade.....A..score.......1.+... date..L<.>....grade.....A..score.......2.+... date...$|7....grade.....A..score.......3.+... date.....4....grade.....A..score.......4.+... date..x..2....grade.....A..score. -......name.....Seville Restaurant..restaurant_id. ...40364439......._id.U..._.Ve...i.address.f....building.....100..coord......0...N-..R..1..!q..[D@..street.....Centre Street..zipcode.....10013...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..p=.D....grade.....A..score.......1.+... date...LG=....grade.....A..score.......2.+... date..@..5....grade.....A..score.......3.+... date...5 /....grade.....A..score........name.....Criminal Court Bldg Cafeteria..restaurant_id. ...40364443......._id.U..._.Ve...j.address.h....building.....657..coord......0....v.yR..1.Y...tZD@..street.....Fairview Avenue..zipcode.....11385...borough.....Queens..cuisine.....German..grades......0.+... date... .D....grade.....A..score.......1.+... date....[=....grade.....A..score.......2.+... date..h..8....grade.....B..score.......3.+... date.....3....grade.....B..score.......4.+... date..t..0....grade.....B..score........name.....Gottscheer Hall..restaurant_id. ...40364449......._id.U..._.Ve...l.address.a....building.....108..coord......0.B..=.~R..1......\D@..street. ...Avenue B..zipcode.....10009...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..<+2G....grade.....B..score.......1.+... date..(.EC....grade.....A..score.......2.+... date.....:....grade.....A..score.......3.+... date..t.$7....grade.....A..score.......4.+... date..l..2....grade.....A..score........name.....7B Bar..restaurant_id. ...40364518......._id.U..._.Ve...k.address.k....building.....180..coord......0......~R..1..^-..bD@..street.....Central Park South..zipcode.....10019...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...>KJ....grade.....A..score.......1.+... date.....G....grade.....C..score.(.....2.+... date..t.'@....grade.....A..score.......3.+... date.....;....grade.....A..score.......4.+... date.....8....grade.....C..score.......5.+... date..Xu.5....grade.....A..score........name.....Nyac Main Dining Room..restaurant_id. ...40364467......._id.U..._.Ve...m.address.k....building.....96-40..coord......0.6... wR..1....3\]D@..street.....Queens Boulevard..zipcode.....11374...borough.....Queens..cuisine.....Jewish/Kosher..grades......0.+... date.....D....grade.....A..score.......1.+... date...*lA....grade.....A..score. .....2.+... date....C>....grade.....A..score.......3.+... date.....9....grade.....B..score.......4.+... date..p.~2....grade.....A..score. -.....5.+... date..........grade.....A..score........name.....Ben-Best Deli & Restaurant..restaurant_id. ...40364529......._id.U..._.Ve...n.address.g....building.....215..coord......0.y....~R..1.NJ.p -bD@..street.....West 57 Street..zipcode.....10019...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..x..H....grade.....A..score.......1.+... date..T.-D....grade.....A..sc -22:09:43.610439 IP 127.0.0.1.27017 > 127.0.0.1.57203: tcp 13403 -E.4.%.@.@..R........i..s"_#o........2...... -........ore.......2.+... date..h_`@....grade.....A..score.......3.+... date...3.<....grade.....B..score.......4.+... date..Xu.5....grade.....A..score........name.#...Cafe Atelier (Art Students League)..restaurant_id. ...40364531......._id.U..._.Ve...o.address.i....building.....845..coord......0....B.}R..1..,...aD@..street.....Lexington Avenue..zipcode.....10065...borough. -...Manhattan..cuisine.....Steak..grades......0.+... date.....D....grade.....A..score.......1.+... date...>.=....grade.....A..score.......2.+... date..d+q:....grade.....A..score. .....3.+... date..t.$7....grade.....A..score.......4.+... date...@./....grade.....A..score........name.....Donohue'S Steak House..restaurant_id. ...40364572......._id.U..._.Ve...p.address.i....building.....311..coord......0.g.K6..R..1....I.aD@..street.....West 51 Street..zipcode.....10019...borough. -...Manhattan..cuisine.....French..grades......0.+... date.....I....grade.....B..score.......1.+... date....$E....grade.....A..score.......2.+... date..$..?....grade.....C..score.$.....3.+... date..H..<....grade.....B..score.......4.+... date.....8....grade.....C..score.$.....5.+... date.....5....grade.....C..score........name.....Tout Va Bien..restaurant_id. ...40364576......._id.U..._.Ve...q.address.r....building.....386..coord......0.zM.P.~R..1..ca.UXD@..street.....Flatbush Avenue Extension..zipcode.....11201...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date..p..I....grade.....A..score.......1.+... date...I.D....grade.....A..score.......2.+... date....!<....grade.....A..score.......3.+... date.....9....grade.....A..score........name. ...Junior'S..restaurant_id. ...40364581......._id.U..._.Ve...r.address.c....building.....37..coord......0....M..R..1..,...ED@..street.....Mansion Ave..zipcode.....10308...borough.....Staten Island..cuisine. -...American ..grades......0.+... date..h..E....grade.....A..score. -.....1.+... date..LjRA....grade.....A..score.......2.+... date..P..7....grade.....A..score........name.....Great Kills Yacht Club..restaurant_id. ...40364610......._id.U..._.Ve...s.address.i....building.....251..coord......0.9..C.~R..1..*.:!_D@..street.....East 31 Street..zipcode.....10016...borough. -...Manhattan..cuisine.....Italian..grades......0.+... date..h..E....grade.....A..score.......1.+... date....Y?....grade.....C..score. .....2.+... date....q7....grade.....A..score........name.....Marchis Restaurant..restaurant_id. ...40364668......._id.U..._.Ve...t.address.j....building.....2602..coord......0...X..}R..1.L...=KD@..street.....East 15 Street..zipcode.....11235...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date..P..E....grade.....A..score.......1.+... date....H>....grade.....A..score. .....2.+... date..T.*;....grade.....B..score.......3.+... date.....6....grade.....B..score.......4.+... date..\.%1....grade.....B..score........name.....Towne Cafe..restaurant_id. ...40364681......._id.U..._.Ve...u.address.d....building......1-A..coord......0...-..xH..1...]}.Z0...street.....East 77 St..zipcode.....10021...borough. -...Manhattan..cuisine.....Continental..grades......0.+... date.....I....grade.....A..score.......1.+... date.....A....grade.....A..score.......2.+... date..h1.=....grade.....B..score.......3.+... date.....:....grade.....A..score. .....4.+... date..$..6....grade.....A..score........name.....Dining Room..restaurant_id. ...40364691......._id.U..._.Ve...v.address.`....building.....56..coord......0.=...M.R..1.|H...^D@..street. ...9 Avenue..zipcode.....10011...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date.....F....grade.....A..score. .....1.+... date...a+?....grade.....A..score.......2.+... date..|..9....grade.....B..score.......3.+... date...$|7....grade.....A..score.......4.+... date..._.3....grade.....A..score........name.....Old Homestead..restaurant_id. ...40364715......._id.U..._.Ve...w.address.n....building.....156-71..coord......0.KVE..uR..1.4e...TD@..street.....Crossbay Boulevard..zipcode.....11414...borough.....Queens..cuisine.....Pizza/Italian..grades......0.+... date...3YI....grade.....A..score.......1.+... date.....B....grade.....A..score. .....2.+... date....5?....grade.....A..score.......3.+... date...sQ6....grade.....A..score. ......name.....New Park Pizzeria & Restaurant..restaurant_id. ...40364744......._id.U..._.Ve...x.address.c....building.....600..coord......0.....$pR..1..?W.jcD@..street.....West Drive..zipcode.....11363...borough.....Queens..cuisine. -...American ..grades......0.+... date..t..B....grade.....A..score.......1.+... date....:?....grade.....A..score.......2.+... date....m;....grade.....A..score. .....3.+... date..x..6....grade.....A..score.......4.+... date...Wx1....grade.....B..score........name.....Douglaston Club..restaurant_id. ...40364858......._id.U..._.Ve...y.address.i....building.....225..coord......0.,..;.}R..1.t....aD@..street.....East 60 Street..zipcode.....10022...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..L].G....grade.....A..score. -.....1.+... date..d..D....grade.....A..score.......2.+... date....@<....grade.....A..score. -.....3.+... date..,|x8....grade.....A..score. ......name.....Serendipity 3..restaurant_id. ...40364863......._id.U..._.Ve...z.address.b....building.....461..coord......0....<0.R..1..A$C.SD@..street. -...37 Street..zipcode.....11232...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date..x..I....grade.....A..score.......1.+... date..t..B....grade.....A..score.......2.+... date....m;....grade.....A..score.......3.+... date..x..4....grade.....A..score........name.....Melody Lanes..restaurant_id. ...40364889......._id.U..._.Ve...{.address.j....building.....30-13..coord......0.\..'.zR..1.a.hV.aD@..street.....Steinway Street..zipcode.....11103...borough.....Queens..cuisine.....Pizza..grades......0.+... date..l..H....grade.....A..score.......1.+... date.....A....grade.....A..score.......2.+... date.....:....grade.....A..score. -.....3.+... date...#.7....grade.....A..score.......4.+... date.....4....grade.....A..score. -......name.....Rizzo'S Fine Pizza..restaurant_id. ...40364920......._id.U..._.Ve...|.address.i....building.....2222..coord......0.z...avR..1..9f4MjD@..street.....Haviland Avenue..zipcode.....10462...borough.....Bronx..cuisine. -...American ..grades......0.+... date....ZJ....grade.....A..score.......1.+... date.....E....grade.....B..score.......2.+... date..82f=....grade.....A..score.......3.+... date..T..9....grade.....A..score. .....4.+... date..xBZ5....grade.....B..score........name.,...The New Starling Athletic Club Of The Bronx..restaurant_id. ...40364956......._id.U..._.Ve...}.address.f....building.....567..coord......0.....e.R..1.J..4*^D@..street.....Hudson Street..zipcode.....10014...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date..DDzG....grade.....A..score.......1.+... date... .@....grade.....A..score.......2.+... date.....<....grade.....A..score.......3.+... date..\..7....grade.....A..score.......4.+... date...7h4....grade.....A..score........name.....White Horse Tavern..restaurant_id. ...40364958......._id.U..._.Ve...~.address.i....building.....67..coord......0.......R..1..|.}.KD@..street.....Olympia Boulevard..zipcode.....10305...borough.....Staten Island..cuisine.....Italian..grades......0.+... date.. ..E....grade.....A..score.......1.+... date...W.=....grade.....A..score.......2.+... date..P\;5....grade.....A..score.......3.+... date...JT1....grade.....A..score........name.....Crystal Room..restaurant_id. ...40365013......._id.U..._.Ve.....address.h....building.....390..coord......0.6Zi...R..1...'^ -ND@..street.....Hylan Boulevard..zipcode.....10305...borough.....Staten Island..cuisine. -...American ..grades......0.+... date.....F....grade.....A..score. .....1.+... date...!E?....grade.....A..score.......2.+... date....t9....grade.....A..score. -.....3.+... date.....2....grade.....B..score........name.....Labetti'S Post # 2159..restaurant_id. ...40365022......._id.U..._.Ve.....address.b....building.....1..coord......0.....A~R..1.d..._KD@..street.....Bouck Court..zipcode.....11223...borough. ...Brooklyn..cuisine. -...American ..grades......0.+... date...~1J....grade.....A..score.......1.+... date..hR.F....grade.....A..score. .....2.+... date..\..A....grade.....C..score.......3.+... date...>.=....grade.....C..score.......4.+... date..xp.7....grade.....A..score. -......name.....Shell Lanes..restaurant_id. ...40365043......._id.U..._.Ve.....address.h....building.....15..coord......0...J.V.R..1..2.?I]D@..street.....East 7 Street..zipcode.....10003...borough. -...Manhattan..cuisine.....Irish..grades......0.+... date....sF....grade.....A..score.......1.+... date..dLtC....grade.....A..score. -.....2.+... date....5?....grade.....A..score.......3.+... date..|.l7....grade.....A..score.......4.+... date..h..4....grade.....A..score. .....5.+... date..l#.1....grade.....B..score........name.....Mcsorley'S Old Ale House..restaurant_id. ...40365075......._id.U..._.Ve.....address.e....building.....93..coord......0.Elf...R..1......[D@..street.....Baxter Street..zipcode.....10013...borough. -...Manhattan..cuisine.....Italian..grades......0.+... date...>KJ....grade.....A..score. .....1.+... date..,4.B....grade.....A..score. -.....2.+... date..0..:....grade.....A..score.......3.+... date.....7....grade.....A..score.......4.+... date...6.4....grade.....A..score........name.....Forlinis Restaurant..restaurant_id. ...40365098......._id.U..._.Ve.....address.i....building.....6736..coord......0..A.C..R..1..E...@D@..street.....Hylan Boulevard..zipcode.....10309...borough.....Staten Island..cuisine. -...American ..grades......0.+... date.....G....grade.....A..score.......1.+... date..\..@....grade.....B..score.......2.+... date.....7....grade.....A..score........name.....South Shore Swimming Club..restaurant_id. ...40365120......._id.U..._.Ve.....address.i....building.....331..coord......0.o...=.R..1.F../w^D@..street.....West 4 Street..zipcode.....10014...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date.....I....grade.....A..score.......1.+... date.. ..F....grade.....A..score.......2.+... date...|.>....grade.....A..score.......3.+... date..,..7....grade.....A..score........name.....Corner Bistro..restaurant_id. ...40365166......._id.U..._.Ve.....address.i....building.....1449..coord......0..w...|R..1./cTcSSD@..street.....Nostrand Avenue..zipcode.....11226...borough. ...Brooklyn..cuisine.....Donuts..grades......0.+... date....0I....grade.....B..score.......1.+... date.....F....grade.....A..score.......2.+... date....b>....grade.....A..score. -.....3.+... date....^;....grade.....A..score. .....4.+... date..,..7....grade.....B..score.......5.+... date..X.94....grade.....A..score........name.....Nostrand Donut Shop..restaurant_id. ...40365226......._id.U..._.Ve.....address.b....building.....1616..coord......0......|R..1.....^cD@..street. ...2 Avenue..zipcode.....10028...borough. -...Manhattan..cuisine.....Irish..grades......0.+... date..\.uD....grade.....A..score.......1.+... date.....@....grade.....A..score.......2.+... date...`e9....grade.....A..score.......3.+... date...;e2....grade.....A..score. ......name.....Dorrian'S Red Hand Restaurant..restaurant_id. ...40365239..|...._id.U..._.Ve.....address.g....building.....3..coord......0.....o~R..1...^.=aD@..street.....East 52 Street..zipcode.....10022...borough. -...Manhattan..cuisine.....French..grades......0.+... date....CE....grade.....A..score. -.....1.+... date....7=....grade.....A..score. .....2.+... date..P\;5....grade.....A..score........name.....La Grenouille..restaurant_id. ...40365264......._id.U..._.Ve.....address.b....building.....4035..coord......0.....!|R..1...aN.kD@..street. ...Broadway..zipcode.....10032...borough. -...Manhattan..cuisine.....Pizza..grades......0.+... date.....D....grade.....A..score. -.....1.+... date.....<....grade.....A..score. -.....2.+... date.....4....grade.....A..score.......3.+... date...bj2....grade.....C..score.<......name.....Como Pizza..restaurant_id. ...40365280......._id.U..._.Ve.....address.a....building.....842..coord......0.h....~R..1.....0`D@..street. ...2 Avenue..zipcode.....10017...borough. -...Manhattan..cuisine. -...American ..grades......0.+... date...^[G....grade.....A..score.......1.+... date..,o.>....grade.....A..score.......2.+... date..\..7....grade.....A..score.......3.+... date..@*.4....grade.....A..score. .....4.+... date.....1....grade.....B..score........name.....Keats Restaurant..restaurant_id. ...40365288......._id.U..._.Ve.....address.h....building.....146..coord......0..j....R..1......\D@..street.....Mulberry Street..zipcode.....10013...borough. -...Manhattan..cuisine.....Italian..grades......0.+... date...;.E....grade.....A..score.......1.+... date..82f=....grade.....A..score.......2.+... date..|..9....grade.....A..score. .....3.+... date...N~5....grade.....A..score.......4.+... date...bj2....grade.....A..score........name.....Angelo Of Mulberry St...restaurant_id. ...40365293......._id.U..._.Ve.....address.i....building.....103..coord......0.......R..1.e.,.i]D@..street.....Macdougal Street..zipcode.....10012...borough. -...Manhattan..cuisine.....Mexican..grades......0.+... date..0:!F....grade.....A..score. -.....1.+... date.....A....grade.....A..score. .....2.+... date..`..=....grade.....A..score.......3.+... date....X7....grade.....B..score........name.....Panchito'S..restaurant_id. ...40365348.. From dfb8ea9983fc334923f0eb31c5952905a72af9f5 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Tue, 26 Mar 2019 07:32:22 +0800 Subject: [PATCH 09/13] trigger cla From f4e2725e16901f8a248c2df7ff090522b0bd26d6 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Tue, 26 Mar 2019 07:41:37 +0800 Subject: [PATCH 10/13] trigger cla From 804f53b8e028cc3f7ee80c3f2939cb6aa34f2a08 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Tue, 26 Mar 2019 07:42:47 +0800 Subject: [PATCH 11/13] trigger cla From 029cfbe8ae5c5c6c6eea70953ffaf7b34032719b Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Wed, 27 Mar 2019 09:41:55 +0800 Subject: [PATCH 12/13] add pcap test --- libbeat/tests/system/beat/beat.py | 24 ++++++++++++++ .../pcaps/mongodb_invalid_opcode_2269.pcap | Bin 0 -> 2319 bytes .../tests/system/test_0025_mongodb_basic.py | 31 +++++++++++++++--- 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 packetbeat/tests/system/pcaps/mongodb_invalid_opcode_2269.pcap diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 452d3ed9287..587c2692caf 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -408,6 +408,30 @@ def log_contains_count(self, msg, logfile=None, ignore_case=False): return counter + def log_contains_countmap(self, pattern, capture_group, logfile=None): + """ + Returns a map of the number of appearances of each captured group in the log file + """ + counts = {} + + if logfile is None: + logfile = self.beat_name + ".log" + + try: + with open(os.path.join(self.working_dir, logfile), "r") as f: + for line in f: + res = pattern.search(line) + if res is not None: + capt = res.group(capture_group) + if capt in counts: + counts[capt] += 1 + else: + counts[capt] = 1 + except IOError: + pass + + return counts + def output_lines(self, output_file=None): """ Count number of lines in a file.""" if output_file is None: diff --git a/packetbeat/tests/system/pcaps/mongodb_invalid_opcode_2269.pcap b/packetbeat/tests/system/pcaps/mongodb_invalid_opcode_2269.pcap new file mode 100644 index 0000000000000000000000000000000000000000..1070e23da3d66873d65b8c5fd8105f6dc1b6ebe0 GIT binary patch literal 2319 zcmb8vZ%7ky7zgm*-OjZvV@5~>z4JvdP+zon;#Lrt)QeOiq8Pb9Fq;$FqQ$^QqZfN| z8!~!PxCB52lseZ9~N?qkZ$>plQ_)}i`NJd@RS$)iIuylCj924 zI6b}DMTjVE%}b)C*%9xgfM7E1KzWTT6M}O0QzrWp^1|Hi@yDQ>C->INs56-iwUDFm zTXeYNVO?=Qh&Q%H z0mbuG#21VDp%<>AKN#{}3X^8I4|_vlmwPl2AY!{`h?w9JTqMt^`|^l47!Hv(cw*`z zd*EVq`91WF!9B)xaTk#8#klvcvbgGMbFf->%>pp4fZ{DLZ>8zuRE+3~t(ui?i+Qdn zZ_t&_GEsd=Zg@%gn1NX2is?R@@d^{wsq-+kM{nvJO#S49$SHrahh~g2QC{>X1`C<@ z@Dg;&?T7Q29q1EI-C=g}BBG7<-0sXPQ`Cr~$6Ff3T44Fj~N8zWAJq zlCG$!YGtKAxMK1MU1^MoO0lvXkaIv={)+gED~`?3j3y?kQxCw@oqAIjV2?M4cXG;a zdxfI)HxuP0e`2}ZlhiMN#Lmi(@)TDl_2pkWxE8zvdaAzs5xXNEv19pBegLxZySKfj zng7*fPJuq5-GZW`xDH|#!sA(eDF6JnSt@3AMNRn?v7zz@8!SJ@-Qq4FI*)N*d}nc? z2DsV@SnaCO0+uiGRvP_C#T8vqQ~t_I_XT*O?G0V&1{2j{twPR>w)_?G09TBDq8Yz2 NQJp#iQ~%VP`ai6~71sa& literal 0 HcmV?d00001 diff --git a/packetbeat/tests/system/test_0025_mongodb_basic.py b/packetbeat/tests/system/test_0025_mongodb_basic.py index b1ff493377a..25453193015 100644 --- a/packetbeat/tests/system/test_0025_mongodb_basic.py +++ b/packetbeat/tests/system/test_0025_mongodb_basic.py @@ -1,6 +1,6 @@ +import re from packetbeat import BaseTest - class Test(BaseTest): """ Basic MongoDB tests @@ -220,14 +220,35 @@ def test_request_after_reply(self): assert o["type"] == "mongodb" assert o["event.duration"] >= 0 - def test_unknown_opcode_flood(self): + def test_opmsg(self): """ - Tests that a repeated unknown opcode is reported just once. + Tests parser works with opcode 2013 (OP_MSG). """ self.render_config_template( mongodb_ports=[9991] ) self.run_packetbeat(pcap="mongodb_op_msg_opcode.pcap", debug_selectors=["mongodb"]) - num_msgs = self.log_contains_count('Unknown operation code: ') - assert num_msgs == 1, "Unknown opcode reported more than once: {0}".format(num_msgs) + + objs = self.read_output() + o = objs[0] + assert o["type"] == "mongodb" + + count = self.log_contains_count('Unknown operation code: ') + assert count == 0 + + def test_unknown_opcode_flood(self): + """ + Tests that any repeated unknown opcodes are reported just once. + """ + self.render_config_template( + mongodb_ports=[27017] + ) + self.run_packetbeat(pcap="mongodb_invalid_opcode_2269.pcap", + debug_selectors=["mongodb"]) + + unknown_counts = self.log_contains_countmap(re.compile(r'Unknown operation code: (\d+)'), 1) + + assert len(unknown_counts) > 0 + for k, v in unknown_counts.items(): + assert v == 1, "Unknown opcode reported more than once: opcode={0}, count={1}".format(k, v) \ No newline at end of file From 4c512a4c850268f30d6ec5cab2d08888404d8376 Mon Sep 17 00:00:00 2001 From: pohzipohzi Date: Wed, 27 Mar 2019 21:52:14 +0800 Subject: [PATCH 13/13] run autopep8 --- libbeat/tests/system/beat/beat.py | 30 ++++++++++++------- .../tests/system/test_0025_mongodb_basic.py | 7 +++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 587c2692caf..1118088bb17 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -131,7 +131,8 @@ def setUpClass(self): # Path to test binary if not hasattr(self, 'test_binary'): - self.test_binary = os.path.abspath(self.beat_path + "/" + self.beat_name + ".test") + self.test_binary = os.path.abspath( + self.beat_path + "/" + self.beat_name + ".test") template_paths = [ self.beat_path, @@ -282,7 +283,8 @@ def read_output_json(self, output_file=None): # hit EOF break - event = json.loads(line, object_pairs_hook=self.json_raise_on_duplicates) + event = json.loads( + line, object_pairs_hook=self.json_raise_on_duplicates) del event['@metadata'] jsons.append(event) return jsons @@ -319,7 +321,8 @@ def setUp(self): fields_yml = os.path.join(self.beat_path, "fields.yml") # Only add it if it exists if os.path.isfile(fields_yml): - shutil.copyfile(fields_yml, os.path.join(self.working_dir, "fields.yml")) + shutil.copyfile(fields_yml, os.path.join( + self.working_dir, "fields.yml")) try: # update the last_run link @@ -365,7 +368,8 @@ def wait_log_contains(self, msg, logfile=None, name="log_contains", ignore_case=False): self.wait_until( - cond=lambda: self.log_contains(msg, logfile, ignore_case=ignore_case), + cond=lambda: self.log_contains( + msg, logfile, ignore_case=ignore_case), max_timeout=max_timeout, poll_interval=poll_interval, name=name) @@ -535,7 +539,8 @@ def extract_fields(doc_list, name): newName = field["name"] if field.get("type") == "group": - subfields, subdictfields, subaliases = extract_fields(field["fields"], newName) + subfields, subdictfields, subaliases = extract_fields( + field["fields"], newName) fields.extend(subfields) dictfields.extend(subdictfields) aliases.extend(subaliases) @@ -554,9 +559,11 @@ def extract_fields(doc_list, name): # TODO: Make fields_doc path more generic to work with beat-generator. If it can't find file # "fields.yml" you should run "make update" on metricbeat folder with open(fields_doc, "r") as f: - path = os.path.abspath(os.path.dirname(__file__) + "../../../../fields.yml") + path = os.path.abspath(os.path.dirname( + __file__) + "../../../../fields.yml") if not os.path.isfile(path): - path = os.path.abspath(os.path.dirname(__file__) + "../../../../_meta/fields.common.yml") + path = os.path.abspath(os.path.dirname( + __file__) + "../../../../_meta/fields.common.yml") with open(path) as f2: content = f2.read() @@ -575,7 +582,8 @@ def extract_fields(doc_list, name): aliases = [] for item in doc: - subfields, subdictfields, subaliases = extract_fields(item["fields"], "") + subfields, subdictfields, subaliases = extract_fields( + item["fields"], "") fields.extend(subfields) dictfields.extend(subdictfields) aliases.extend(subaliases) @@ -668,6 +676,8 @@ def is_documented(key, docs): for key in flat.keys(): metaKey = key.startswith('@metadata.') if not(is_documented(key, expected_fields) or metaKey): - raise Exception("Key '{}' found in event is not documented!".format(key)) + raise Exception( + "Key '{}' found in event is not documented!".format(key)) if is_documented(key, aliases): - raise Exception("Key '{}' found in event is documented as an alias!".format(key)) + raise Exception( + "Key '{}' found in event is documented as an alias!".format(key)) diff --git a/packetbeat/tests/system/test_0025_mongodb_basic.py b/packetbeat/tests/system/test_0025_mongodb_basic.py index 25453193015..46a1fb441a9 100644 --- a/packetbeat/tests/system/test_0025_mongodb_basic.py +++ b/packetbeat/tests/system/test_0025_mongodb_basic.py @@ -1,6 +1,7 @@ import re from packetbeat import BaseTest + class Test(BaseTest): """ Basic MongoDB tests @@ -247,8 +248,10 @@ def test_unknown_opcode_flood(self): self.run_packetbeat(pcap="mongodb_invalid_opcode_2269.pcap", debug_selectors=["mongodb"]) - unknown_counts = self.log_contains_countmap(re.compile(r'Unknown operation code: (\d+)'), 1) + unknown_counts = self.log_contains_countmap( + re.compile(r'Unknown operation code: (\d+)'), 1) assert len(unknown_counts) > 0 for k, v in unknown_counts.items(): - assert v == 1, "Unknown opcode reported more than once: opcode={0}, count={1}".format(k, v) \ No newline at end of file + assert v == 1, "Unknown opcode reported more than once: opcode={0}, count={1}".format( + k, v)