diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..fb19733 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,5 @@ +Copyright 2017,2018 Farsight Security, Inc. + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..14e2f77 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md new file mode 100644 index 0000000..446ce4c --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +# Pure Golang NMSG Library + +`go-nmsg` is a pure go implementation of the NMSG container and payload +format used by the C (nmsg)[https://github.com/farsightsec/nmsg] toolkit +and library. + +## Synopsis + + import "github.com/farsightsec/go-nmsg" + import "github.com/farsightsec/go-nmsg/nmsg_base" + + var r io.Reader + var w io.Writer + ... + input := nmsg.NewInput(r, mtu) + output := nmsg.BufferedOutput(w) + output.SetMaxSize(nmsg.MaxContainerSize, 0) + + for { + payload, err := inp.Recv() + if err != nil { + if nmsg.IsDataError(err) { + continue + } + break + } + message := payload.Message() + + switch message.(type) { + case *nmsg_base.Dnstap: + // process dnstap + // write copy to output + output.Send(payload) + } + } + + output.Close() + + +## Requirements + +`go-nmsg` requires the following open source libraries: + + "github.com/golang/protobuf/proto" + "github.com/dnstap/golang-dnstap" + +## Limitations + +`go-nmsg` can pack and unpack the protobuf structure of an NMSG payload, +and the protobuf structure of the data contained in the payload. It does +not implement the full functionality of the C libnmsg message +modules, such as: + + * Advanced field types (e.g., a protobuf []byte as an IP address) + * Generated fields + * Formatting of fields for presentation and JSON output + +Applications needing such functionality in go should use the +`cgo-nmsg` package included in this distribution under: + + "github.com/farsightsec/go-nmsg/cgo-nmsg" diff --git a/cgo-nmsg/README.md b/cgo-nmsg/README.md new file mode 100644 index 0000000..7d812e1 --- /dev/null +++ b/cgo-nmsg/README.md @@ -0,0 +1,10 @@ +# Golang bindings for NMSG + +`cgo-nmsg` provides Golang bindings to the C libnmsg library. + +The NMSG network message encapsulation library format is an efficient +encoding of typed, structured data into payloads which are packed into +containers which can be transmitted over the network or stored to disk. +For more information, see https://github.com/farsightsec/nmsg/. + +A pure but limited Golang NMSG library is available with `go-nmsg`. diff --git a/cgo-nmsg/callbacks.go b/cgo-nmsg/callbacks.go new file mode 100644 index 0000000..f8b2605 --- /dev/null +++ b/cgo-nmsg/callbacks.go @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include +*/ +import "C" +import ( + "fmt" + "runtime" + "sync" + "unsafe" +) + +type outCbEntry struct { + index int + Output +} + +type inCbEntry struct { + index int + Input +} + +var cbm sync.Mutex +var outCbTable []Output +var inCbTable []Input + +// The C library may not hold a pointer to a Go variable, but we +// need to store enough context in the callback user data to find +// the go object which registered the callback. We solve this by +// allocating memory on the C side (with C.malloc, C.calloc) and +// storing a value in that memory which we can use to look up the +// Go value on the Go side. +// +// The approach we take here is to have a package-global list of +// Output and Input, and store the index in the list as a C.int +// in C-allocated memory. The location of this memory is returned +// as an unsafe.Pointer suitable for passing to the (void *user) +// argument of libnmsg callback registration functions. + +func registerOutput(o Output) unsafe.Pointer { + cbm.Lock() + defer cbm.Unlock() + idx := len(outCbTable) + outCbTable = append(outCbTable, o) + idxptr := C.calloc(C.size_t(1), C.size_t(unsafe.Sizeof(C.int(1)))) + *(*C.int)(idxptr) = C.int(idx) + return idxptr +} + +func registerInput(i Input) unsafe.Pointer { + cbm.Lock() + defer cbm.Unlock() + idx := len(inCbTable) + inCbTable = append(inCbTable, i) + idxptr := C.calloc(C.size_t(1), C.size_t(unsafe.Sizeof(C.int(1)))) + *(*C.int)(idxptr) = C.int(idx) + return idxptr +} + +//export outputCallback +func outputCallback(msg C.nmsg_message_t, user unsafe.Pointer) { + idx := int(*(*C.int)(user)) + if idx < len(outCbTable) { + o := outCbTable[idx] + o.Write(messageFromC(msg)) + return + } + panic(fmt.Sprintf("outputCallback: invalid index %d", idx)) +} + +//export inputCallback +func inputCallback(msg, user unsafe.Pointer) C.nmsg_res { + idx := int(*(*C.int)(user)) + if idx < len(inCbTable) { + i := inCbTable[idx] + for { + m, err := i.Read() + + if ErrorRetry(err) { + continue + } + if err != nil { + *(*C.nmsg_message_t)(msg) = nil + if e, ok := err.(nmsgResError); ok { + return C.nmsg_res(e) + } + return C.nmsg_res_failure + } + runtime.SetFinalizer(m, nil) + *(*C.nmsg_message_t)(msg) = m.message + return C.nmsg_res_success + } + } + panic(fmt.Sprintf("inputCallback: invalid index %d", idx)) +} diff --git a/cgo-nmsg/container.go b/cgo-nmsg/container.go new file mode 100644 index 0000000..a7a10e7 --- /dev/null +++ b/cgo-nmsg/container.go @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include +*/ +import "C" +import ( + "crypto/rand" + "encoding/binary" + "io" + "runtime" + "sync" + "unsafe" +) + +// A Container is a collection of NMSG payloads with a target size. +type Container struct { + config ContainerConfig + sequenceID uint64 + sequenceNumber uint32 + container C.nmsg_container_t +} + +// ContainerConfig contains +type ContainerConfig struct { + Compress bool + Sequence bool + Size int +} + +// NewContainer creates a container with the given target size. +func NewContainer(conf *ContainerConfig) *Container { + c := &Container{config: *conf, container: C.nmsg_container_init(C.size_t(conf.Size))} + runtime.SetFinalizer(c, func(c *Container) { + C.nmsg_container_destroy(&c.container) + }) + if conf.Sequence { + C.nmsg_container_set_sequence(c.container, C.bool(true)) + binary.Read(rand.Reader, binary.BigEndian, &c.sequenceID) + } + return c +} + +// ErrorFull returns true if the container is full. If the Container Add() +// method returns such an error, the message will need to be added to the +// next container. +func ErrorFull(err error) bool { + t, ok := err.(nmsgResError) + return ok && t == nmsgResError(C.nmsg_res_container_full) +} + +// ErrorOverfull returns true if the container contains a single payload +// and its size is greater than the target size. +func ErrorOverfull(err error) bool { + t, ok := err.(nmsgResError) + return ok && t == nmsgResError(C.nmsg_res_container_overfull) +} + +// Add adds the supplied Message to the Container. +func (c *Container) Add(m *Message) error { + return nmsgError(C.nmsg_container_add(c.container, m.message)) +} + +// Bytes returns the serialized container and resets the container. +func (c *Container) Bytes() []byte { + var pbuf *C.uint8_t + var pbufLen C.size_t + res := C.nmsg_container_serialize(c.container, + &pbuf, &pbufLen, + C.bool(true), + C.bool(c.config.Compress), + C.uint32_t(c.sequenceNumber), + C.uint64_t(c.sequenceID), + ) + defer C.free(unsafe.Pointer(pbuf)) + if err := nmsgError(res); err != nil { + return nil + } + c.sequenceID++ + C.nmsg_container_destroy(&c.container) + c.container = C.nmsg_container_init(C.size_t(c.config.Size)) + if c.config.Sequence { + C.nmsg_container_set_sequence(c.container, C.bool(true)) + } + + return C.GoBytes(unsafe.Pointer(pbuf), C.int(pbufLen)) +} + +// UnpackContainer returns the messages the container contains. +func UnpackContainer(b []byte) ([]*Message, error) { + var msgarray *C.nmsg_message_t + var nmsgs C.size_t + + res := C.nmsg_container_deserialize( + (*C.uint8_t)(unsafe.Pointer(&b[0])), + C.size_t(len(b)), + &msgarray, + &nmsgs) + if err := nmsgError(res); err != nil { + return nil, err + } + msgs := make([]*Message, 0, int(nmsgs)) + p := unsafe.Pointer(msgarray) + for i := 0; i < int(nmsgs); i++ { + mp := unsafe.Pointer(uintptr(p) + uintptr(i)*unsafe.Sizeof(*msgarray)) + msgs = append(msgs, messageFromC(*(*C.nmsg_message_t)(mp))) + } + + C.free(unsafe.Pointer(msgarray)) + return msgs, nil +} + +// A ContainerOutput writes containers to a generic io.Writer. No fragmentation +// of oversize containers is performed. +type containerOutput struct { + mu sync.Mutex + w io.Writer + c *Container + rate *Rate + buffered bool + empty bool + filtervendor uint32 + filtermsgtype uint32 + source uint32 + operator uint32 + group uint32 +} + +// NewContainerOutput creates a ContainerOutput writing to the supplied +// io.Writer with the given buffer size. +func newContainerOutput(w io.Writer, size int) *containerOutput { + return &containerOutput{ + c: NewContainer(&ContainerConfig{ + Size: size, + Sequence: true, + }), + buffered: true, + empty: true, + w: w, + } +} + +func (co *containerOutput) Write(m *Message) error { + for { + vid, msgtype := m.GetMsgtype() + if co.filtervendor > 0 && co.filtervendor != vid { + return nil + } + if co.filtermsgtype > 0 && co.filtermsgtype != msgtype { + return nil + } + if co.source > 0 { + m.SetSource(co.source) + } + if co.operator > 0 { + m.SetOperator(co.operator) + } + if co.group > 0 { + m.SetGroup(co.group) + } + + co.mu.Lock() + err := co.c.Add(m) + if co.buffered && err == nil { + co.empty = false + co.mu.Unlock() + return nil + } + _, werr := co.w.Write(co.c.Bytes()) + co.empty = true + r := co.rate + co.mu.Unlock() + if r != nil { + r.Sleep() + } + if werr == nil && ErrorFull(err) { + continue + } + return werr + } +} + +// SetFilterMsgtype instructs the output to only accept Messages +// with the given vendor and messagetype, specified by id. +func (co *containerOutput) SetFilterMsgtype(vendor, msgtype uint32) { + co.filtervendor = vendor + co.filtermsgtype = msgtype +} + +// SetFilterMsgtypeByname instructs the output to only accept Messages +// with the given vendor and messagetype, specified by name. +func (co *containerOutput) SetFilterMsgtypeByname(vendor, msgtype string) { + cvendor := C.CString(vendor) + cmsgtype := C.CString(msgtype) + defer C.free(unsafe.Pointer(cvendor)) + defer C.free(unsafe.Pointer(cmsgtype)) + cvid := C.nmsg_msgmod_vname_to_vid(cvendor) + co.filtervendor = uint32(cvid) + co.filtermsgtype = uint32(C.nmsg_msgmod_mname_to_msgtype(cvid, cmsgtype)) +} + +func (co *containerOutput) SetRate(r *Rate) { + co.mu.Lock() + co.rate = r + co.mu.Unlock() +} + +func (co *containerOutput) SetSource(source uint32) { + co.source = source +} + +func (co *containerOutput) SetOperator(op uint32) { + co.operator = op +} + +func (co *containerOutput) SetGroup(group uint32) { + co.group = group +} + +// Flush writes any buffered output to the underlying writer. +func (co *containerOutput) Flush() error { + co.mu.Lock() + written := false + defer func() { + r := co.rate + co.mu.Unlock() + if written && r != nil { + r.Sleep() + } + }() + if !co.empty { + _, werr := co.w.Write(co.c.Bytes()) + co.empty = true + written = true + return werr + } + return nil +} + +// SetBuffered controls whether the ContainerOutput collects +// multiple messages into a container (buffered == true, the +// default), or sends a container per message (buffered == false). +func (co *containerOutput) SetBuffered(buffered bool) { + co.buffered = buffered +} + +// SetCompression controls whether the containers are compressed +// before sending. +func (co *containerOutput) SetCompression(compress bool) { + co.c.config.Compress = compress +} diff --git a/cgo-nmsg/error.go b/cgo-nmsg/error.go new file mode 100644 index 0000000..048f4d2 --- /dev/null +++ b/cgo-nmsg/error.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +*/ +import "C" + +// NmsgError encapsulates an error condition +type nmsgResError C.nmsg_res + +func (n nmsgResError) Error() string { + return C.GoString(C.nmsg_res_lookup(uint32(n))) +} + +func nmsgError(res C.nmsg_res) error { + if res == C.nmsg_res_success { + return nil + } + return nmsgResError(res) +} + +// ErrorRetry returns true if the error indicates that the nmsg +// operation should be retried. +func ErrorRetry(err error) bool { + if ne, ok := err.(nmsgResError); ok { + return ne == nmsgResError(C.nmsg_res_again) + } + return false +} diff --git a/cgo-nmsg/input.go b/cgo-nmsg/input.go new file mode 100644 index 0000000..d9d3356 --- /dev/null +++ b/cgo-nmsg/input.go @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include + +extern nmsg_res inputCallback(nmsg_message_t *msg, void *user); + +nmsg_res input_callback(nmsg_message_t *msg, void *user) { + return inputCallback(msg, user); +} +*/ +import "C" +import ( + "io" + "net" + "os" + "unsafe" +) + +// An Input is a source of NMSG payloads (Messages). +type Input interface { + // Read returns a Message or nil, and an error if any. + Read() (*Message, error) + + // SetFilterMsgtype instructs the input to discard all Messages + // not of the given vendor id and msgtype, specified by number. + SetFilterMsgtype(vendor, msgtype uint32) + + // SetFilterMsgtypeByname instructs the input to discard all Messages + // not of the given vendor id and msgtype, specified by name. + SetFilterMsgtypeByname(vendor, msgtype string) + + // SetFilterSource instructs the input to discard all Messages not + // from the supplied source. + SetFilterSource(source uint32) + + // SetFilterOperator instructs the input to discard all Messages not + // from the supplied operator. + SetFilterOperator(operator uint32) + + // SetFilterGroup instructs the input to discard all Messages not + // in the supplied group. + SetFilterGroup(group uint32) +} + +// NmsgInput is an Input managed by libnmsg. It satisfies +// the Input interface, and has +type nmsgInput struct { + file *os.File + input C.nmsg_input_t +} + +func (i *nmsgInput) Read() (*Message, error) { + var msg C.nmsg_message_t + res := C.nmsg_input_read(i.input, &msg) + if res == C.nmsg_res_success { + return messageFromC(msg), nil + } + return nil, nmsgError(res) +} + +func (i *nmsgInput) SetFilterMsgtype(vid, msgtype uint32) { + C.nmsg_input_set_filter_msgtype(i.input, C.uint(vid), C.uint(msgtype)) +} + +func (i *nmsgInput) SetFilterMsgtypeByname(vendor, msgtype string) { + cname := C.CString(vendor) + ctype := C.CString(msgtype) + C.nmsg_input_set_filter_msgtype_byname(i.input, cname, ctype) + C.free(unsafe.Pointer(cname)) + C.free(unsafe.Pointer(ctype)) +} + +func (i *nmsgInput) SetFilterSource(source uint32) { + C.nmsg_input_set_filter_source(i.input, C.uint(source)) +} + +func (i *nmsgInput) SetFilterOperator(operator uint32) { + C.nmsg_input_set_filter_operator(i.input, C.uint(operator)) +} + +func (i *nmsgInput) SetFilterGroup(group uint32) { + C.nmsg_input_set_filter_group(i.input, C.uint(group)) +} + +// NewInput creates a new Input from an io.Reader. +// Currently, the reader must be a *net.UDPConn or a *os.File +func NewInput(r io.Reader) Input { + switch r := r.(type) { + case *net.UDPConn: + f, err := r.File() + if err != nil { + return nil + } + return &nmsgInput{f, C.nmsg_input_open_sock(C.int(f.Fd()))} + case *os.File: + return &nmsgInput{r, C.nmsg_input_open_file(C.int(r.Fd()))} + default: + return nil + // return &containerReader{Reader: r} + } +} + +// NewCallbackInput creates an NmsgInput which calls the supplied InputFunc. +func NewCallbackInput(i InputFunc) Input { + return &nmsgInput{ + file: nil, + input: C.nmsg_input_open_callback(C.nmsg_cb_message_read(C.input_callback), registerInput(i)), + } +} + +// An InputFunc is a function with the same signature as Input.Read(), usable +// directly as an Input. +// +// When used directly as an Input, only the Read() method is implemented. All +// others are no-ops. If the functionality of the other methods is desired, +// the InputFunc can be passed to NewCallbackInput. +type InputFunc func() (*Message, error) + +// Read calls the underlying function to return the next message. +func (i InputFunc) Read() (*Message, error) { return i() } + +// SetFilterMsgtype satisfies the Input interface with a no-op +func (i InputFunc) SetFilterMsgtype(vendor, msgtype uint32) {} + +// SetFilterMsgtypeByname satisfies the Input interface with a no-op +func (i InputFunc) SetFilterMsgtypeByname(vendor, msgtype string) {} + +// SetFilterSource satisfies the Input interface with a no-op +func (i InputFunc) SetFilterSource(source uint32) {} + +// SetFilterOperator satisfies the Input interface with a no-op +func (i InputFunc) SetFilterOperator(operator uint32) {} + +// SetFilterGroup satisfies the Input interface with a no-op +func (i InputFunc) SetFilterGroup(group uint32) {} diff --git a/cgo-nmsg/io.go b/cgo-nmsg/io.go new file mode 100644 index 0000000..dab4d16 --- /dev/null +++ b/cgo-nmsg/io.go @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include +*/ +import "C" +import "unsafe" + +// IO is a handle to a libnmsg io loop connecting one or more Inputs +// with one ore more Outputs. +type IO struct { + nmsgIO C.nmsg_io_t +} + +// NewIO creates and returns a new IO +func NewIO() *IO { + io := C.nmsg_io_init() + if io != nil { + return &IO{io} + } + return nil +} + +// AddInputChannel opens an NMSG channel and adds it as an Input to the +// IO. +func (io *IO) AddInputChannel(channel string) error { + cchan := C.CString(channel) + res := C.nmsg_io_add_input_channel(io.nmsgIO, cchan, nil) + C.free(unsafe.Pointer(cchan)) + return nmsgError(res) +} + +// AddInputSockSpec opens one or more sockets based on the sockspec +// (add/port ,or addr/lowport..highport) and adds it to the IO +// as an input. +func (io *IO) AddInputSockSpec(sockspec string) error { + css := C.CString(sockspec) + res := C.nmsg_io_add_input_sockspec(io.nmsgIO, css, nil) + C.free(unsafe.Pointer(css)) + return nmsgError(res) +} + +// AddInput adds a separately created Input to the IO as an input. +func (io *IO) AddInput(i Input) error { + ni, ok := i.(*nmsgInput) + if !ok { + ni = NewCallbackInput(i.Read).(*nmsgInput) + } + return nmsgError(C.nmsg_io_add_input(io.nmsgIO, ni.input, nil)) +} + +// AddOutput adds a separately created Output to the IO as an output. +func (io *IO) AddOutput(o Output) error { + nout, ok := o.(*nmsgOutput) + if !ok { + nout = NewCallbackOutput(o.Write).(*nmsgOutput) + } + return nmsgError(C.nmsg_io_add_output(io.nmsgIO, nout.output, nil)) +} + +// SetMirrored controls whether the IO mirrors output to all outputs +// (mirrored = true) or stripes its output across all outputs. +func (io *IO) SetMirrored(mirrored bool) { + if mirrored { + C.nmsg_io_set_output_mode(io.nmsgIO, C.nmsg_io_output_mode_mirror) + return + } + C.nmsg_io_set_output_mode(io.nmsgIO, C.nmsg_io_output_mode_stripe) +} + +// SetDebug sets the debug print level of the underlying io. +// Larger numbers are more verbose. +func (io *IO) SetDebug(debug int) { + C.nmsg_io_set_debug(io.nmsgIO, C.int(debug)) +} + +// Run starts the IO loop, returning when it is finished or broken +// with Break() +func (io *IO) Run() error { + return nmsgError(C.nmsg_io_loop(io.nmsgIO)) +} + +// Break stops the IO main loop. +func (io *IO) Break() { + C.nmsg_io_breakloop(io.nmsgIO) +} diff --git a/cgo-nmsg/message.go b/cgo-nmsg/message.go new file mode 100644 index 0000000..24b32b4 --- /dev/null +++ b/cgo-nmsg/message.go @@ -0,0 +1,380 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include + +const char *endline="\n"; +unsigned flag_repeated = NMSG_MSGMOD_FIELD_REPEATED; +*/ +import "C" +import ( + "fmt" + "net" + "runtime" + "unsafe" +) + +// MessageMod something something +type MessageMod struct { + nmsgMsgMod C.nmsg_msgmod_t +} + +// MessageModLookupByName something something +func MessageModLookupByName(vname, mname string) *MessageMod { + vstr := C.CString(vname) + mstr := C.CString(mname) + defer C.free(unsafe.Pointer(vstr)) + defer C.free(unsafe.Pointer(mstr)) + return &MessageMod{C.nmsg_msgmod_lookup_byname(vstr, mstr)} +} + +// MessageModLookup something something +func MessageModLookup(v, m uint32) *MessageMod { + return &MessageMod{C.nmsg_msgmod_lookup(C.uint(v), C.uint(m))} +} + +// A Message is a unit of NMSG data. +type Message struct { + message C.nmsg_message_t +} + +// NewMessage initializes a message of a type given by +// the supplied MessageMod +func NewMessage(mod *MessageMod) *Message { + return messageFromC(C.nmsg_message_init(mod.nmsgMsgMod)) +} + +// NewMessageFromPayload encapsulates a byte buffer in a payload with +// the supplied vendor and message type. +func NewMessageFromPayload(payload []byte, vendor uint32, msgtype uint32) *Message { + Csiz := C.size_t(len(payload)) + // C.CString allocates a buffer to hold the copy of payload + // built by string. This buffer is passed to nmsg_message_from_raw_payload, + // which takes ownership of the buffer. It will be freed when + // nmsg_message_destroy() is called by the Message finalizer. + Cbuf := unsafe.Pointer(C.CString(string(payload))) + return messageFromC(C.nmsg_message_from_raw_payload( + C.unsigned(vendor), C.unsigned(msgtype), + (*C.uint8_t)(Cbuf), Csiz, nil)) +} + +func messageDestroy(m *Message) { + C.nmsg_message_destroy(&m.message) +} + +// GetMsgtype returns the vendor and payload type of the message. +func (msg *Message) GetMsgtype() (vendor, msgtype uint32) { + vendor = uint32(C.nmsg_message_get_vid(msg.message)) + msgtype = uint32(C.nmsg_message_get_msgtype(msg.message)) + return +} + +// Source returns the source id of the message, or zero if the source id +// is not set. +func (msg *Message) Source() uint32 { + return uint32(C.nmsg_message_get_source(msg.message)) +} + +// SetSource sets the source id of the message. +func (msg *Message) SetSource(source uint32) { + C.nmsg_message_set_source(msg.message, C.uint32_t(source)) +} + +// Operator returns the operator id of the message, or zero if the operator id +// is not set. +func (msg *Message) Operator() uint32 { + return uint32(C.nmsg_message_get_operator(msg.message)) +} + +// SetOperator sets the operator id of the message. +func (msg *Message) SetOperator(operator uint32) { + C.nmsg_message_set_operator(msg.message, C.uint32_t(operator)) +} + +// Group returns the group id of the message, or zero if the group id +// is not set. +func (msg *Message) Group() uint32 { + return uint32(C.nmsg_message_get_group(msg.message)) +} + +// SetGroup sets the group id of the message. +func (msg *Message) SetGroup(group uint32) { + C.nmsg_message_set_group(msg.message, C.uint32_t(group)) +} + +func messageFromC(message C.nmsg_message_t) *Message { + msg := &Message{message} + runtime.SetFinalizer(msg, messageDestroy) + return msg +} + +// MarshalJSON formats a JSON representation of the Message +func (msg *Message) MarshalJSON() ([]byte, error) { + var jsonCstr *C.char + err := nmsgError(C.nmsg_message_to_json(msg.message, &jsonCstr)) + defer C.free(unsafe.Pointer(jsonCstr)) + if err != nil { + return nil, err + } + return []byte(C.GoString(jsonCstr)), nil +} + +// UnmarshalJSON parses a JSON representation of the Message +func (msg *Message) UnmarshalJSON(b []byte) error { + jsonCstr := C.CString(string(b)) + defer C.free(unsafe.Pointer(jsonCstr)) + return nmsgError(C.nmsg_message_from_json(jsonCstr, &msg.message)) +} + +// MarshalText converts a Message to presentation format. +func (msg *Message) MarshalText() ([]byte, error) { + var presCstr *C.char + err := nmsgError(C.nmsg_message_to_pres(msg.message, &presCstr, C.endline)) + defer C.free(unsafe.Pointer(presCstr)) + if err != nil { + return nil, err + } + return []byte(C.GoString(presCstr)), nil +} + +// Enum contains both the numeric Value and the string Description of +// an enumerated NMSG field value. +type Enum struct { + Value uint32 + Description string +} + +type fieldValue struct { + typ C.nmsg_msgmod_field_type + buf unsafe.Pointer + size C.int +} + +func (msg *Message) getFieldValue(name string, idx int) (fv fieldValue, err error) { + var Csize C.size_t + + Cname := C.CString(name) + defer C.free(unsafe.Pointer(Cname)) + + Cidx := C.uint(uint(idx)) + + res := C.nmsg_message_get_field_type(msg.message, Cname, &fv.typ) + if err = nmsgError(res); err != nil { + return + } + + res = C.nmsg_message_get_field(msg.message, Cname, Cidx, &fv.buf, &Csize) + if err = nmsgError(res); err != nil { + return + } + + fv.size = C.int(Csize) + return +} + +func (msg *Message) setFieldValue(name string, idx int, buf unsafe.Pointer, size int) error { + Cname := C.CString(name) + defer C.free(unsafe.Pointer(Cname)) + + Cidx := C.uint(uint(idx)) + Csize := C.size_t(size) + return nmsgError(C.nmsg_message_set_field(msg.message, Cname, Cidx, + (*C.uint8_t)(buf), Csize)) +} + +// GetUintField retrieves the named field of a unsigned int type from a Message. +// If the field has an enumerated type, the numeric value is retrieved. +func (msg *Message) GetUintField(name string, idx int) (uint64, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return 0, err + } + + switch fv.typ { + case C.nmsg_msgmod_ft_uint16: + return uint64(*(*uint16)(fv.buf)), nil + case C.nmsg_msgmod_ft_uint32: + fallthrough + case C.nmsg_msgmod_ft_enum: + return uint64(*(*uint32)(fv.buf)), nil + case C.nmsg_msgmod_ft_uint64: + return *(*uint64)(fv.buf), nil + default: + return 0, fmt.Errorf("Field %s not of uint type", name) + } + +} + +// SetUintField sets the value of a field of type uint16, uint32, or uint64. +// The bitsize parameter specifies which type, and must be 16, 32, or 64 +func (msg *Message) SetUintField(name string, idx, bitsize int, val uint64) error { + switch bitsize { + case 16: + v := uint16(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + case 32: + v := uint32(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + case 64: + v := uint64(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + default: + return fmt.Errorf("Invalid bitsize %d", bitsize) + } +} + +// GetIntField retrieves the value of a named field of integer type from +// a Message. +func (msg *Message) GetIntField(name string, idx int) (int64, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return 0, err + } + + switch fv.typ { + case C.nmsg_msgmod_ft_int16: + return int64(*(*int16)(fv.buf)), nil + case C.nmsg_msgmod_ft_int32: + return int64(*(*int32)(fv.buf)), nil + case C.nmsg_msgmod_ft_int64: + return *(*int64)(fv.buf), nil + default: + return 0, fmt.Errorf("Field %s not of int type", name) + } +} + +// SetIntField sets the value of an int16, int32, or int64 field in the message. +// The bitsize field specifies which size, and must by 16, 32, or 64 +func (msg *Message) SetIntField(name string, idx, bitsize int, val int64) error { + switch bitsize { + case 16: + v := int16(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + case 32: + v := int32(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + case 64: + v := int64(val) + return msg.setFieldValue(name, idx, unsafe.Pointer(&v), bitsize) + default: + return fmt.Errorf("Invalid bitsize %d", bitsize) + } +} + +// GetBytesField retrieves a field of type bytes from a Message. +func (msg *Message) GetBytesField(name string, idx int) ([]byte, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return nil, err + } + if fv.typ != C.nmsg_msgmod_ft_bytes { + return nil, fmt.Errorf("Field %s not of bytes type", name) + } + return C.GoBytes(fv.buf, fv.size), nil +} + +// SetBytesField sets the value of a bytes field in a Message +func (msg *Message) SetBytesField(name string, idx int, val []byte) error { + Cbuf := unsafe.Pointer(&val[0]) + return msg.setFieldValue(name, idx, Cbuf, len(val)) +} + +// GetStringField retrieves the value of a string field in a Message +func (msg *Message) GetStringField(name string, idx int) (string, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return "", err + } + return C.GoStringN((*C.char)(fv.buf), fv.size), nil +} + +// SetStringField sets the value of a string field in a Message +func (msg *Message) SetStringField(name string, idx int, val string) error { + b := []byte(val) + Cbuf := unsafe.Pointer(&b[0]) + return msg.setFieldValue(name, idx, Cbuf, len(val)) +} + +// GetIPField retrieves the value of an IP field in a Message +func (msg *Message) GetIPField(name string, idx int) (net.IP, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return nil, err + } + if fv.typ != C.nmsg_msgmod_ft_ip { + return nil, fmt.Errorf("Field %s not of iptype", name) + } + return net.IP(C.GoBytes(fv.buf, fv.size)), nil +} + +// SetIPField sets the value of an IP field in a Message +func (msg *Message) SetIPField(name string, idx int, val net.IP) error { + Cbuf := unsafe.Pointer(&val[0]) + return msg.setFieldValue(name, idx, Cbuf, len(val)) +} + +// GetDoubleField retrieves the value of a double field in a Message +func (msg *Message) GetDoubleField(name string, idx int) (float64, error) { + fv, err := msg.getFieldValue(name, idx) + if err != nil { + return 0, err + } + if fv.typ != C.nmsg_msgmod_ft_double { + return 0, fmt.Errorf("Field %s is not of double type", name) + } + return *(*float64)(fv.buf), nil +} + +// SetDoubleField sets the value of a double field in a Message +func (msg *Message) SetDoubleField(name string, idx int, val float64) error { + Cbuf := unsafe.Pointer(&val) + return msg.setFieldValue(name, idx, Cbuf, 8) +} + +// GetEnumField returns the string description of a Message field +// with an enumerated type. +func (msg *Message) GetEnumField(name string, idx int) (string, error) { + enumValue, err := msg.GetUintField(name, idx) + if err != nil { + return "", err + } + + Cname := C.CString(name) + defer C.free(unsafe.Pointer(Cname)) + var Ename *C.char + res := C.nmsg_message_enum_value_to_name( + msg.message, Cname, C.unsigned(enumValue), + &Ename, + ) + if err = nmsgError(res); err != nil { + return "", err + } + return C.GoString(Ename), nil +} + +// SetEnumField sets the value of the named Message field to the value +// corresponding to the supplied description. +func (msg *Message) SetEnumField(name string, idx int, vname string) error { + Cname := C.CString(name) + defer C.free(unsafe.Pointer(Cname)) + Cvname := C.CString(vname) + defer C.free(unsafe.Pointer(Cvname)) + + var v C.uint + res := C.nmsg_message_enum_name_to_value(msg.message, Cname, Cvname, &v) + if err := nmsgError(res); err != nil { + return err + } + return msg.SetUintField(name, idx, 32, uint64(v)) +} diff --git a/cgo-nmsg/nmsg.go b/cgo-nmsg/nmsg.go new file mode 100644 index 0000000..996d5d1 --- /dev/null +++ b/cgo-nmsg/nmsg.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include + +int nmsg_wbufsiz_min = NMSG_WBUFSZ_MIN; +int nmsg_wbufsiz_max = NMSG_WBUFSZ_MAX; +int nmsg_wbufsiz_ether = NMSG_WBUFSZ_ETHER; +int nmsg_wbufsiz_jumbo = NMSG_WBUFSZ_JUMBO; +*/ +import "C" + +func init() { + if C.nmsg_init() != C.nmsg_res_success { + panic("failed to initialize nmsg library") + } +} + +// Buffer Size constants from libnmsg +var ( + BufferSizeMax = int(C.nmsg_wbufsiz_max) + BufferSizeMin = int(C.nmsg_wbufsiz_min) + BufferSizeEther = int(C.nmsg_wbufsiz_ether) + BufferSizeJumbo = int(C.nmsg_wbufsiz_jumbo) +) + +// SetDebug sets the debug print level for the nmsg library. +// Debugging messages are sent to stderr. Higher debug values +// increase verbosity. +func SetDebug(debug int) { + C.nmsg_set_debug(C.int(debug)) +} diff --git a/cgo-nmsg/output.go b/cgo-nmsg/output.go new file mode 100644 index 0000000..cd2d0e3 --- /dev/null +++ b/cgo-nmsg/output.go @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include + +extern void outputCallback(nmsg_message_t, void *); + +void output_callback(nmsg_message_t msg, void *user) { + outputCallback(msg, user); +} +*/ +import "C" +import ( + "io" + "net" + "os" + "unsafe" +) + +// An Output is a destination for NMSG data (Messages) +type Output interface { + // Write sends the supplied message to the Output. + Write(*Message) error + + // SetBuffered controls whether the output buffers Messages into containers + // before sending them. NmsgOutputs are buffered by default, but low volume + // sources may choose to turn this off to reduce latency. + SetBuffered(bool) + + // SetCompression controls whether the output compresses + // the container data prior to sending. + SetCompression(bool) + + // Flush writes any buffered data to the Output. + Flush() error + + // SetFilterMsgtype instructs the output to discard all Messages + // not of the supplied vendor and type, specified by number. + SetFilterMsgtype(vendor, msgtype uint32) + + // SetFilterMsgtypeByname instructs the output to discard all Messages + // not of the supplied vendor and type, specified by name. + SetFilterMsgtypeByname(vendor, msgtype string) + + // SetRate sets an output rate limit. The rate is specified + // in containers per second, and is checked every freq pauses. + // The freq parameter should be about 10-15% of the rate. + SetRate(rate *Rate) + + // SetSource instructs the output to set the source parameter + // of all outbound messages to the supplied value. + SetSource(source uint32) + + // SetOperator instructs the output to set the operator parameter + // of all outbound messages to the supplied value. + SetOperator(group uint32) + + // SetGroup instructs the output to set the group parameter + // of all outbound messages to the supplied value. + SetGroup(group uint32) +} + +// An NmsgOutput is an output managed by the nmsg library. +type nmsgOutput struct { + file *os.File + rate *Rate + output C.nmsg_output_t +} + +func (o *nmsgOutput) Write(m *Message) error { + return nmsgError(C.nmsg_output_write(o.output, m.message)) +} + +func (o *nmsgOutput) SetBuffered(buffered bool) { + C.nmsg_output_set_buffered(o.output, C.bool(buffered)) +} + +func (o *nmsgOutput) SetFilterMsgtype(vid, msgtype uint32) { + C.nmsg_output_set_filter_msgtype(o.output, C.uint(vid), C.uint(msgtype)) +} + +func (o *nmsgOutput) SetFilterMsgtypeByname(vendor, msgtype string) { + cname := C.CString(vendor) + ctype := C.CString(msgtype) + C.nmsg_output_set_filter_msgtype_byname(o.output, cname, ctype) + C.free(unsafe.Pointer(cname)) + C.free(unsafe.Pointer(ctype)) +} + +func (o *nmsgOutput) SetRate(r *Rate) { + if r == nil { + C.nmsg_output_set_rate(o.output, nil) + } else { + C.nmsg_output_set_rate(o.output, r.rate) + } + // keep a reference to avoid calling the finalizer + o.rate = r +} + +func (o *nmsgOutput) SetSource(source uint32) { + C.nmsg_output_set_source(o.output, C.uint(source)) +} + +func (o *nmsgOutput) SetOperator(operator uint32) { + C.nmsg_output_set_operator(o.output, C.uint(operator)) +} + +func (o *nmsgOutput) SetGroup(group uint32) { + C.nmsg_output_set_group(o.output, C.uint(group)) +} + +func (o *nmsgOutput) SetCompression(compress bool) { + C.nmsg_output_set_zlibout(o.output, C.bool(compress)) +} + +func (o *nmsgOutput) Flush() error { + return nmsgError(C.nmsg_output_flush(o.output)) +} + +// NewOutput creates an output writing to w, with target +// container size of bufsiz. The Writer currently must be a +// *os.File or *net.UDPConn. +func NewOutput(w io.Writer, bufsiz int) Output { + switch w := w.(type) { + case *net.UDPConn: + f, err := w.File() + if err != nil { + return nil + } + return &nmsgOutput{f, nil, C.nmsg_output_open_sock(C.int(f.Fd()), C.size_t(bufsiz))} + case *os.File: + return &nmsgOutput{w, nil, C.nmsg_output_open_file(C.int(w.Fd()), C.size_t(bufsiz))} + default: + return newContainerOutput(w, bufsiz) + } +} + +// NewCallbackOutput creates an NmsgOutput which calls o.Send() +// on every message. +func NewCallbackOutput(o OutputFunc) Output { + return &nmsgOutput{ + file: nil, + output: C.nmsg_output_open_callback(C.nmsg_cb_message(C.output_callback), registerOutput(o)), + } +} + +// An OutputFunc is a function with the same signature as Output.Write, usable +// directly as an Output. +// +// When used directly as an Output, only the Write() method is defined. All others +// are no-ops. +type OutputFunc func(*Message) error + +// Write calls the underlying function with the supplied message +func (o OutputFunc) Write(m *Message) error { return o(m) } + +// Flush satisfies the Output interface with a no-op +func (o OutputFunc) Flush() error { return nil } + +// SetBuffered satisfies the Output interface with a no-op +func (o OutputFunc) SetBuffered(bool) {} + +// SetCompression satisfies the Output interface with a no-op +func (o OutputFunc) SetCompression(bool) {} + +// SetFilterMsgtype satisfies the Output interface with a no-op +func (o OutputFunc) SetFilterMsgtype(vendor, msgtype uint32) {} + +// SetFilterMsgtypeByname satisfies the Output interface with a no-op +func (o OutputFunc) SetFilterMsgtypeByname(vendor, msgtype string) {} + +// SetRate satisfies the Output interface with a no-op +func (o OutputFunc) SetRate(r *Rate) {} + +// SetSource satisfies the Output interface with a no-op +func (o OutputFunc) SetSource(source uint32) {} + +// SetOperator satisfies the Output interface with a no-op +func (o OutputFunc) SetOperator(group uint32) {} + +// SetGroup satisfies the Output interface with a no-op +func (o OutputFunc) SetGroup(group uint32) {} diff --git a/cgo-nmsg/rate.go b/cgo-nmsg/rate.go new file mode 100644 index 0000000..ebf8e38 --- /dev/null +++ b/cgo-nmsg/rate.go @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg +#cgo LDFLAGS: -lnmsg +#include +#include +*/ +import "C" +import "runtime" + +// A Rate provides Rate limiting across one or more outputs. +type Rate struct{ rate C.nmsg_rate_t } + +// NewRate initializes and returns a rate context. The rate parameter +// specifies the target rate of packets (containers and fragments) sent +// on all outputs using the Rate. The freq parameter specifies how often +// (in packets) to check the rate limit. +func NewRate(rate, freq uint) *Rate { + r := &Rate{C.nmsg_rate_init(C.uint(rate), C.uint(freq))} + runtime.SetFinalizer(r, func(r *Rate) { + C.nmsg_rate_destroy(&r.rate) + }) + return r +} + +// Sleep pauses for an appropriate amount of time to maintain the given +// output rate. +func (r *Rate) Sleep() { + C.nmsg_rate_sleep(r.rate) +} diff --git a/cgo-nmsg/xs.go b/cgo-nmsg/xs.go new file mode 100644 index 0000000..a92d932 --- /dev/null +++ b/cgo-nmsg/xs.go @@ -0,0 +1,49 @@ +// +build libxs + +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +/* +#cgo pkg-config: libnmsg libxs +#cgo LDFLAGS: -lnmsg -lxs +#include +#include +#include +*/ +import "C" +import "unsafe" + +var xsContext unsafe.Pointer + +func init() { + xsContext = C.xs_init() +} + +// NewXSInput opens an Input reading from the given XS endpoint. +func NewXSInput(xep string) Input { + cxep := C.CString(xep) + defer C.free(unsafe.Pointer(cxep)) + inp := C.nmsg_input_open_xs_endpoint(xsContext, cxep) + if inp == nil { + return nil + } + return &nmsgInput{input: inp} +} + +// NewXSOutput creates an output writing to the given XS endpoint. +func NewXSOutput(xep string, bufsiz int) Output { + cxep := C.CString(xep) + defer C.free(unsafe.Pointer(cxep)) + outp := C.nmsg_output_open_xs_endpoint(xsContext, cxep, C.size_t(bufsiz)) + if outp == nil { + return nil + } + return &nmsgOutput{output: outp} +} diff --git a/container.go b/container.go new file mode 100644 index 0000000..b7c32e8 --- /dev/null +++ b/container.go @@ -0,0 +1,363 @@ +/* + * Copyright (c) 2017,2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/rand" + + "github.com/golang/protobuf/proto" +) + +const ( + nmsgVersion = 2 + nmsgFlagZlib = 1 + nmsgFlagFragment = 2 + headerSize = 10 +) + +var ( + nmsgMagic = [4]byte{'N', 'M', 'S', 'G'} + errBadMagic = errors.New("Bad NMSG Magic Number") + u32max uint32 = (1 << 31) + containerOverhead = 10 + fragmentOverhead = 10 + 4 + proto.Size( + &NmsgFragment{ + Id: &u32max, + Current: &u32max, + Last: &u32max, + Crc: &u32max, + }) +) + +type containerHeader struct { + Magic [4]byte + Flags, Version byte + Length uint32 +} + +// isCompressed() and isFragmented() are helper functions for readability. +func (h *containerHeader) isCompressed() bool { + return h.Flags&nmsgFlagZlib != 0 +} + +func (h *containerHeader) isFragmented() bool { + return h.Flags&nmsgFlagFragment != 0 +} + +// A Container encapsulates an Nmsg envelope, and maintains metadata for +// sizing containers as payloads are added. +type Container struct { + // Maximum size of a container. AddPayload attempts to keep the container + // under this size. + maxSize int + // Maximum size of fragment or container. Any containers larger than this + // will be fragmented by WriteTo. + writeSize int + // If true, compress container contents before writing. + compress bool + // If true, container was populated from compressed data + // This is primarily used in fragment reassembly to detect whether the + // fragmented data was compressed prior to fragmentation. + isCompressed bool + // If nonzero, an estimate of the effectiveness of compression, expressed + // as compressedSize / uncompressedSize. Default: 0.5 + compressionRatio float32 + // The current estimated size of the serialized data, before compression + size int + Nmsg + *NmsgFragment +} + +// NewContainer creates a new empty NMSG container. +func NewContainer() *Container { + c := &Container{size: containerOverhead} + c.SetMaxSize(0, 0) + return c +} + +// SetMaxSize sets the maximum size (including Marshaling overhead, +// container header, and anticipated compression ratio) of a container. +// AddPayload attempts to keep the container within this size. +// +// writeSize specifies the maximum size of containers or fragments. +// Containers larger than writeSize will be written as fragments instead +// of single containers. +// +// A writeSize value of 0 is treated as equal to size. +func (c *Container) SetMaxSize(size, writeSize int) { + if size < MinContainerSize { + size = MinContainerSize + } + if size > MaxContainerSize { + size = MaxContainerSize + } + if writeSize < size { + writeSize = size + } + c.maxSize = size + c.writeSize = writeSize +} + +// SetCompression instructs WriteTo to write containers with compressed +// (if true) or uncompressed (if false) contents. +func (c *Container) SetCompression(compress bool) { + c.compress = compress +} + +// SetCompressionRatio sets an estimated compression ratio for the data. +// The default value is 2.0 +func (c *Container) SetCompressionRatio(ratio float32) { + c.compressionRatio = ratio +} + +// SetSequenced sets or unsets sequencing on the container stream. +// The sequence number is updated every time WriteTo() is called. +func (c *Container) SetSequenced(sequenced bool) { + if sequenced { + seqid := uint64(rand.Uint32()) << 32 + seqid |= uint64(rand.Uint32()) + c.Nmsg.SequenceId = proto.Uint64(seqid) + c.Nmsg.Sequence = proto.Uint32(0) + } else { + c.Nmsg.SequenceId = nil + c.Nmsg.Sequence = nil + } +} + +// AddPayload adds the supplied NmsgPayload to the Container if possible. +// +// The return value 'full' is true if the container is full and needs to +// be emptied with WriteTo(). +// +// The return value 'ok' is true if the payload was successfully added to +// the container, otherwise, AddPayload() must be called again after WriteTo(). +// +// Both ok and full may be true if the payload is larger than the container's +// MaxSize, or if the container is full after adding the payload. +func (c *Container) AddPayload(p *NmsgPayload) (ok, full bool) { + limit := c.maxSize + if c.compress { + if c.compressionRatio > 0 { + limit = int(float32(limit) * c.compressionRatio) + } else { + limit *= 2 + } + } + ps := p.payloadSize() + if c.size+ps >= limit { + full = true + } + + if !full || c.size == containerOverhead || c.size+ps == limit { + ok = true + c.size += ps + c.Nmsg.Payloads = append(c.Nmsg.Payloads, p) + c.Nmsg.PayloadCrcs = append(c.Nmsg.PayloadCrcs, nmsgCRC(p.Payload)) + } + + return +} + +// Reset discards payloads and crcs from the Container +func (c *Container) Reset() { + c.Nmsg.Payloads = c.Nmsg.Payloads[:0] + c.Nmsg.PayloadCrcs = c.Nmsg.PayloadCrcs[:0] + c.NmsgFragment = nil +} + +// WriteTo writes the Container to Writer w. If the +// container requires fragmentation, it will call +// w.Write() multiple times. +func (c *Container) WriteTo(w io.Writer) (int64, error) { + var buf bytes.Buffer + + header := containerHeader{ + Magic: nmsgMagic, + Version: nmsgVersion, + } + + defer c.Reset() + + b, err := proto.Marshal(&c.Nmsg) + if err != nil { + return 0, err + } + + if c.compress { + b, err = zbufDeflate(b) + if err != nil { + return 0, err + } + header.Flags |= nmsgFlagZlib + } + + header.Length = uint32(len(b)) + if c.Nmsg.Sequence != nil { + *c.Nmsg.Sequence++ + } + c.size = containerOverhead + + if len(b)+containerOverhead > c.writeSize { + return c.writeFragments(w, b) + } + + if err = binary.Write(&buf, binary.BigEndian, &header); err != nil { + return 0, err + } + + if _, err = buf.Write(b); err != nil { + return 0, err + } + + return buf.WriteTo(w) +} + +func (c *Container) writeFragments(w io.Writer, b []byte) (int64, error) { + header := containerHeader{ + Magic: nmsgMagic, + Version: nmsgVersion, + Flags: nmsgFlagFragment, + } + + if c.compress { + header.Flags |= nmsgFlagZlib + } + + fragSize := c.writeSize - fragmentOverhead + lastFrag := len(b) / fragSize + fragID := rand.Uint32() + + nf := NmsgFragment{ + Id: proto.Uint32(fragID), + Current: proto.Uint32(uint32(0)), + Last: proto.Uint32(uint32(lastFrag)), + Crc: proto.Uint32(nmsgCRC(b)), + } + + var written int64 + for i := 0; i <= lastFrag; i++ { + var buf bytes.Buffer + + fblen := len(b) + if fblen > fragSize { + fblen = fragSize + } + + *nf.Current = uint32(i) + nf.Fragment = b[:fblen] + b = b[fblen:] + + fbytes, err := proto.Marshal(&nf) + if err != nil { + return written, err + } + + header.Length = uint32(len(fbytes)) + if err = binary.Write(&buf, binary.BigEndian, header); err != nil { + return written, err + } + + if _, err = buf.Write(fbytes); err != nil { + return written, err + } + + n, err := buf.WriteTo(w) + if err != nil { + return written, err + } + written += n + } + return written, nil +} + +// ReadFrom Reads a Container from the given io.Reader. It returns the +// number of container bytes read on success. +func (c *Container) ReadFrom(r io.Reader) (n int64, err error) { + /* + * The bytes.Buffer Grow() method may panic with ErrTooLarge. + * We catch this panic (and any other error panic()s and return + * an error. + */ + defer func() { + if r := recover(); r != nil { + var ok bool + if err, ok = r.(error); !ok { + err = fmt.Errorf("nmsg.Container ReadFrom: panic %v", r) + } + } + }() + var buf bytes.Buffer + var h containerHeader + if n, err = io.CopyN(&buf, r, headerSize); err != nil { + return n, err + } + + err = binary.Read(&buf, binary.BigEndian, &h) + if err != nil { + return n, &dataError{err} + } + if h.Magic != nmsgMagic { + return 0, &dataError{errBadMagic} + } + + buf.Grow(int(h.Length)) + if n, err = io.CopyN(&buf, r, int64(h.Length)); err != nil { + return int64(buf.Len()), err + } + + // err = c.fromBytesHeader(buf.Bytes(), &h) + err = c.fromNmsgBytes(buf.Bytes(), h.isCompressed(), h.isFragmented()) + if err != nil { + err = &dataError{err} + } + return int64(buf.Len()), err +} + +// FromBytes parses the given buffer as an NMSG container and stores +// the result in the receiver *Container. +func (c *Container) FromBytes(b []byte) error { + var h containerHeader + buf := bytes.NewBuffer(b) + err := binary.Read(buf, binary.BigEndian, &h) + if err != nil { + return err + } + if h.Magic != nmsgMagic { + return errBadMagic + } + + return c.fromNmsgBytes(buf.Bytes(), h.isCompressed(), h.isFragmented()) +} + +// fromNmsgBytes parses the contents (b) of an NMSG container, according to +// whether the container contents are compressed, fragmented, or both. +func (c *Container) fromNmsgBytes(b []byte, compressed, fragmented bool) error { + var err error + cbytes := b + c.isCompressed = compressed + if compressed { + cbytes, err = zbufInflate(b) + if err != nil { + return err + } + } + + if fragmented { + c.NmsgFragment = &NmsgFragment{} + return proto.Unmarshal(cbytes, c.NmsgFragment) + } + + c.NmsgFragment = nil + return proto.Unmarshal(cbytes, &c.Nmsg) +} diff --git a/container_test.go b/container_test.go new file mode 100644 index 0000000..b70436c --- /dev/null +++ b/container_test.go @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_test + +// These tests verify container compatibility between C libnmsg (wrapped in +// cgo-nmsg) and go-nmsg, both with and without compression. + +import ( + "bytes" + "log" + "testing" + + cnmsg "github.com/farsightsec/go-nmsg/cgo-nmsg" + "github.com/farsightsec/go-nmsg" + "github.com/farsightsec/go-nmsg/nmsg_base" +) + +func compare(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestContainerGoCgoUnpack(t *testing.T) { + b := new(bytes.Buffer) + c := nmsg.NewContainer() + c.SetMaxSize(nmsg.MinContainerSize, nmsg.MinContainerSize) + c.AddPayload(testGoMessage(100)) + c.WriteTo(b) + + m, err := cnmsg.UnpackContainer(b.Bytes()) + if err != nil { + t.Fatal(err) + } + + if len(m) != 1 { + t.Fatalf("message count mismatch %d != 1", len(m)) + } + + if checkCgoMessage(m[0], 100) { + return + } + + t.Error("payload mismatch") +} + +func TestContainerGoCgoUnpackCompress(t *testing.T) { + b := new(bytes.Buffer) + c := nmsg.NewContainer() + c.SetCompression(true) + c.SetMaxSize(nmsg.MinContainerSize, nmsg.MinContainerSize) + c.AddPayload(testGoMessage(100)) + c.WriteTo(b) + + byt := b.Bytes() + + m, err := cnmsg.UnpackContainer(byt) + if err != nil { + t.Fatal(err) + } + + if len(m) != 1 { + t.Fatalf("message count mismatch %d != 1", len(m)) + } + + if checkCgoMessage(m[0], 100) { + return + } + + t.Error("payload mismatch") +} + +func testCgoMessage(size int) *cnmsg.Message { + mod := cnmsg.MessageModLookupByName("base", "encode") + if mod == nil { + log.Fatal("module not found") + } + msg := cnmsg.NewMessage(mod) + if err := msg.SetEnumField("type", 0, "TEXT"); err != nil { + log.Fatal(err) + } + + if err := msg.SetBytesField("payload", 0, make([]byte, size)); err != nil { + log.Fatal(err) + } + return msg +} + +func checkCgoMessage(m *cnmsg.Message, size int) bool { + b, err := m.GetBytesField("payload", 0) + if err != nil { + return false + } + return compare(b, make([]byte, size)) +} + +func testGoMessage(size int) *nmsg.NmsgPayload { + m := new(nmsg_base.Encode) + m.Payload = make([]byte, size) + m.Type = nmsg_base.EncodeType_TEXT.Enum() + p, err := nmsg.Payload(m) + if err != nil { + log.Fatal(err) + } + return p +} + +func checkGoMessage(m nmsg.Message, size int) bool { + enc, ok := m.(*nmsg_base.Encode) + + if !ok { + log.Printf("type mismatch: %T != *nmsg_base.Encode", m) + return false + } + return compare(enc.GetPayload(), make([]byte, size)) +} + +func TestContainerCgoGoUnpack(t *testing.T) { + c := cnmsg.NewContainer(&cnmsg.ContainerConfig{ + Size: cnmsg.BufferSizeMin, + }) + c.Add(testCgoMessage(100)) + + i := nmsg.NewInput(bytes.NewReader(c.Bytes()), cnmsg.BufferSizeMin) + p, err := i.Recv() + if err != nil { + t.Fatal(err) + } + + m, err := p.Message() + if err != nil { + t.Fatal(err) + } + + if checkGoMessage(m, 100) { + return + } + + t.Error("payload mismatch") +} + +func TestContainerCgoGoUnpackCompress(t *testing.T) { + c := cnmsg.NewContainer(&cnmsg.ContainerConfig{ + Size: cnmsg.BufferSizeMin, + Compress: true, + }) + c.Add(testCgoMessage(100)) + + byt := c.Bytes() + i := nmsg.NewInput(bytes.NewReader(byt), cnmsg.BufferSizeMin) + p, err := i.Recv() + if err != nil { + t.Fatal(err) + } + + m, err := p.Message() + if err != nil { + t.Fatal(err) + } + + if checkGoMessage(m, 100) { + return + } + + t.Error("payload mismatch") +} diff --git a/endian.go b/endian.go new file mode 100644 index 0000000..16a2bbc --- /dev/null +++ b/endian.go @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +// The nmsg C library renders checksums in network byte order before presenting +// them to the protobuf-c library as uint32 values. While Go's encoding/binary +// library can format and parse uint32 values as BigEndian or LittleEndian byte +// arrays, this is not sufficient to calculate an integer that will represent +// a BigEndian (network) byte array in the host's native byte order. This +// requires determining the host's byte order, a task which Go's type system +// makes cumbersome. +// +// This file uses the "unsafe" package to defeat Go's type system for the +// purposes of determining whether the package is running on a BigEndian or +// LittleEndian machine, and uses this information to implement htonl. + +import ( + "encoding/binary" + "unsafe" +) + +var hostEndian binary.ByteOrder + +func init() { + n := uint32(1) + b := *(*[4]byte)(unsafe.Pointer(&n)) + if b[0] == 1 { + hostEndian = binary.LittleEndian + } else { + hostEndian = binary.BigEndian + } +} + +func htonl(n uint32) uint32 { + var buf [4]byte + hostEndian.PutUint32(buf[:], n) + return binary.BigEndian.Uint32(buf[:]) +} diff --git a/input.go b/input.go new file mode 100644 index 0000000..77ae9f8 --- /dev/null +++ b/input.go @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2017,2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "bufio" + "fmt" + "io" + "time" +) + +// An Input is a source of NMSG Payloads. +type Input interface { + // Recv() returns the next Nmsg Payload from the input, + // blocking if none is available. + Recv() (*NmsgPayload, error) + // Stats() returns interface statistics + Stats() *InputStatistics +} + +// InputStatistics holds useful metrics for input performance. +type InputStatistics struct { + // Count of total container received, including fragments + InputContainers uint64 + // Count of total bytes received and processed + InputBytes uint64 + // Count of containers marked lost by sequence tracking + LostContainers uint64 + // Count of fragment containers received + InputFragments uint64 + // Count of fragments expired from cache + ExpiredFragments uint64 + // Count of containers dropped due to incomplete fragments + PartialContainers uint64 +} + +type dataError struct{ error } + +func (d *dataError) Error() string { return d.error.Error() } + +// IsDataError returns true of the supplied error is an error unpacking +// or decoding the NMSG data rather than an I/O error with the input. +func IsDataError(err error) bool { + _, ok := err.(*dataError) + return ok +} + +type input struct { + r io.Reader + n Nmsg + fcache *fragCache + scache *seqCache + stats InputStatistics +} + +func (i *input) Stats() *InputStatistics { + res := &InputStatistics{} + *res = i.stats + return res +} + +// NewInput constructs an input from the supplied Reader. +// The size parameter sizes the input buffer, and should +// be greater than the maximum anticipated container size +// for datagram inputs. +func NewInput(r io.Reader, size int) Input { + return &input{ + r: bufio.NewReaderSize(r, size), + n: Nmsg{}, + fcache: newFragmentCache(2 * time.Minute), + scache: newSequenceCache(2 * time.Minute), + } +} + +type checksumError struct { + calc, wire uint32 +} + +func (c *checksumError) Error() string { + return fmt.Sprintf("checksum mismatch: %x != %x", c.calc, c.wire) +} + +func (i *input) Recv() (*NmsgPayload, error) { + for len(i.n.Payloads) == 0 { + var c Container + n, err := c.ReadFrom(i.r) + if err != nil { + return nil, err + } + if n == 0 { + return nil, io.EOF + } + + i.stats.InputBytes += uint64(n) + + if c.NmsgFragment != nil { + i.stats.InputFragments++ + var b []byte + if b = i.fcache.Insert(c.NmsgFragment); b == nil { + continue + } + err = c.fromNmsgBytes(b, c.isCompressed, false) + if err != nil { + return nil, &dataError{err} + } + } + + i.stats.InputContainers++ + i.stats.LostContainers += uint64(i.scache.Update(&c.Nmsg)) + i.scache.Expire() + i.n = c.Nmsg + } + ccount, fcount := i.fcache.Expire() + i.stats.PartialContainers += uint64(ccount) + i.stats.ExpiredFragments += uint64(fcount) + p := i.n.Payloads[0] + i.n.Payloads = i.n.Payloads[1:] + + var err error + if len(i.n.PayloadCrcs) > 0 { + wire := i.n.PayloadCrcs[0] + calc := nmsgCRC(p.Payload) + if wire != calc { + err = &dataError{&checksumError{calc, wire}} + } + i.n.PayloadCrcs = i.n.PayloadCrcs[1:] + } + + return p, err +} diff --git a/input_frag.go b/input_frag.go new file mode 100644 index 0000000..56d5b9a --- /dev/null +++ b/input_frag.go @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +// NMSG Fragment Cache. + +import ( + "bytes" + "container/list" + "sort" + "time" +) + +type fragCacheEntry struct { + lastUsed time.Time + id uint32 + frags fragList +} + +// fragList implements sort.Interface to support sorting fragments on +// their "Current" field prior to reassembly. +type fragList []*NmsgFragment + +func (fl fragList) Len() int { return len(fl) } +func (fl fragList) Less(i, j int) bool { return fl[i].GetCurrent() < fl[j].GetCurrent() } +func (fl fragList) Swap(i, j int) { fl[i], fl[j] = fl[j], fl[i] } + +type fragCache struct { + expiry time.Duration + idmap map[uint32]*list.Element + lru *list.List +} + +func newFragmentCache(expiry time.Duration) *fragCache { + return &fragCache{ + expiry: expiry, + idmap: make(map[uint32]*list.Element), + lru: list.New(), + } +} + +// Expire too-old entries from the fragment cache, returning the number +// of incomplete containers and fragments dropped. +func (fc *fragCache) Expire() (containers, frags int) { + for fc.lru.Len() > 0 { + lruent := fc.lru.Front() + ent := lruent.Value.(*fragCacheEntry) + if time.Since(ent.lastUsed) <= fc.expiry { + break + } + containers++ + frags += len(ent.frags) + fc.lru.Remove(lruent) + delete(fc.idmap, ent.id) + } + return +} + +// Inserts a fragment into the cache. If the fragment completes a fragmented +// container, Insert returns the reassembled container body. Otherwise, returns +// nil. +func (fc *fragCache) Insert(f *NmsgFragment) []byte { + id := f.GetId() + lruent, ok := fc.idmap[id] + if !ok { + fc.idmap[id] = fc.lru.PushBack( + &fragCacheEntry{ + lastUsed: time.Now(), + id: id, + frags: fragList{f}, + }) + return nil + } + + ent := lruent.Value.(*fragCacheEntry) + for i := range ent.frags { + if ent.frags[i].GetCurrent() == f.GetCurrent() { + /* duplicate fragment */ + return nil + } + } + ent.frags = append(ent.frags, f) + if ent.frags.Len() <= int(f.GetLast()) { + ent.lastUsed = time.Now() + fc.lru.MoveToBack(lruent) + return nil + } + fc.lru.Remove(lruent) + delete(fc.idmap, id) + + /* sort and reassemble fragments */ + sort.Sort(ent.frags) + var b bytes.Buffer + for i := range ent.frags { + b.Write(ent.frags[i].GetFragment()) + } + return b.Bytes() +} diff --git a/input_seq.go b/input_seq.go new file mode 100644 index 0000000..f252833 --- /dev/null +++ b/input_seq.go @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "container/list" + "time" +) + +type seqCacheEntry struct { + lastUsed time.Time + seqid uint64 + nextSeq uint32 +} + +type seqCache struct { + expiry time.Duration + idmap map[uint64]*list.Element + lru *list.List +} + +func newSequenceCache(expiry time.Duration) *seqCache { + return &seqCache{ + expiry: expiry, + idmap: make(map[uint64]*list.Element), + lru: list.New(), + } +} + +const maxDrop = 1048576 + +func (sc *seqCache) Update(n *Nmsg) (missed int) { + if n.Sequence == nil || n.SequenceId == nil { + return + } + seqid := n.GetSequenceId() + lruent, ok := sc.idmap[seqid] + if !ok { + sc.idmap[seqid] = sc.lru.PushBack( + &seqCacheEntry{ + lastUsed: time.Now(), + seqid: seqid, + nextSeq: n.GetSequence() + 1, + }) + return 0 + } + seq := n.GetSequence() + ent := lruent.Value.(*seqCacheEntry) + + ent.lastUsed = time.Now() + sc.lru.MoveToBack(lruent) + + if seq == ent.nextSeq { + ent.nextSeq++ + return 0 + } + + if seq > ent.nextSeq { + if seq-ent.nextSeq < maxDrop { + missed = int(seq - ent.nextSeq) + } + ent.nextSeq = seq + 1 + return missed + } + + delta := int64(int64(seq) + (1 << 32) - int64(ent.nextSeq)) + if delta < maxDrop { + missed = int(delta) + } + + ent.nextSeq = seq + 1 + return missed +} + +func (sc *seqCache) Expire() { + for sc.lru.Len() > 0 { + lruent := sc.lru.Front() + ent := lruent.Value.(*seqCacheEntry) + if time.Since(ent.lastUsed) <= sc.expiry { + break + } + sc.lru.Remove(lruent) + delete(sc.idmap, ent.seqid) + } +} diff --git a/input_test.go b/input_test.go new file mode 100644 index 0000000..7145d9a --- /dev/null +++ b/input_test.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_test + +import ( + "bytes" + "io" + "math" + "testing" + + "github.com/farsightsec/go-nmsg" +) + +func testReader(t *testing.T, n, size, mtu int) io.Reader { + buf := new(bytes.Buffer) + // nw := nmsg.NewWriter(w, mtu) + o := nmsg.BufferedOutput(buf) + o.SetMaxSize(mtu, 0) + o.SetSequenced(true) + + p, err := nmsg.Payload(testMessage(size)) + if err != nil { + t.Error(err.Error()) + return nil + } + + for i := 0; i < n; i++ { + o.Send(p) + } + + o.Close() + + t.Logf("testReader: buf = %d bytes (%d, %d, %d)", buf.Len(), n, size, mtu) + return buf +} + +func TestInput(t *testing.T) { + for _, mtu := range []int{0, 512, 1500} { + for _, n := range []int{1, 10, 100} { + for _, size := range []int{64, 256, 4096} { + i := nmsg.NewInput(testReader(t, n, size, mtu), mtu) + if i != nil { + c := 0 + for { + _, err := i.Recv() + if err != nil { + if err != io.EOF { + t.Error(err) + } + break + } + c++ + } + if c < n { + t.Errorf("(%d,%d,%d) expected %d, received %d", n, size, mtu, n, c) + } + } + } + } + } +} + +func TestInputFragExpire(t *testing.T) { + // Fragment expiration is not checked here, only in + // coverage. + var readers []io.Reader + npayloads := 10 + payloadSize := 512 + mtu := 512 + for i := 0; i < 1000; i++ { + readers = append(readers, testReader(t, npayloads, + payloadSize, mtu)) + } + inp := nmsg.NewInput(io.MultiReader(readers...), 512) + var count int + for ; ; count++ { + _, err := inp.Recv() + if err != nil { + break + } + } + if count != npayloads*1000 { + t.Errorf("missed input, received %d payloads", count) + } +} + +func testLoss(t *testing.T, r io.Reader, loss uint64, title string) { + t.Helper() + i := nmsg.NewInput(r, nmsg.MaxContainerSize) + for { + if _, err := i.Recv(); err != nil { + break + } + } + stats := i.Stats() + if stats.LostContainers != loss { + t.Errorf("%s: lost %d (expected %d)", title, stats.LostContainers, loss) + } +} + +func TestInputSequenceLoss1(t *testing.T) { + var buf bytes.Buffer + c := nmsg.NewContainer() + + c.SetSequenced(true) + c.WriteTo(&buf) + c.WriteTo(&buf) + *c.Nmsg.Sequence++ // skip one + c.WriteTo(&buf) + + testLoss(t, &buf, 1, "drop 1") +} + +func TestInputSequenceInterleaveLoss1(t *testing.T) { + var buf bytes.Buffer + + c1 := nmsg.NewContainer() + c2 := nmsg.NewContainer() + c1.SetSequenced(true) + c2.SetSequenced(true) + + c1.WriteTo(&buf) + c2.WriteTo(&buf) + c2.WriteTo(&buf) + c1.WriteTo(&buf) + c2.WriteTo(&buf) + *c1.Nmsg.Sequence++ + c1.WriteTo(&buf) + c2.WriteTo(&buf) + testLoss(t, &buf, 1, "interleaved, drop 1") +} + +func TestInputSequenceWrap(t *testing.T) { + var buf bytes.Buffer + + c := nmsg.NewContainer() + c.SetSequenced(true) + *c.Nmsg.Sequence = math.MaxUint32 - 1 + t.Log("sequence", c.Nmsg.GetSequence()) + c.WriteTo(&buf) + t.Log("sequence", c.Nmsg.GetSequence()) + *c.Nmsg.Sequence++ + t.Log("sequence", c.Nmsg.GetSequence()) + c.WriteTo(&buf) + t.Log("sequence", c.Nmsg.GetSequence()) + testLoss(t, &buf, 1, "wrapped, drop 1") +} diff --git a/nmsg.go b/nmsg.go new file mode 100644 index 0000000..c605f8b --- /dev/null +++ b/nmsg.go @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +//go:generate protoc --go_out=. nmsg.proto + +package nmsg + +import ( + "hash/crc32" + + "github.com/golang/protobuf/proto" +) + +// Container size limits to avoid silly fragmentation and memory +// exhaustion. +const ( + MinContainerSize = 512 + MaxContainerSize = 1048576 + EtherContainerSize = 1280 + invalidContainerSize = MaxContainerSize * 16 +) + +var crc32c = crc32.MakeTable(crc32.Castagnoli) + +// nmsgCRC calculates a crc32 checksum compatible with that used by +// the nmsg C library. +// +// As in the C library, the checksum is converted to network byte order +// before eventually being encoded as a protocol buffers integer. This +// defeats the endian neutrality of protocol buffers, but is necessary +// for compatibility with the C library operating on little endian machines. +func nmsgCRC(b []byte) uint32 { + return htonl(crc32.Checksum(b, crc32c)) +} + +// Message encapsulates a protobuf-encoded payload. +// +// The values returned by the GetVid() and GetMsgtype() methods return +// identify the format of the payload. +type Message interface { + proto.Message + GetVid() uint32 + GetMsgtype() uint32 +} diff --git a/nmsg.pb.go b/nmsg.pb.go new file mode 100644 index 0000000..6b05995 --- /dev/null +++ b/nmsg.pb.go @@ -0,0 +1,223 @@ +// Code generated by protoc-gen-go. +// source: nmsg.proto +// DO NOT EDIT! + +/* +Package nmsg is a generated protocol buffer package. + +It is generated from these files: + nmsg.proto + +It has these top-level messages: + Nmsg + NmsgFragment + NmsgPayload +*/ +package nmsg + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Nmsg struct { + Payloads []*NmsgPayload `protobuf:"bytes,1,rep,name=payloads" json:"payloads,omitempty"` + PayloadCrcs []uint32 `protobuf:"varint,2,rep,name=payload_crcs" json:"payload_crcs,omitempty"` + Sequence *uint32 `protobuf:"varint,3,opt,name=sequence" json:"sequence,omitempty"` + SequenceId *uint64 `protobuf:"varint,4,opt,name=sequence_id" json:"sequence_id,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Nmsg) Reset() { *m = Nmsg{} } +func (m *Nmsg) String() string { return proto.CompactTextString(m) } +func (*Nmsg) ProtoMessage() {} +func (*Nmsg) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *Nmsg) GetPayloads() []*NmsgPayload { + if m != nil { + return m.Payloads + } + return nil +} + +func (m *Nmsg) GetPayloadCrcs() []uint32 { + if m != nil { + return m.PayloadCrcs + } + return nil +} + +func (m *Nmsg) GetSequence() uint32 { + if m != nil && m.Sequence != nil { + return *m.Sequence + } + return 0 +} + +func (m *Nmsg) GetSequenceId() uint64 { + if m != nil && m.SequenceId != nil { + return *m.SequenceId + } + return 0 +} + +type NmsgFragment struct { + Id *uint32 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` + Current *uint32 `protobuf:"varint,2,req,name=current" json:"current,omitempty"` + Last *uint32 `protobuf:"varint,3,req,name=last" json:"last,omitempty"` + Fragment []byte `protobuf:"bytes,4,req,name=fragment" json:"fragment,omitempty"` + Crc *uint32 `protobuf:"varint,5,opt,name=crc" json:"crc,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *NmsgFragment) Reset() { *m = NmsgFragment{} } +func (m *NmsgFragment) String() string { return proto.CompactTextString(m) } +func (*NmsgFragment) ProtoMessage() {} +func (*NmsgFragment) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } + +func (m *NmsgFragment) GetId() uint32 { + if m != nil && m.Id != nil { + return *m.Id + } + return 0 +} + +func (m *NmsgFragment) GetCurrent() uint32 { + if m != nil && m.Current != nil { + return *m.Current + } + return 0 +} + +func (m *NmsgFragment) GetLast() uint32 { + if m != nil && m.Last != nil { + return *m.Last + } + return 0 +} + +func (m *NmsgFragment) GetFragment() []byte { + if m != nil { + return m.Fragment + } + return nil +} + +func (m *NmsgFragment) GetCrc() uint32 { + if m != nil && m.Crc != nil { + return *m.Crc + } + return 0 +} + +type NmsgPayload struct { + Vid *uint32 `protobuf:"varint,1,req,name=vid" json:"vid,omitempty"` + Msgtype *uint32 `protobuf:"varint,2,req,name=msgtype" json:"msgtype,omitempty"` + TimeSec *int64 `protobuf:"varint,3,req,name=time_sec" json:"time_sec,omitempty"` + TimeNsec *uint32 `protobuf:"fixed32,4,req,name=time_nsec" json:"time_nsec,omitempty"` + Payload []byte `protobuf:"bytes,5,opt,name=payload" json:"payload,omitempty"` + Source *uint32 `protobuf:"varint,7,opt,name=source" json:"source,omitempty"` + Operator *uint32 `protobuf:"varint,8,opt,name=operator" json:"operator,omitempty"` + Group *uint32 `protobuf:"varint,9,opt,name=group" json:"group,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *NmsgPayload) Reset() { *m = NmsgPayload{} } +func (m *NmsgPayload) String() string { return proto.CompactTextString(m) } +func (*NmsgPayload) ProtoMessage() {} +func (*NmsgPayload) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } + +func (m *NmsgPayload) GetVid() uint32 { + if m != nil && m.Vid != nil { + return *m.Vid + } + return 0 +} + +func (m *NmsgPayload) GetMsgtype() uint32 { + if m != nil && m.Msgtype != nil { + return *m.Msgtype + } + return 0 +} + +func (m *NmsgPayload) GetTimeSec() int64 { + if m != nil && m.TimeSec != nil { + return *m.TimeSec + } + return 0 +} + +func (m *NmsgPayload) GetTimeNsec() uint32 { + if m != nil && m.TimeNsec != nil { + return *m.TimeNsec + } + return 0 +} + +func (m *NmsgPayload) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *NmsgPayload) GetSource() uint32 { + if m != nil && m.Source != nil { + return *m.Source + } + return 0 +} + +func (m *NmsgPayload) GetOperator() uint32 { + if m != nil && m.Operator != nil { + return *m.Operator + } + return 0 +} + +func (m *NmsgPayload) GetGroup() uint32 { + if m != nil && m.Group != nil { + return *m.Group + } + return 0 +} + +func init() { + proto.RegisterType((*Nmsg)(nil), "nmsg.Nmsg") + proto.RegisterType((*NmsgFragment)(nil), "nmsg.NmsgFragment") + proto.RegisterType((*NmsgPayload)(nil), "nmsg.NmsgPayload") +} + +func init() { proto.RegisterFile("nmsg.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 259 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0x8f, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0x86, 0x15, 0xc7, 0x25, 0xe9, 0xc5, 0x05, 0x6a, 0x18, 0x3c, 0x46, 0x61, 0xe9, 0xd4, 0x81, + 0x87, 0x60, 0x44, 0x8c, 0x6c, 0x91, 0xe5, 0x9a, 0x50, 0xa9, 0x89, 0x8d, 0xed, 0x20, 0xf5, 0x35, + 0x78, 0x62, 0xee, 0xdc, 0x54, 0xea, 0x94, 0xf8, 0xfb, 0xef, 0xbe, 0xbb, 0x03, 0x98, 0xc6, 0x38, + 0xec, 0x7d, 0x70, 0xc9, 0x49, 0x4e, 0xff, 0xdd, 0x37, 0xf0, 0x77, 0xfc, 0xca, 0x17, 0xa8, 0xbd, + 0x3e, 0x9f, 0x9c, 0x3e, 0x44, 0x55, 0xb4, 0xe5, 0xae, 0x79, 0xdd, 0xee, 0x73, 0x31, 0xa5, 0x1f, + 0x97, 0x44, 0x3e, 0x83, 0x58, 0x8a, 0x7a, 0x13, 0x4c, 0x54, 0x0c, 0x0b, 0x37, 0xf2, 0x11, 0xea, + 0x68, 0x7f, 0x66, 0x3b, 0x19, 0xab, 0xca, 0xb6, 0x40, 0xf2, 0x04, 0xcd, 0x95, 0xf4, 0xc7, 0x83, + 0xe2, 0x08, 0x79, 0xf7, 0x09, 0x82, 0x5c, 0x6f, 0x41, 0x0f, 0xa3, 0x9d, 0x92, 0x04, 0x60, 0x98, + 0x15, 0x2d, 0xc3, 0x86, 0x07, 0xa8, 0xcc, 0x1c, 0x02, 0x62, 0x74, 0x12, 0x10, 0xc0, 0x4f, 0x3a, + 0x26, 0xf4, 0xb1, 0xcb, 0x84, 0xaf, 0xa5, 0x0d, 0x65, 0x6c, 0x27, 0x64, 0x03, 0x25, 0x6e, 0xa0, + 0x56, 0x34, 0xae, 0xfb, 0x2b, 0xa0, 0xb9, 0x5d, 0x13, 0xc3, 0xdf, 0x5b, 0x35, 0x46, 0xe9, 0xec, + 0xed, 0xa2, 0x46, 0x59, 0x3a, 0x8e, 0xb6, 0x8f, 0xd6, 0x64, 0x7d, 0x29, 0xb7, 0xb0, 0xce, 0x64, + 0x22, 0x44, 0xfe, 0x8a, 0xba, 0x96, 0x4b, 0xf3, 0x0c, 0x21, 0xef, 0xe1, 0x2e, 0xba, 0x39, 0xe0, + 0x89, 0x55, 0x3e, 0x11, 0x2d, 0xce, 0xdb, 0xa0, 0x93, 0x0b, 0xaa, 0xce, 0x64, 0x03, 0xab, 0x21, + 0xb8, 0xd9, 0xab, 0x35, 0x3d, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x73, 0x07, 0x52, 0x34, 0x6b, + 0x01, 0x00, 0x00, +} diff --git a/nmsg.proto b/nmsg.proto new file mode 100644 index 0000000..a3d0435 --- /dev/null +++ b/nmsg.proto @@ -0,0 +1,28 @@ +syntax = "proto2"; +package nmsg; + +message Nmsg { + repeated NmsgPayload payloads = 1; + repeated uint32 payload_crcs = 2; + optional uint32 sequence = 3; + optional uint64 sequence_id = 4; +} + +message NmsgFragment { + required uint32 id = 1; + required uint32 current = 2; + required uint32 last = 3; + required bytes fragment = 4; + optional uint32 crc = 5; +} + +message NmsgPayload { + required uint32 vid = 1; + required uint32 msgtype = 2; + required int64 time_sec = 3; + required fixed32 time_nsec = 4; + optional bytes payload = 5; + optional uint32 source = 7; + optional uint32 operator = 8; + optional uint32 group = 9; +} diff --git a/nmsg_base/base.go b/nmsg_base/base.go new file mode 100644 index 0000000..750d8ce --- /dev/null +++ b/nmsg_base/base.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_base + +import ( + "github.com/dnstap/golang-dnstap" + "github.com/farsightsec/go-nmsg" + "github.com/golang/protobuf/proto" +) + +func (p *Ncap) GetVid() uint32 { return 1 } +func (p *Ncap) GetMsgtype() uint32 { return 1 } + +func (p *Email) GetVid() uint32 { return 1 } +func (p *Email) GetMsgtype() uint32 { return 2 } + +func (p *Linkpair) GetVid() uint32 { return 1 } +func (p *Linkpair) GetMsgtype() uint32 { return 3 } + +func (p *Http) GetVid() uint32 { return 1 } +func (p *Http) GetMsgtype() uint32 { return 4 } + +func (p *IPConn) GetVid() uint32 { return 1 } +func (p *IPConn) GetMsgtype() uint32 { return 5 } + +func (p *LogLine) GetVid() uint32 { return 1 } +func (p *LogLine) GetMsgtype() uint32 { return 6 } + +func (p *Dns) GetVid() uint32 { return 1 } +func (p *Dns) GetMsgtype() uint32 { return 7 } + +func (p *Pkt) GetVid() uint32 { return 1 } +func (p *Pkt) GetMsgtype() uint32 { return 8 } + +func (p *DnsQR) GetVid() uint32 { return 1 } +func (p *DnsQR) GetMsgtype() uint32 { return 9 } + +func (p *Xml) GetVid() uint32 { return 1 } +func (p *Xml) GetMsgtype() uint32 { return 10 } + +func (p *Encode) GetVid() uint32 { return 1 } +func (p *Encode) GetMsgtype() uint32 { return 11 } + +func (p *Packet) GetVid() uint32 { return 1 } +func (p *Packet) GetMsgtype() uint32 { return 12 } + +type Dnstap struct { + dnstap.Dnstap +} + +func (d *Dnstap) GetVid() uint32 { return 1 } +func (d *Dnstap) GetMsgtype() uint32 { return 13 } + +func (d *Dnstap) Marshal() ([]byte, error) { + return proto.Marshal(&d.Dnstap) +} +func (d *Dnstap) Unmarshal(b []byte) error { + return proto.Unmarshal(b, &d.Dnstap) +} + +func init() { + nmsg.Register(&Ncap{}) + nmsg.Register(&Email{}) + nmsg.Register(&Linkpair{}) + nmsg.Register(&Http{}) + nmsg.Register(&IPConn{}) + nmsg.Register(&LogLine{}) + nmsg.Register(&Dns{}) + nmsg.Register(&Pkt{}) + nmsg.Register(&DnsQR{}) + nmsg.Register(&Xml{}) + nmsg.Register(&Encode{}) + nmsg.Register(&Packet{}) + nmsg.Register(&Dnstap{}) +} diff --git a/nmsg_base/dns.pb.go b/nmsg_base/dns.pb.go new file mode 100644 index 0000000..a51317d --- /dev/null +++ b/nmsg_base/dns.pb.go @@ -0,0 +1,152 @@ +// Code generated by protoc-gen-go. +// source: dns.proto +// DO NOT EDIT! + +/* +Package nmsg_base is a generated protocol buffer package. + +It is generated from these files: + dns.proto + dnsqr.proto + email.proto + encode.proto + http.proto + ipconn.proto + linkpair.proto + logline.proto + ncap.proto + packet.proto + pkt.proto + xml.proto + +It has these top-level messages: + Dns + DnsQR + Email + Encode + Http + IPConn + Linkpair + LogLine + Ncap + Packet + Pkt + Xml +*/ +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type Dns struct { + Section *uint32 `protobuf:"varint,6,opt,name=section" json:"section,omitempty"` + Qname []byte `protobuf:"bytes,7,opt,name=qname" json:"qname,omitempty"` + Qtype *uint32 `protobuf:"varint,8,opt,name=qtype" json:"qtype,omitempty"` + Qclass *uint32 `protobuf:"varint,9,opt,name=qclass" json:"qclass,omitempty"` + Rrname []byte `protobuf:"bytes,1,opt,name=rrname" json:"rrname,omitempty"` + Rrtype *uint32 `protobuf:"varint,2,opt,name=rrtype" json:"rrtype,omitempty"` + Rrclass *uint32 `protobuf:"varint,3,opt,name=rrclass" json:"rrclass,omitempty"` + Rrttl *uint32 `protobuf:"varint,4,opt,name=rrttl" json:"rrttl,omitempty"` + Rdata [][]byte `protobuf:"bytes,5,rep,name=rdata" json:"rdata,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Dns) Reset() { *m = Dns{} } +func (m *Dns) String() string { return proto.CompactTextString(m) } +func (*Dns) ProtoMessage() {} +func (*Dns) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *Dns) GetSection() uint32 { + if m != nil && m.Section != nil { + return *m.Section + } + return 0 +} + +func (m *Dns) GetQname() []byte { + if m != nil { + return m.Qname + } + return nil +} + +func (m *Dns) GetQtype() uint32 { + if m != nil && m.Qtype != nil { + return *m.Qtype + } + return 0 +} + +func (m *Dns) GetQclass() uint32 { + if m != nil && m.Qclass != nil { + return *m.Qclass + } + return 0 +} + +func (m *Dns) GetRrname() []byte { + if m != nil { + return m.Rrname + } + return nil +} + +func (m *Dns) GetRrtype() uint32 { + if m != nil && m.Rrtype != nil { + return *m.Rrtype + } + return 0 +} + +func (m *Dns) GetRrclass() uint32 { + if m != nil && m.Rrclass != nil { + return *m.Rrclass + } + return 0 +} + +func (m *Dns) GetRrttl() uint32 { + if m != nil && m.Rrttl != nil { + return *m.Rrttl + } + return 0 +} + +func (m *Dns) GetRdata() [][]byte { + if m != nil { + return m.Rdata + } + return nil +} + +func init() { + proto.RegisterType((*Dns)(nil), "nmsg.base.Dns") +} + +func init() { proto.RegisterFile("dns.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 148 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0xc9, 0x2b, 0xd6, + 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, 0x4a, 0x2c, 0x4e, + 0x55, 0x9a, 0xcc, 0xc8, 0xc5, 0xec, 0x92, 0x57, 0x2c, 0xc4, 0xcf, 0xc5, 0x5e, 0x9c, 0x9a, 0x5c, + 0x92, 0x99, 0x9f, 0x27, 0xc1, 0xa6, 0xc0, 0xa8, 0xc1, 0x2b, 0xc4, 0xcb, 0xc5, 0x5a, 0x98, 0x97, + 0x98, 0x9b, 0x2a, 0xc1, 0x0e, 0xe4, 0xf2, 0x80, 0xb9, 0x25, 0x95, 0x05, 0xa9, 0x12, 0x1c, 0x60, + 0x59, 0x3e, 0x2e, 0xb6, 0xc2, 0xe4, 0x9c, 0xc4, 0xe2, 0x62, 0x09, 0x4e, 0x18, 0xbf, 0xa8, 0x08, + 0xac, 0x9c, 0x11, 0xac, 0x1c, 0xcc, 0x07, 0xab, 0x67, 0x02, 0xcb, 0x03, 0x8d, 0x2f, 0x2a, 0x82, + 0x68, 0x60, 0x86, 0x19, 0x0f, 0x54, 0x50, 0x92, 0x23, 0xc1, 0x02, 0xe7, 0xa6, 0x24, 0x96, 0x24, + 0x4a, 0xb0, 0x2a, 0x30, 0x6b, 0xf0, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0x75, 0x4a, 0x12, 0x4c, + 0xac, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/dns.proto b/nmsg_base/dns.proto new file mode 100644 index 0000000..0e3cfd9 --- /dev/null +++ b/nmsg_base/dns.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; +package nmsg.base; + +message Dns { + optional uint32 section = 6; + optional bytes qname = 7; + optional uint32 qtype = 8; + optional uint32 qclass = 9; + optional bytes rrname = 1; + optional uint32 rrtype = 2; + optional uint32 rrclass = 3; + optional uint32 rrttl = 4; + repeated bytes rdata = 5; +} diff --git a/nmsg_base/dnsqr.pb.go b/nmsg_base/dnsqr.pb.go new file mode 100644 index 0000000..9452635 --- /dev/null +++ b/nmsg_base/dnsqr.pb.go @@ -0,0 +1,334 @@ +// Code generated by protoc-gen-go. +// source: dnsqr.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type DnsQRType int32 + +const ( + DnsQRType_UDP_INVALID DnsQRType = 0 + DnsQRType_UDP_QUERY_RESPONSE DnsQRType = 1 + DnsQRType_UDP_UNANSWERED_QUERY DnsQRType = 2 + DnsQRType_UDP_UNSOLICITED_RESPONSE DnsQRType = 3 + DnsQRType_TCP DnsQRType = 4 + DnsQRType_ICMP DnsQRType = 5 + DnsQRType_UDP_QUERY_ONLY DnsQRType = 6 + DnsQRType_UDP_RESPONSE_ONLY DnsQRType = 7 +) + +var DnsQRType_name = map[int32]string{ + 0: "UDP_INVALID", + 1: "UDP_QUERY_RESPONSE", + 2: "UDP_UNANSWERED_QUERY", + 3: "UDP_UNSOLICITED_RESPONSE", + 4: "TCP", + 5: "ICMP", + 6: "UDP_QUERY_ONLY", + 7: "UDP_RESPONSE_ONLY", +} +var DnsQRType_value = map[string]int32{ + "UDP_INVALID": 0, + "UDP_QUERY_RESPONSE": 1, + "UDP_UNANSWERED_QUERY": 2, + "UDP_UNSOLICITED_RESPONSE": 3, + "TCP": 4, + "ICMP": 5, + "UDP_QUERY_ONLY": 6, + "UDP_RESPONSE_ONLY": 7, +} + +func (x DnsQRType) Enum() *DnsQRType { + p := new(DnsQRType) + *p = x + return p +} +func (x DnsQRType) String() string { + return proto.EnumName(DnsQRType_name, int32(x)) +} +func (x *DnsQRType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(DnsQRType_value, data, "DnsQRType") + if err != nil { + return err + } + *x = DnsQRType(value) + return nil +} +func (DnsQRType) EnumDescriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +type UdpChecksum int32 + +const ( + UdpChecksum_ERROR UdpChecksum = 0 + UdpChecksum_ABSENT UdpChecksum = 1 + UdpChecksum_INCORRECT UdpChecksum = 2 + UdpChecksum_CORRECT UdpChecksum = 3 +) + +var UdpChecksum_name = map[int32]string{ + 0: "ERROR", + 1: "ABSENT", + 2: "INCORRECT", + 3: "CORRECT", +} +var UdpChecksum_value = map[string]int32{ + "ERROR": 0, + "ABSENT": 1, + "INCORRECT": 2, + "CORRECT": 3, +} + +func (x UdpChecksum) Enum() *UdpChecksum { + p := new(UdpChecksum) + *p = x + return p +} +func (x UdpChecksum) String() string { + return proto.EnumName(UdpChecksum_name, int32(x)) +} +func (x *UdpChecksum) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(UdpChecksum_value, data, "UdpChecksum") + if err != nil { + return err + } + *x = UdpChecksum(value) + return nil +} +func (UdpChecksum) EnumDescriptor() ([]byte, []int) { return fileDescriptor1, []int{1} } + +type DnsQR struct { + Type *DnsQRType `protobuf:"varint,1,req,name=type,enum=nmsg.base.DnsQRType" json:"type,omitempty"` + QueryIp []byte `protobuf:"bytes,2,req,name=query_ip" json:"query_ip,omitempty"` + ResponseIp []byte `protobuf:"bytes,3,req,name=response_ip" json:"response_ip,omitempty"` + Proto *uint32 `protobuf:"varint,4,req,name=proto" json:"proto,omitempty"` + QueryPort *uint32 `protobuf:"varint,5,req,name=query_port" json:"query_port,omitempty"` + ResponsePort *uint32 `protobuf:"varint,6,req,name=response_port" json:"response_port,omitempty"` + Id *uint32 `protobuf:"varint,7,req,name=id" json:"id,omitempty"` + Qname []byte `protobuf:"bytes,8,opt,name=qname" json:"qname,omitempty"` + Qtype *uint32 `protobuf:"varint,9,opt,name=qtype" json:"qtype,omitempty"` + Qclass *uint32 `protobuf:"varint,10,opt,name=qclass" json:"qclass,omitempty"` + Rcode *uint32 `protobuf:"varint,11,opt,name=rcode" json:"rcode,omitempty"` + QueryPacket [][]byte `protobuf:"bytes,12,rep,name=query_packet" json:"query_packet,omitempty"` + QueryTimeSec []int64 `protobuf:"varint,13,rep,name=query_time_sec" json:"query_time_sec,omitempty"` + QueryTimeNsec []int32 `protobuf:"fixed32,14,rep,name=query_time_nsec" json:"query_time_nsec,omitempty"` + ResponsePacket [][]byte `protobuf:"bytes,15,rep,name=response_packet" json:"response_packet,omitempty"` + ResponseTimeSec []int64 `protobuf:"varint,16,rep,name=response_time_sec" json:"response_time_sec,omitempty"` + ResponseTimeNsec []int32 `protobuf:"fixed32,17,rep,name=response_time_nsec" json:"response_time_nsec,omitempty"` + Tcp []byte `protobuf:"bytes,18,opt,name=tcp" json:"tcp,omitempty"` + Icmp []byte `protobuf:"bytes,19,opt,name=icmp" json:"icmp,omitempty"` + Timeout *float64 `protobuf:"fixed64,20,opt,name=timeout" json:"timeout,omitempty"` + UdpChecksum *UdpChecksum `protobuf:"varint,21,opt,name=udp_checksum,enum=nmsg.base.UdpChecksum" json:"udp_checksum,omitempty"` + ResolverAddressZeroed *bool `protobuf:"varint,22,opt,name=resolver_address_zeroed" json:"resolver_address_zeroed,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *DnsQR) Reset() { *m = DnsQR{} } +func (m *DnsQR) String() string { return proto.CompactTextString(m) } +func (*DnsQR) ProtoMessage() {} +func (*DnsQR) Descriptor() ([]byte, []int) { return fileDescriptor1, []int{0} } + +func (m *DnsQR) GetType() DnsQRType { + if m != nil && m.Type != nil { + return *m.Type + } + return DnsQRType_UDP_INVALID +} + +func (m *DnsQR) GetQueryIp() []byte { + if m != nil { + return m.QueryIp + } + return nil +} + +func (m *DnsQR) GetResponseIp() []byte { + if m != nil { + return m.ResponseIp + } + return nil +} + +func (m *DnsQR) GetProto() uint32 { + if m != nil && m.Proto != nil { + return *m.Proto + } + return 0 +} + +func (m *DnsQR) GetQueryPort() uint32 { + if m != nil && m.QueryPort != nil { + return *m.QueryPort + } + return 0 +} + +func (m *DnsQR) GetResponsePort() uint32 { + if m != nil && m.ResponsePort != nil { + return *m.ResponsePort + } + return 0 +} + +func (m *DnsQR) GetId() uint32 { + if m != nil && m.Id != nil { + return *m.Id + } + return 0 +} + +func (m *DnsQR) GetQname() []byte { + if m != nil { + return m.Qname + } + return nil +} + +func (m *DnsQR) GetQtype() uint32 { + if m != nil && m.Qtype != nil { + return *m.Qtype + } + return 0 +} + +func (m *DnsQR) GetQclass() uint32 { + if m != nil && m.Qclass != nil { + return *m.Qclass + } + return 0 +} + +func (m *DnsQR) GetRcode() uint32 { + if m != nil && m.Rcode != nil { + return *m.Rcode + } + return 0 +} + +func (m *DnsQR) GetQueryPacket() [][]byte { + if m != nil { + return m.QueryPacket + } + return nil +} + +func (m *DnsQR) GetQueryTimeSec() []int64 { + if m != nil { + return m.QueryTimeSec + } + return nil +} + +func (m *DnsQR) GetQueryTimeNsec() []int32 { + if m != nil { + return m.QueryTimeNsec + } + return nil +} + +func (m *DnsQR) GetResponsePacket() [][]byte { + if m != nil { + return m.ResponsePacket + } + return nil +} + +func (m *DnsQR) GetResponseTimeSec() []int64 { + if m != nil { + return m.ResponseTimeSec + } + return nil +} + +func (m *DnsQR) GetResponseTimeNsec() []int32 { + if m != nil { + return m.ResponseTimeNsec + } + return nil +} + +func (m *DnsQR) GetTcp() []byte { + if m != nil { + return m.Tcp + } + return nil +} + +func (m *DnsQR) GetIcmp() []byte { + if m != nil { + return m.Icmp + } + return nil +} + +func (m *DnsQR) GetTimeout() float64 { + if m != nil && m.Timeout != nil { + return *m.Timeout + } + return 0 +} + +func (m *DnsQR) GetUdpChecksum() UdpChecksum { + if m != nil && m.UdpChecksum != nil { + return *m.UdpChecksum + } + return UdpChecksum_ERROR +} + +func (m *DnsQR) GetResolverAddressZeroed() bool { + if m != nil && m.ResolverAddressZeroed != nil { + return *m.ResolverAddressZeroed + } + return false +} + +func init() { + proto.RegisterType((*DnsQR)(nil), "nmsg.base.DnsQR") + proto.RegisterEnum("nmsg.base.DnsQRType", DnsQRType_name, DnsQRType_value) + proto.RegisterEnum("nmsg.base.UdpChecksum", UdpChecksum_name, UdpChecksum_value) +} + +func init() { proto.RegisterFile("dnsqr.proto", fileDescriptor1) } + +var fileDescriptor1 = []byte{ + // 491 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x54, 0x92, 0xdb, 0x6e, 0x9b, 0x40, + 0x10, 0x86, 0x6b, 0xe3, 0xe3, 0x70, 0xf4, 0xc6, 0x71, 0xb6, 0x55, 0xa5, 0x5a, 0xb9, 0xaa, 0xa2, + 0xca, 0x17, 0x79, 0x82, 0xba, 0x98, 0x0b, 0x24, 0x17, 0x1c, 0xc0, 0xad, 0x72, 0x85, 0x28, 0xac, + 0x5a, 0x2b, 0xe1, 0xb8, 0xb8, 0x52, 0xfa, 0x48, 0x7d, 0xb1, 0xbe, 0x46, 0x87, 0x25, 0xc1, 0xce, + 0x1d, 0xf3, 0xfd, 0x33, 0xff, 0xfc, 0xcb, 0x2e, 0xc8, 0x49, 0xc6, 0xcb, 0x6a, 0x55, 0x54, 0x79, + 0x9d, 0x93, 0x69, 0x96, 0xf2, 0x9f, 0xab, 0x1f, 0x11, 0x67, 0xd7, 0xff, 0x24, 0x18, 0x6e, 0x32, + 0x7e, 0xe7, 0x91, 0x6b, 0x18, 0xd4, 0x4f, 0x05, 0xa3, 0xbd, 0x65, 0xff, 0xa3, 0x76, 0x3b, 0x5f, + 0x75, 0x3d, 0x2b, 0xa1, 0x07, 0xa8, 0x11, 0x03, 0x26, 0xe5, 0x91, 0x55, 0x4f, 0xe1, 0xa1, 0xa0, + 0x7d, 0xec, 0x53, 0xc8, 0x05, 0xc8, 0x15, 0xe3, 0x45, 0x9e, 0x71, 0xd6, 0x40, 0x49, 0x40, 0x15, + 0x86, 0x62, 0x11, 0x1d, 0x60, 0xa9, 0x12, 0x02, 0xd0, 0x4e, 0x15, 0x79, 0x55, 0xd3, 0xa1, 0x60, + 0x97, 0xa0, 0x76, 0x73, 0x02, 0x8f, 0x04, 0x06, 0xe8, 0x1f, 0x12, 0x3a, 0x16, 0xdf, 0xe8, 0x52, + 0x66, 0x51, 0xca, 0xe8, 0x64, 0xd9, 0x6b, 0x4d, 0x4b, 0x11, 0x70, 0x8a, 0xa5, 0x4a, 0x34, 0x18, + 0x95, 0xf1, 0x63, 0xc4, 0x39, 0x05, 0x51, 0xa3, 0x5c, 0xc5, 0x79, 0xc2, 0xa8, 0x2c, 0xca, 0x39, + 0x28, 0xcf, 0x3b, 0xa3, 0xf8, 0x81, 0xd5, 0x54, 0x59, 0x4a, 0xe8, 0xb1, 0x00, 0xad, 0xa5, 0xf5, + 0x21, 0x65, 0x21, 0x67, 0x31, 0x55, 0x91, 0x4b, 0xe4, 0x0a, 0xf4, 0x33, 0x9e, 0x35, 0x82, 0x86, + 0x82, 0xde, 0x08, 0xa7, 0x98, 0xad, 0x93, 0x2e, 0x9c, 0xde, 0xc2, 0xac, 0x13, 0x3a, 0x33, 0x43, + 0x98, 0xbd, 0x03, 0xf2, 0x5a, 0x12, 0x7e, 0x33, 0xe1, 0x27, 0x83, 0x54, 0xc7, 0x05, 0x25, 0xe2, + 0x44, 0x0a, 0x0c, 0x0e, 0x71, 0x5a, 0xd0, 0x0b, 0x51, 0xe9, 0x30, 0x6e, 0xba, 0xf3, 0x63, 0x4d, + 0xe7, 0x08, 0x7a, 0xe4, 0x13, 0x28, 0xc7, 0xa4, 0x08, 0xe3, 0x5f, 0x2c, 0x7e, 0xe0, 0xc7, 0x94, + 0x5e, 0x22, 0xd5, 0x6e, 0x17, 0x67, 0x17, 0xb3, 0x4f, 0x0a, 0xf3, 0x59, 0x25, 0x1f, 0xe0, 0x0a, + 0xb7, 0xe6, 0x8f, 0xbf, 0x59, 0x15, 0x46, 0x49, 0x82, 0xdf, 0x3c, 0xfc, 0xc3, 0xaa, 0x9c, 0x25, + 0x74, 0x81, 0x83, 0x93, 0x9b, 0xbf, 0x3d, 0x98, 0x9e, 0x6e, 0x52, 0x07, 0x79, 0xbf, 0xd9, 0x85, + 0xb6, 0xf3, 0x6d, 0xbd, 0xb5, 0x37, 0xc6, 0x1b, 0xfc, 0x35, 0xa4, 0x01, 0x77, 0x7b, 0xcb, 0xbb, + 0x0f, 0x3d, 0xcb, 0xdf, 0xb9, 0x8e, 0x6f, 0x19, 0x3d, 0x42, 0x61, 0xde, 0xf0, 0xbd, 0xb3, 0x76, + 0xfc, 0xef, 0x96, 0x67, 0x6d, 0xda, 0x16, 0xa3, 0x4f, 0xde, 0x03, 0x6d, 0x15, 0xdf, 0xdd, 0xda, + 0xa6, 0x1d, 0xa0, 0xd4, 0xcd, 0x49, 0x64, 0x0c, 0x52, 0x60, 0xee, 0x8c, 0x01, 0x99, 0xc0, 0xc0, + 0x36, 0xbf, 0xee, 0x8c, 0x21, 0xbe, 0x03, 0xed, 0xb4, 0xc2, 0x75, 0xb6, 0xf7, 0xc6, 0x08, 0xdf, + 0xc1, 0xac, 0x61, 0x2f, 0x83, 0x2d, 0x1e, 0xdf, 0x7c, 0xc6, 0x78, 0x67, 0x87, 0x9b, 0xc2, 0xd0, + 0xf2, 0x3c, 0xd7, 0xc3, 0x9c, 0x00, 0xa3, 0xf5, 0x17, 0xdf, 0x72, 0x02, 0xcc, 0xa6, 0xc2, 0xd4, + 0x76, 0x4c, 0xd7, 0xf3, 0x2c, 0x33, 0xc0, 0x40, 0x32, 0x8c, 0x5f, 0x0a, 0xe9, 0x7f, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x04, 0x82, 0x9b, 0xdb, 0xf1, 0x02, 0x00, 0x00, +} diff --git a/nmsg_base/dnsqr.proto b/nmsg_base/dnsqr.proto new file mode 100644 index 0000000..e98826b --- /dev/null +++ b/nmsg_base/dnsqr.proto @@ -0,0 +1,76 @@ +syntax = "proto2"; +package nmsg.base; + +enum DnsQRType { + UDP_INVALID = 0; + UDP_QUERY_RESPONSE = 1; + UDP_UNANSWERED_QUERY = 2; + UDP_UNSOLICITED_RESPONSE = 3; + TCP = 4; + ICMP = 5; + UDP_QUERY_ONLY = 6; + UDP_RESPONSE_ONLY = 7; +} + +enum UdpChecksum { + ERROR = 0; + ABSENT = 1; + INCORRECT = 2; + CORRECT = 3; +} + +message DnsQR { + required DnsQRType type = 1; + + // the 9-tuple + + required bytes query_ip = 2; + required bytes response_ip = 3; + required uint32 proto = 4; + required uint32 query_port = 5; + required uint32 response_port = 6; + + required uint32 id = 7; + optional bytes qname = 8; + optional uint32 qtype = 9; + optional uint32 qclass = 10; + + // rcode from the response + + optional uint32 rcode = 11; + + // packet data + + repeated bytes query_packet = 12; + repeated int64 query_time_sec = 13; + repeated sfixed32 query_time_nsec = 14; + + repeated bytes response_packet = 15; + repeated int64 response_time_sec = 16; + repeated sfixed32 response_time_nsec = 17; + + // only used if type = TCP + + optional bytes tcp = 18; + + // only used if type = ICMP + + optional bytes icmp = 19; + + // only set for UDP_UNANSWERED_QUERY + + optional double timeout = 20; + + // the result of UDP checksum verification of the response datagram. + // note that the query datagram isn't checksummed, since a) the relevant + // information from the query is almost always included in the response, + // b) when capturing from the perspective of an initiator, the outbound + // query is commonly subject to UDP checksum offload and will be incorrect + // anyway. + + optional UdpChecksum udp_checksum = 21; + + // set if the address of the initiator was zeroed. + + optional bool resolver_address_zeroed = 22; +} diff --git a/nmsg_base/email.pb.go b/nmsg_base/email.pb.go new file mode 100644 index 0000000..45d1524 --- /dev/null +++ b/nmsg_base/email.pb.go @@ -0,0 +1,163 @@ +// Code generated by protoc-gen-go. +// source: email.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type EmailType int32 + +const ( + EmailType_unknown EmailType = 0 + EmailType_spamtrap EmailType = 1 + EmailType_rej_network EmailType = 2 + EmailType_rej_content EmailType = 3 + EmailType_rej_user EmailType = 4 +) + +var EmailType_name = map[int32]string{ + 0: "unknown", + 1: "spamtrap", + 2: "rej_network", + 3: "rej_content", + 4: "rej_user", +} +var EmailType_value = map[string]int32{ + "unknown": 0, + "spamtrap": 1, + "rej_network": 2, + "rej_content": 3, + "rej_user": 4, +} + +func (x EmailType) Enum() *EmailType { + p := new(EmailType) + *p = x + return p +} +func (x EmailType) String() string { + return proto.EnumName(EmailType_name, int32(x)) +} +func (x *EmailType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(EmailType_value, data, "EmailType") + if err != nil { + return err + } + *x = EmailType(value) + return nil +} +func (EmailType) EnumDescriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +type Email struct { + Type *EmailType `protobuf:"varint,8,opt,name=type,enum=nmsg.base.EmailType" json:"type,omitempty"` + Headers []byte `protobuf:"bytes,2,opt,name=headers" json:"headers,omitempty"` + Srcip []byte `protobuf:"bytes,3,opt,name=srcip" json:"srcip,omitempty"` + Srchost []byte `protobuf:"bytes,4,opt,name=srchost" json:"srchost,omitempty"` + Helo []byte `protobuf:"bytes,5,opt,name=helo" json:"helo,omitempty"` + From []byte `protobuf:"bytes,6,opt,name=from" json:"from,omitempty"` + Rcpt [][]byte `protobuf:"bytes,7,rep,name=rcpt" json:"rcpt,omitempty"` + Bodyurl [][]byte `protobuf:"bytes,9,rep,name=bodyurl" json:"bodyurl,omitempty"` + Body []byte `protobuf:"bytes,10,opt,name=body" json:"body,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Email) Reset() { *m = Email{} } +func (m *Email) String() string { return proto.CompactTextString(m) } +func (*Email) ProtoMessage() {} +func (*Email) Descriptor() ([]byte, []int) { return fileDescriptor2, []int{0} } + +func (m *Email) GetType() EmailType { + if m != nil && m.Type != nil { + return *m.Type + } + return EmailType_unknown +} + +func (m *Email) GetHeaders() []byte { + if m != nil { + return m.Headers + } + return nil +} + +func (m *Email) GetSrcip() []byte { + if m != nil { + return m.Srcip + } + return nil +} + +func (m *Email) GetSrchost() []byte { + if m != nil { + return m.Srchost + } + return nil +} + +func (m *Email) GetHelo() []byte { + if m != nil { + return m.Helo + } + return nil +} + +func (m *Email) GetFrom() []byte { + if m != nil { + return m.From + } + return nil +} + +func (m *Email) GetRcpt() [][]byte { + if m != nil { + return m.Rcpt + } + return nil +} + +func (m *Email) GetBodyurl() [][]byte { + if m != nil { + return m.Bodyurl + } + return nil +} + +func (m *Email) GetBody() []byte { + if m != nil { + return m.Body + } + return nil +} + +func init() { + proto.RegisterType((*Email)(nil), "nmsg.base.Email") + proto.RegisterEnum("nmsg.base.EmailType", EmailType_name, EmailType_value) +} + +func init() { proto.RegisterFile("email.proto", fileDescriptor2) } + +var fileDescriptor2 = []byte{ + // 222 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x44, 0x8e, 0x41, 0x4e, 0xc3, 0x30, + 0x10, 0x45, 0x49, 0x93, 0x90, 0x66, 0x52, 0xc0, 0x8a, 0x58, 0xcc, 0xb2, 0xea, 0x0a, 0xb1, 0xc8, + 0x82, 0x3b, 0x70, 0x03, 0xc4, 0x16, 0xb9, 0xe9, 0x40, 0x4a, 0x13, 0x8f, 0x35, 0x76, 0x54, 0xf5, + 0x40, 0xdc, 0x13, 0xdb, 0x28, 0x74, 0xf9, 0xdf, 0x7b, 0xb6, 0x06, 0x1a, 0x9a, 0xf4, 0x71, 0xec, + 0xac, 0xb0, 0xe7, 0xb6, 0x36, 0x93, 0xfb, 0xea, 0xf6, 0xda, 0xd1, 0xee, 0x27, 0x83, 0xf2, 0x35, + 0xaa, 0x76, 0x07, 0x85, 0xbf, 0x58, 0xc2, 0xf5, 0x36, 0x7b, 0xba, 0x7f, 0x79, 0xec, 0xfe, 0x9b, + 0x2e, 0xf9, 0xb7, 0xe0, 0xda, 0x07, 0xa8, 0x06, 0xd2, 0x07, 0x12, 0x87, 0xab, 0x90, 0x6d, 0xda, + 0x3b, 0x28, 0x9d, 0xf4, 0x47, 0x8b, 0x79, 0x9a, 0xc1, 0x87, 0x39, 0xb0, 0xf3, 0x58, 0x24, 0xb0, + 0x81, 0x62, 0xa0, 0x91, 0xb1, 0x5c, 0xd6, 0xa7, 0xf0, 0x84, 0xb7, 0xcb, 0x92, 0xde, 0x7a, 0xac, + 0xb6, 0xf9, 0xdf, 0xd3, 0x3d, 0x1f, 0x2e, 0xb3, 0x8c, 0x58, 0x27, 0x10, 0x74, 0x04, 0x08, 0x31, + 0x7e, 0x7e, 0x87, 0xfa, 0x7a, 0x46, 0x03, 0xd5, 0x6c, 0x4e, 0x86, 0xcf, 0x46, 0xdd, 0x84, 0x6e, + 0xed, 0xac, 0x9e, 0xbc, 0x68, 0xab, 0xb2, 0xf0, 0x4d, 0x23, 0xf4, 0xfd, 0x61, 0xc8, 0x9f, 0x59, + 0x4e, 0x6a, 0xb5, 0x80, 0x9e, 0x8d, 0x27, 0xe3, 0x55, 0x1e, 0xfb, 0x08, 0x66, 0x47, 0xa2, 0x8a, + 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6b, 0xe2, 0x02, 0x36, 0x18, 0x01, 0x00, 0x00, +} diff --git a/nmsg_base/email.proto b/nmsg_base/email.proto new file mode 100644 index 0000000..a1af375 --- /dev/null +++ b/nmsg_base/email.proto @@ -0,0 +1,22 @@ +syntax = "proto2"; +package nmsg.base; + +enum EmailType { + unknown = 0; + spamtrap = 1; + rej_network = 2; + rej_content = 3; + rej_user = 4; +} + +message Email { + optional EmailType type = 8; + optional bytes headers = 2; + optional bytes srcip = 3; + optional bytes srchost = 4; + optional bytes helo = 5; + optional bytes from = 6; + repeated bytes rcpt = 7; + repeated bytes bodyurl = 9; + optional bytes body = 10; +} diff --git a/nmsg_base/encode.pb.go b/nmsg_base/encode.pb.go new file mode 100644 index 0000000..d00ada9 --- /dev/null +++ b/nmsg_base/encode.pb.go @@ -0,0 +1,103 @@ +// Code generated by protoc-gen-go. +// source: encode.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type EncodeType int32 + +const ( + EncodeType_TEXT EncodeType = 0 + EncodeType_JSON EncodeType = 1 + EncodeType_YAML EncodeType = 2 + EncodeType_MSGPACK EncodeType = 3 + EncodeType_XML EncodeType = 4 +) + +var EncodeType_name = map[int32]string{ + 0: "TEXT", + 1: "JSON", + 2: "YAML", + 3: "MSGPACK", + 4: "XML", +} +var EncodeType_value = map[string]int32{ + "TEXT": 0, + "JSON": 1, + "YAML": 2, + "MSGPACK": 3, + "XML": 4, +} + +func (x EncodeType) Enum() *EncodeType { + p := new(EncodeType) + *p = x + return p +} +func (x EncodeType) String() string { + return proto.EnumName(EncodeType_name, int32(x)) +} +func (x *EncodeType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(EncodeType_value, data, "EncodeType") + if err != nil { + return err + } + *x = EncodeType(value) + return nil +} +func (EncodeType) EnumDescriptor() ([]byte, []int) { return fileDescriptor3, []int{0} } + +type Encode struct { + Type *EncodeType `protobuf:"varint,1,req,name=type,enum=nmsg.base.EncodeType" json:"type,omitempty"` + Payload []byte `protobuf:"bytes,2,req,name=payload" json:"payload,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Encode) Reset() { *m = Encode{} } +func (m *Encode) String() string { return proto.CompactTextString(m) } +func (*Encode) ProtoMessage() {} +func (*Encode) Descriptor() ([]byte, []int) { return fileDescriptor3, []int{0} } + +func (m *Encode) GetType() EncodeType { + if m != nil && m.Type != nil { + return *m.Type + } + return EncodeType_TEXT +} + +func (m *Encode) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func init() { + proto.RegisterType((*Encode)(nil), "nmsg.base.Encode") + proto.RegisterEnum("nmsg.base.EncodeType", EncodeType_name, EncodeType_value) +} + +func init() { proto.RegisterFile("encode.proto", fileDescriptor3) } + +var fileDescriptor3 = []byte{ + // 148 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x49, 0xcd, 0x4b, 0xce, + 0x4f, 0x49, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, + 0x4a, 0x2c, 0x4e, 0x55, 0xb2, 0xe3, 0x62, 0x73, 0x05, 0x4b, 0x09, 0x29, 0x73, 0xb1, 0x94, 0x54, + 0x16, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x69, 0xf0, 0x19, 0x89, 0xea, 0xc1, 0xd5, 0xe8, 0x41, 0x14, + 0x84, 0x00, 0x25, 0x85, 0xf8, 0xb9, 0xd8, 0x0b, 0x12, 0x2b, 0x73, 0xf2, 0x13, 0x53, 0x24, 0x98, + 0x80, 0xea, 0x78, 0xb4, 0x1c, 0xb8, 0xb8, 0x90, 0xa4, 0x39, 0xb8, 0x58, 0x42, 0x5c, 0x23, 0x42, + 0x04, 0x18, 0x40, 0x2c, 0xaf, 0x60, 0x7f, 0x3f, 0x01, 0x46, 0x10, 0x2b, 0xd2, 0xd1, 0xd7, 0x47, + 0x80, 0x49, 0x88, 0x9b, 0x8b, 0xdd, 0x37, 0xd8, 0x3d, 0xc0, 0xd1, 0xd9, 0x5b, 0x80, 0x59, 0x88, + 0x9d, 0x8b, 0x39, 0x02, 0x28, 0xca, 0x02, 0x08, 0x00, 0x00, 0xff, 0xff, 0xb9, 0x9e, 0xfe, 0xff, + 0x9b, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/encode.proto b/nmsg_base/encode.proto new file mode 100644 index 0000000..896b04e --- /dev/null +++ b/nmsg_base/encode.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; +package nmsg.base; + +enum EncodeType { + TEXT = 0; + JSON = 1; + YAML = 2; + MSGPACK = 3; + XML = 4; +} + +message Encode { + required EncodeType type = 1; + required bytes payload = 2; +} diff --git a/nmsg_base/http.pb.go b/nmsg_base/http.pb.go new file mode 100644 index 0000000..a969f90 --- /dev/null +++ b/nmsg_base/http.pb.go @@ -0,0 +1,227 @@ +// Code generated by protoc-gen-go. +// source: http.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type HttpType int32 + +const ( + // unknown = 0; + HttpType_sinkhole HttpType = 1 +) + +var HttpType_name = map[int32]string{ + 1: "sinkhole", +} +var HttpType_value = map[string]int32{ + "sinkhole": 1, +} + +func (x HttpType) Enum() *HttpType { + p := new(HttpType) + *p = x + return p +} +func (x HttpType) String() string { + return proto.EnumName(HttpType_name, int32(x)) +} +func (x *HttpType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(HttpType_value, data, "HttpType") + if err != nil { + return err + } + *x = HttpType(value) + return nil +} +func (HttpType) EnumDescriptor() ([]byte, []int) { return fileDescriptor4, []int{0} } + +type Http struct { + Type *HttpType `protobuf:"varint,1,req,name=type,enum=nmsg.base.HttpType" json:"type,omitempty"` + Srcip []byte `protobuf:"bytes,2,opt,name=srcip" json:"srcip,omitempty"` + Srchost []byte `protobuf:"bytes,3,opt,name=srchost" json:"srchost,omitempty"` + Srcport *uint32 `protobuf:"varint,4,opt,name=srcport" json:"srcport,omitempty"` + Dstip []byte `protobuf:"bytes,5,opt,name=dstip" json:"dstip,omitempty"` + Dstport *uint32 `protobuf:"varint,6,opt,name=dstport" json:"dstport,omitempty"` + Request []byte `protobuf:"bytes,7,opt,name=request" json:"request,omitempty"` + P0FGenre []byte `protobuf:"bytes,65,opt,name=p0f_genre" json:"p0f_genre,omitempty"` + P0FDetail []byte `protobuf:"bytes,66,opt,name=p0f_detail" json:"p0f_detail,omitempty"` + P0FDist *int32 `protobuf:"varint,67,opt,name=p0f_dist" json:"p0f_dist,omitempty"` + P0FLink []byte `protobuf:"bytes,68,opt,name=p0f_link" json:"p0f_link,omitempty"` + P0FTos []byte `protobuf:"bytes,69,opt,name=p0f_tos" json:"p0f_tos,omitempty"` + P0FFw *uint32 `protobuf:"varint,70,opt,name=p0f_fw" json:"p0f_fw,omitempty"` + P0FNat *uint32 `protobuf:"varint,71,opt,name=p0f_nat" json:"p0f_nat,omitempty"` + P0FReal *uint32 `protobuf:"varint,72,opt,name=p0f_real" json:"p0f_real,omitempty"` + P0FScore *int32 `protobuf:"varint,73,opt,name=p0f_score" json:"p0f_score,omitempty"` + P0FMflags *uint32 `protobuf:"varint,74,opt,name=p0f_mflags" json:"p0f_mflags,omitempty"` + P0FUptime *int32 `protobuf:"varint,75,opt,name=p0f_uptime" json:"p0f_uptime,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Http) Reset() { *m = Http{} } +func (m *Http) String() string { return proto.CompactTextString(m) } +func (*Http) ProtoMessage() {} +func (*Http) Descriptor() ([]byte, []int) { return fileDescriptor4, []int{0} } + +func (m *Http) GetType() HttpType { + if m != nil && m.Type != nil { + return *m.Type + } + return HttpType_sinkhole +} + +func (m *Http) GetSrcip() []byte { + if m != nil { + return m.Srcip + } + return nil +} + +func (m *Http) GetSrchost() []byte { + if m != nil { + return m.Srchost + } + return nil +} + +func (m *Http) GetSrcport() uint32 { + if m != nil && m.Srcport != nil { + return *m.Srcport + } + return 0 +} + +func (m *Http) GetDstip() []byte { + if m != nil { + return m.Dstip + } + return nil +} + +func (m *Http) GetDstport() uint32 { + if m != nil && m.Dstport != nil { + return *m.Dstport + } + return 0 +} + +func (m *Http) GetRequest() []byte { + if m != nil { + return m.Request + } + return nil +} + +func (m *Http) GetP0FGenre() []byte { + if m != nil { + return m.P0FGenre + } + return nil +} + +func (m *Http) GetP0FDetail() []byte { + if m != nil { + return m.P0FDetail + } + return nil +} + +func (m *Http) GetP0FDist() int32 { + if m != nil && m.P0FDist != nil { + return *m.P0FDist + } + return 0 +} + +func (m *Http) GetP0FLink() []byte { + if m != nil { + return m.P0FLink + } + return nil +} + +func (m *Http) GetP0FTos() []byte { + if m != nil { + return m.P0FTos + } + return nil +} + +func (m *Http) GetP0FFw() uint32 { + if m != nil && m.P0FFw != nil { + return *m.P0FFw + } + return 0 +} + +func (m *Http) GetP0FNat() uint32 { + if m != nil && m.P0FNat != nil { + return *m.P0FNat + } + return 0 +} + +func (m *Http) GetP0FReal() uint32 { + if m != nil && m.P0FReal != nil { + return *m.P0FReal + } + return 0 +} + +func (m *Http) GetP0FScore() int32 { + if m != nil && m.P0FScore != nil { + return *m.P0FScore + } + return 0 +} + +func (m *Http) GetP0FMflags() uint32 { + if m != nil && m.P0FMflags != nil { + return *m.P0FMflags + } + return 0 +} + +func (m *Http) GetP0FUptime() int32 { + if m != nil && m.P0FUptime != nil { + return *m.P0FUptime + } + return 0 +} + +func init() { + proto.RegisterType((*Http)(nil), "nmsg.base.Http") + proto.RegisterEnum("nmsg.base.HttpType", HttpType_name, HttpType_value) +} + +func init() { proto.RegisterFile("http.proto", fileDescriptor4) } + +var fileDescriptor4 = []byte{ + // 268 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x4c, 0xd0, 0x4d, 0x4f, 0xf3, 0x30, + 0x0c, 0x07, 0x70, 0xb5, 0x4f, 0xbb, 0x75, 0xd6, 0xb6, 0x67, 0x94, 0x8b, 0x8f, 0x83, 0xd3, 0xc4, + 0xa1, 0x42, 0x7c, 0x03, 0xde, 0x07, 0x5c, 0xb9, 0xa3, 0xb2, 0xa5, 0x2f, 0x22, 0x6d, 0x42, 0xe2, + 0x09, 0xf1, 0x41, 0xf9, 0x3e, 0x38, 0x56, 0x3b, 0x71, 0xf4, 0xef, 0x6f, 0xc7, 0x56, 0x00, 0x1a, + 0x22, 0x5b, 0x58, 0x67, 0xc8, 0xe4, 0xb3, 0xbe, 0xf3, 0x75, 0xf1, 0x5e, 0x7a, 0x75, 0xfe, 0x13, + 0x43, 0xb2, 0xe5, 0x24, 0x3f, 0x83, 0x84, 0xbe, 0xad, 0xc2, 0x68, 0x1d, 0x6f, 0x96, 0x57, 0xa7, + 0xc5, 0xb1, 0xa5, 0x08, 0xf1, 0x2b, 0x47, 0xf9, 0x02, 0x52, 0xef, 0x76, 0xad, 0xc5, 0x78, 0x1d, + 0x6d, 0xe6, 0xf9, 0x7f, 0x98, 0x72, 0xd9, 0x18, 0x4f, 0xf8, 0xef, 0x0f, 0x58, 0xe3, 0x08, 0x13, + 0x86, 0x45, 0x18, 0xd8, 0x7b, 0xe2, 0x81, 0x74, 0xcc, 0xb9, 0x94, 0x7c, 0x22, 0x39, 0x83, 0x53, + 0x9f, 0x07, 0xc5, 0x2f, 0x4c, 0xa5, 0xe3, 0x04, 0x66, 0xf6, 0xb2, 0x7a, 0xab, 0x55, 0xef, 0x14, + 0x5e, 0x0b, 0xe5, 0x00, 0x81, 0xf6, 0x8a, 0xca, 0x56, 0xe3, 0x8d, 0xd8, 0x0a, 0x32, 0xb1, 0x96, + 0x07, 0x6f, 0x59, 0xd2, 0x51, 0x74, 0xdb, 0x7f, 0xe0, 0xdd, 0xb8, 0x2c, 0x08, 0x19, 0x8f, 0xf7, + 0x02, 0x4b, 0x98, 0x04, 0xa8, 0xbe, 0xf0, 0x61, 0x5c, 0x1e, 0xea, 0xbe, 0x24, 0x7c, 0x14, 0x18, + 0xde, 0x70, 0xaa, 0xd4, 0xb8, 0x15, 0x19, 0xce, 0xf1, 0x3b, 0xc3, 0xe7, 0x3c, 0xc9, 0xa2, 0xe1, + 0x9c, 0xae, 0xd2, 0x65, 0xed, 0xf1, 0x59, 0xda, 0x06, 0x3b, 0x58, 0x6a, 0x3b, 0x85, 0x2f, 0xa1, + 0xef, 0x02, 0x21, 0x3b, 0xfe, 0xdb, 0x1c, 0x32, 0xcf, 0x87, 0x35, 0x46, 0xab, 0x55, 0xf4, 0x1b, + 0x00, 0x00, 0xff, 0xff, 0xa3, 0x64, 0x11, 0x56, 0x89, 0x01, 0x00, 0x00, +} diff --git a/nmsg_base/http.proto b/nmsg_base/http.proto new file mode 100644 index 0000000..4c35c15 --- /dev/null +++ b/nmsg_base/http.proto @@ -0,0 +1,29 @@ +syntax = "proto2"; +package nmsg.base; + +enum HttpType { + // unknown = 0; + sinkhole = 1; +} + +message Http { + required HttpType type = 1; + optional bytes srcip = 2; + optional bytes srchost = 3; + optional uint32 srcport = 4; + optional bytes dstip = 5; + optional uint32 dstport = 6; + optional bytes request = 7; + + optional bytes p0f_genre = 65; + optional bytes p0f_detail = 66; + optional int32 p0f_dist = 67; + optional bytes p0f_link = 68; + optional bytes p0f_tos = 69; + optional uint32 p0f_fw = 70; + optional uint32 p0f_nat = 71; + optional uint32 p0f_real = 72; + optional int32 p0f_score = 73; + optional uint32 p0f_mflags = 74; + optional int32 p0f_uptime = 75; +} diff --git a/nmsg_base/ipconn.pb.go b/nmsg_base/ipconn.pb.go new file mode 100644 index 0000000..33cfa1c --- /dev/null +++ b/nmsg_base/ipconn.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go. +// source: ipconn.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type IPConn struct { + Proto *uint32 `protobuf:"varint,1,opt,name=proto" json:"proto,omitempty"` + Srcip []byte `protobuf:"bytes,2,opt,name=srcip" json:"srcip,omitempty"` + Srcport *uint32 `protobuf:"varint,3,opt,name=srcport" json:"srcport,omitempty"` + Dstip []byte `protobuf:"bytes,4,opt,name=dstip" json:"dstip,omitempty"` + Dstport *uint32 `protobuf:"varint,5,opt,name=dstport" json:"dstport,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *IPConn) Reset() { *m = IPConn{} } +func (m *IPConn) String() string { return proto.CompactTextString(m) } +func (*IPConn) ProtoMessage() {} +func (*IPConn) Descriptor() ([]byte, []int) { return fileDescriptor5, []int{0} } + +func (m *IPConn) GetProto() uint32 { + if m != nil && m.Proto != nil { + return *m.Proto + } + return 0 +} + +func (m *IPConn) GetSrcip() []byte { + if m != nil { + return m.Srcip + } + return nil +} + +func (m *IPConn) GetSrcport() uint32 { + if m != nil && m.Srcport != nil { + return *m.Srcport + } + return 0 +} + +func (m *IPConn) GetDstip() []byte { + if m != nil { + return m.Dstip + } + return nil +} + +func (m *IPConn) GetDstport() uint32 { + if m != nil && m.Dstport != nil { + return *m.Dstport + } + return 0 +} + +func init() { + proto.RegisterType((*IPConn)(nil), "nmsg.base.IPConn") +} + +func init() { proto.RegisterFile("ipconn.proto", fileDescriptor5) } + +var fileDescriptor5 = []byte{ + // 109 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x2c, 0x48, 0xce, + 0xcf, 0xcb, 0xd3, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, + 0x4a, 0x2c, 0x4e, 0x55, 0x0a, 0xe7, 0x62, 0xf3, 0x0c, 0x70, 0x06, 0x4a, 0x09, 0xf1, 0x72, 0xb1, + 0x82, 0x65, 0x25, 0x18, 0x15, 0x18, 0x35, 0x78, 0x41, 0xdc, 0xe2, 0xa2, 0xe4, 0xcc, 0x02, 0x09, + 0x26, 0x20, 0x97, 0x47, 0x88, 0x9f, 0x8b, 0x1d, 0xc8, 0x2d, 0xc8, 0x2f, 0x2a, 0x91, 0x60, 0x86, + 0xc9, 0xa7, 0x14, 0x97, 0x00, 0xe5, 0x59, 0x60, 0xf2, 0x40, 0x2e, 0x58, 0x9e, 0x15, 0x24, 0x0f, + 0x08, 0x00, 0x00, 0xff, 0xff, 0x83, 0x37, 0xd2, 0x7d, 0x72, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/ipconn.proto b/nmsg_base/ipconn.proto new file mode 100644 index 0000000..9dd11ff --- /dev/null +++ b/nmsg_base/ipconn.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; +package nmsg.base; + +message IPConn { + optional uint32 proto = 1; + optional bytes srcip = 2; + optional uint32 srcport = 3; + optional bytes dstip = 4; + optional uint32 dstport = 5; +} diff --git a/nmsg_base/linkpair.pb.go b/nmsg_base/linkpair.pb.go new file mode 100644 index 0000000..ceb8b0e --- /dev/null +++ b/nmsg_base/linkpair.pb.go @@ -0,0 +1,110 @@ +// Code generated by protoc-gen-go. +// source: linkpair.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Linktype int32 + +const ( + Linktype_anchor Linktype = 0 + Linktype_redirect Linktype = 1 +) + +var Linktype_name = map[int32]string{ + 0: "anchor", + 1: "redirect", +} +var Linktype_value = map[string]int32{ + "anchor": 0, + "redirect": 1, +} + +func (x Linktype) Enum() *Linktype { + p := new(Linktype) + *p = x + return p +} +func (x Linktype) String() string { + return proto.EnumName(Linktype_name, int32(x)) +} +func (x *Linktype) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(Linktype_value, data, "Linktype") + if err != nil { + return err + } + *x = Linktype(value) + return nil +} +func (Linktype) EnumDescriptor() ([]byte, []int) { return fileDescriptor6, []int{0} } + +type Linkpair struct { + Type *Linktype `protobuf:"varint,1,req,name=type,enum=nmsg.base.Linktype" json:"type,omitempty"` + Src []byte `protobuf:"bytes,2,req,name=src" json:"src,omitempty"` + Dst []byte `protobuf:"bytes,3,req,name=dst" json:"dst,omitempty"` + Headers []byte `protobuf:"bytes,5,opt,name=headers" json:"headers,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Linkpair) Reset() { *m = Linkpair{} } +func (m *Linkpair) String() string { return proto.CompactTextString(m) } +func (*Linkpair) ProtoMessage() {} +func (*Linkpair) Descriptor() ([]byte, []int) { return fileDescriptor6, []int{0} } + +func (m *Linkpair) GetType() Linktype { + if m != nil && m.Type != nil { + return *m.Type + } + return Linktype_anchor +} + +func (m *Linkpair) GetSrc() []byte { + if m != nil { + return m.Src + } + return nil +} + +func (m *Linkpair) GetDst() []byte { + if m != nil { + return m.Dst + } + return nil +} + +func (m *Linkpair) GetHeaders() []byte { + if m != nil { + return m.Headers + } + return nil +} + +func init() { + proto.RegisterType((*Linkpair)(nil), "nmsg.base.Linkpair") + proto.RegisterEnum("nmsg.base.Linktype", Linktype_name, Linktype_value) +} + +func init() { proto.RegisterFile("linkpair.proto", fileDescriptor6) } + +var fileDescriptor6 = []byte{ + // 149 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcb, 0xc9, 0xcc, 0xcb, + 0x2e, 0x48, 0xcc, 0x2c, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, + 0xd7, 0x4b, 0x4a, 0x2c, 0x4e, 0x55, 0x8a, 0xe0, 0xe2, 0xf0, 0x81, 0x4a, 0x0a, 0x29, 0x72, 0xb1, + 0x94, 0x54, 0x16, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x69, 0xf0, 0x19, 0x09, 0xeb, 0xc1, 0x55, 0xe9, + 0x81, 0x94, 0x80, 0xa4, 0x84, 0xb8, 0xb9, 0x98, 0x8b, 0x8b, 0x92, 0x25, 0x98, 0x80, 0x2a, 0x78, + 0x40, 0x9c, 0x94, 0xe2, 0x12, 0x09, 0x66, 0x30, 0x87, 0x9f, 0x8b, 0x3d, 0x23, 0x35, 0x31, 0x25, + 0xb5, 0xa8, 0x58, 0x82, 0x55, 0x81, 0x51, 0x83, 0x47, 0x4b, 0x05, 0x62, 0x32, 0x58, 0x1b, 0x17, + 0x17, 0x5b, 0x62, 0x5e, 0x72, 0x46, 0x7e, 0x91, 0x00, 0x83, 0x10, 0x0f, 0x17, 0x47, 0x51, 0x6a, + 0x4a, 0x66, 0x51, 0x6a, 0x72, 0x89, 0x00, 0x23, 0x20, 0x00, 0x00, 0xff, 0xff, 0x64, 0x13, 0x33, + 0xdf, 0x9b, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/linkpair.proto b/nmsg_base/linkpair.proto new file mode 100644 index 0000000..04f10a8 --- /dev/null +++ b/nmsg_base/linkpair.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; +package nmsg.base; + +enum Linktype { + anchor = 0; + redirect = 1; +} + +message Linkpair { + required Linktype type = 1; + required bytes src = 2; + required bytes dst = 3; + optional bytes headers = 5; +} diff --git a/nmsg_base/logline.pb.go b/nmsg_base/logline.pb.go new file mode 100644 index 0000000..aac48be --- /dev/null +++ b/nmsg_base/logline.pb.go @@ -0,0 +1,55 @@ +// Code generated by protoc-gen-go. +// source: logline.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type LogLine struct { + Category []byte `protobuf:"bytes,1,opt,name=category" json:"category,omitempty"` + Message []byte `protobuf:"bytes,2,opt,name=message" json:"message,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *LogLine) Reset() { *m = LogLine{} } +func (m *LogLine) String() string { return proto.CompactTextString(m) } +func (*LogLine) ProtoMessage() {} +func (*LogLine) Descriptor() ([]byte, []int) { return fileDescriptor7, []int{0} } + +func (m *LogLine) GetCategory() []byte { + if m != nil { + return m.Category + } + return nil +} + +func (m *LogLine) GetMessage() []byte { + if m != nil { + return m.Message + } + return nil +} + +func init() { + proto.RegisterType((*LogLine)(nil), "nmsg.base.LogLine") +} + +func init() { proto.RegisterFile("logline.proto", fileDescriptor7) } + +var fileDescriptor7 = []byte{ + // 93 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0xc9, 0x4f, 0xcf, + 0xc9, 0xcc, 0x4b, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, + 0x4b, 0x4a, 0x2c, 0x4e, 0x55, 0xd2, 0xe1, 0x62, 0xf7, 0xc9, 0x4f, 0xf7, 0x01, 0xca, 0x09, 0x09, + 0x70, 0x71, 0x24, 0x27, 0x96, 0xa4, 0xa6, 0xe7, 0x17, 0x55, 0x4a, 0x30, 0x2a, 0x30, 0x6a, 0xf0, + 0x08, 0xf1, 0x73, 0xb1, 0xe7, 0xa6, 0x16, 0x17, 0x27, 0xa6, 0xa7, 0x4a, 0x30, 0x81, 0x04, 0x00, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x4e, 0xe7, 0x48, 0x48, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/logline.proto b/nmsg_base/logline.proto new file mode 100644 index 0000000..0328413 --- /dev/null +++ b/nmsg_base/logline.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +package nmsg.base; + +message LogLine { + optional bytes category = 1; + optional bytes message = 2; +} diff --git a/nmsg_base/ncap.pb.go b/nmsg_base/ncap.pb.go new file mode 100644 index 0000000..ea6ee57 --- /dev/null +++ b/nmsg_base/ncap.pb.go @@ -0,0 +1,180 @@ +// Code generated by protoc-gen-go. +// source: ncap.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type NcapType int32 + +const ( + NcapType_IPV4 NcapType = 0 + NcapType_IPV6 NcapType = 1 + NcapType_Legacy NcapType = 2 +) + +var NcapType_name = map[int32]string{ + 0: "IPV4", + 1: "IPV6", + 2: "Legacy", +} +var NcapType_value = map[string]int32{ + "IPV4": 0, + "IPV6": 1, + "Legacy": 2, +} + +func (x NcapType) Enum() *NcapType { + p := new(NcapType) + *p = x + return p +} +func (x NcapType) String() string { + return proto.EnumName(NcapType_name, int32(x)) +} +func (x *NcapType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(NcapType_value, data, "NcapType") + if err != nil { + return err + } + *x = NcapType(value) + return nil +} +func (NcapType) EnumDescriptor() ([]byte, []int) { return fileDescriptor8, []int{0} } + +type NcapLegacyType int32 + +const ( + NcapLegacyType_Ncap_UDP NcapLegacyType = 0 + NcapLegacyType_Ncap_TCP NcapLegacyType = 1 + NcapLegacyType_Ncap_ICMP NcapLegacyType = 2 +) + +var NcapLegacyType_name = map[int32]string{ + 0: "Ncap_UDP", + 1: "Ncap_TCP", + 2: "Ncap_ICMP", +} +var NcapLegacyType_value = map[string]int32{ + "Ncap_UDP": 0, + "Ncap_TCP": 1, + "Ncap_ICMP": 2, +} + +func (x NcapLegacyType) Enum() *NcapLegacyType { + p := new(NcapLegacyType) + *p = x + return p +} +func (x NcapLegacyType) String() string { + return proto.EnumName(NcapLegacyType_name, int32(x)) +} +func (x *NcapLegacyType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(NcapLegacyType_value, data, "NcapLegacyType") + if err != nil { + return err + } + *x = NcapLegacyType(value) + return nil +} +func (NcapLegacyType) EnumDescriptor() ([]byte, []int) { return fileDescriptor8, []int{1} } + +type Ncap struct { + Type *NcapType `protobuf:"varint,1,req,name=type,enum=nmsg.base.NcapType" json:"type,omitempty"` + Payload []byte `protobuf:"bytes,2,req,name=payload" json:"payload,omitempty"` + // legacy NCAP fields + Ltype *NcapLegacyType `protobuf:"varint,3,opt,name=ltype,enum=nmsg.base.NcapLegacyType" json:"ltype,omitempty"` + Srcip []byte `protobuf:"bytes,4,opt,name=srcip" json:"srcip,omitempty"` + Dstip []byte `protobuf:"bytes,5,opt,name=dstip" json:"dstip,omitempty"` + Lint0 *uint32 `protobuf:"varint,6,opt,name=lint0" json:"lint0,omitempty"` + Lint1 *uint32 `protobuf:"varint,7,opt,name=lint1" json:"lint1,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Ncap) Reset() { *m = Ncap{} } +func (m *Ncap) String() string { return proto.CompactTextString(m) } +func (*Ncap) ProtoMessage() {} +func (*Ncap) Descriptor() ([]byte, []int) { return fileDescriptor8, []int{0} } + +func (m *Ncap) GetType() NcapType { + if m != nil && m.Type != nil { + return *m.Type + } + return NcapType_IPV4 +} + +func (m *Ncap) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *Ncap) GetLtype() NcapLegacyType { + if m != nil && m.Ltype != nil { + return *m.Ltype + } + return NcapLegacyType_Ncap_UDP +} + +func (m *Ncap) GetSrcip() []byte { + if m != nil { + return m.Srcip + } + return nil +} + +func (m *Ncap) GetDstip() []byte { + if m != nil { + return m.Dstip + } + return nil +} + +func (m *Ncap) GetLint0() uint32 { + if m != nil && m.Lint0 != nil { + return *m.Lint0 + } + return 0 +} + +func (m *Ncap) GetLint1() uint32 { + if m != nil && m.Lint1 != nil { + return *m.Lint1 + } + return 0 +} + +func init() { + proto.RegisterType((*Ncap)(nil), "nmsg.base.Ncap") + proto.RegisterEnum("nmsg.base.NcapType", NcapType_name, NcapType_value) + proto.RegisterEnum("nmsg.base.NcapLegacyType", NcapLegacyType_name, NcapLegacyType_value) +} + +func init() { proto.RegisterFile("ncap.proto", fileDescriptor8) } + +var fileDescriptor8 = []byte{ + // 218 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xca, 0x4b, 0x4e, 0x2c, + 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, 0x4a, 0x2c, + 0x4e, 0x55, 0x5a, 0xc0, 0xc8, 0xc5, 0xe2, 0x07, 0x94, 0x11, 0x52, 0xe4, 0x62, 0x29, 0xa9, 0x2c, + 0x48, 0x95, 0x60, 0x54, 0x60, 0xd2, 0xe0, 0x33, 0x12, 0xd6, 0x83, 0x2b, 0xd1, 0x03, 0x49, 0x87, + 0x00, 0xa5, 0x84, 0xf8, 0xb9, 0xd8, 0x0b, 0x12, 0x2b, 0x73, 0xf2, 0x13, 0x53, 0x24, 0x98, 0x80, + 0xaa, 0x78, 0x84, 0x34, 0xb8, 0x58, 0x73, 0xc0, 0x9a, 0x98, 0x15, 0x18, 0x81, 0x9a, 0x24, 0xd1, + 0x34, 0xf9, 0xa4, 0xa6, 0x27, 0x26, 0x57, 0x82, 0xb5, 0xf2, 0x72, 0xb1, 0x16, 0x17, 0x25, 0x67, + 0x16, 0x48, 0xb0, 0x00, 0x55, 0xf2, 0x80, 0xb8, 0x29, 0xc5, 0x25, 0x40, 0x2e, 0x2b, 0x8c, 0x9b, + 0x93, 0x99, 0x57, 0x62, 0x20, 0xc1, 0x06, 0xe4, 0xf2, 0xc2, 0xb8, 0x86, 0x12, 0xec, 0x20, 0xae, + 0x96, 0x16, 0x17, 0x07, 0xdc, 0x09, 0x1c, 0x5c, 0x2c, 0x9e, 0x01, 0x61, 0x26, 0x02, 0x0c, 0x50, + 0x96, 0x99, 0x00, 0xa3, 0x10, 0x17, 0x17, 0x1b, 0xc4, 0x26, 0x01, 0x26, 0x2d, 0x6b, 0x2e, 0x3e, + 0x34, 0x9b, 0x79, 0x20, 0xba, 0xe3, 0x43, 0x5d, 0x02, 0x80, 0xba, 0x60, 0xbc, 0x10, 0xe7, 0x00, + 0xa0, 0x4e, 0x5e, 0x2e, 0x4e, 0x30, 0xcf, 0xd3, 0xd9, 0x37, 0x40, 0x80, 0x09, 0x10, 0x00, 0x00, + 0xff, 0xff, 0x36, 0x82, 0xd0, 0xf3, 0x23, 0x01, 0x00, 0x00, +} diff --git a/nmsg_base/ncap.proto b/nmsg_base/ncap.proto new file mode 100644 index 0000000..aec20da --- /dev/null +++ b/nmsg_base/ncap.proto @@ -0,0 +1,26 @@ +syntax = "proto2"; +package nmsg.base; + +enum NcapType { + IPV4 = 0; + IPV6 = 1; + Legacy = 2; +} + +enum NcapLegacyType { + Ncap_UDP = 0; + Ncap_TCP = 1; + Ncap_ICMP = 2; +} + +message Ncap { + required NcapType type = 1; + required bytes payload = 2; + + // legacy NCAP fields + optional NcapLegacyType ltype = 3; + optional bytes srcip = 4; + optional bytes dstip = 5; + optional uint32 lint0 = 6; + optional uint32 lint1 = 7; +} diff --git a/nmsg_base/packet.pb.go b/nmsg_base/packet.pb.go new file mode 100644 index 0000000..51fcc85 --- /dev/null +++ b/nmsg_base/packet.pb.go @@ -0,0 +1,92 @@ +// Code generated by protoc-gen-go. +// source: packet.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type PacketType int32 + +const ( + // An IPv4 or IPv6 packet. The packet begins immediately with the IP + // header and contains the complete packet payload. Distinguishing between + // IPv4 and IPv6 packets is done by examining the IP version field in the + // IP header. + PacketType_IP PacketType = 1 +) + +var PacketType_name = map[int32]string{ + 1: "IP", +} +var PacketType_value = map[string]int32{ + "IP": 1, +} + +func (x PacketType) Enum() *PacketType { + p := new(PacketType) + *p = x + return p +} +func (x PacketType) String() string { + return proto.EnumName(PacketType_name, int32(x)) +} +func (x *PacketType) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(PacketType_value, data, "PacketType") + if err != nil { + return err + } + *x = PacketType(value) + return nil +} +func (PacketType) EnumDescriptor() ([]byte, []int) { return fileDescriptor9, []int{0} } + +type Packet struct { + PayloadType *PacketType `protobuf:"varint,1,req,name=payload_type,enum=nmsg.base.PacketType" json:"payload_type,omitempty"` + Payload []byte `protobuf:"bytes,2,req,name=payload" json:"payload,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Packet) Reset() { *m = Packet{} } +func (m *Packet) String() string { return proto.CompactTextString(m) } +func (*Packet) ProtoMessage() {} +func (*Packet) Descriptor() ([]byte, []int) { return fileDescriptor9, []int{0} } + +func (m *Packet) GetPayloadType() PacketType { + if m != nil && m.PayloadType != nil { + return *m.PayloadType + } + return PacketType_IP +} + +func (m *Packet) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func init() { + proto.RegisterType((*Packet)(nil), "nmsg.base.Packet") + proto.RegisterEnum("nmsg.base.PacketType", PacketType_name, PacketType_value) +} + +func init() { proto.RegisterFile("packet.proto", fileDescriptor9) } + +var fileDescriptor9 = []byte{ + // 111 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x48, 0x4c, 0xce, + 0x4e, 0x2d, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, + 0x4a, 0x2c, 0x4e, 0x55, 0x72, 0xe3, 0x62, 0x0b, 0x00, 0x4b, 0x09, 0x69, 0x83, 0x14, 0x55, 0xe6, + 0xe4, 0x27, 0xa6, 0xc4, 0x97, 0x54, 0x16, 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x69, 0xf0, 0x19, 0x89, + 0xea, 0xc1, 0xd5, 0xea, 0x41, 0x14, 0x86, 0x00, 0x25, 0x85, 0xf8, 0xb9, 0xd8, 0xa1, 0x8a, 0x25, + 0x98, 0x80, 0xea, 0x78, 0xb4, 0x44, 0xb8, 0xb8, 0x90, 0xa4, 0xd9, 0xb8, 0x98, 0x3c, 0x03, 0x04, + 0x18, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x3e, 0xce, 0x93, 0x77, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/packet.proto b/nmsg_base/packet.proto new file mode 100644 index 0000000..0444e48 --- /dev/null +++ b/nmsg_base/packet.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; +package nmsg.base; + +enum PacketType { + // An IPv4 or IPv6 packet. The packet begins immediately with the IP + // header and contains the complete packet payload. Distinguishing between + // IPv4 and IPv6 packets is done by examining the IP version field in the + // IP header. + IP = 1; +} + +message Packet { + required PacketType payload_type = 1; + required bytes payload = 2; +} diff --git a/nmsg_base/pkt.pb.go b/nmsg_base/pkt.pb.go new file mode 100644 index 0000000..e4466a3 --- /dev/null +++ b/nmsg_base/pkt.pb.go @@ -0,0 +1,55 @@ +// Code generated by protoc-gen-go. +// source: pkt.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Pkt struct { + Payload []byte `protobuf:"bytes,1,req,name=payload" json:"payload,omitempty"` + LenFrame *uint32 `protobuf:"varint,2,opt,name=len_frame" json:"len_frame,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Pkt) Reset() { *m = Pkt{} } +func (m *Pkt) String() string { return proto.CompactTextString(m) } +func (*Pkt) ProtoMessage() {} +func (*Pkt) Descriptor() ([]byte, []int) { return fileDescriptor10, []int{0} } + +func (m *Pkt) GetPayload() []byte { + if m != nil { + return m.Payload + } + return nil +} + +func (m *Pkt) GetLenFrame() uint32 { + if m != nil && m.LenFrame != nil { + return *m.LenFrame + } + return 0 +} + +func init() { + proto.RegisterType((*Pkt)(nil), "nmsg.base.Pkt") +} + +func init() { proto.RegisterFile("pkt.proto", fileDescriptor10) } + +var fileDescriptor10 = []byte{ + // 89 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0x2c, 0xc8, 0x2e, 0xd1, + 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, 0x4a, 0x2c, 0x4e, + 0x55, 0xd2, 0xe4, 0x62, 0x0e, 0xc8, 0x2e, 0x11, 0xe2, 0xe7, 0x62, 0x2f, 0x48, 0xac, 0xcc, 0xc9, + 0x4f, 0x4c, 0x91, 0x60, 0x54, 0x60, 0xd2, 0xe0, 0x11, 0x12, 0xe4, 0xe2, 0xcc, 0x49, 0xcd, 0x8b, + 0x4f, 0x2b, 0x4a, 0xcc, 0x4d, 0x95, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x05, 0x04, 0x00, 0x00, 0xff, + 0xff, 0x9d, 0xda, 0xa2, 0xb6, 0x41, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/pkt.proto b/nmsg_base/pkt.proto new file mode 100644 index 0000000..250ca5f --- /dev/null +++ b/nmsg_base/pkt.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +package nmsg.base; + +message Pkt { + required bytes payload = 1; + optional uint32 len_frame = 2; +} diff --git a/nmsg_base/xml.pb.go b/nmsg_base/xml.pb.go new file mode 100644 index 0000000..5608860 --- /dev/null +++ b/nmsg_base/xml.pb.go @@ -0,0 +1,55 @@ +// Code generated by protoc-gen-go. +// source: xml.proto +// DO NOT EDIT! + +package nmsg_base + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +type Xml struct { + Xmltype []byte `protobuf:"bytes,1,req,name=xmltype" json:"xmltype,omitempty"` + Xmlpayload []byte `protobuf:"bytes,2,req,name=xmlpayload" json:"xmlpayload,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Xml) Reset() { *m = Xml{} } +func (m *Xml) String() string { return proto.CompactTextString(m) } +func (*Xml) ProtoMessage() {} +func (*Xml) Descriptor() ([]byte, []int) { return fileDescriptor11, []int{0} } + +func (m *Xml) GetXmltype() []byte { + if m != nil { + return m.Xmltype + } + return nil +} + +func (m *Xml) GetXmlpayload() []byte { + if m != nil { + return m.Xmlpayload + } + return nil +} + +func init() { + proto.RegisterType((*Xml)(nil), "nmsg.base.Xml") +} + +func init() { proto.RegisterFile("xml.proto", fileDescriptor11) } + +var fileDescriptor11 = []byte{ + // 86 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xac, 0xc8, 0xcd, 0xd1, + 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xcc, 0xcb, 0x2d, 0x4e, 0xd7, 0x4b, 0x4a, 0x2c, 0x4e, + 0x55, 0xd2, 0xe2, 0x62, 0x8e, 0xc8, 0xcd, 0x11, 0xe2, 0xe7, 0x62, 0x07, 0x4a, 0x97, 0x54, 0x16, + 0xa4, 0x4a, 0x30, 0x2a, 0x30, 0x69, 0xf0, 0x08, 0x09, 0x71, 0x71, 0x01, 0x05, 0x0a, 0x12, 0x2b, + 0x73, 0xf2, 0x13, 0x53, 0x24, 0x98, 0x40, 0x62, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb0, 0x92, + 0x10, 0x81, 0x42, 0x00, 0x00, 0x00, +} diff --git a/nmsg_base/xml.proto b/nmsg_base/xml.proto new file mode 100644 index 0000000..e82b3aa --- /dev/null +++ b/nmsg_base/xml.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +package nmsg.base; + +message Xml { + required bytes xmltype = 1; + required bytes xmlpayload = 2; +} diff --git a/output.go b/output.go new file mode 100644 index 0000000..4c472e3 --- /dev/null +++ b/output.go @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "io" + "sync" + "time" +) + +// An Output encapsulates NmsgPayloads in Nmsg containers and writes them to +// an io.Writer. +type Output interface { + // Send sends an Nmsg Payload along the output. Implementations + // may queue the payload for later sending, so the caller should + // not modify the payload after calling Send. + // + // Send may be safely called from multiple goroutines. + Send(*NmsgPayload) error + // SetSequenced controls whether the Nmsg containers generated by + // the Output have sequence numbers. + SetSequenced(bool) + // SetCompression controls whether the Output generates compressed + // containers or uncompressed. + SetCompression(bool) + // SetCompressionRatio sets the anticipated compression ratio for + // compressed containers. + SetCompressionRatio(float32) + // SetMaxSize sets the maximum size of a container the Output will + // buffer, and the maximum size of a container or fragment the Output + // will write. For Ethernet, consider using nmsg.EtherContainerSize. + SetMaxSize(size int, writeSize int) + // Close shuts down the output, flushing any queued payloads. + // It will not close the underlying io.Writer. + Close() error +} + +type output struct { + w io.Writer + *Container +} + +func (o *output) Send(p *NmsgPayload) error { + c := o.Container + c.AddPayload(p) + _, err := c.WriteTo(o.w) + return err +} + +func (o *output) Close() error { + return nil +} + +// UnbufferedOutput returns an Output which writes an Nmsg container for +// each payload. +func UnbufferedOutput(w io.Writer) Output { + return &output{w: w, Container: NewContainer()} +} + +type bufferedOutput struct { + output + mu sync.Mutex +} + +func (o *bufferedOutput) Send(p *NmsgPayload) error { + o.mu.Lock() + defer o.mu.Unlock() + var ok, full bool + for !ok { + ok, full = o.AddPayload(p) + if !full { + return nil + } + + _, err := o.WriteTo(o.w) + if err != nil { + return err + } + } + return nil +} + +func (o *bufferedOutput) Close() error { + o.mu.Lock() + defer o.mu.Unlock() + if len(o.Nmsg.Payloads) > 0 { + _, err := o.WriteTo(o.w) + return err + } + return nil +} + +// BufferedOutput creates an Output which collects NmsgPayloads and sends +// them in containers as close as possible to the size set by SetMaxSize() +func BufferedOutput(w io.Writer) Output { + o := new(bufferedOutput) + o.output = output{w: w, Container: NewContainer()} + return o +} + +type timedBufferedOutput struct { + bufferedOutput + timer *time.Timer + d time.Duration + err error +} + +func (t *timedBufferedOutput) Send(p *NmsgPayload) error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.err != nil { + return t.err + } + + // We are sending the first payload on a new or recently-flushed + // output. Reset or restart flush timer. + if len(t.Nmsg.Payloads) == 0 && !t.timer.Reset(t.d) { + t.timer = time.AfterFunc(t.d, t.flush) + } + + var ok, full bool + for !ok { + ok, full = t.AddPayload(p) + if !full { + break + } + + t.timer.Reset(t.d) + + _, err := t.WriteTo(t.w) + if err != nil { + t.err = err + return err + } + } + return nil +} + +func (t *timedBufferedOutput) Close() error { + t.timer.Stop() + return t.bufferedOutput.Close() +} + +func (t *timedBufferedOutput) flush() { + t.mu.Lock() + defer t.mu.Unlock() + if len(t.Nmsg.Payloads) > 0 { + _, t.err = t.WriteTo(t.w) + } +} + +// TimedBufferedOutput creates an Output which collects NmsgPayloads +// and sends them in containers as close as possible to the size provided to +// SetMaxSize or after the given Duration, whichever comes first. +func TimedBufferedOutput(w io.Writer, d time.Duration) Output { + t := &timedBufferedOutput{d: d} + t.bufferedOutput.output = output{w: w, Container: NewContainer()} + t.timer = time.AfterFunc(d, t.flush) + + return t +} diff --git a/output_test.go b/output_test.go new file mode 100644 index 0000000..1cc1ba8 --- /dev/null +++ b/output_test.go @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_test + +import ( + "bytes" + "errors" + "testing" + "time" + + "github.com/farsightsec/go-nmsg" +) + +type countWriter struct { + count, total int + closed bool + t *testing.T +} + +// Testing output. + +// countWriter Implements io.WriteCloser, plus a Count() method returning +// how many times it has been called and a Total() method returning the +// number of bytes written. + +func (w *countWriter) Count() int { return w.count } +func (w *countWriter) Total() int { return w.total } + +func (w *countWriter) Write(b []byte) (int, error) { + w.t.Logf("Writing %d bytes", len(b)) + w.count++ + w.total += len(b) + return len(b), nil +} + +func newCountWriter(t *testing.T) *countWriter { + return &countWriter{t: t} +} + +// bufWriter augments bytes.Buffer with a Clos() method to +// satisfy io.WriteCloser +type bufWriter struct { + *bytes.Buffer +} + +func newBufWriter() *bufWriter { + return &bufWriter{new(bytes.Buffer)} +} + +func TestUnBufferedOutput(t *testing.T) { + c := newCountWriter(t) + p, err := nmsg.Payload(testMessage(1000)) + if err != nil { + t.Errorf(err.Error()) + } + o := nmsg.UnbufferedOutput(c) + o.SetMaxSize(1500, 0) + if err := o.Send(p); err != nil { + t.Errorf(err.Error()) + } + if c.Count() < 1 { + t.Errorf("No write occurred") + } + if c.Total() < 1000 { + t.Errorf("Write was too short") + } + if err := o.Close(); err != nil { + t.Errorf("Close failed") + } + +} + +func TestBufferedOutput(t *testing.T) { + c := newCountWriter(t) + o := nmsg.BufferedOutput(c) + o.SetMaxSize(1500, 0) + o.SetSequenced(true) + + // this should go in the buffer, and not be written + if err := o.Send(testPayload(800)); err != nil { + t.Errorf(err.Error()) + } + if c.Count() > 0 { + t.Error("Buffer did not suppress write") + } + + // this should flush the buffer, causing one write, + // then go into the buffer, not causing a second write. + if err := o.Send(testPayload(800)); err != nil { + t.Errorf(err.Error()) + } + if c.Count() < 1 { + t.Error("Buffer did not write") + } + if c.Count() > 1 { + t.Error("Buffer did not suppress write") + } + + // this should flush the buffer, causing one write, + // then bypass the buffer and be written in two fragments + if err := o.Send(testPayload(1700)); err != nil { + t.Errorf(err.Error()) + } + if err := o.Close(); err != nil { + t.Errorf(err.Error()) + } + if c.Count() < 4 { + t.Errorf("Missing writes: %d should be 4", c.Count()) + } + if c.Count() > 4 { + t.Error("Extra writes") + } +} + +func TestBufferedOutputNoConfig(t *testing.T) { + c := newCountWriter(t) + o := nmsg.BufferedOutput(c) + + // this should go in the buffer with the default + // MinContainerSize maximum, and not be written + if err := o.Send(testPayload(300)); err != nil { + t.Errorf(err.Error()) + } + if c.Count() > 0 { + t.Error("Buffer did not suppress write") + } + + // this should flush the buffer, causing one write, + // then go into the buffer, not causing a second write. + if err := o.Send(testPayload(300)); err != nil { + t.Errorf(err.Error()) + } + if c.Count() < 1 { + t.Error("Buffer did not write") + } + if c.Count() > 1 { + t.Error("Buffer did not suppress write") + } + + // this should flush the buffer, causing one write, + // then bypass the buffer and be written in two fragments + if err := o.Send(testPayload(600)); err != nil { + t.Errorf(err.Error()) + } + if err := o.Close(); err != nil { + t.Errorf(err.Error()) + } + if c.Count() < 4 { + t.Errorf("Missing writes: %d should be 4", c.Count()) + } + if c.Count() > 4 { + t.Error("Extra writes") + } +} + +func TestTimedBufferedOutput(t *testing.T) { + c := newCountWriter(t) + o := nmsg.TimedBufferedOutput(c, 100*time.Millisecond) + o.SetMaxSize(1500, 0) + o.SetSequenced(true) + + // This should wait about 100ms to send + if err := o.Send(testPayload(100)); err != nil { + t.Error(err.Error()) + } + if c.Count() > 0 { + t.Error("Write not delayed") + } + + time.Sleep(110 * time.Millisecond) + + if c.Count() < 1 { + t.Error("Write timed out.") + } + + if err := o.Close(); err != nil { + t.Error(err.Error()) + } +} + +func TestTimedBufferedOutputNoConfig(t *testing.T) { + c := newCountWriter(t) + o := nmsg.TimedBufferedOutput(c, 100*time.Millisecond) + o.SetSequenced(true) + + // This should wait about 100ms to send + if err := o.Send(testPayload(100)); err != nil { + t.Error(err.Error()) + } + if c.Count() > 0 { + t.Error("Write not delayed") + } + + time.Sleep(110 * time.Millisecond) + + if c.Count() < 1 { + t.Error("Write timed out.") + } + + if err := o.Close(); err != nil { + t.Error(err.Error()) + } + + for i := 0; i < 10; i++ { + o.Send(testPayload(100)) + } + time.Sleep(110 * time.Millisecond) + + if c.Count() < 2 { + t.Error("Writes timed out") + } +} + +func TestTimedBufferReset(t *testing.T) { + c := newCountWriter(t) + o := nmsg.TimedBufferedOutput(c, 100*time.Millisecond) + o.SetMaxSize(1500, 0) + o.SetSequenced(true) + + if err := o.Send(testPayload(750)); err != nil { + t.Error(err.Error()) + } + time.Sleep(50 * time.Millisecond) + // This should trigger a write, leave this payload in + // the buffer, and reset the timer for another 100ms. + if err := o.Send(testPayload(750)); err != nil { + t.Error(err.Error()) + } + + time.Sleep(25 * time.Millisecond) + + if c.Count() < 1 { + t.Error("Write failed to happen") + } + if c.Count() > 1 { + t.Error("Spurious write happened") + } + + // Check at start + 100ms, to make sure the buffer didn't fire twice + time.Sleep(25 * time.Millisecond) + if c.Count() > 1 { + t.Error("premature second write") + } + + // Check in after start + 150ms, second write should have happened. + time.Sleep(55 * time.Millisecond) + if c.Count() < 2 { + t.Error("second write late") + } + + time.Sleep(55 * time.Millisecond) + // The previous write caused the timer to expire, and it will need to + // be restarted. Test that code path with one more sequence of Sends + // which will force a flush. + for i := 0; i < 3; i++ { + if err := o.Send(testPayload(750)); err != nil { + t.Error(err.Error()) + } + } + + time.Sleep(25 * time.Millisecond) + + if c.Count() < 3 { + t.Error("third write late") + } + + time.Sleep(80 * time.Millisecond) + if c.Count() < 4 { + t.Error("Final write late") + } + + o.Close() +} + +type countdownWriter int + +func (c *countdownWriter) Write(b []byte) (int, error) { + if *c > 0 { + (*c)-- + return len(b), nil + } + return 0, errors.New("writer finished") +} + +func newCountdownWriter(n int) *countdownWriter { + c := countdownWriter(n) + return &c +} + +func TestTimedBufferedOutputError(t *testing.T) { + cw := newCountdownWriter(1) + + o := nmsg.TimedBufferedOutput(cw, 100*time.Millisecond) + o.SetMaxSize(1500, 0) + if err := o.Send(testPayload(750)); err != nil { + t.Error(err.Error()) + } + if err := o.Send(testPayload(750)); err != nil { + t.Error(err.Error()) + } + // write should occur above, and leave one payload in buffer, + // to be flushed by the next, which should return an error + if err := o.Send(testPayload(750)); err == nil { + t.Error("no error") + } +} + +func TestTimedBufferedOutputTimedError(t *testing.T) { + cw := newCountdownWriter(0) + o := nmsg.TimedBufferedOutput(cw, 100*time.Millisecond) + if err := o.Send(testPayload(100)); err != nil { + t.Error(err) + } + <-time.After(110 * time.Millisecond) + // At this point, a timer-driven flush should have triggered + // a writer error, which should be returned on the next Send. + if err := o.Send(testPayload(100)); err == nil { + t.Error("no error") + } +} + +type nullwriter struct{} + +func (n nullwriter) Write(b []byte) (int, error) { return len(b), nil } + +func BenchmarkUnbufferedOutput(b *testing.B) { + var w nullwriter + p, err := nmsg.Payload(testMessage(1000)) + if err != nil { + b.Error(err.Error()) + } + o := nmsg.UnbufferedOutput(w) + o.SetMaxSize(1500, 0) + for i := 0; i < b.N; i++ { + if err := o.Send(p); err != nil { + b.Error(err.Error()) + return + } + } + o.Close() +} + +func BenchmarkBufferedOutput(b *testing.B) { + var w nullwriter + p, err := nmsg.Payload(testMessage(1000)) + if err != nil { + b.Error(err.Error()) + } + o := nmsg.BufferedOutput(w) + o.SetMaxSize(1500, 0) + o.SetSequenced(true) + for i := 0; i < b.N; i++ { + if err := o.Send(p); err != nil { + b.Error(err.Error()) + return + } + } + o.Close() +} + +func BenchmarkTimedBufferedOutput(b *testing.B) { + var w nullwriter + p, err := nmsg.Payload(testMessage(1000)) + if err != nil { + b.Error(err.Error()) + } + o := nmsg.TimedBufferedOutput(w, 100*time.Millisecond) + o.SetMaxSize(1500, 0) + o.SetSequenced(true) + for i := 0; i < b.N; i++ { + if err := o.Send(p); err != nil { + b.Error(err.Error()) + return + } + } + o.Close() +} diff --git a/payload.go b/payload.go new file mode 100644 index 0000000..70180bb --- /dev/null +++ b/payload.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "encoding/binary" + "time" + + "github.com/golang/protobuf/proto" +) + +// Payload encapsulates an nmsg message in a NmsgPayload, suitable for sending to +// an Output +func Payload(m Message) (*NmsgPayload, error) { + mbytes, err := proto.Marshal(m) + if err != nil { + return nil, err + } + now := time.Now().UnixNano() + return &NmsgPayload{ + Vid: proto.Uint32(m.GetVid()), + Msgtype: proto.Uint32(m.GetMsgtype()), + TimeSec: proto.Int64(now / 1000000000), + TimeNsec: proto.Uint32(uint32(now % 1000000000)), + Payload: mbytes, + }, nil +} + +// SetSource sets the NmsgPayload source identifier. +func (p *NmsgPayload) SetSource(s uint32) { + p.Source = proto.Uint32(s) +} + +// SetOperator sets the NmsgPayload operator identifier. +func (p *NmsgPayload) SetOperator(o uint32) { + p.Operator = proto.Uint32(o) +} + +// SetGroup sets the NmsgPayload group identifier. +func (p *NmsgPayload) SetGroup(g uint32) { + p.Group = proto.Uint32(g) +} + +// Message returns the message encapsulated in the NmsgPayload, +// Unmarshaled +func (p *NmsgPayload) Message() (Message, error) { + m, err := NewMessage(*p.Vid, *p.Msgtype) + if err != nil { + return nil, err + } + err = proto.Unmarshal(p.Payload, m) + if err != nil { + return nil, err + } + return m, nil +} + +func (p *NmsgPayload) payloadSize() int { + var ibuf [binary.MaxVarintLen64]byte + + psiz := proto.Size(p) + // tag + varint length of encoded p + psiz += 1 + binary.PutUvarint(ibuf[:], uint64(psiz)) + // tag + varint CRC32 + psiz += 1 + binary.MaxVarintLen32 + return psiz +} diff --git a/payload_test.go b/payload_test.go new file mode 100644 index 0000000..8ca470a --- /dev/null +++ b/payload_test.go @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_test + +import ( + "testing" + + "github.com/farsightsec/go-nmsg" + "github.com/golang/protobuf/proto" +) + +func testMessage(length int) nmsg.Message { + return &testMsg{Bytes: make([]byte, length)} +} + +func testPayload(length int) *nmsg.NmsgPayload { + p, err := nmsg.Payload(testMessage(length)) + if err != nil { + return nil + } + return p +} + +type testMsg struct { + Bytes []byte `protobuf:"bytes,2,opt,name=bytes"` +} + +func (t *testMsg) GetVid() uint32 { return 10 } +func (t *testMsg) GetMsgtype() uint32 { return 20 } + +func (t *testMsg) Reset() { *t = testMsg{} } +func (t *testMsg) String() string { return proto.CompactTextString(t) } +func (t *testMsg) ProtoMessage() {} + +func init() { + nmsg.Register(&testMsg{}) +} + +func TestRegister(t *testing.T) { + msg, err := nmsg.NewMessage(10, 20) + if err != nil { + t.Error(err) + } + if _, ok := msg.(*testMsg); !ok { + t.Errorf("NewMessage returned wrong type") + } +} + +func TestPayload(t *testing.T) { + p, err := nmsg.Payload(testMessage(1000)) + if err != nil { + t.Errorf("nmsg.Payload(): %s", err) + } + + m, err := p.Message() + if err != nil { + t.Error(err) + } + + if tp, ok := m.(*testMsg); !ok { + t.Errorf("Wrong type from payload") + } else if len(tp.Bytes) != 1000 { + t.Error("decode failed") + } +} diff --git a/register.go b/register.go new file mode 100644 index 0000000..9b319ac --- /dev/null +++ b/register.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "fmt" + "reflect" +) + +var types map[uint32]map[uint32]reflect.Type + +// Register records the supplied message's type, indexed by its MessageType +// and VendorID, for the purposes of decoding protobuf-encoded payloads. +// +// Register should be called from the init() function of the module defining +// the payload type. It is not safe to call from multiple goroutines, and +// may not be called if any goroutine is concurrently decoding NMSG payloads. +func Register(m Message) { + if types == nil { + types = make(map[uint32]map[uint32]reflect.Type) + } + vid := m.GetVid() + v, ok := types[vid] + if !ok { + v = make(map[uint32]reflect.Type) + types[vid] = v + } + + msgtype := m.GetMsgtype() + v[msgtype] = reflect.TypeOf(m) +} + +type unknownVendor uint32 + +func (v unknownVendor) Error() string { + return fmt.Sprintf("Vendor %d has no registered Msgtypes.", v) +} + +type unknownMsgtype struct{ vid, msgtype uint32 } + +func (t unknownMsgtype) Error() string { + return fmt.Sprintf("Msgtype %d is not registered for vendor %d.", t.msgtype, t.vid) +} + +// NewMessage creates a new Message with an underlying type identified +// by vid, msgtype. +func NewMessage(vid, msgtype uint32) (Message, error) { + v, ok := types[vid] + if !ok { + return nil, unknownVendor(vid) + } + + t, ok := v[msgtype] + if !ok { + return nil, unknownMsgtype{vid, msgtype} + } + + return reflect.New(t.Elem()).Interface().(Message), nil +} diff --git a/sockspec.go b/sockspec.go new file mode 100644 index 0000000..f4fcd1a --- /dev/null +++ b/sockspec.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "encoding/json" + "fmt" + "net" + "strconv" + "strings" +) + +// A Sockspec is an address of a single socket (addr/port) or a series of +// sockets with contiguous port numbers (addr/loport..hiport) +type Sockspec struct { + Addr *net.UDPAddr + Hiport int +} + +// ParseSockspec creates a Sockspec from its text representaion v. +func ParseSockspec(v string) (*Sockspec, error) { + s := &Sockspec{} + return s, s.Set(v) +} + +// Set initializes a Sockspec from its text representation v. Set satisfies +// flag.Value allowing a sockspec to be conveniently specified as a command +// line parameter. +func (s *Sockspec) Set(v string) error { + l := strings.SplitN(v, "/", 2) + if len(l) != 2 { + return fmt.Errorf("Invalid sockspec: %s", v) + } + p := strings.SplitN(l[1], "..", 2) + addr := fmt.Sprintf("%s:%s", l[0], p[0]) + uaddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + return fmt.Errorf("Invalid addr %s: %v", addr, err) + } + s.Addr = uaddr + if len(p) == 1 { + s.Hiport = uaddr.Port + return nil + } + + hiport, err := strconv.ParseUint(p[1], 10, 16) + if err != nil { + return fmt.Errorf("Invalid high port %s: %v", p[1], err) + } + + if int(hiport) <= uaddr.Port { + return fmt.Errorf("Invalid port range %s", l[1]) + } + s.Hiport = int(hiport) + return nil +} + +// UnmarshalJSON satisifies json.Unmarshaler allowing Sockspecs to be parsed +// from JSON configurations. +func (s *Sockspec) UnmarshalJSON(b []byte) error { + var v string + if err := json.Unmarshal(b, &v); err != nil { + return err + } + return s.Set(v) +} + +// UnmarshalYAML satisifies yaml.Unmarshaler allowing Sockspecs to be parsed +// from YAML configurations. +func (s *Sockspec) UnmarshalYAML(u func(interface{}) error) error { + var v string + if err := u(&v); err != nil { + return err + } + return s.Set(v) +} + +// Addrs returns the list of UDP socket addresses of the Sockspec, or nil +// if the Sockspec is uninitialized. +func (s *Sockspec) Addrs() []*net.UDPAddr { + var addrs []*net.UDPAddr + if s.Addr == nil { + return nil + } + for i := s.Addr.Port; i <= s.Hiport; i++ { + a := &net.UDPAddr{} + *a = *s.Addr + a.Port = i + addrs = append(addrs, a) + } + return addrs +} + +// String returns the string representation of the Sockspec. If the Sockspec +// is uninitialized, String returns the empty string. +func (s *Sockspec) String() string { + if s.Addr == nil { + return "" + } + if s.Hiport > s.Addr.Port { + return fmt.Sprintf("%s/%d..%d", s.Addr.IP.String(), + s.Addr.Port, s.Hiport) + } + return fmt.Sprintf("%s/%d", s.Addr.IP.String(), s.Addr.Port) +} diff --git a/sockspec_test.go b/sockspec_test.go new file mode 100644 index 0000000..9cef0b5 --- /dev/null +++ b/sockspec_test.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "encoding/json" + "testing" + + yaml "gopkg.in/yaml.v2" +) + +var sockspecTestCases = []struct { + sockspec string + valid bool + naddrs int +}{ + {"127.0.0.1/382", true, 1}, + {"127.0.0.1/382..390", true, 9}, + {"foobar", false, 0}, + {"127.0.0.1/foobar", false, 0}, + {"127.0.0.1/390..382", false, 0}, + {"127.0.0.1/390..foobar", false, 0}, + {"invalid_hostname/381", false, 0}, +} + +func testSockSpecCommon(t *testing.T, parse func(string, *Sockspec) error) { + t.Helper() + for _, tc := range sockspecTestCases { + var ss Sockspec + if err := parse(tc.sockspec, &ss); err != nil { + if tc.valid { + t.Errorf("%s: %v", tc.sockspec, err) + } + continue + } + if !tc.valid { + t.Errorf("parsed invalid sockspec %s", tc.sockspec) + continue + } + if len(ss.Addrs()) != tc.naddrs { + t.Errorf("%s: expected %d addrs, got %d", tc.sockspec, + tc.naddrs, len(ss.Addrs())) + } + if ss.String() != tc.sockspec { + t.Errorf("%s parsed to %s (%#v)", tc.sockspec, &ss, ss) + } + } +} + +func TestSockSpecSet(t *testing.T) { + testSockSpecCommon(t, func(s string, ss *Sockspec) error { + return ss.Set(s) + }) +} + +func TestSockSpecParse(t *testing.T) { + testSockSpecCommon(t, func(s string, ss *Sockspec) error { + parsed, err := ParseSockspec(s) + *ss = *parsed + return err + }) +} + +func TestSockSpecJSON(t *testing.T) { + testSockSpecCommon(t, func(s string, ss *Sockspec) error { + b, err := json.Marshal(s) + if err != nil { + return err + } + return json.Unmarshal(b, ss) + }) +} + +func TestSockSpecYAML(t *testing.T) { + testSockSpecCommon(t, func(s string, ss *Sockspec) error { + b, err := yaml.Marshal(s) + if err != nil { + return err + } + return yaml.Unmarshal(b, ss) + }) +} diff --git a/zbuf.go b/zbuf.go new file mode 100644 index 0000000..e44531d --- /dev/null +++ b/zbuf.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "io" +) + +func zbufDeflate(b []byte) ([]byte, error) { + buf := new(bytes.Buffer) + binary.Write(buf, binary.BigEndian, uint32(len(b))) + w := zlib.NewWriter(buf) + if _, err := w.Write(b); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func zbufInflate(b []byte) ([]byte, error) { + br := bytes.NewReader(b) + var ilen uint32 + binary.Read(br, binary.BigEndian, &ilen) + buf := bytes.NewBuffer(make([]byte, 0, int(ilen))) + r, err := zlib.NewReader(br) + if err != nil { + return nil, err + } + if _, err = io.Copy(buf, r); err != nil { + return nil, err + } + r.Close() + return buf.Bytes(), nil +} diff --git a/zlib_test.go b/zlib_test.go new file mode 100644 index 0000000..25b7be2 --- /dev/null +++ b/zlib_test.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2017 by Farsight Security, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package nmsg_test + +import ( + "bytes" + "testing" + + "github.com/farsightsec/go-nmsg" +) + +func TestZlib(t *testing.T) { + b := new(bytes.Buffer) + m := testMessage(20) + p, err := nmsg.Payload(m) + if err != nil { + t.Fatal(err) + } + out := nmsg.UnbufferedOutput(b) + out.SetCompression(true) + out.SetMaxSize(1500, 0) + if err := out.Send(p); err != nil { + t.Fatal(err) + } + + inp := nmsg.NewInput(b, 1500) + p, err = inp.Recv() + if err != nil { + t.Fatal(err) + } + mm, err := p.Message() + if err != nil { + t.Fatal(err) + } + mi, ok := mm.(*testMsg) + if !ok { + t.Error("received message of wrong type") + } + if len(mi.Bytes) != len(m.(*testMsg).Bytes) { + t.Error("received message of wrong length") + } + for i := range mi.Bytes { + if mi.Bytes[i] != 0 { + t.Fatal("received message with wrong data") + } + } + +}