diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1377554 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.swp diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..169e848 --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,14 @@ +Copyright (c) 2013-2014 by Farsight Security, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/FrameStreamInput.go b/FrameStreamInput.go new file mode 100644 index 0000000..8403c61 --- /dev/null +++ b/FrameStreamInput.go @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2013-2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "io" + "log" + "os" + "time" + + "github.com/farsightsec/golang-framestream" +) + +// MaxPayloadSize sets the upper limit on input Dnstap payload sizes. If an Input +// receives a Dnstap payload over this size limit, ReadInto will log an error and +// return. +// +// EDNS0 and DNS over TCP use 2 octets for DNS message size, imposing a maximum +// size of 65535 octets for the DNS message, which is the bulk of the data carried +// in a Dnstap message. Protobuf encoding overhead and metadata with some size +// guidance (e.g., identity and version being DNS strings, which have a maximum +// length of 255) add up to less than 1KB. The default 96KiB size of the buffer +// allows a bit over 30KB space for "extra" metadata. +// +var MaxPayloadSize uint32 = 96 * 1024 + +// A FrameStreamInput reads dnstap data from an io.ReadWriter. +type FrameStreamInput struct { + wait chan bool + decoder *framestream.Decoder + timeout time.Duration +} + +// NewFrameStreamInput creates a FrameStreamInput reading data from the given +// io.ReadWriter. If bi is true, the input will use the bidirectional +// framestream protocol suitable for TCP and unix domain socket connections. +func NewFrameStreamInput(r io.ReadWriter, bi bool) (input *FrameStreamInput, err error) { + return NewFrameStreamInputTimeout(r, bi, 0) +} + +// NewFrameStreamInputTimeout creates a FramestreamInput reading data from the +// given io.ReadWriter with a timeout applied to reading and (for bidirectional +// inputs) writing control messages. +func NewFrameStreamInputTimeout(r io.ReadWriter, bi bool, timeout time.Duration) (input *FrameStreamInput, err error) { + input = new(FrameStreamInput) + decoderOptions := framestream.DecoderOptions{ + MaxPayloadSize: MaxPayloadSize, + ContentType: FSContentType, + Bidirectional: bi, + Timeout: timeout, + } + input.decoder, err = framestream.NewDecoder(r, &decoderOptions) + if err != nil { + return + } + input.wait = make(chan bool) + return +} + +// NewFrameStreamInputFromFilename creates a FrameStreamInput reading from +// the named file. +func NewFrameStreamInputFromFilename(fname string) (input *FrameStreamInput, err error) { + file, err := os.Open(fname) + if err != nil { + return nil, err + } + input, err = NewFrameStreamInput(file, false) + return +} + +// ReadInto reads data from the FrameStreamInput into the output channel. +// +// ReadInto satisfies the dnstap Input interface. +func (input *FrameStreamInput) ReadInto(output chan []byte) { + for { + buf, err := input.decoder.Decode() + if err != nil { + if err != io.EOF { + log.Printf("framestream.Decoder.Decode() failed: %s\n", err) + } + break + } + newbuf := make([]byte, len(buf)) + copy(newbuf, buf) + output <- newbuf + } + close(input.wait) +} + +// Wait reeturns when ReadInto has finished. +// +// Wait satisfies the dnstap Input interface. +func (input *FrameStreamInput) Wait() { + <-input.wait +} diff --git a/FrameStreamOutput.go b/FrameStreamOutput.go new file mode 100644 index 0000000..5955b8b --- /dev/null +++ b/FrameStreamOutput.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "io" + "log" + "os" + + "github.com/farsightsec/golang-framestream" +) + +// FrameStreamOutput implements a dnstap Output to an io.Writer. +type FrameStreamOutput struct { + outputChannel chan []byte + wait chan bool + enc *framestream.Encoder +} + +// NewFrameStreamOutput creates a FrameStreamOutput writing dnstap data to +// the given io.Writer. +func NewFrameStreamOutput(w io.Writer) (o *FrameStreamOutput, err error) { + o = new(FrameStreamOutput) + o.outputChannel = make(chan []byte, outputChannelSize) + o.enc, err = framestream.NewEncoder(w, &framestream.EncoderOptions{ContentType: FSContentType}) + if err != nil { + return + } + o.wait = make(chan bool) + return +} + +// NewFrameStreamOutputFromFilename creates a file with the namee fname, +// truncates it if it exists, and returns a FrameStreamOutput writing to +// the newly created or truncated file. +func NewFrameStreamOutputFromFilename(fname string) (o *FrameStreamOutput, err error) { + if fname == "" || fname == "-" { + return NewFrameStreamOutput(os.Stdout) + } + w, err := os.Create(fname) + if err != nil { + return + } + return NewFrameStreamOutput(w) +} + +// GetOutputChannel returns the channel on which the FrameStreamOutput accepts +// data. +// +// GetOutputData satisfies the dnstap Output interface. +func (o *FrameStreamOutput) GetOutputChannel() chan []byte { + return o.outputChannel +} + +// RunOutputLoop processes data received on the channel returned by +// GetOutputChannel, returning after the CLose method is called. +// If there is an error writing to the Output's writer, RunOutputLoop() +// logs a fatal error exits the program. +// +// RunOutputLoop satisfies the dnstap Output interface. +func (o *FrameStreamOutput) RunOutputLoop() { + for frame := range o.outputChannel { + if _, err := o.enc.Write(frame); err != nil { + log.Fatalf("framestream.Encoder.Write() failed: %s\n", err) + break + } + } + close(o.wait) +} + +// Close closes the channel returned from GetOutputChannel, and flushes +// all pending output. +// +// Close satisifies the dnstap Output interface. +func (o *FrameStreamOutput) Close() { + close(o.outputChannel) + <-o.wait + o.enc.Flush() + o.enc.Close() +} diff --git a/FrameStreamSockInput.go b/FrameStreamSockInput.go new file mode 100644 index 0000000..116fa2d --- /dev/null +++ b/FrameStreamSockInput.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2013-2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "log" + "net" + "os" + "time" +) + +// A FrameStreamSockInput collects dnstap data from one or more clients of +// a listening socket. +type FrameStreamSockInput struct { + wait chan bool + listener net.Listener + timeout time.Duration +} + +// NewFrameStreamSockInput creates a FrameStreamSockInput collecting dnstap +// data from clients which connect to the given listener. +func NewFrameStreamSockInput(listener net.Listener) (input *FrameStreamSockInput) { + input = new(FrameStreamSockInput) + input.listener = listener + return +} + +// SetTimeout sets the timeout for reading the initial handshake and writing +// response control messages to clients of the FrameStreamSockInput's listener. +// +// The timeout is effective only for connections accepted after the call to +// FrameStreamSockInput. +func (input *FrameStreamSockInput) SetTimeout(timeout time.Duration) { + input.timeout = timeout +} + +// NewFrameStreamSockInputFromPath creates a unix domain socket at the +// given socketPath and returns a FrameStreamSockInput collecting dnstap +// data from clients connecting to this socket. +// +// If a socket or other file already exists at socketPath, +// NewFrameStreamSockInputFromPath removes it before creating the socket. +func NewFrameStreamSockInputFromPath(socketPath string) (input *FrameStreamSockInput, err error) { + os.Remove(socketPath) + listener, err := net.Listen("unix", socketPath) + if err != nil { + return + } + return NewFrameStreamSockInput(listener), nil +} + +// ReadInto accepts connections to the FrameStreamSockInput's listening +// socket and sends all dnstap data read from these connections to the +// output channel. +// +// ReadInto satisfies the dnstap Input interface. +func (input *FrameStreamSockInput) ReadInto(output chan []byte) { + for { + conn, err := input.listener.Accept() + if err != nil { + log.Printf("net.Listener.Accept() failed: %s\n", err) + continue + } + i, err := NewFrameStreamInputTimeout(conn, true, input.timeout) + if err != nil { + log.Printf("dnstap.NewFrameStreamInput() failed: %s\n", err) + continue + } + log.Printf("dnstap.FrameStreamSockInput: accepted a socket connection\n") + go i.ReadInto(output) + } +} + +// Wait satisfies the dnstap Input interface. +// +// The FrameSTreamSocketInput Wait method never returns, because the +// corresponding Readinto method also never returns. +func (input *FrameStreamSockInput) Wait() { + select {} +} diff --git a/FrameStreamSockOutput.go b/FrameStreamSockOutput.go new file mode 100644 index 0000000..66d689f --- /dev/null +++ b/FrameStreamSockOutput.go @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2019 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "log" + "net" + "time" + + "github.com/farsightsec/golang-framestream" +) + +// A FrameStreamSockOutput manages a socket connection and sends dnstap +// data over a framestream connection on that socket. +type FrameStreamSockOutput struct { + outputChannel chan []byte + address net.Addr + wait chan bool + dialer *net.Dialer + timeout time.Duration + retry time.Duration + flushTimeout time.Duration +} + +// NewFrameStreamSockOutput creates a FrameStreamSockOutput manaaging a +// connection to the given address. +func NewFrameStreamSockOutput(address net.Addr) (*FrameStreamSockOutput, error) { + return &FrameStreamSockOutput{ + outputChannel: make(chan []byte, outputChannelSize), + address: address, + wait: make(chan bool), + retry: 10 * time.Second, + flushTimeout: 5 * time.Second, + dialer: &net.Dialer{ + Timeout: 30 * time.Second, + }, + }, nil +} + +// SetTimeout sets the write timeout for data and control messages and the +// read timeout for handshake responses on the FrameStreamSockOutput's +// connection. The default timeout is zero, for no timeout. +func (o *FrameStreamSockOutput) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// SetFlushTimeout sets the maximum time data will be kept in the output +// buffer. +// +// The default flush timeout is five seconds. +func (o *FrameStreamSockOutput) SetFlushTimeout(timeout time.Duration) { + o.flushTimeout = timeout +} + +// SetRetryInterval specifies how long the FrameStreamSockOutput will wait +// before re-establishing a failed connection. The default retry interval +// is 10 seconds. +func (o *FrameStreamSockOutput) SetRetryInterval(retry time.Duration) { + o.retry = retry +} + +// SetDialer replaces the default net.Dialer for re-establishing the +// the FrameStreamSockOutput connection. This can be used to set the +// timeout for connection establishment and enable keepalives +// new connections. +// +// FrameStreamSockOutput uses a default dialer with a 30 second +// timeout. +func (o *FrameStreamSockOutput) SetDialer(dialer *net.Dialer) { + o.dialer = dialer +} + +// GetOutputChannel returns the channel on which the +// FrameStreamSockOutput accepts data. +// +// GetOutputChannel satisifes the dnstap Output interface. +func (o *FrameStreamSockOutput) GetOutputChannel() chan []byte { + return o.outputChannel +} + +// A timedConn resets an associated timer on each Write to the underlying +// connection, and is used to implement the output's flush timeout. +type timedConn struct { + net.Conn + timer *time.Timer + timeout time.Duration + + // idle is true if the timer has fired and we have consumed + // the time from its channel. We use this to prevent deadlocking + // when resetting or stopping an already fired timer. + idle bool +} + +// SetIdle informs the timedConn that the associated timer is idle, i.e. +// it has fired and has not been reset. +func (t *timedConn) SetIdle() { + t.idle = true +} + +// Stop stops the underlying timer, consuming any time value if the timer +// had fired before Stop was called. +func (t *timedConn) StopTimer() { + if !t.timer.Stop() && !t.idle { + <-t.timer.C + } + t.idle = true +} + +func (t *timedConn) Write(b []byte) (int, error) { + t.StopTimer() + t.timer.Reset(t.timeout) + t.idle = false + return t.Conn.Write(b) +} + +func (t *timedConn) Close() error { + t.StopTimer() + return t.Conn.Close() +} + +// RunOutputLoop reads data from the output channel and sends it over +// a connections to the FrameStreamSockOutput's address, establishing +// the connection as needed. +// +// RunOutputLoop satisifes the dnstap Output interface. +func (o *FrameStreamSockOutput) RunOutputLoop() { + var enc *framestream.Encoder + var err error + + // Start with the connection flush timer in a stopped state. + // It will be reset by the first Write call on a new connection. + conn := &timedConn{ + timer: time.NewTimer(0), + timeout: o.flushTimeout, + } + conn.StopTimer() + + defer func() { + if enc != nil { + enc.Flush() + enc.Close() + } + if conn != nil { + conn.Close() + } + close(o.wait) + }() + + for { + select { + case frame, ok := <-o.outputChannel: + if !ok { + return + } + + // the retry loop + for ;; time.Sleep(o.retry) { + if enc == nil { + // connect the socket + conn.Conn, err = o.dialer.Dial(o.address.Network(), o.address.String()) + if err != nil { + log.Printf("Dial() failed: %v", err) + continue // = retry + } + + // create the encoder + eopt := framestream.EncoderOptions{ + ContentType: FSContentType, + Bidirectional: true, + Timeout: o.timeout, + } + enc, err = framestream.NewEncoder(conn, &eopt) + if err != nil { + log.Printf("framestream.NewEncoder() failed: %v", err) + conn.Close() + enc = nil + continue // = retry + } + } + + // try writing + if _, err = enc.Write(frame); err != nil { + log.Printf("framestream.Encoder.Write() failed: %v", err) + enc.Close() + enc = nil + conn.Close() + continue // = retry + } + + break // success! + } + + case <-conn.timer.C: + conn.SetIdle() + if enc == nil { + continue + } + if err := enc.Flush(); err != nil { + log.Printf("framestream.Encoder.Flush() failed: %s", err) + enc.Close() + enc = nil + conn.Close() + time.Sleep(o.retry) + } + } + } +} + +// Close shuts down the FrameStreamSockOutput's output channel and returns +// after all pending data has been flushed and the connection has been closed. +// +// Close satisifes the dnstap Output interface +func (o *FrameStreamSockOutput) Close() { + close(o.outputChannel) + <-o.wait +} diff --git a/JsonFormat.go b/JsonFormat.go new file mode 100644 index 0000000..092026a --- /dev/null +++ b/JsonFormat.go @@ -0,0 +1,140 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "bytes" + "encoding/json" + "fmt" + "net" + "time" + + "github.com/miekg/dns" +) + +type jsonTime time.Time + +func (jt *jsonTime) MarshalJSON() ([]byte, error) { + stamp := time.Time(*jt).Format(time.RFC3339Nano) + return []byte(fmt.Sprintf("\"%s\"", stamp)), nil +} + +type jsonDnstap struct { + Type string `json:"type"` + Identity string `json:"identity,omitempty"` + Version string `json:"version,omitempty"` + Message jsonMessage `json:"message"` +} + +type jsonMessage struct { + Type string `json:"type"` + QueryTime *jsonTime `json:"query_time,omitempty"` + ResponseTime *jsonTime `json:"response_time,omitempty"` + SocketFamily string `json:"socket_family,omitempty"` + SocketProtocol string `json:"socket_protocol,omitempty"` + QueryAddress *net.IP `json:"query_address,omitempty"` + ResponseAddress *net.IP `json:"response_address,omitempty"` + QueryPort uint32 `json:"query_port,omitempty"` + ResponsePort uint32 `json:"response_port,omitempty"` + QueryZone string `json:"query_zone,omitempty"` + QueryMessage string `json:"query_message,omitempty"` + ResponseMessage string `json:"response_message,omitempty"` +} + +func convertJSONMessage(m *Message) jsonMessage { + jMsg := jsonMessage{ + Type: fmt.Sprint(m.Type), + SocketFamily: fmt.Sprint(m.SocketFamily), + SocketProtocol: fmt.Sprint(m.SocketProtocol), + } + + if m.QueryTimeSec != nil && m.QueryTimeNsec != nil { + qt := jsonTime(time.Unix(int64(*m.QueryTimeSec), int64(*m.QueryTimeNsec)).UTC()) + jMsg.QueryTime = &qt + } + + if m.ResponseTimeSec != nil && m.ResponseTimeNsec != nil { + rt := jsonTime(time.Unix(int64(*m.ResponseTimeSec), int64(*m.ResponseTimeNsec)).UTC()) + jMsg.ResponseTime = &rt + } + + if m.QueryAddress != nil { + qa := net.IP(m.QueryAddress) + jMsg.QueryAddress = &qa + } + + if m.ResponseAddress != nil { + ra := net.IP(m.ResponseAddress) + jMsg.ResponseAddress = &ra + } + + if m.QueryPort != nil { + jMsg.QueryPort = *m.QueryPort + } + + if m.ResponsePort != nil { + jMsg.ResponsePort = *m.ResponsePort + } + + if m.QueryZone != nil { + name, _, err := dns.UnpackDomainName(m.QueryZone, 0) + if err != nil { + jMsg.QueryZone = fmt.Sprintf("parse failed: %v", err) + } else { + jMsg.QueryZone = string(name) + } + } + + if m.QueryMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.QueryMessage) + if err != nil { + jMsg.QueryMessage = fmt.Sprintf("parse failed: %v", err) + } else { + jMsg.QueryMessage = msg.String() + } + } + + if m.ResponseMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.ResponseMessage) + if err != nil { + jMsg.ResponseMessage = fmt.Sprintf("parse failed: %v", err) + } else { + jMsg.ResponseMessage = msg.String() + } + } + return jMsg +} + +// JSONFormat renders a Dnstap message in JSON format. Any encapsulated +// DNS messages are rendered as strings in a format similar to 'dig' output. +func JSONFormat(dt *Dnstap) (out []byte, ok bool) { + var s bytes.Buffer + + j, err := json.Marshal(jsonDnstap{ + Type: fmt.Sprint(dt.Type), + Identity: string(dt.Identity), + Version: string(dt.Version), + Message: convertJSONMessage(dt.Message), + }) + if err != nil { + return nil, false + } + + s.WriteString(string(j) + "\n") + + return s.Bytes(), true +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/QuietTextFormat.go b/QuietTextFormat.go new file mode 100644 index 0000000..4dc8909 --- /dev/null +++ b/QuietTextFormat.go @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2013-2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "bytes" + "fmt" + "net" + "strconv" + "time" + + "github.com/miekg/dns" +) + +const quietTimeFormat = "15:04:05" + +func textConvertTime(s *bytes.Buffer, secs *uint64, nsecs *uint32) { + if secs != nil { + s.WriteString(time.Unix(int64(*secs), 0).Format(quietTimeFormat)) + } else { + s.WriteString("??:??:??") + } + if nsecs != nil { + s.WriteString(fmt.Sprintf(".%06d", *nsecs/1000)) + } else { + s.WriteString(".??????") + } +} + +func textConvertIP(s *bytes.Buffer, ip []byte) { + if ip != nil { + s.WriteString(net.IP(ip).String()) + } else { + s.WriteString("MISSING_ADDRESS") + } +} + +func textConvertMessage(m *Message, s *bytes.Buffer) { + isQuery := false + printQueryAddress := false + + switch *m.Type { + case Message_CLIENT_QUERY, + Message_RESOLVER_QUERY, + Message_AUTH_QUERY, + Message_FORWARDER_QUERY, + Message_TOOL_QUERY: + isQuery = true + case Message_CLIENT_RESPONSE, + Message_RESOLVER_RESPONSE, + Message_AUTH_RESPONSE, + Message_FORWARDER_RESPONSE, + Message_TOOL_RESPONSE: + isQuery = false + default: + s.WriteString("[unhandled Message.Type]\n") + return + } + + if isQuery { + textConvertTime(s, m.QueryTimeSec, m.QueryTimeNsec) + } else { + textConvertTime(s, m.ResponseTimeSec, m.ResponseTimeNsec) + } + s.WriteString(" ") + + switch *m.Type { + case Message_CLIENT_QUERY, + Message_CLIENT_RESPONSE: + { + s.WriteString("C") + } + case Message_RESOLVER_QUERY, + Message_RESOLVER_RESPONSE: + { + s.WriteString("R") + } + case Message_AUTH_QUERY, + Message_AUTH_RESPONSE: + { + s.WriteString("A") + } + case Message_FORWARDER_QUERY, + Message_FORWARDER_RESPONSE: + { + s.WriteString("F") + } + case Message_STUB_QUERY, + Message_STUB_RESPONSE: + { + s.WriteString("S") + } + case Message_TOOL_QUERY, + Message_TOOL_RESPONSE: + { + s.WriteString("T") + } + } + + if isQuery { + s.WriteString("Q ") + } else { + s.WriteString("R ") + } + + switch *m.Type { + case Message_CLIENT_QUERY, + Message_CLIENT_RESPONSE, + Message_AUTH_QUERY, + Message_AUTH_RESPONSE: + printQueryAddress = true + } + + if printQueryAddress { + textConvertIP(s, m.QueryAddress) + } else { + textConvertIP(s, m.ResponseAddress) + } + s.WriteString(" ") + + if m.SocketProtocol != nil { + s.WriteString(m.SocketProtocol.String()) + } + s.WriteString(" ") + + var err error + msg := new(dns.Msg) + if isQuery { + s.WriteString(strconv.Itoa(len(m.QueryMessage))) + s.WriteString("b ") + err = msg.Unpack(m.QueryMessage) + } else { + s.WriteString(strconv.Itoa(len(m.ResponseMessage))) + s.WriteString("b ") + err = msg.Unpack(m.ResponseMessage) + } + + if err != nil || len(msg.Question) == 0 { + s.WriteString("X ") + } else { + s.WriteString("\"" + msg.Question[0].Name + "\" ") + s.WriteString(dns.Class(msg.Question[0].Qclass).String() + " ") + s.WriteString(dns.Type(msg.Question[0].Qtype).String()) + } + + s.WriteString("\n") +} + +// TextFormat renders a dnstap message in a compact human-readable text +// form. +func TextFormat(dt *Dnstap) (out []byte, ok bool) { + var s bytes.Buffer + + if *dt.Type == Dnstap_MESSAGE { + textConvertMessage(dt.Message, &s) + return s.Bytes(), true + } + + return nil, false +} diff --git a/README b/README new file mode 100644 index 0000000..e59effb --- /dev/null +++ b/README @@ -0,0 +1,15 @@ +dnstap: flexible, structured event replication format for DNS servers +--------------------------------------------------------------------- + +dnstap implements an encoding format for DNS server events. It uses a +lightweight framing on top of event payloads encoded using Protocol Buffers and +is transport neutral. + +dnstap can represent internal state inside a DNS server that is difficult to +obtain using techniques based on traditional packet capture or unstructured +textual format logging. + +This repository contains a command-line tool named "dnstap" developed in the +Go programming language. It can be installed with the following command: + + go get -u github.com/dnstap/golang-dnstap/dnstap diff --git a/TextOutput.go b/TextOutput.go new file mode 100644 index 0000000..edaa11a --- /dev/null +++ b/TextOutput.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "bufio" + "io" + "log" + "os" + + "github.com/golang/protobuf/proto" +) + +// A TextFormatFunc renders a dnstap message into a human readable format. +type TextFormatFunc func(*Dnstap) ([]byte, bool) + +// TextOutput implements a dnstap Output rendering dnstap data as text. +type TextOutput struct { + format TextFormatFunc + outputChannel chan []byte + wait chan bool + writer *bufio.Writer +} + +// NewTextOutput creates a TextOutput writing dnstap data to the given io.Writer +// in the text format given by the TextFormatFunc format. +func NewTextOutput(writer io.Writer, format TextFormatFunc) (o *TextOutput) { + o = new(TextOutput) + o.format = format + o.outputChannel = make(chan []byte, outputChannelSize) + o.writer = bufio.NewWriter(writer) + o.wait = make(chan bool) + return +} + +// NewTextOutputFromFilename creates a TextOutput writing dnstap data to a +// file with the given filename in the format given by format. If doAppend +// is false, the file is truncated if it already exists, otherwise the file +// is opened for appending. +func NewTextOutputFromFilename(fname string, format TextFormatFunc, doAppend bool) (o *TextOutput, err error) { + if fname == "" || fname == "-" { + return NewTextOutput(os.Stdout, format), nil + } + var writer io.Writer + if doAppend { + writer, err = os.OpenFile(fname, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) + } else { + writer, err = os.Create(fname) + } + if err != nil { + return + } + return NewTextOutput(writer, format), nil +} + +// GetOutputChannel returns the channel on which the TextOutput accepts dnstap data. +// +// GetOutputChannel satisfies the dnstap Output interface. +func (o *TextOutput) GetOutputChannel() chan []byte { + return o.outputChannel +} + +// RunOutputLoop receives dnstap data sent on the output channel, formats it +// with the configured TextFormatFunc, and writes it to the file or io.Writer +// of the TextOutput. +// +// RunOutputLoop satisfies the dnstap Output interface. +func (o *TextOutput) RunOutputLoop() { + dt := &Dnstap{} + for frame := range o.outputChannel { + if err := proto.Unmarshal(frame, dt); err != nil { + log.Fatalf("dnstap.TextOutput: proto.Unmarshal() failed: %s\n", err) + break + } + buf, ok := o.format(dt) + if !ok { + log.Fatalf("dnstap.TextOutput: text format function failed\n") + break + } + if _, err := o.writer.Write(buf); err != nil { + log.Fatalf("dnstap.TextOutput: write failed: %s\n", err) + break + } + o.writer.Flush() + } + close(o.wait) +} + +// Close closes the output channel and returns when all pending data has been +// written. +// +// Close satisfies the dnstap Output interface. +func (o *TextOutput) Close() { + close(o.outputChannel) + <-o.wait + o.writer.Flush() +} diff --git a/YamlFormat.go b/YamlFormat.go new file mode 100644 index 0000000..73f94f7 --- /dev/null +++ b/YamlFormat.go @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2013-2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +import ( + "bytes" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/miekg/dns" +) + +const yamlTimeFormat = "2006-01-02 15:04:05.999999999" + +func yamlConvertMessage(m *Message, s *bytes.Buffer) { + s.WriteString(fmt.Sprint(" type: ", m.Type, "\n")) + + if m.QueryTimeSec != nil && m.QueryTimeNsec != nil { + t := time.Unix(int64(*m.QueryTimeSec), int64(*m.QueryTimeNsec)).UTC() + s.WriteString(fmt.Sprint(" query_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) + } + + if m.ResponseTimeSec != nil && m.ResponseTimeNsec != nil { + t := time.Unix(int64(*m.ResponseTimeSec), int64(*m.ResponseTimeNsec)).UTC() + s.WriteString(fmt.Sprint(" response_time: !!timestamp ", t.Format(yamlTimeFormat), "\n")) + } + + if m.SocketFamily != nil { + s.WriteString(fmt.Sprint(" socket_family: ", m.SocketFamily, "\n")) + } + + if m.SocketProtocol != nil { + s.WriteString(fmt.Sprint(" socket_protocol: ", m.SocketProtocol, "\n")) + } + + if m.QueryAddress != nil { + s.WriteString(fmt.Sprint(" query_address: ", net.IP(m.QueryAddress), "\n")) + } + + if m.ResponseAddress != nil { + s.WriteString(fmt.Sprint(" response_address: ", net.IP(m.ResponseAddress), "\n")) + } + + if m.QueryPort != nil { + s.WriteString(fmt.Sprint(" query_port: ", *m.QueryPort, "\n")) + } + + if m.ResponsePort != nil { + s.WriteString(fmt.Sprint(" response_port: ", *m.ResponsePort, "\n")) + } + + if m.QueryZone != nil { + name, _, err := dns.UnpackDomainName(m.QueryZone, 0) + if err != nil { + fmt.Fprintf(s, " # query_zone: parse failed: %v\n", err) + } else { + s.WriteString(fmt.Sprint(" query_zone: ", strconv.Quote(name), "\n")) + } + } + + if m.QueryMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.QueryMessage) + if err != nil { + fmt.Fprintf(s, " # query_message: parse failed: %v\n", err) + } else { + s.WriteString(" query_message: |\n") + s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") + } + } + if m.ResponseMessage != nil { + msg := new(dns.Msg) + err := msg.Unpack(m.ResponseMessage) + if err != nil { + fmt.Fprintf(s, " # response_message: parse failed: %v\n", err) + } else { + s.WriteString(" response_message: |\n") + s.WriteString(" " + strings.Replace(strings.TrimSpace(msg.String()), "\n", "\n ", -1) + "\n") + } + } + s.WriteString("---\n") +} + +// YamlFormat renders a dnstap message in YAML format. Any encapsulated DNS +// messages are rendered as strings in a format similar to 'dig' output. +func YamlFormat(dt *Dnstap) (out []byte, ok bool) { + var s bytes.Buffer + + s.WriteString(fmt.Sprint("type: ", dt.Type, "\n")) + if dt.Identity != nil { + s.WriteString(fmt.Sprint("identity: ", strconv.Quote(string(dt.Identity)), "\n")) + } + if dt.Version != nil { + s.WriteString(fmt.Sprint("version: ", strconv.Quote(string(dt.Version)), "\n")) + } + if *dt.Type == Dnstap_MESSAGE { + s.WriteString("message:\n") + yamlConvertMessage(dt.Message, &s) + } + return s.Bytes(), true +} diff --git a/dnstap/dnstap.8 b/dnstap/dnstap.8 new file mode 100644 index 0000000..2e80dd3 --- /dev/null +++ b/dnstap/dnstap.8 @@ -0,0 +1,157 @@ +.TH dnstap 8 + +.SH NAME + +dnstap \- Capture, display, and relay Dnstap data. + +.SH SYNOPSIS + +.B dnstap [ -u \fIsocket-path\fB [ -u \fIsocket2-path\fB ... ] ] +.br +.B " [ -l \fIhost:port\fB [ -l \fIhost2:port2\fB ... ] ]" +.br +.B " [ -r \fIfile\fB [ -r \fIfile2\fB ... ] ]" +.br +.B " [ -U \fIsocket-path\fB [ -U \fIsocket2-path\fB ... ] ]" +.br +.B " [ -T \fIhost:port\fB [ -T \fIhost2:port2\fB ... ] ]" +.br +.B " [ -w \fIfile\fB ] [ -q | -y | -j ] [-a]" +.br +.B " [ -t \fItimeout\fB ]" +.br + +.SH DESCRIPTION + +.B dnstap +reads data in the Dnstap export format from Frame Streams files or +receives data on Frame Streams connections to TCP/IP or unix domain +socket addresses. +.B dnstap +can display this data in a compact text (the default), JSON, or YAML +formats. It can also save data to a file in display or Frame Streams +binary format, or relay the data to other Dnstap processes over unix +domain socket or TCP/IP connections. + +.SH OPTIONS + +.TP +.B -a +When opening an file (\fB-w\fR) for text format output +(\fB-j\fR, \fB-q\fR, or \fB-y\fR), append to the file rather +truncating. + +.B -a +does not apply when writing binary Frame Streams data to a file. + +.TP +.B -j +Write data in JSON format. Encapsulated DNS messages are +rendered in text form similar to the output of \fBdig(1)\fR. + +At most one text format (\fB-j\fR, \fB-q\fR, or \fB-y\fR) option may be +given. + +.TP +.B -l \fIhost:port\fR +Listen for Dnstap data on TCP/IP port \fBport\fR on address \fIhost\fR. + +The \fB-l\fR option may be given multiple times to listen on multiple +addresses. + +At least one input (\fB-l\fR, \fB-r\fR, or \fB-u\fR) option must be given. + +.TP +.B -q +Write or display data in compact (quiet) text format. + +At most one text format (\fB-j\fR, \fB-q\fR, or \fB-y\fR) option may be given. + +.TP +.B -r \fIfile\fR +Read Dnstap data from the given \fIfile\fR. The \fB-r\fR option +may be given multiple times to read from multiple files. + +At least one input (\fB-l\fR, \fB-r\fR, or \fB-u\fR) option must be given. + +.TP +.B -T \fIhost:port\fR +Relay Dnstap data over a TCP/IP connection to \fIhost:port\fR. +\fBdnstap\fR will establish or re-establish this connection as needed. + +The \fB-T\fR option may be given multiple times to relay Dnstap data +to multiple addresses. + +.TP +.B -t \fItimeout\fR +Apply i/o \fItimeout\fR to TCP/IP and unix domain socket +connections. \fItimeout\fR is given as a number followed by a unit +abbreviation (e.g., \fIms\fR for milliseconds, \fIs\fR for seconds, +\fIm\fR for minutes). + +.TP +.B -u \fIsocket-path\fR +Listen for Dnstap data on the unix domain socket at +\fIsocket-path\fR. \fBdnstap\fR will remove any file or socket +\fIsocket-path\fR before listening. + +The \fB-u\fR option may be given multiple times to listen on multiple +socket paths. + +At least one input (\fB-l\fR, \fB-r\fR, or \fB-u\fR) option must be given. + +.TP +.B -U \fIsocket-path\fR +Relay Dnstap data over a unix domain socket connection to +\fIsocket-path\fR. \fBdnstap\fR will establish or re-establish this +connection as needed. + +The \fB-U\fR option may be given multiple times to relay Dnstap data to +multiple socket paths. + + +.TP +.B -w \fIfile\fR +Write Dnstap data to \fIfile\fR. + +If \fIfile\fR is "-" or no \fB-w\fR, \fB-T\fR, or \fB-U\fR output +options are present, data will be written to standard output in quiet +text format (\fB-q\fR), unless the YAML or JSON format is specified +with the \fB-y\fR or \fB-j\fR options, respectively. + +If \fIfile\fR is a filename other than "-", Dnstap data is written to the +named file in Frame Streams binary format by default, unless quiet text, +JSON, or YAML formats are specified. + +.B dnstap +will reopen \fIfile\fR on \fBSIGHUP\fR, for file rotation purposes. + + +.TP +.B -y +Write Dnstap output in YAML format. Encapsulated DNS messages are rendered in text +form similar to the output of \fBdig(1)\fR. + +At most one text format (\fB-j\fR, \fB-q\fR, or \fB-y\fR) option may be given. + + +.SH EXAMPLES + +Listen for Dnstap data from a local name server and print quiet text format +to standard output. + +.nf + dnstap -u /var/named/dnstap.sock +.fi + +Listen for Dnstap data from a local name server, save a local binary copy, and +relay it to a remote host over TCP. + +.nf + dnstap -u /usr/local/unbound/dnstap.sock -w dnstap.fstrm \\ + -T dns-admin.example.com:5353 +.fi + +.SH SEE ALSO + +.B dig(1) diff --git a/dnstap/fileoutput.go b/dnstap/fileoutput.go new file mode 100644 index 0000000..5145adc --- /dev/null +++ b/dnstap/fileoutput.go @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2019 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "errors" + "fmt" + "os" + "os/signal" + "syscall" + + dnstap "github.com/dnstap/golang-dnstap" +) + +// Output channel buffer size value from main dnstap package. +const outputChannelSize = 32 + +// +// A fileOutput implements a dnstap.Output which writes frames to a file +// and closes and reopens the file on SIGHUP. +// +// Data frames are written in binary fstrm format unless a text formatting +// function (dnstp.TextFormatFunc) is given or the filename is blank or "-". +// In the latter case, data is written in compact (quiet) text format unless +// an alternate text format is given on the assumption that stdout is a terminal. +// +type fileOutput struct { + formatter dnstap.TextFormatFunc + filename string + doAppend bool + output dnstap.Output + data chan []byte + done chan struct{} +} + +func openOutputFile(filename string, formatter dnstap.TextFormatFunc, doAppend bool) (o dnstap.Output, err error) { + if formatter == nil { + if filename == "-" || filename == "" { + o = dnstap.NewTextOutput(os.Stdout, dnstap.TextFormat) + return + } + o, err = dnstap.NewFrameStreamOutputFromFilename(filename) + } else { + if filename == "-" || filename == "" { + if doAppend { + return nil, errors.New("cannot append to stdout (-)") + } + o = dnstap.NewTextOutput(os.Stdout, formatter) + return + } + o, err = dnstap.NewTextOutputFromFilename(filename, formatter, doAppend) + } + return +} + +func newFileOutput(filename string, formatter dnstap.TextFormatFunc, doAppend bool) (*fileOutput, error) { + o, err := openOutputFile(filename, formatter, doAppend) + if err != nil { + return nil, err + } + return &fileOutput{ + formatter: formatter, + filename: filename, + doAppend: doAppend, + output: o, + data: make(chan []byte, outputChannelSize), + done: make(chan struct{}), + }, nil +} + +func (fo *fileOutput) GetOutputChannel() chan []byte { + return fo.data +} + +func (fo *fileOutput) Close() { + close(fo.data) + <-fo.done +} + +func (fo *fileOutput) RunOutputLoop() { + sigch := make(chan os.Signal, 1) + signal.Notify(sigch, os.Interrupt, syscall.SIGHUP) + o := fo.output + go o.RunOutputLoop() + defer func() { + o.Close() + close(fo.done) + }() + for { + select { + case b, ok := <-fo.data: + if !ok { + return + } + o.GetOutputChannel() <- b + case sig := <-sigch: + if sig == syscall.SIGHUP { + o.Close() + newo, err := openOutputFile(fo.filename, fo.formatter, fo.doAppend) + if err != nil { + fmt.Fprintf(os.Stderr, + "dnstap: Error: failed to reopen %s: %v\n", + fo.filename, err) + os.Exit(1) + } + o = newo + go o.RunOutputLoop() + continue + } + os.Exit(0) + } + } +} diff --git a/dnstap/main.go b/dnstap/main.go new file mode 100644 index 0000000..1a8ec5f --- /dev/null +++ b/dnstap/main.go @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2013-2019 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "net" + "os" + "runtime" + "strings" + "sync" + + "github.com/dnstap/golang-dnstap" +) + +type stringList []string + +func (sl *stringList) Set(s string) error { + *sl = append(*sl, s) + return nil +} +func (sl *stringList) String() string { + return strings.Join(*sl, ", ") +} + +var ( + flagTimeout = flag.Duration("t", 0, "I/O timeout for tcp/ip and unix domain sockets") + flagWriteFile = flag.String("w", "", "write output to file") + flagAppendFile = flag.Bool("a", false, "append to the given file, do not overwrite. valid only when outputting a text or YAML file.") + flagQuietText = flag.Bool("q", false, "use quiet text output") + flagYamlText = flag.Bool("y", false, "use verbose YAML output") + flagJSONText = flag.Bool("j", false, "use verbose JSON output") +) + +func usage() { + fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]...\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, ` +Quiet text output format mnemonics: + AQ: AUTH_QUERY + AR: AUTH_RESPONSE + RQ: RESOLVER_QUERY + RR: RESOLVER_RESPONSE + CQ: CLIENT_QUERY + CR: CLIENT_RESPONSE + FQ: FORWARDER_QUERY + FR: FORWARDER_RESPONSE + SQ: STUB_QUERY + SR: STUB_RESPONSE + TQ: TOOL_QUERY + TR: TOOL_RESPONSE +`) +} + +func main() { + var tcpOutputs, unixOutputs stringList + var fileInputs, tcpInputs, unixInputs stringList + + flag.Var(&tcpOutputs, "T", "write dnstap payloads to tcp/ip address") + flag.Var(&unixOutputs, "U", "write dnstap payloads to unix socket") + flag.Var(&fileInputs, "r", "read dnstap payloads from file") + flag.Var(&tcpInputs, "l", "read dnstap payloads from tcp/ip") + flag.Var(&unixInputs, "u", "read dnstap payloads from unix socket") + + runtime.GOMAXPROCS(runtime.NumCPU()) + log.SetFlags(0) + flag.Usage = usage + + // Handle command-line arguments. + flag.Parse() + + if len(fileInputs)+len(unixInputs)+len(tcpInputs) == 0 { + fmt.Fprintf(os.Stderr, "dnstap: Error: no inputs specified.\n") + os.Exit(1) + } + + haveFormat := false + for _, f := range []bool{*flagQuietText, *flagYamlText, *flagJSONText} { + if haveFormat && f { + fmt.Fprintf(os.Stderr, "dnstap: Error: specify at most one of -q, -y, or -j.\n") + os.Exit(1) + } + haveFormat = haveFormat || f + } + + output := newMirrorOutput() + if err := addSockOutputs(output, "tcp", tcpOutputs); err != nil { + fmt.Fprintf(os.Stderr, "dnstap: TCP error: %v\n", err) + os.Exit(1) + } + if err := addSockOutputs(output, "unix", unixOutputs); err != nil { + fmt.Fprintf(os.Stderr, "dnstap: Unix socket error: %v\n", err) + os.Exit(1) + } + if *flagWriteFile != "" || len(tcpOutputs)+len(unixOutputs) == 0 { + var format dnstap.TextFormatFunc + + switch { + case *flagYamlText: + format = dnstap.YamlFormat + case *flagQuietText: + format = dnstap.TextFormat + case *flagJSONText: + format = dnstap.JSONFormat + } + + o, err := newFileOutput(*flagWriteFile, format, *flagAppendFile) + if err != nil { + fmt.Fprintf(os.Stderr, "dnstap: File output error on '%s': %v\n", + *flagWriteFile, err) + os.Exit(1) + } + go o.RunOutputLoop() + output.Add(o) + } + + go output.RunOutputLoop() + + var iwg sync.WaitGroup + // Open the input and start the input loop. + for _, fname := range fileInputs { + i, err := dnstap.NewFrameStreamInputFromFilename(fname) + if err != nil { + fmt.Fprintf(os.Stderr, "dnstap: Failed to open input file %s: %v\n", fname, err) + os.Exit(1) + } + fmt.Fprintf(os.Stderr, "dnstap: opened input file %s\n", fname) + iwg.Add(1) + go runInput(i, output, &iwg) + } + for _, path := range unixInputs { + i, err := dnstap.NewFrameStreamSockInputFromPath(path) + if err != nil { + fmt.Fprintf(os.Stderr, "dnstap: Failed to open input socket %s: %v\n", path, err) + os.Exit(1) + } + i.SetTimeout(*flagTimeout) + fmt.Fprintf(os.Stderr, "dnstap: opened input socket %s\n", path) + iwg.Add(1) + go runInput(i, output, &iwg) + } + for _, addr := range tcpInputs { + l, err := net.Listen("tcp", addr) + if err != nil { + fmt.Fprintf(os.Stderr, "dnstap: Failed to listen on %s: %v\n", addr, err) + os.Exit(1) + } + i := dnstap.NewFrameStreamSockInput(l) + i.SetTimeout(*flagTimeout) + iwg.Add(1) + go runInput(i, output, &iwg) + } + iwg.Wait() + + output.Close() +} + +func runInput(i dnstap.Input, o dnstap.Output, wg *sync.WaitGroup) { + go i.ReadInto(o.GetOutputChannel()) + i.Wait() + wg.Done() +} + +func addSockOutputs(mo *mirrorOutput, network string, addrs stringList) error { + var naddr net.Addr + var err error + for _, addr := range addrs { + switch network { + case "tcp": + naddr, err = net.ResolveTCPAddr(network, addr) + case "unix": + naddr, err = net.ResolveUnixAddr(network, addr) + default: + return fmt.Errorf("invalid network '%s'", network) + } + if err != nil { + return err + } + + o, err := dnstap.NewFrameStreamSockOutput(naddr) + if err != nil { + return err + } + o.SetTimeout(*flagTimeout) + go o.RunOutputLoop() + mo.Add(o) + } + return nil +} diff --git a/dnstap/mirroroutput.go b/dnstap/mirroroutput.go new file mode 100644 index 0000000..d368357 --- /dev/null +++ b/dnstap/mirroroutput.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + + +import ( + dnstap "github.com/dnstap/golang-dnstap" +) + +type mirrorOutput struct { + outputs []dnstap.Output + data chan []byte + done chan struct{} +} + +func newMirrorOutput() *mirrorOutput { + return &mirrorOutput{ + data: make(chan []byte, outputChannelSize), + done: make(chan struct{}), + } +} + +func (mo *mirrorOutput) Add(o dnstap.Output) { + mo.outputs = append(mo.outputs, o) +} + +func (mo *mirrorOutput) RunOutputLoop() { + for b := range mo.data { + for _, o := range mo.outputs { + select { + case o.GetOutputChannel() <- b: + default: + } + } + } + for _, o := range mo.outputs { + o.Close() + } + close(mo.done) +} + +func (mo *mirrorOutput) Close() { + close(mo.data) + <-mo.done +} + +func (mo *mirrorOutput) GetOutputChannel() chan []byte { + return mo.data +} diff --git a/dnstap.go b/dnstap.go new file mode 100644 index 0000000..3058ecc --- /dev/null +++ b/dnstap.go @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2014 by Farsight Security, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dnstap + +const outputChannelSize = 32 + +// FSContentType is the FrameStream content type for dnstap protobuf data. +var FSContentType = []byte("protobuf:dnstap.Dnstap") + +// An Input is a source of dnstap data. It provides validation of the +// content type and will present any data read or received on the channel +// provided to the ReadInto method. +type Input interface { + ReadInto(chan []byte) + Wait() +} + +// An Output is a desintaion for dnstap data. It accepts data on the channel +// returned from the GetOutputChannel method. The RunOutputLoop() method +// processes data received on this channel, and returns after the Close() +// method is called. +type Output interface { + GetOutputChannel() chan []byte + RunOutputLoop() + Close() +} diff --git a/dnstap.pb/.gitignore b/dnstap.pb/.gitignore new file mode 100644 index 0000000..8ee78ff --- /dev/null +++ b/dnstap.pb/.gitignore @@ -0,0 +1,10 @@ +.deps/ +.dirstamp +.libs/ +*.pb-c.c +*.pb-c.h +*.pb.cc +*.pb.h +*.pb.go +*_pb2.py +*_pb2.pyc diff --git a/dnstap.pb/LICENSE b/dnstap.pb/LICENSE new file mode 100644 index 0000000..0e259d4 --- /dev/null +++ b/dnstap.pb/LICENSE @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/dnstap.pb/README b/dnstap.pb/README new file mode 100644 index 0000000..7f44551 --- /dev/null +++ b/dnstap.pb/README @@ -0,0 +1,5 @@ +dnstap: flexible, structured event replication format for DNS software +---------------------------------------------------------------------- + +This directory contains only the protobuf schemas for dnstap, and is the root of +a repository named "dnstap.pb". diff --git a/dnstap.pb/dnstap.proto b/dnstap.pb/dnstap.proto new file mode 100644 index 0000000..1ed1bb0 --- /dev/null +++ b/dnstap.pb/dnstap.proto @@ -0,0 +1,268 @@ +// dnstap: flexible, structured event replication format for DNS software +// +// This file contains the protobuf schemas for the "dnstap" structured event +// replication format for DNS software. + +// Written in 2013-2014 by Farsight Security, Inc. +// +// To the extent possible under law, the author(s) have dedicated all +// copyright and related and neighboring rights to this file to the public +// domain worldwide. This file is distributed without any warranty. +// +// You should have received a copy of the CC0 Public Domain Dedication along +// with this file. If not, see: +// +// . + +package dnstap; + +// "Dnstap": this is the top-level dnstap type, which is a "union" type that +// contains other kinds of dnstap payloads, although currently only one type +// of dnstap payload is defined. +// See: https://developers.google.com/protocol-buffers/docs/techniques#union +message Dnstap { + // DNS server identity. + // If enabled, this is the identity string of the DNS server which generated + // this message. Typically this would be the same string as returned by an + // "NSID" (RFC 5001) query. + optional bytes identity = 1; + + // DNS server version. + // If enabled, this is the version string of the DNS server which generated + // this message. Typically this would be the same string as returned by a + // "version.bind" query. + optional bytes version = 2; + + // Extra data for this payload. + // This field can be used for adding an arbitrary byte-string annotation to + // the payload. No encoding or interpretation is applied or enforced. + optional bytes extra = 3; + + // Identifies which field below is filled in. + enum Type { + MESSAGE = 1; + } + required Type type = 15; + + // One of the following will be filled in. + optional Message message = 14; +} + +// SocketFamily: the network protocol family of a socket. This specifies how +// to interpret "network address" fields. +enum SocketFamily { + INET = 1; // IPv4 (RFC 791) + INET6 = 2; // IPv6 (RFC 2460) +} + +// SocketProtocol: the transport protocol of a socket. This specifies how to +// interpret "transport port" fields. +enum SocketProtocol { + UDP = 1; // User Datagram Protocol (RFC 768) + TCP = 2; // Transmission Control Protocol (RFC 793) +} + +// Message: a wire-format (RFC 1035 section 4) DNS message and associated +// metadata. Applications generating "Message" payloads should follow +// certain requirements based on the MessageType, see below. +message Message { + + // There are eight types of "Message" defined that correspond to the + // four arrows in the following diagram, slightly modified from RFC 1035 + // section 2: + + // +---------+ +----------+ +--------+ + // | | query | | query | | + // | Stub |-SQ--------CQ->| Recursive|-RQ----AQ->| Auth. | + // | Resolver| | Server | | Name | + // | |<-SR--------CR-| |<-RR----AR-| Server | + // +---------+ response | | response | | + // +----------+ +--------+ + + // Each arrow has two Type values each, one for each "end" of each arrow, + // because these are considered to be distinct events. Each end of each + // arrow on the diagram above has been marked with a two-letter Type + // mnemonic. Clockwise from upper left, these mnemonic values are: + // + // SQ: STUB_QUERY + // CQ: CLIENT_QUERY + // RQ: RESOLVER_QUERY + // AQ: AUTH_QUERY + // AR: AUTH_RESPONSE + // RR: RESOLVER_RESPONSE + // CR: CLIENT_RESPONSE + // SR: STUB_RESPONSE + + // Two additional types of "Message" have been defined for the + // "forwarding" case where an upstream DNS server is responsible for + // further recursion. These are not shown on the diagram above, but have + // the following mnemonic values: + + // FQ: FORWARDER_QUERY + // FR: FORWARDER_RESPONSE + + // The "Message" Type values are defined below. + + enum Type { + // AUTH_QUERY is a DNS query message received from a resolver by an + // authoritative name server, from the perspective of the authorative + // name server. + AUTH_QUERY = 1; + + // AUTH_RESPONSE is a DNS response message sent from an authoritative + // name server to a resolver, from the perspective of the authoritative + // name server. + AUTH_RESPONSE = 2; + + // RESOLVER_QUERY is a DNS query message sent from a resolver to an + // authoritative name server, from the perspective of the resolver. + // Resolvers typically clear the RD (recursion desired) bit when + // sending queries. + RESOLVER_QUERY = 3; + + // RESOLVER_RESPONSE is a DNS response message received from an + // authoritative name server by a resolver, from the perspective of + // the resolver. + RESOLVER_RESPONSE = 4; + + // CLIENT_QUERY is a DNS query message sent from a client to a DNS + // server which is expected to perform further recursion, from the + // perspective of the DNS server. The client may be a stub resolver or + // forwarder or some other type of software which typically sets the RD + // (recursion desired) bit when querying the DNS server. The DNS server + // may be a simple forwarding proxy or it may be a full recursive + // resolver. + CLIENT_QUERY = 5; + + // CLIENT_RESPONSE is a DNS response message sent from a DNS server to + // a client, from the perspective of the DNS server. The DNS server + // typically sets the RA (recursion available) bit when responding. + CLIENT_RESPONSE = 6; + + // FORWARDER_QUERY is a DNS query message sent from a downstream DNS + // server to an upstream DNS server which is expected to perform + // further recursion, from the perspective of the downstream DNS + // server. + FORWARDER_QUERY = 7; + + // FORWARDER_RESPONSE is a DNS response message sent from an upstream + // DNS server performing recursion to a downstream DNS server, from the + // perspective of the downstream DNS server. + FORWARDER_RESPONSE = 8; + + // STUB_QUERY is a DNS query message sent from a stub resolver to a DNS + // server, from the perspective of the stub resolver. + STUB_QUERY = 9; + + // STUB_RESPONSE is a DNS response message sent from a DNS server to a + // stub resolver, from the perspective of the stub resolver. + STUB_RESPONSE = 10; + + // TOOL_QUERY is a DNS query message sent from a DNS software tool to a + // DNS server, from the perspective of the tool. + TOOL_QUERY = 11; + + // TOOL_RESPONSE is a DNS response message received by a DNS software + // tool from a DNS server, from the perspective of the tool. + TOOL_RESPONSE = 12; + } + + // One of the Type values described above. + required Type type = 1; + + // One of the SocketFamily values described above. + optional SocketFamily socket_family = 2; + + // One of the SocketProtocol values described above. + optional SocketProtocol socket_protocol = 3; + + // The network address of the message initiator. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + optional bytes query_address = 4; + + // The network address of the message responder. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + optional bytes response_address = 5; + + // The transport port of the message initiator. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + optional uint32 query_port = 6; + + // The transport port of the message responder. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + optional uint32 response_port = 7; + + // The time at which the DNS query message was sent or received, depending + // on whether this is an AUTH_QUERY, RESOLVER_QUERY, or CLIENT_QUERY. + // This is the number of seconds since the UNIX epoch. + optional uint64 query_time_sec = 8; + + // The time at which the DNS query message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + optional fixed32 query_time_nsec = 9; + + // The initiator's original wire-format DNS query message, verbatim. + optional bytes query_message = 10; + + // The "zone" or "bailiwick" pertaining to the DNS query message. + // This is a wire-format DNS domain name. + optional bytes query_zone = 11; + + // The time at which the DNS response message was sent or received, + // depending on whether this is an AUTH_RESPONSE, RESOLVER_RESPONSE, or + // CLIENT_RESPONSE. + // This is the number of seconds since the UNIX epoch. + optional uint64 response_time_sec = 12; + + // The time at which the DNS response message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + optional fixed32 response_time_nsec = 13; + + // The responder's original wire-format DNS response message, verbatim. + optional bytes response_message = 14; +} + +// All fields except for 'type' in the Message schema are optional. +// It is recommended that at least the following fields be filled in for +// particular types of Messages. + +// AUTH_QUERY: +// socket_family, socket_protocol +// query_address, query_port +// query_message +// query_time_sec, query_time_nsec + +// AUTH_RESPONSE: +// socket_family, socket_protocol +// query_address, query_port +// query_time_sec, query_time_nsec +// response_message +// response_time_sec, response_time_nsec + +// RESOLVER_QUERY: +// socket_family, socket_protocol +// query_message +// query_time_sec, query_time_nsec +// query_zone +// response_address, response_port + +// RESOLVER_RESPONSE: +// socket_family, socket_protocol +// query_time_sec, query_time_nsec +// query_zone +// response_address, response_port +// response_message +// response_time_sec, response_time_nsec + +// CLIENT_QUERY: +// socket_family, socket_protocol +// query_message +// query_time_sec, query_time_nsec + +// CLIENT_RESPONSE: +// socket_family, socket_protocol +// query_time_sec, query_time_nsec +// response_message +// response_time_sec, response_time_nsec diff --git a/dnstap.pb.go b/dnstap.pb.go new file mode 100644 index 0000000..43b5fd1 --- /dev/null +++ b/dnstap.pb.go @@ -0,0 +1,448 @@ +// Code generated by protoc-gen-go. +// source: dnstap.proto +// DO NOT EDIT! + +/* +Package dnstap is a generated protocol buffer package. + +It is generated from these files: + dnstap.proto + +It has these top-level messages: + Dnstap + Message +*/ +package dnstap + +import proto "github.com/golang/protobuf/proto" +import json "encoding/json" +import math "math" + +// Reference proto, json, and math imports to suppress error if they are not otherwise used. +var _ = proto.Marshal +var _ = &json.SyntaxError{} +var _ = math.Inf + +// SocketFamily: the network protocol family of a socket. This specifies how +// to interpret "network address" fields. +type SocketFamily int32 + +const ( + SocketFamily_INET SocketFamily = 1 + SocketFamily_INET6 SocketFamily = 2 +) + +var SocketFamily_name = map[int32]string{ + 1: "INET", + 2: "INET6", +} +var SocketFamily_value = map[string]int32{ + "INET": 1, + "INET6": 2, +} + +func (x SocketFamily) Enum() *SocketFamily { + p := new(SocketFamily) + *p = x + return p +} +func (x SocketFamily) String() string { + return proto.EnumName(SocketFamily_name, int32(x)) +} +func (x *SocketFamily) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(SocketFamily_value, data, "SocketFamily") + if err != nil { + return err + } + *x = SocketFamily(value) + return nil +} + +// SocketProtocol: the transport protocol of a socket. This specifies how to +// interpret "transport port" fields. +type SocketProtocol int32 + +const ( + SocketProtocol_UDP SocketProtocol = 1 + SocketProtocol_TCP SocketProtocol = 2 +) + +var SocketProtocol_name = map[int32]string{ + 1: "UDP", + 2: "TCP", +} +var SocketProtocol_value = map[string]int32{ + "UDP": 1, + "TCP": 2, +} + +func (x SocketProtocol) Enum() *SocketProtocol { + p := new(SocketProtocol) + *p = x + return p +} +func (x SocketProtocol) String() string { + return proto.EnumName(SocketProtocol_name, int32(x)) +} +func (x *SocketProtocol) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(SocketProtocol_value, data, "SocketProtocol") + if err != nil { + return err + } + *x = SocketProtocol(value) + return nil +} + +// Identifies which field below is filled in. +type Dnstap_Type int32 + +const ( + Dnstap_MESSAGE Dnstap_Type = 1 +) + +var Dnstap_Type_name = map[int32]string{ + 1: "MESSAGE", +} +var Dnstap_Type_value = map[string]int32{ + "MESSAGE": 1, +} + +func (x Dnstap_Type) Enum() *Dnstap_Type { + p := new(Dnstap_Type) + *p = x + return p +} +func (x Dnstap_Type) String() string { + return proto.EnumName(Dnstap_Type_name, int32(x)) +} +func (x *Dnstap_Type) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(Dnstap_Type_value, data, "Dnstap_Type") + if err != nil { + return err + } + *x = Dnstap_Type(value) + return nil +} + +type Message_Type int32 + +const ( + // AUTH_QUERY is a DNS query message received from a resolver by an + // authoritative name server, from the perspective of the authorative + // name server. + Message_AUTH_QUERY Message_Type = 1 + // AUTH_RESPONSE is a DNS response message sent from an authoritative + // name server to a resolver, from the perspective of the authoritative + // name server. + Message_AUTH_RESPONSE Message_Type = 2 + // RESOLVER_QUERY is a DNS query message sent from a resolver to an + // authoritative name server, from the perspective of the resolver. + // Resolvers typically clear the RD (recursion desired) bit when + // sending queries. + Message_RESOLVER_QUERY Message_Type = 3 + // RESOLVER_RESPONSE is a DNS response message received from an + // authoritative name server by a resolver, from the perspective of + // the resolver. + Message_RESOLVER_RESPONSE Message_Type = 4 + // CLIENT_QUERY is a DNS query message sent from a client to a DNS + // server which is expected to perform further recursion, from the + // perspective of the DNS server. The client may be a stub resolver or + // forwarder or some other type of software which typically sets the RD + // (recursion desired) bit when querying the DNS server. The DNS server + // may be a simple forwarding proxy or it may be a full recursive + // resolver. + Message_CLIENT_QUERY Message_Type = 5 + // CLIENT_RESPONSE is a DNS response message sent from a DNS server to + // a client, from the perspective of the DNS server. The DNS server + // typically sets the RA (recursion available) bit when responding. + Message_CLIENT_RESPONSE Message_Type = 6 + // FORWARDER_QUERY is a DNS query message sent from a downstream DNS + // server to an upstream DNS server which is expected to perform + // further recursion, from the perspective of the downstream DNS + // server. + Message_FORWARDER_QUERY Message_Type = 7 + // FORWARDER_RESPONSE is a DNS response message sent from an upstream + // DNS server performing recursion to a downstream DNS server, from the + // perspective of the downstream DNS server. + Message_FORWARDER_RESPONSE Message_Type = 8 + // STUB_QUERY is a DNS query message sent from a stub resolver to a DNS + // server, from the perspective of the stub resolver. + Message_STUB_QUERY Message_Type = 9 + // STUB_RESPONSE is a DNS response message sent from a DNS server to a + // stub resolver, from the perspective of the stub resolver. + Message_STUB_RESPONSE Message_Type = 10 + // TOOL_QUERY is a DNS query message sent from a DNS software tool to a + // DNS server, from the perspective of the tool. + Message_TOOL_QUERY Message_Type = 11 + // TOOL_RESPONSE is a DNS response message received by a DNS software + // tool from a DNS server, from the perspective of the tool. + Message_TOOL_RESPONSE Message_Type = 12 +) + +var Message_Type_name = map[int32]string{ + 1: "AUTH_QUERY", + 2: "AUTH_RESPONSE", + 3: "RESOLVER_QUERY", + 4: "RESOLVER_RESPONSE", + 5: "CLIENT_QUERY", + 6: "CLIENT_RESPONSE", + 7: "FORWARDER_QUERY", + 8: "FORWARDER_RESPONSE", + 9: "STUB_QUERY", + 10: "STUB_RESPONSE", + 11: "TOOL_QUERY", + 12: "TOOL_RESPONSE", +} +var Message_Type_value = map[string]int32{ + "AUTH_QUERY": 1, + "AUTH_RESPONSE": 2, + "RESOLVER_QUERY": 3, + "RESOLVER_RESPONSE": 4, + "CLIENT_QUERY": 5, + "CLIENT_RESPONSE": 6, + "FORWARDER_QUERY": 7, + "FORWARDER_RESPONSE": 8, + "STUB_QUERY": 9, + "STUB_RESPONSE": 10, + "TOOL_QUERY": 11, + "TOOL_RESPONSE": 12, +} + +func (x Message_Type) Enum() *Message_Type { + p := new(Message_Type) + *p = x + return p +} +func (x Message_Type) String() string { + return proto.EnumName(Message_Type_name, int32(x)) +} +func (x *Message_Type) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(Message_Type_value, data, "Message_Type") + if err != nil { + return err + } + *x = Message_Type(value) + return nil +} + +// "Dnstap": this is the top-level dnstap type, which is a "union" type that +// contains other kinds of dnstap payloads, although currently only one type +// of dnstap payload is defined. +// See: https://developers.google.com/protocol-buffers/docs/techniques#union +type Dnstap struct { + // DNS server identity. + // If enabled, this is the identity string of the DNS server which generated + // this message. Typically this would be the same string as returned by an + // "NSID" (RFC 5001) query. + Identity []byte `protobuf:"bytes,1,opt,name=identity" json:"identity,omitempty"` + // DNS server version. + // If enabled, this is the version string of the DNS server which generated + // this message. Typically this would be the same string as returned by a + // "version.bind" query. + Version []byte `protobuf:"bytes,2,opt,name=version" json:"version,omitempty"` + // Extra data for this payload. + // This field can be used for adding an arbitrary byte-string annotation to + // the payload. No encoding or interpretation is applied or enforced. + Extra []byte `protobuf:"bytes,3,opt,name=extra" json:"extra,omitempty"` + Type *Dnstap_Type `protobuf:"varint,15,req,name=type,enum=dnstap.Dnstap_Type" json:"type,omitempty"` + // One of the following will be filled in. + Message *Message `protobuf:"bytes,14,opt,name=message" json:"message,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Dnstap) Reset() { *m = Dnstap{} } +func (m *Dnstap) String() string { return proto.CompactTextString(m) } +func (*Dnstap) ProtoMessage() {} + +func (m *Dnstap) GetIdentity() []byte { + if m != nil { + return m.Identity + } + return nil +} + +func (m *Dnstap) GetVersion() []byte { + if m != nil { + return m.Version + } + return nil +} + +func (m *Dnstap) GetExtra() []byte { + if m != nil { + return m.Extra + } + return nil +} + +func (m *Dnstap) GetType() Dnstap_Type { + if m != nil && m.Type != nil { + return *m.Type + } + return Dnstap_MESSAGE +} + +func (m *Dnstap) GetMessage() *Message { + if m != nil { + return m.Message + } + return nil +} + +// Message: a wire-format (RFC 1035 section 4) DNS message and associated +// metadata. Applications generating "Message" payloads should follow +// certain requirements based on the MessageType, see below. +type Message struct { + // One of the Type values described above. + Type *Message_Type `protobuf:"varint,1,req,name=type,enum=dnstap.Message_Type" json:"type,omitempty"` + // One of the SocketFamily values described above. + SocketFamily *SocketFamily `protobuf:"varint,2,opt,name=socket_family,enum=dnstap.SocketFamily" json:"socket_family,omitempty"` + // One of the SocketProtocol values described above. + SocketProtocol *SocketProtocol `protobuf:"varint,3,opt,name=socket_protocol,enum=dnstap.SocketProtocol" json:"socket_protocol,omitempty"` + // The network address of the message initiator. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + QueryAddress []byte `protobuf:"bytes,4,opt,name=query_address" json:"query_address,omitempty"` + // The network address of the message responder. + // For SocketFamily INET, this field is 4 octets (IPv4 address). + // For SocketFamily INET6, this field is 16 octets (IPv6 address). + ResponseAddress []byte `protobuf:"bytes,5,opt,name=response_address" json:"response_address,omitempty"` + // The transport port of the message initiator. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + QueryPort *uint32 `protobuf:"varint,6,opt,name=query_port" json:"query_port,omitempty"` + // The transport port of the message responder. + // This is a 16-bit UDP or TCP port number, depending on SocketProtocol. + ResponsePort *uint32 `protobuf:"varint,7,opt,name=response_port" json:"response_port,omitempty"` + // The time at which the DNS query message was sent or received, depending + // on whether this is an AUTH_QUERY, RESOLVER_QUERY, or CLIENT_QUERY. + // This is the number of seconds since the UNIX epoch. + QueryTimeSec *uint64 `protobuf:"varint,8,opt,name=query_time_sec" json:"query_time_sec,omitempty"` + // The time at which the DNS query message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + QueryTimeNsec *uint32 `protobuf:"fixed32,9,opt,name=query_time_nsec" json:"query_time_nsec,omitempty"` + // The initiator's original wire-format DNS query message, verbatim. + QueryMessage []byte `protobuf:"bytes,10,opt,name=query_message" json:"query_message,omitempty"` + // The "zone" or "bailiwick" pertaining to the DNS query message. + // This is a wire-format DNS domain name. + QueryZone []byte `protobuf:"bytes,11,opt,name=query_zone" json:"query_zone,omitempty"` + // The time at which the DNS response message was sent or received, + // depending on whether this is an AUTH_RESPONSE, RESOLVER_RESPONSE, or + // CLIENT_RESPONSE. + // This is the number of seconds since the UNIX epoch. + ResponseTimeSec *uint64 `protobuf:"varint,12,opt,name=response_time_sec" json:"response_time_sec,omitempty"` + // The time at which the DNS response message was sent or received. + // This is the seconds fraction, expressed as a count of nanoseconds. + ResponseTimeNsec *uint32 `protobuf:"fixed32,13,opt,name=response_time_nsec" json:"response_time_nsec,omitempty"` + // The responder's original wire-format DNS response message, verbatim. + ResponseMessage []byte `protobuf:"bytes,14,opt,name=response_message" json:"response_message,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} + +func (m *Message) GetType() Message_Type { + if m != nil && m.Type != nil { + return *m.Type + } + return Message_AUTH_QUERY +} + +func (m *Message) GetSocketFamily() SocketFamily { + if m != nil && m.SocketFamily != nil { + return *m.SocketFamily + } + return SocketFamily_INET +} + +func (m *Message) GetSocketProtocol() SocketProtocol { + if m != nil && m.SocketProtocol != nil { + return *m.SocketProtocol + } + return SocketProtocol_UDP +} + +func (m *Message) GetQueryAddress() []byte { + if m != nil { + return m.QueryAddress + } + return nil +} + +func (m *Message) GetResponseAddress() []byte { + if m != nil { + return m.ResponseAddress + } + return nil +} + +func (m *Message) GetQueryPort() uint32 { + if m != nil && m.QueryPort != nil { + return *m.QueryPort + } + return 0 +} + +func (m *Message) GetResponsePort() uint32 { + if m != nil && m.ResponsePort != nil { + return *m.ResponsePort + } + return 0 +} + +func (m *Message) GetQueryTimeSec() uint64 { + if m != nil && m.QueryTimeSec != nil { + return *m.QueryTimeSec + } + return 0 +} + +func (m *Message) GetQueryTimeNsec() uint32 { + if m != nil && m.QueryTimeNsec != nil { + return *m.QueryTimeNsec + } + return 0 +} + +func (m *Message) GetQueryMessage() []byte { + if m != nil { + return m.QueryMessage + } + return nil +} + +func (m *Message) GetQueryZone() []byte { + if m != nil { + return m.QueryZone + } + return nil +} + +func (m *Message) GetResponseTimeSec() uint64 { + if m != nil && m.ResponseTimeSec != nil { + return *m.ResponseTimeSec + } + return 0 +} + +func (m *Message) GetResponseTimeNsec() uint32 { + if m != nil && m.ResponseTimeNsec != nil { + return *m.ResponseTimeNsec + } + return 0 +} + +func (m *Message) GetResponseMessage() []byte { + if m != nil { + return m.ResponseMessage + } + return nil +} + +func init() { + proto.RegisterEnum("dnstap.SocketFamily", SocketFamily_name, SocketFamily_value) + proto.RegisterEnum("dnstap.SocketProtocol", SocketProtocol_name, SocketProtocol_value) + proto.RegisterEnum("dnstap.Dnstap_Type", Dnstap_Type_name, Dnstap_Type_value) + proto.RegisterEnum("dnstap.Message_Type", Message_Type_name, Message_Type_value) +} diff --git a/sock_test.go b/sock_test.go new file mode 100644 index 0000000..f650d10 --- /dev/null +++ b/sock_test.go @@ -0,0 +1,197 @@ +package dnstap + +import ( + "fmt" + "net" + "testing" + "time" +) + +func dialAndSend(t *testing.T, network, address string) *FrameStreamSockOutput { + var addr net.Addr + var err error + switch network { + case "unix": + addr, err = net.ResolveUnixAddr(network, address) + case "tcp", "tcp4", "tcp6": + addr, err = net.ResolveTCPAddr(network, address) + default: + err = fmt.Errorf("invalid network %s", network) + } + if err != nil { + t.Fatal(err) + } + + out, err := NewFrameStreamSockOutput(addr) + if err != nil { + t.Fatal(err) + } + + out.SetDialer(&net.Dialer{Timeout: time.Second}) + out.SetTimeout(time.Second) + out.SetFlushTimeout(100 * time.Millisecond) + out.SetRetryInterval(time.Second) + + go out.RunOutputLoop() + <-time.After(500 * time.Millisecond) + out.GetOutputChannel() <- []byte("frame") + return out +} + +func readOne(t *testing.T, out chan []byte) { + select { + case <-out: + case <-time.After(time.Second): + t.Fatal("timed out waiting for frame") + } +} + +// Test if dnstap can accept multiple connections on the socket +func TestMultiConn(t *testing.T) { + in, err := NewFrameStreamSockInputFromPath("dnstap.sock") + if err != nil { + t.Fatal(err) + } + + out := make(chan []byte) + go in.ReadInto(out) + + // send two framestream messages on different connections + defer dialAndSend(t, "unix", "dnstap.sock").Close() + defer dialAndSend(t, "unix", "dnstap.sock").Close() + + readOne(t, out) + readOne(t, out) +} + +func TestReconnect(t *testing.T) { + // Find an open port on localhost by opening a listener on an + // unspecified port, querying its address, then closing it. + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + laddr := l.Addr() + l.Close() + + defer dialAndSend(t, laddr.Network(), laddr.String()).Close() + defer dialAndSend(t, laddr.Network(), laddr.String()).Close() + time.Sleep(1500 * time.Millisecond) + l, err = net.Listen(laddr.Network(), laddr.String()) + if err != nil { + t.Fatal(err) + } + + in := NewFrameStreamSockInput(l) + out := make(chan []byte) + go in.ReadInto(out) + readOne(t, out) + readOne(t, out) +} + +func BenchmarkConnectUnidirectional(b *testing.B) { + b.StopTimer() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + b.Fatal(err) + } + + // read from tcp socket into outch + outch := make(chan []byte, 32) + go func() { + // wait for connection + s, err := l.Accept() + if err != nil { + b.Error(err) + close(outch) + return + } + + // start rewriter + in, err := NewFrameStreamInput(s, false) + if err != nil { + b.Error(err) + close(outch) + return + } + + // read ASAP into outch + in.ReadInto(outch) + close(outch) + }() + + // read from outch exactly b.N frames + // this is separate from the above, because the process of rewriting tcp into outch + // must run in parallel with reading b.N frames from outch + readDone := make(chan struct{}) + go func() { + // wait for the first frame before starting the timer + <-outch + i := 1 + + b.StartTimer() + for _ = range outch { i++ } + if i != b.N { + b.Error("invalid frame count") + } + close(readDone) + }() + + // connect to tcp socket and start the output loop + c, err := net.Dial(l.Addr().Network(), l.Addr().String()) + if err != nil { + b.Fatal(err) + } + out, err := NewFrameStreamOutput(c) + if err != nil { + b.Fatal(err) + } + go out.RunOutputLoop() + + // write to the output channel exactly b.N frames + for i := 0; i < b.N; i++ { + out.GetOutputChannel() <- []byte("frame") + } + out.Close() + + // wait for the reader + <-readDone +} + +func BenchmarkConnectBidirectional(b *testing.B) { + b.StopTimer() + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + b.Fatal(err) + } + + // start an infinite tcp socket reader + in := NewFrameStreamSockInput(l) + outch := make(chan []byte, 32) + go in.ReadInto(outch) + + // read up to b.N frames in background + readDone := make(chan struct{}) + go func() { + <-outch + b.StartTimer() + for i := 1; i < b.N; i++ { <-outch } // NB: read never fails + close(readDone) + }() + + // connect to tcp socket and start the output loop + out, err := NewFrameStreamSockOutput(l.Addr()) + if err != nil { + b.Fatal(err) + } + go out.RunOutputLoop() + + // write to the output channel exactly b.N frames + for i := 0; i < b.N; i++ { + out.GetOutputChannel() <- []byte("frame") + } + out.Close() + + // wait for the reader + <-readDone +}