diff --git a/.golangci.yml b/.golangci.yml
index 2d69a91370..e00d3670be 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -23,6 +23,11 @@ linters:
- predeclared
- tagliatelle
+run:
+ skip-dirs:
+ - dev
+ - doc
+
linters-settings:
misspell:
ignore-words:
@@ -37,9 +42,3 @@ linters-settings:
# allow md5
- G401
- G501
-issues:
- exclude-rules:
- - path: dev/.*\.go
- linters:
- # ignore main re-declared errors
- - typecheck
diff --git a/README.md b/README.md
index 568f8b4d7b..b2d2659213 100644
--- a/README.md
+++ b/README.md
@@ -130,6 +130,11 @@ Basic usage is `fq . file`.
For details see [usage.md](doc/usage.md)
+## Presentations
+
+- "fq - jq for binary formats" at [Binary Tools Summit 2022](https://binary-tools.net/summit.html) - (recording soon) - [slides](doc/presentations/bts2022/fq-bts2022-v1.pdf)
+
+
## Install
Use one of the methods listed below or download [release](https://github.com/wader/fq/releases) for your platform. Unarchive it and move the executable to `PATH` etc.
diff --git a/doc/presentations/bts2022/README.md b/doc/presentations/bts2022/README.md
new file mode 100644
index 0000000000..e65c8defc0
--- /dev/null
+++ b/doc/presentations/bts2022/README.md
@@ -0,0 +1,21 @@
+fq presentation from Binary Tools Summit 2022 https://binary-tools.net/
+
+[fq-bts2022-v1.pdf](fq-bts2022-v1.pdf)
+
+Will update with link to recording when availabe.
+
+Was done at the time of ~fq 0.0.5, things might have changed since.
+
+How to build:
+
+```
+go install golang.org/x/tools/cmd/present
+present -notes -content doc/presentations/bts2022 -base ~/go/pkg/mod/golang.org/x/tools@v0.1.9/cmd/present
+```
+
+```
+./usage.sh | ansisvg > usage.svg
+```
+
+Export to PDF via browser.
+
diff --git a/doc/presentations/bts2022/abstract.txt b/doc/presentations/bts2022/abstract.txt
new file mode 100644
index 0000000000..521a308b4f
--- /dev/null
+++ b/doc/presentations/bts2022/abstract.txt
@@ -0,0 +1,15 @@
+fq is inspired by the well known jq tool and language and allows you
+to work with binary formats the same way you would using jq. In
+addition it can also present data similar to a hex viewer, transform,
+slice and concatenate binary data, supports nested formats and has an
+interactive REPL with auto-completion.
+
+It was originally designed to query, inspect and debug codecs and
+metadata in media files and containers like mp4, flac, mp3, jpeg. But
+has since been extended to support a variety of formats like
+executables, packet captures including TCP reassembly and
+serialization formats like ASN1 BER, Avro, CBOR, protobuf and a lot
+more.
+
+In summary it aims to be something like jq, hexdump, dd and gdb
+combined into one.
diff --git a/doc/presentations/bts2022/avc_sps_hdr_params.go b/doc/presentations/bts2022/avc_sps_hdr_params.go
new file mode 100644
index 0000000000..0ab6de62e6
--- /dev/null
+++ b/doc/presentations/bts2022/avc_sps_hdr_params.go
@@ -0,0 +1,18 @@
+func avcHdrParameters(d *decode.D) {
+ cpbCnt := d.FieldUFn("cpb_cnt", uEV, scalar.UAdd(1))
+ d.FieldU4("bit_rate_scale")
+ d.FieldU4("cpb_size_scale")
+ d.FieldArray("sched_sels", func(d *decode.D) {
+ for i := uint64(0); i < cpbCnt; i++ {
+ d.FieldStruct("sched_sel", func(d *decode.D) {
+ d.FieldUFn("bit_rate_value", uEV, scalar.UAdd(1))
+ d.FieldUFn("cpb_size_value", uEV, scalar.UAdd(1))
+ d.FieldBool("cbr_flag")
+ })
+ }
+ })
+ d.FieldU5("initial_cpb_removal_delay_length", scalar.UAdd(1))
+ d.FieldU5("cpb_removal_delay_length", scalar.UAdd(1))
+ d.FieldU5("dpb_output_delay_length", scalar.UAdd(1))
+ d.FieldU5("time_offset_length")
+}
\ No newline at end of file
diff --git a/doc/presentations/bts2022/avc_sps_hdr_params.png b/doc/presentations/bts2022/avc_sps_hdr_params.png
new file mode 100644
index 0000000000..431e9bbd86
Binary files /dev/null and b/doc/presentations/bts2022/avc_sps_hdr_params.png differ
diff --git a/doc/presentations/bts2022/file.mp3 b/doc/presentations/bts2022/file.mp3
new file mode 100644
index 0000000000..b325d0e3e6
Binary files /dev/null and b/doc/presentations/bts2022/file.mp3 differ
diff --git a/doc/presentations/bts2022/file.png b/doc/presentations/bts2022/file.png
new file mode 100644
index 0000000000..a6ecff21a4
Binary files /dev/null and b/doc/presentations/bts2022/file.png differ
diff --git a/doc/presentations/bts2022/formats.svg b/doc/presentations/bts2022/formats.svg
new file mode 100644
index 0000000000..dd809a8907
--- /dev/null
+++ b/doc/presentations/bts2022/formats.svg
@@ -0,0 +1,1621 @@
+
+
+
+
+
diff --git a/doc/presentations/bts2022/fq-bts2022-v1.pdf b/doc/presentations/bts2022/fq-bts2022-v1.pdf
new file mode 100644
index 0000000000..d450bf19ad
Binary files /dev/null and b/doc/presentations/bts2022/fq-bts2022-v1.pdf differ
diff --git a/doc/presentations/bts2022/fq.slide b/doc/presentations/bts2022/fq.slide
new file mode 100644
index 0000000000..6e1a52d635
--- /dev/null
+++ b/doc/presentations/bts2022/fq.slide
@@ -0,0 +1,238 @@
+# fq
+
+jq for binary formats
+
+Mattias Wadman
+mattias.wadman@gmail.com
+https://github.com/wader/fq
+@mwader
+
+## Background
+
+.html style.html
+
+- Use various tools to extract data
+ - ffprobe, gm identify, mp4dump, mediainfo, wireshark, one off programs, ...
+- Convert to usable format and do queries
+ - jq, grep, sqlite, sort, awk, sed, one off programs, ...
+- Digging into and slicing binaries
+ - Hexfiend, hexdump, dd, cat, one off programs, ...
+
+
+## Wishlist
+
+"Want to see everything about this picture except the picture"
+
+- A very verbose version of file(1)
+- gdb for files
+- Select and query things using a language
+- Make all parts of a file symbolically addressable
+- Nested formats and binaries
+- Convenient bit-oriented decoder DSL
+
+
+## Experiments and prototypes
+
+- Decoder DSL
+ - TCL, lisp, tengo, Starlark, JavaScript, Go
+- Query language
+ - JSONPath, SQL, jq, JavaScript
+- How to use
+ - IR-JSON: `fq file | jq ... | fq`
+ - Extend existing project
+ - Decode and query in same tool
+
+
+## Result
+
+Go
+
+- Tests showed fast enough to decode big files
+- Found gojq
+- Previous good experience
+- Good tooling
+
+
+## jq
+
+"The JSON indenter"
+
+- JSON in/out
+- Syntax kind of a superset of JSON with same types
+- Functional language based on generators and backtracking
+ - Expressions can return or "output" zero, one or more values
+ - No more outputs backtracks
+- Implicit input and output similar to shell pipes
+- Extraordinary iteration and combinatorial abilities
+- Great for traversing tree structures
+
+
+## Examples
+
+.code jq1
+
+## Examples
+
+.code jq2
+
+## Examples
+
+.code jq3
+
+
+## Examples
+
+.code jq4
+
+
+## fq
+
+"The binary indenter"
+
+- Superset of jq
+- Re-implements most of jq's CLI interface
+- 83 input formats, 22 supports probe
+- Additional standard library functions
+- Additional types that act as standard jq types but has special abilities
+ - _Decode value_ has bit range, actual and symbolic value, description, ...
+ - _Binary_ has a unit size, bit or bytes, and can be sliced
+- Output fancy hexdump, JSON and binary
+- Interactive REPL with completion and sub-REPL support
+
+
+##
+
+.image formats.svg _ 1024
+
+## Usage
+
+- Basic usage
+ - `fq . file`, `cat file | fq`
+- Multiple input files
+ - `fq 'grep_by(format == "exif")' *.png *.jpeg`
+- Hexdump, JSON and binary output
+ - `fq '.frames[10] | d' file.mp3`
+ - `fq '[grep_by(format == "dns").questions[].name.value]' file.pcap`
+ - `fq 'first(grep_by(format == "jpeg")) | tobytes' file > file.jpeg`
+- Interactive REPL
+ - `fq -i . *.png`
+
+
+##
+
+.background data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAAXNSR0IArs4c6QAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAABNJREFUCB1jZGBg+A/EDEwgAgQADigBA//q6GsAAAAASUVORK5CYII%3D
+.image usage.svg _ 900
+
+
+## fq specific functions
+
+- Standard library
+ - `streaks`, `count`, `delta`, `chunk`, `diff`, `grep`, `grep_by`, ...
+ - `toradix`, `fromradix`, `hex`, `base64`, ...
+- Decode value
+ - `display` (alias `d`, `dv`, `da` ...)
+ - `parent`, `format`, ...
+ - `tobytes`, `tovalue`, `toactual`, ...
+ - `torepr`, ...
+- Binary
+ - Regexp functions `test`, `match`, ...
+ - Decode functions `probe`, `mp3_frame`, ...
+
+
+## Binary and binary array
+
+- A binary is created using `tobits`, `tobytes`, `tobitsrange` or `tobytesrange`.
+ - From decode value `.frames[1] | tobytes`
+ - String or number `"hello" | tobits`
+ - Binary array `[0xab, ["hello", .name]] | tobytes`
+- Can be sliced using normal jq slice syntax.
+ - `"hello" | tobits[8:8+16]` are the bits for `"el"`
+- Can be decoded
+ - `[tobytes[-10:], 0, 0, 0, 0] | flac_frame`
+
+
+## Example queries
+
+- Slice and decode
+ - `tobits[8:8+8000] | mp3_frame | d`
+ - `match([0xff,0xd8]) as $m | tobytes[$m.offset:] | jpeg`
+- ASN1 BER, CBOR, msgpack, BSON, ... has `torepr` support
+ - `fq -d cbor torepr file.cbor`
+ - `fq -d msgpack '[torepr.items[].name]' file.msgpack`
+- PCAP with TCP reassembly, look for GET requests
+ - `fq 'grep("GET .*")' file.pcap`
+- Parent of scalar value that includes bit 100
+ - `grep_by(scalars and in_bits_range(100)) | parent`
+
+
+## Use as script interpreter
+
+.code fqscript
+
+
+## Use as script interpreter
+
+.code fqscriptout
+
+
+## Implementation
+
+- Library of jq function implemented in Go
+ - Decoders, decode value, binary, bit reader, IO, tty, ...
+- CLI and REPL is mostly written in jq
+ ```
+ ( open
+ | decode
+ | if $repl then repeat(read as $expr | eval($expr) | print)
+ else eval($arg) | print
+ end
+ )
+ ```
+- All current decoders in Go
+- Uses a forked version of gojq
+ - Helped add native functions and iterators support
+ - JQValue interface, bin/hex/oct literals, reflection, query AST functions, ...
+
+## Decode API
+
+SPS HRD parameters from ITU-T H.264 specification
+
+.code avc_sps_hdr_params.go
+
+## Decode API
+
+.image avc_sps_hdr_params.png _ 900
+
+
+## Decode API
+
+Formats can use other formats. Simplified version of mp3 decoder:
+
+.code mp3.go
+
+
+## Future
+
+- Declarative decoding like kaitai struct, decoder in jq
+- Nicer way to handle checksums, encoding, validation etc
+- Schemas for ASN1, protobuf, ...
+- Better support for modifying data
+- More formats like tls, http, http2, grpc, filesystems, ...
+- Encoders
+- More efficient, lazy decoding, smarter representation
+- GUI
+- Streaming input, read network traffic `tap("eth0") | select(...)`?
+- Hope for more contributors
+
+
+## Thanks and useful tools
+
+- @itchyny for gojq
+- Stephen Dolan and others for jq
+- HexFiend
+- GNU poke
+- Kaitai struct
+- Wireshark
+- [vscode-jq](https://github.com/wader/vscode-jq)
+- [jq-lsp](https://github.com/wader/jq-lsp)
+
diff --git a/doc/presentations/bts2022/fqscript b/doc/presentations/bts2022/fqscript
new file mode 100755
index 0000000000..5b93cd957c
--- /dev/null
+++ b/doc/presentations/bts2022/fqscript
@@ -0,0 +1,22 @@
+#!/usr/bin/env fq -d mp4 -f
+
+( first(.boxes[] | select(.type == "moov")?)
+| first(.boxes[] | select(.type == "mvhd")?) as $mvhd
+| { time_scale: $mvhd.time_scale,
+ duration: ($mvhd.duration / $mvhd.time_scale),
+ tracks:
+ [ .boxes[]
+ | select(.type == "trak")
+ | [("mdhd", "stsd", "elst") as $t | first(grep_by(.type == $t))] as [$mdhd, $stsd, $elst]
+ | { data_format: $stsd.boxes[0].type,
+ media_scale: $mdhd.time_scale,
+ edit_list:
+ [ $elst.entries[]
+ | { track_duration: (.segment_duration / $mvhd.time_scale),
+ media_time: (.media_time / $mdhd.time_scale)
+ }
+ ]
+ }
+ ]
+ }
+)
\ No newline at end of file
diff --git a/doc/presentations/bts2022/fqscriptout b/doc/presentations/bts2022/fqscriptout
new file mode 100644
index 0000000000..10fa96b93d
--- /dev/null
+++ b/doc/presentations/bts2022/fqscriptout
@@ -0,0 +1,23 @@
+$ ./editlist file.mp4
+{
+ "duration": 60.095,
+ "time_scale": 600,
+ "tracks": [
+ {
+ "data_format": "mp4a",
+ "edit_list": [
+ {
+ "media_time": 0,
+ "track_duration": 60.095
+ }
+ ],
+ "media_scale": 22050
+ },
+ {
+ "data_format": "avc1",
+ "edit_list": [
+ {
+ "media_time": 0,
+ "track_duration": 60.095
+ }
+...
\ No newline at end of file
diff --git a/doc/presentations/bts2022/jq1 b/doc/presentations/bts2022/jq1
new file mode 100644
index 0000000000..33e838c211
--- /dev/null
+++ b/doc/presentations/bts2022/jq1
@@ -0,0 +1,23 @@
+# Literals
+> 123
+123
+
+> "abc"
+"abc"
+
+> [1,2,3]
+[
+ 1,
+ 2,
+ 3
+]
+
+> {a: (1+2+3), b: ["abc", false, null]}
+{
+ "a": 6,
+ "b": [
+ "abc",
+ false,
+ null
+ ]
+}
\ No newline at end of file
diff --git a/doc/presentations/bts2022/jq2 b/doc/presentations/bts2022/jq2
new file mode 100644
index 0000000000..e99ef7103c
--- /dev/null
+++ b/doc/presentations/bts2022/jq2
@@ -0,0 +1,20 @@
+# Pipeline using pipe operator "|" and identity function "." for current input
+> "hello" | length | . * 2
+10
+
+# Multiple outputs using output operator ","
+> 1, 2 | . * 2
+2
+4
+
+# Index array or object using .[key/index] or just .key for objects
+> [1,2,3][1]
+2
+
+# Collect outputs into array using [...]
+> [1,empty,2]
+[1,2]
+
+# Iterate array or object using .[]
+> [[1,2,3][]]
+[1,2,3]
diff --git a/doc/presentations/bts2022/jq3 b/doc/presentations/bts2022/jq3
new file mode 100644
index 0000000000..3632a5b692
--- /dev/null
+++ b/doc/presentations/bts2022/jq3
@@ -0,0 +1,22 @@
+# Generators and backtracking
+> 1, (2, 3 | . * 2), 4
+1
+4
+6
+4
+
+# Conditional, boolean operators and comparsion
+> if 1 == 2 and true then "a" else "b" end
+"b"
+
+# Reduce and foreach
+> reduce (1,2,3) as $i (0; . + $i)
+6
+> foreach (1,2,3) as $i (0; . + $i; .)
+1
+3
+6
+
+# Bindings (variables)
+> 1 as $a | 2 as $b | $a + $b
+3
diff --git a/doc/presentations/bts2022/jq4 b/doc/presentations/bts2022/jq4
new file mode 100644
index 0000000000..29342a4d89
--- /dev/null
+++ b/doc/presentations/bts2022/jq4
@@ -0,0 +1,20 @@
+# Function using lambda argument. map from standard library:
+def map(f): [.[] | f];
+> [1,2,3] | map(. * 2)
+[
+ 2,
+ 4,
+ 6
+]
+# select from standard library:
+def select(f): if f then . else empty end;
+> [1,2,3] | map(select(. % 2 == 0))
+[
+ 2
+]
+
+# Function using argument binding and recursion to output multiple values
+def down($n):
+ if $n >= 0 then $n, down($n-1)
+ else empty
+ end;
diff --git a/doc/presentations/bts2022/mp3.go b/doc/presentations/bts2022/mp3.go
new file mode 100644
index 0000000000..83b421698c
--- /dev/null
+++ b/doc/presentations/bts2022/mp3.go
@@ -0,0 +1,21 @@
+func decode(d *decode.D, in interface{}) interface{} {
+ d.FieldArray("headers", func(d *decode.D) {
+ for !d.End() {
+ d.TryFieldFormat("header", headerGroup)
+ }
+ })
+
+ d.FieldArray("frames", func(d *decode.D) {
+ for !d.End() {
+ d.TryFieldFormat("frame", mp3Group)
+ }
+ })
+
+ d.FieldArray("footers", func(d *decode.D) {
+ for !d.End() {
+ d.TryFieldFormat("footer", footerGroup)
+ }
+ })
+
+ return nil
+}
diff --git a/doc/presentations/bts2022/style.html b/doc/presentations/bts2022/style.html
new file mode 100644
index 0000000000..bd8799e6f5
--- /dev/null
+++ b/doc/presentations/bts2022/style.html
@@ -0,0 +1,9 @@
+
\ No newline at end of file
diff --git a/doc/presentations/bts2022/usage.sh b/doc/presentations/bts2022/usage.sh
new file mode 100755
index 0000000000..9048ebef11
--- /dev/null
+++ b/doc/presentations/bts2022/usage.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+c() {
+ echo -e "\x1b[97m# $1\x1b[0m"
+}
+
+s() {
+ echo "\$ $1"
+ sh -c "${1/fq/fq -o line_bytes=16 -o unicode=true -C}"
+}
+
+n() {
+ sh -c "${1/fq/fq -o line_bytes=16 -o unicode=true -C}"
+}
+
+c "display a decode value"
+s "fq . file.mp3"
+echo
+echo
+
+c "expression returning a number"
+s "fq '.frames | length' file.mp3"
+echo
+echo
+
+c "raw bytes"
+s "fq 'grep_by(format == \"png\") | tobytes' file.mp3 >file.png"
+s "file file.png"
+echo
+echo
+
+c "interactve REPL"
+echo "\$ fq -i . file.mp3"
+echo "mp3> .frames | length"
+n "fq '.frames | length' file.mp3"
+echo "mp3> .header[0] | repl"
+echo "> .headers[0] id3v2> .frames[0].text"
+n "fq '.headers[0].frames[0].text' file.mp3"
+echo "> .headers[0] id3v2> .frames[0].text | tovalue"
+n "fq '.headers[0].frames[0].text | tovalue' file.mp3"
+echo "> .headers[0] id3v2> ^D"
+echo "mp3> ^D"
+echo "$"
diff --git a/doc/presentations/bts2022/usage.svg b/doc/presentations/bts2022/usage.svg
new file mode 100644
index 0000000000..0a0eb4db5c
--- /dev/null
+++ b/doc/presentations/bts2022/usage.svg
@@ -0,0 +1,911 @@
+