Skip to content

Commit c0c56bc

Browse files
authored
Merge pull request #257 from coroot/fix_segfaults
replace uretprobes with uprobes using return offsets to avoid segmentation faults
2 parents 34373d2 + 96c530a commit c0c56bc

File tree

6 files changed

+346
-197
lines changed

6 files changed

+346
-197
lines changed

ebpftracer/elf.go

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
package ebpftracer
2+
3+
import (
4+
"debug/elf"
5+
"fmt"
6+
"io"
7+
8+
"github.com/cilium/ebpf"
9+
"github.com/cilium/ebpf/link"
10+
"golang.org/x/arch/arm64/arm64asm"
11+
"golang.org/x/arch/x86/x86asm"
12+
)
13+
14+
type Symbol struct {
15+
s *elf.Symbol
16+
f *ELFFile
17+
address uint64
18+
}
19+
20+
func (s *Symbol) Name() string {
21+
return s.s.Name
22+
}
23+
24+
func (s *Symbol) Address() uint64 {
25+
if s.address == 0 {
26+
s.address = s.s.Value
27+
for _, p := range s.f.elf.Progs {
28+
if p.Type != elf.PT_LOAD || (p.Flags&elf.PF_X) == 0 {
29+
continue
30+
}
31+
if p.Vaddr <= s.s.Value && s.s.Value < (p.Vaddr+p.Memsz) {
32+
s.address = s.s.Value - p.Vaddr + p.Off
33+
break
34+
}
35+
}
36+
}
37+
return s.address
38+
}
39+
40+
func (s *Symbol) ReturnOffsets() ([]int, error) {
41+
text, reader, err := s.f.getTextSectionAndReader()
42+
if err != nil {
43+
return nil, err
44+
}
45+
46+
sStart := s.s.Value - text.Addr
47+
_, err = reader.Seek(int64(sStart), io.SeekStart)
48+
if err != nil {
49+
return nil, err
50+
}
51+
sBytes := make([]byte, s.s.Size)
52+
_, err = reader.Read(sBytes)
53+
if err != nil {
54+
return nil, err
55+
}
56+
57+
offsets := getReturnOffsets(s.f.elf.Machine, sBytes)
58+
if len(offsets) == 0 {
59+
return nil, fmt.Errorf("no offsets found")
60+
}
61+
return offsets, nil
62+
}
63+
64+
func (s *Symbol) AttachUprobe(exe *link.Executable, prog *ebpf.Program, pid uint32) (link.Link, error) {
65+
return exe.Uprobe(s.Name(), prog, &link.UprobeOptions{Address: s.Address(), PID: int(pid)})
66+
}
67+
68+
func (s *Symbol) AttachUretprobes(exe *link.Executable, prog *ebpf.Program, pid uint32) ([]link.Link, error) {
69+
returnOffsets, err := s.ReturnOffsets()
70+
if err != nil {
71+
return nil, err
72+
}
73+
var links []link.Link
74+
for _, offset := range returnOffsets {
75+
l, err := exe.Uprobe("pthread_cond_timedwait", prog, &link.UprobeOptions{Address: s.Address(), Offset: uint64(offset), PID: int(pid)})
76+
if err != nil {
77+
return links, err
78+
}
79+
links = append(links, l)
80+
}
81+
82+
return links, nil
83+
}
84+
85+
type ELFFile struct {
86+
elf *elf.File
87+
symbols []elf.Symbol
88+
textSection *elf.Section
89+
textSectionReader io.ReadSeeker
90+
}
91+
92+
func OpenELFFile(path string) (*ELFFile, error) {
93+
file, err := elf.Open(path)
94+
if err != nil {
95+
return nil, err
96+
}
97+
return &ELFFile{elf: file}, nil
98+
}
99+
100+
func (f *ELFFile) readSymbols() error {
101+
symbols, _ := f.elf.Symbols()
102+
dyn, _ := f.elf.DynamicSymbols()
103+
104+
if len(symbols) == 0 && len(dyn) == 0 {
105+
return fmt.Errorf("no symbols found")
106+
}
107+
f.symbols = append(symbols, dyn...)
108+
return nil
109+
}
110+
111+
func (f *ELFFile) GetSymbol(name string) (*Symbol, error) {
112+
if f.symbols == nil {
113+
if err := f.readSymbols(); err != nil {
114+
return nil, err
115+
}
116+
}
117+
var es *elf.Symbol
118+
for _, s := range f.symbols {
119+
if elf.ST_TYPE(s.Info) != elf.STT_FUNC || s.Size == 0 || s.Value == 0 {
120+
continue
121+
}
122+
if s.Name == name && s.VersionIndex&0x8000 == 0 {
123+
es = &s
124+
break
125+
}
126+
}
127+
if es == nil {
128+
return nil, fmt.Errorf("symbol %s not found", name)
129+
}
130+
return &Symbol{s: es, f: f}, nil
131+
}
132+
133+
func (f *ELFFile) getTextSectionAndReader() (*elf.Section, io.ReadSeeker, error) {
134+
if f.textSection == nil {
135+
f.textSection = f.elf.Section(".text")
136+
if f.textSection == nil {
137+
return nil, nil, fmt.Errorf("no .text")
138+
}
139+
f.textSectionReader = f.textSection.Open()
140+
}
141+
return f.textSection, f.textSectionReader, nil
142+
}
143+
144+
func (f *ELFFile) Close() error {
145+
return f.elf.Close()
146+
}
147+
148+
func getReturnOffsets(machine elf.Machine, instructions []byte) []int {
149+
var res []int
150+
switch machine {
151+
case elf.EM_X86_64:
152+
for i := 0; i < len(instructions); {
153+
ins, err := x86asm.Decode(instructions[i:], 64)
154+
if err == nil && ins.Op == x86asm.RET {
155+
res = append(res, i)
156+
}
157+
i += ins.Len
158+
}
159+
case elf.EM_AARCH64:
160+
for i := 0; i < len(instructions); {
161+
ins, err := arm64asm.Decode(instructions[i:])
162+
if err == nil && ins.Op == arm64asm.RET {
163+
res = append(res, i)
164+
}
165+
i += 4
166+
}
167+
}
168+
return res
169+
}

ebpftracer/nodejs.go

Lines changed: 52 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -25,55 +25,65 @@ func (t *Tracer) AttachNodejsProbes(pid uint32, exe string) []link.Link {
2525
klog.InfofDepth(1, "pid=%d lib=%s: %s", pid, libPath, msg)
2626
}
2727

28-
var (
29-
lastErr error
30-
links []link.Link
31-
libPath string
32-
)
33-
for _, libPath = range append(getLibuv(pid), proc.Path(pid, "root", exe)) {
34-
exe, err := link.OpenExecutable(libPath)
35-
if err != nil {
36-
log(libPath, "failed to open executable", err)
37-
return nil
38-
}
39-
options := &link.UprobeOptions{PID: int(pid)}
40-
var uprobe, uretprobe link.Link
41-
uprobe, lastErr = exe.Uprobe("uv__io_poll", t.uprobes["uv_io_poll_enter"], options)
42-
if lastErr != nil {
43-
continue
28+
for _, libPath := range append(getLibuv(pid), proc.Path(pid, "root", exe)) {
29+
if links, err := t.attachNodejsUprobes(libPath, pid); err == nil {
30+
log(libPath, "nodejs uprobes attached", nil)
31+
return links
32+
} else {
33+
log(libPath, "failed to attach nodejs uprobes", err)
4434
}
35+
}
36+
return nil
37+
}
4538

46-
links = append(links, uprobe)
47-
uretprobe, lastErr = exe.Uretprobe("uv__io_poll", t.uprobes["uv_io_poll_exit"], options)
48-
if lastErr != nil {
49-
continue
50-
}
39+
func (t *Tracer) attachNodejsUprobes(libPath string, pid uint32) ([]link.Link, error) {
40+
exe, err := link.OpenExecutable(libPath)
41+
if err != nil {
42+
return nil, err
43+
}
44+
ef, err := OpenELFFile(libPath)
45+
if err != nil {
46+
return nil, err
47+
}
48+
defer ef.Close()
49+
50+
s, err := ef.GetSymbol("uv__io_poll")
51+
if err != nil {
52+
return nil, err
53+
}
54+
l, err := s.AttachUprobe(exe, t.uprobes["uv_io_poll_enter"], pid)
55+
if err != nil {
56+
return nil, err
57+
}
58+
var links []link.Link
59+
links = append(links, l)
5160

52-
links = append(links, uretprobe)
61+
ls, err := s.AttachUretprobes(exe, t.uprobes["uv_io_poll_exit"], pid)
62+
links = append(links, ls...)
63+
if err != nil {
64+
for _, l := range links {
65+
_ = l.Close()
66+
}
67+
return nil, err
68+
}
5369

54-
for _, cb := range []string{"uv__stream_io", "uv__async_io", "uv__poll_io", "uv__server_io", "uv__udp_io"} {
55-
uprobe, lastErr = exe.Uprobe(cb, t.uprobes["uv_io_cb_enter"], options)
56-
if lastErr != nil {
57-
break
58-
}
59-
links = append(links, uprobe)
60-
uretprobe, lastErr = exe.Uretprobe(cb, t.uprobes["uv_io_cb_exit"], options)
61-
if lastErr != nil {
62-
break
63-
}
64-
links = append(links, uretprobe)
70+
for _, cb := range []string{"uv__stream_io", "uv__async_io", "uv__poll_io", "uv__server_io", "uv__udp_io"} {
71+
s, err = ef.GetSymbol(cb)
72+
if err != nil {
73+
break
6574
}
66-
if lastErr != nil {
67-
continue
75+
l, err = s.AttachUprobe(exe, t.uprobes["uv_io_cb_enter"], pid)
76+
if err != nil {
77+
break
78+
}
79+
links = append(links, l)
80+
ls, err = s.AttachUretprobes(exe, t.uprobes["uv_io_cb_exit"], pid)
81+
links = append(links, ls...)
82+
if err != nil {
83+
break
6884
}
69-
70-
log(libPath, "nodejs uprobes attached", nil)
71-
break
72-
}
73-
if lastErr != nil {
74-
log(libPath, "failed to attach uprobe", lastErr)
7585
}
76-
return links
86+
return links, nil
7787
}
7888

7989
func getLibuv(pid uint32) []string {

ebpftracer/python.go

Lines changed: 42 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414

1515
var (
1616
libcRegexp = regexp.MustCompile(`libc[\.-]`)
17-
muslRegexp = regexp.MustCompile(`musl[\.-]`)
17+
muslRegexp = regexp.MustCompile(`ld-musl[\.-]`)
1818
)
1919

2020
func (t *Tracer) AttachPythonThreadLockProbes(pid uint32) []link.Link {
@@ -32,38 +32,55 @@ func (t *Tracer) AttachPythonThreadLockProbes(pid uint32) []link.Link {
3232
}
3333

3434
var (
35-
lastErr error
36-
links []link.Link
37-
libPath string
35+
links []link.Link
36+
err error
3837
)
3938

40-
for _, libPath = range getPthreadLibs(pid) {
41-
exe, err := link.OpenExecutable(libPath)
42-
if err != nil {
43-
log(libPath, "failed to open executable", err)
44-
return nil
45-
}
46-
options := &link.UprobeOptions{PID: int(pid)}
47-
var uprobe, uretprobe link.Link
48-
uprobe, lastErr = exe.Uprobe("pthread_cond_timedwait", t.uprobes["pthread_cond_timedwait_enter"], options)
49-
if lastErr != nil {
50-
continue
51-
}
52-
links = append(links, uprobe)
53-
uretprobe, lastErr = exe.Uretprobe("pthread_cond_timedwait", t.uprobes["pthread_cond_timedwait_exit"], options)
54-
if lastErr != nil {
55-
continue
39+
for _, libPath := range getPthreadLibs(pid) {
40+
if links, err = t.attachPythonUprobes(libPath, pid); err == nil {
41+
log(libPath, "python uprobes attached", nil)
42+
return links
43+
} else {
44+
log(libPath, "failed to attach python uprobes", err)
5645
}
57-
links = append(links, uretprobe)
58-
log(libPath, "python uprobes attached", nil)
59-
break
6046
}
61-
if lastErr != nil {
62-
log(libPath, "failed to attach uprobe", lastErr)
47+
if len(links) > 0 {
48+
6349
}
6450
return links
6551
}
6652

53+
func (t *Tracer) attachPythonUprobes(libPath string, pid uint32) ([]link.Link, error) {
54+
exe, err := link.OpenExecutable(libPath)
55+
if err != nil {
56+
return nil, err
57+
}
58+
ef, err := OpenELFFile(libPath)
59+
if err != nil {
60+
return nil, err
61+
}
62+
defer ef.Close()
63+
64+
s, err := ef.GetSymbol("pthread_cond_timedwait")
65+
if err != nil {
66+
return nil, err
67+
}
68+
l, err := s.AttachUprobe(exe, t.uprobes["pthread_cond_timedwait_enter"], pid)
69+
if err != nil {
70+
return nil, err
71+
}
72+
links := []link.Link{l}
73+
ls, err := s.AttachUretprobes(exe, t.uprobes["pthread_cond_timedwait_exit"], pid)
74+
links = append(links, ls...)
75+
if err != nil {
76+
for _, l := range links {
77+
_ = l.Close()
78+
}
79+
return nil, err
80+
}
81+
return links, nil
82+
}
83+
6784
func getPthreadLibs(pid uint32) []string {
6885
f, err := os.Open(proc.Path(pid, "maps"))
6986
if err != nil {

0 commit comments

Comments
 (0)