Skip to content

Commit 4df37d4

Browse files
dnrdnr
authored
Fix disabling utf-8 validation for nested types (#148)
**What changed?** Fix #147 for nested types **Why?** See #147 **How did you test it?** Also added unit test
1 parent b8be610 commit 4df37d4

File tree

26 files changed

+6612
-6331
lines changed

26 files changed

+6612
-6331
lines changed

‎Makefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,7 @@ HELPER_FILES = $(shell find ./cmd/protoc-gen-go-helpers)
5555

5656
go-grpc: clean .go-helpers-installed $(PROTO_OUT)
5757
printf $(COLOR) "Compile for go-gRPC..."
58-
(cd cmd/protogen && go install .)
59-
protogen \
58+
go run ./cmd/protogen \
6059
--root=$(PROTO_ROOT) \
6160
--output=$(PROTO_OUT) \
6261
--exclude=internal \
@@ -130,8 +129,9 @@ gomodtidy:
130129

131130
##### Test #####
132131

132+
# We need to ensure protos are up to date to test our UTF-8 post-processing
133133
test: copy-helpers
134-
go test ./...
134+
go test -tags protolegacy ./...
135135

136136
##### Check #####
137137

‎batch/v1/message.pb.go‎

Lines changed: 79 additions & 79 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cmd/protogen/disable_utf8.go‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,30 @@ type xform struct {
4040
}
4141

4242
// https://github.com/protocolbuffers/protobuf/blob/66ef7bc1330df93c760d52480582efeaed5fe11e/src/google/protobuf/descriptor.proto#L93
43-
var xformFileDescriptorProto = &xform{
44-
xforms: map[protowire.Number]*xform{
45-
4: &xform{ // repeated DescriptorProto message_type = 4;
46-
xforms: map[protowire.Number]*xform{
47-
2: &xform{ // repeated FieldDescriptorProto field = 2;
48-
filters: map[protowire.Number]func([]byte) []byte{
49-
8: processFieldOptionsProto, // optional FieldOptions options = 8;
50-
},
51-
},
52-
},
43+
var (
44+
xformFileDescriptorProto = &xform{
45+
xforms: map[protowire.Number]*xform{
46+
4: xformDescriptorProto, // repeated DescriptorProto message_type = 4;
5347
},
54-
},
48+
}
49+
50+
xformDescriptorProto = &xform{
51+
xforms: map[protowire.Number]*xform{
52+
2: xformFieldDescriptorProto, // repeated FieldDescriptorProto field = 2;
53+
// 3: xformDescriptorProto, // repeated DescriptorProto nested_type = 3; // cycle, create in init()
54+
},
55+
}
56+
57+
xformFieldDescriptorProto = &xform{
58+
filters: map[protowire.Number]func([]byte) []byte{
59+
8: processFieldOptionsProto, // optional FieldOptions options = 8;
60+
},
61+
}
62+
)
63+
64+
func init() {
65+
// create cycle for processing nested types
66+
xformDescriptorProto.xforms[3] = xformDescriptorProto
5567
}
5668

5769
func NewDisableUtf8Validation() *disableUtf8Validation {
@@ -92,8 +104,10 @@ func (v *disableUtf8Validation) visit(n ast.Node) bool {
92104
newElts := make([]ast.Expr, len(newArr))
93105
for i, v := range newArr {
94106
newElts[i] = &ast.BasicLit{
95-
Kind: token.INT,
96-
Value: fmt.Sprintf("%#02x", v),
107+
// steal positions from original list to preserve newlines
108+
ValuePos: lit.Elts[i*(len(lit.Elts)-1)/(len(newElts)-1)].Pos(),
109+
Kind: token.INT,
110+
Value: fmt.Sprintf("%#02x", v),
97111
}
98112
}
99113
lit.Elts = newElts

‎cmd/protogen/disable_utf8_test.go‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
// The MIT License
2+
//
3+
// Copyright (c) 2022 Temporal Technologies Inc. All rights reserved.
4+
//
5+
// Permission is hereby granted, free of charge, to any person obtaining a copy
6+
// of this software and associated documentation files (the "Software"), to deal
7+
// in the Software without restriction, including without limitation the rights
8+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
// copies of the Software, and to permit persons to whom the Software is
10+
// furnished to do so, subject to the following conditions:
11+
//
12+
// The above copyright notice and this permission notice shall be included in
13+
// all copies or substantial portions of the Software.
14+
//
15+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21+
// THE SOFTWARE.
22+
23+
package main_test
24+
25+
import (
26+
"go/format"
27+
"go/parser"
28+
"go/token"
29+
"strings"
30+
"testing"
31+
32+
"github.com/stretchr/testify/require"
33+
protogen "go.temporal.io/api/cmd/protogen"
34+
)
35+
36+
const protoGeneratedMessageDescriptor = `package sdk
37+
38+
import (
39+
reflect "reflect"
40+
sync "sync"
41+
42+
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
43+
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
44+
)
45+
46+
const (
47+
// Verify that this generated code is sufficiently up-to-date.
48+
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
49+
// Verify that runtime/protoimpl is sufficiently up-to-date.
50+
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
51+
)
52+
53+
// ...
54+
55+
var File_temporal_api_sdk_v1_task_complete_metadata_proto protoreflect.FileDescriptor
56+
57+
var file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDesc = []byte{
58+
0x0a, 0x30, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73,
59+
0x64, 0x6b, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c,
60+
0x65, 0x74, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f,
61+
0x74, 0x6f, 0x12, 0x13, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69,
62+
0x2e, 0x73, 0x64, 0x6b, 0x2e, 0x76, 0x31, 0x22, 0xab, 0x01, 0x0a, 0x1d, 0x57, 0x6f, 0x72, 0x6b,
63+
0x66, 0x6c, 0x6f, 0x77, 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65,
64+
0x64, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x6f, 0x72,
65+
0x65, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03,
66+
0x28, 0x0d, 0x52, 0x0d, 0x63, 0x6f, 0x72, 0x65, 0x55, 0x73, 0x65, 0x64, 0x46, 0x6c, 0x61, 0x67,
67+
0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6c, 0x61, 0x6e, 0x67, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x66,
68+
0x6c, 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x6c, 0x61, 0x6e, 0x67,
69+
0x55, 0x73, 0x65, 0x64, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x64, 0x6b,
70+
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x64, 0x6b,
71+
0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x64, 0x6b, 0x5f, 0x76, 0x65, 0x72, 0x73,
72+
0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x64, 0x6b, 0x56, 0x65,
73+
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x87, 0x01, 0x0a, 0x16, 0x69, 0x6f, 0x2e, 0x74, 0x65, 0x6d,
74+
0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x64, 0x6b, 0x2e, 0x76, 0x31,
75+
0x42, 0x19, 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65,
76+
0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x1d, 0x67,
77+
0x6f, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x69, 0x6f, 0x2f, 0x61, 0x70,
78+
0x69, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x64, 0x6b, 0xaa, 0x02, 0x15, 0x54,
79+
0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x6f, 0x2e, 0x41, 0x70, 0x69, 0x2e, 0x53, 0x64,
80+
0x6b, 0x2e, 0x56, 0x31, 0xea, 0x02, 0x18, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69,
81+
0x6f, 0x3a, 0x3a, 0x41, 0x70, 0x69, 0x3a, 0x3a, 0x53, 0x64, 0x6b, 0x3a, 0x3a, 0x56, 0x31, 0x62,
82+
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
83+
}
84+
85+
var (
86+
file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDescOnce sync.Once
87+
file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDescData = file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDesc
88+
)
89+
90+
// ...
91+
`
92+
const afterUtf8Rewrite = `package sdk
93+
94+
import (
95+
reflect "reflect"
96+
sync "sync"
97+
98+
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
99+
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
100+
)
101+
102+
const (
103+
// Verify that this generated code is sufficiently up-to-date.
104+
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
105+
// Verify that runtime/protoimpl is sufficiently up-to-date.
106+
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
107+
)
108+
109+
// ...
110+
111+
var File_temporal_api_sdk_v1_task_complete_metadata_proto protoreflect.FileDescriptor
112+
113+
var file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDesc = []byte{
114+
0x0a, 0x30, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x64,
115+
0x6b, 0x2f, 0x76, 0x31, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74,
116+
0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
117+
0x12, 0x13, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x64,
118+
0x6b, 0x2e, 0x76, 0x31, 0x22, 0xbb, 0x01, 0x0a, 0x1d, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77,
119+
0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4d, 0x65, 0x74,
120+
0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2a, 0x0a, 0x0f, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x75, 0x73, 0x65,
121+
0x64, 0x5f, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x63, 0x6f,
122+
0x72, 0x65, 0x55, 0x73, 0x65, 0x64, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12,
123+
0x2a, 0x0a, 0x0f, 0x6c, 0x61, 0x6e, 0x67, 0x5f, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x66, 0x6c, 0x61, 0x67,
124+
0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x0d, 0x6c, 0x61, 0x6e, 0x67, 0x55, 0x73, 0x65, 0x64,
125+
0x46, 0x6c, 0x61, 0x67, 0x73, 0x42, 0x02, 0x68, 0x00, 0x12, 0x1d, 0x0a, 0x08, 0x73, 0x64, 0x6b,
126+
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x64, 0x6b, 0x4e,
127+
0x61, 0x6d, 0x65, 0x42, 0x02, 0x68, 0x00, 0x12, 0x23, 0x0a, 0x0b, 0x73, 0x64, 0x6b, 0x5f, 0x76, 0x65,
128+
0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x64, 0x6b,
129+
0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x02, 0x68, 0x00, 0x42, 0x87, 0x01, 0x0a, 0x16, 0x69,
130+
0x6f, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x73, 0x64,
131+
0x6b, 0x2e, 0x76, 0x31, 0x42, 0x19, 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65,
132+
0x74, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
133+
0x5a, 0x1d, 0x67, 0x6f, 0x2e, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x2e, 0x69, 0x6f, 0x2f,
134+
0x61, 0x70, 0x69, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x76, 0x31, 0x3b, 0x73, 0x64, 0x6b, 0xaa, 0x02,
135+
0x15, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69, 0x6f, 0x2e, 0x41, 0x70, 0x69, 0x2e, 0x53,
136+
0x64, 0x6b, 0x2e, 0x56, 0x31, 0xea, 0x02, 0x18, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x6c, 0x69,
137+
0x6f, 0x3a, 0x3a, 0x41, 0x70, 0x69, 0x3a, 0x3a, 0x53, 0x64, 0x6b, 0x3a, 0x3a, 0x56, 0x31, 0x62,
138+
0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
139+
}
140+
141+
var (
142+
file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDescOnce sync.Once
143+
file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDescData = file_temporal_api_sdk_v1_task_complete_metadata_proto_rawDesc
144+
)
145+
146+
// ...
147+
`
148+
149+
func TestRewriteDisableUtf8(t *testing.T) {
150+
fset := token.NewFileSet()
151+
f, err := parser.ParseFile(fset, "", protoGeneratedMessageDescriptor, parser.ParseComments)
152+
if err != nil {
153+
t.Errorf("Failed to parse code: %s", err)
154+
t.FailNow()
155+
}
156+
157+
d := protogen.NewDisableUtf8Validation()
158+
d.Process(f)
159+
160+
var b strings.Builder
161+
if err := format.Node(&b, fset, f); err != nil {
162+
t.Errorf("Failed to format AST: %s", err)
163+
t.FailNow()
164+
}
165+
require.Equal(t, afterUtf8Rewrite, b.String())
166+
}

0 commit comments

Comments
 (0)