Merge pull request #295 from noborus/add-text-input

Add extension support for text format

cmd/root.go

@@ -292,6 +292,8 @@ func strToFormat(format string) trdsql.Format {
 		return trdsql.TBLN
 	case "width":
 		return trdsql.WIDTH
+	case "text":
+		return trdsql.TEXT
 	default:
 		return trdsql.GUESS
 	}
@@ -361,9 +363,9 @@ func init() {
 	rootCmd.PersistentFlags().Var(&inNull, "null", "value(string) to convert to null on input.")
 	rootCmd.PersistentFlags().BoolVarP(&inRowNumber, "row-number", "n", false, "add row number.")
 
-	rootCmd.PersistentFlags().StringVarP(&inFormat, "in", "i", "GUESS", "format for input. [CSV|LTSV|JSON|YAML|TBLN|WIDTH]")
+	rootCmd.PersistentFlags().StringVarP(&inFormat, "in", "i", "GUESS", "format for input. [CSV|LTSV|JSON|YAML|TBLN|WIDTH|TEXT]")
 	rootCmd.RegisterFlagCompletionFunc("in", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-		return []string{"CSV", "LTSV", "JSON", "YAML", "TBLN", "WIDTH"}, cobra.ShellCompDirectiveDefault
+		return []string{"CSV", "LTSV", "JSON", "YAML", "TBLN", "WIDTH", "TEXT"}, cobra.ShellCompDirectiveDefault
 	})
 	rootCmd.PersistentFlags().StringVar(&outDelimiter, "out-delimiter", ",", "field delimiter for output.")
 	rootCmd.PersistentFlags().StringVar(&outFile, "out-file", "", "output file name.")

input_text.go

@@ -0,0 +1,65 @@
+package trdsql
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// TextReader provides a reader for text format.
+type TextReader struct {
+	reader *bufio.Reader
+	num    int
+	maxNum int
+}
+
+// NewTextReader returns a new TextReader.
+func NewTextReader(reader io.Reader, opts *ReadOpts) (*TextReader, error) {
+	r := &TextReader{
+		reader: bufio.NewReader(reader),
+	}
+
+	if opts.InSkip > 0 {
+		skipRead(r, opts.InSkip)
+	}
+
+	if opts.InLimitRead {
+		r.maxNum = opts.InPreRead
+	}
+	return r, nil
+}
+
+// Names returns column names.
+func (r *TextReader) Names() ([]string, error) {
+	return []string{"text"}, nil
+}
+
+// Types returns column types.
+func (r *TextReader) Types() ([]string, error) {
+	return []string{"text"}, nil
+}
+
+// PreReadRow returns pre-read rows.
+func (r *TextReader) PreReadRow() [][]any {
+	return nil
+}
+
+// ReadRow reads a row.
+func (r *TextReader) ReadRow([]any) ([]any, error) {
+	var builder strings.Builder
+	for {
+		if r.maxNum > 0 && r.num >= r.maxNum {
+			return []any{""}, io.EOF
+		}
+		line, isPrefix, err := r.reader.ReadLine()
+		if err != nil {
+			return []any{""}, err
+		}
+		builder.Write(line)
+		if isPrefix {
+			continue
+		}
+		r.num++
+		return []any{builder.String()}, nil
+	}
+}

input_text_test.go

@@ -0,0 +1,101 @@
+package trdsql
+
+import (
+	"io"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestNewTextReader(t *testing.T) {
+	type args struct {
+		reader io.Reader
+		opts   *ReadOpts
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "test1",
+			args: args{
+				reader: strings.NewReader("a\nb\nc\n"),
+				opts:   NewReadOpts(),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := NewTextReader(tt.args.reader, tt.args.opts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			names, err := got.Names()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(names, []string{"text"}) {
+				t.Errorf("TextReader.Names() != text %v", names)
+			}
+			types, err := got.Types()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(types, []string{"text"}) {
+				t.Errorf("TextReader.Types() != text %v", types)
+			}
+		})
+	}
+}
+
+func TestTextReaderFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		fileName string
+		opts     *ReadOpts
+		want     []any
+		wantErr  bool
+	}{
+		{
+			name:     "test.csv",
+			fileName: "test.csv",
+			opts:     NewReadOpts(),
+			want:     []any{"1,Orange"},
+			wantErr:  false,
+		},
+		{
+			name:     "test.csv2",
+			fileName: "test.csv",
+			opts:     &ReadOpts{InSkip: 1},
+			want:     []any{"2,Melon"},
+			wantErr:  false,
+		},
+		{
+			name:     "test.csv3",
+			fileName: "test.csv",
+			opts:     &ReadOpts{InLimitRead: true, InPreRead: 1},
+			want:     []any{"1,Orange"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			file, err := singleFileOpen(filepath.Join(dataDir, tt.fileName))
+			if err != nil {
+				t.Error(err)
+			}
+			r, err := NewTextReader(file, tt.opts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := r.ReadRow(nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("TextReader.ReadRow() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("TextReader.ReadRow() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

reader.go

@@ -18,6 +18,7 @@ var extToFormat map[string]Format = map[string]Format{
 	"TSV":   TSV,
 	"PSV":   PSV,
 	"WIDTH": WIDTH,
+	"TEXT":  TEXT,
 }
 
 // ReaderFunc is a function that creates a new Reader.
@@ -49,6 +50,9 @@ var readerFuncs = map[Format]ReaderFunc{
 	WIDTH: func(reader io.Reader, opts *ReadOpts) (Reader, error) {
 		return NewGWReader(reader, opts)
 	},
+	TEXT: func(reader io.Reader, opts *ReadOpts) (Reader, error) {
+		return NewTextReader(reader, opts)
+	},
 }
 
 var (

trdsql.go

@@ -77,6 +77,9 @@ const (
 	// Format using guesswidth library.
 	WIDTH
 
+	// import
+	TEXT
+
 	// export
 	// Output as it is.
 	// Multiple characters can be selected as delimiter.
@@ -142,6 +145,8 @@ func (f Format) String() string {
 		return "PSV"
 	case YAML:
 		return "YAML"
+	case TEXT:
+		return "TEXT"
 	default:
 		return "Unknown"
 	}

trdsql_test.go

@@ -567,6 +567,35 @@ func TestTBLNRun(t *testing.T) {
 	}
 }
 
+func TestTextRun(t *testing.T) {
+	ctx := context.Background()
+	testText := [][]string{
+		{"test.csv", `1,"1,Orange"
+2,"2,Melon"
+3,"3,Apple"
+`},
+		{"aiu.csv", "1,あ\n2,い\n3,う\n"},
+	}
+	outStream := new(bytes.Buffer)
+	importer := NewImporter(
+		InFormat(TEXT),
+		InRowNumber(true),
+	)
+	exporter := NewExporter(NewWriter(OutStream(outStream)))
+	trd := NewTRDSQL(importer, exporter)
+	for _, c := range testText {
+		sqlQuery := "SELECT * FROM " + filepath.Join(dataDir, c[0])
+		err := trd.Exec(ctx, sqlQuery)
+		if err != nil {
+			t.Errorf("trdsql error %s", err)
+		}
+		if outStream.String() != c[1] {
+			t.Fatalf("trdsql error %s:%s:%s", c[0], c[1], outStream)
+		}
+		outStream.Reset()
+	}
+}
+
 func setOutFormatTRDSQL(outFormat Format, outStream io.Writer) *TRDSQL {
 	importer := NewImporter(
 		InFormat(GUESS),