From 95f07b6a774f8742e81465555a59ca57ac474008 Mon Sep 17 00:00:00 2001 From: wangfeng Date: Sun, 5 Feb 2023 10:20:23 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E9=AA=8C=E8=AF=81=E7=9B=B8=E5=90=8C?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E9=95=BF=E5=BA=A6=E7=9A=84=E6=B5=AE=E7=82=B9?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E4=BC=9A=E5=88=A4=E6=96=AD=E4=B8=BAstring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../series1_test.go => dataframe_csv_test.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) rename tests/series1_test.go => dataframe_csv_test.go (74%) diff --git a/tests/series1_test.go b/dataframe_csv_test.go similarity index 74% rename from tests/series1_test.go rename to dataframe_csv_test.go index 6c045f0..9315d84 100644 --- a/tests/series1_test.go +++ b/dataframe_csv_test.go @@ -1,33 +1,32 @@ -package tests +package pandas import ( "bytes" "encoding/json" "fmt" - "gitee.com/quant1x/pandas" "strings" "testing" ) func TestCsv(t *testing.T) { csvStr := ` -Country,Date,Age,Amount,Id -"United States",2012-02-01,50,112.1,01234 -"United States",2012-02-01,32,321.31,54320 -"United Kingdom",2012-02-01,17,18.2,12345 -"United States",2012-02-01,32,321.31,54320 -"United Kingdom",2012-02-01,NA,18.2,12345 -"United States",2012-02-01,32,321.31,54320 -"United States",2012-02-01,32,321.31,54320 -Spain,2012-02-01,66,555.42,00241 +Country,Date,Age,Amount,Id,close +"United States",2012-02-01,50,112.1,01234,1.23 +"United States",2012-02-01,32,321.31,54320,1.23 +"United Kingdom",2012-02-01,17,18.2,12345,1.23 +"United States",2012-02-01,32,321.31,54320,1.23 +"United Kingdom",2012-02-01,NA,18.2,12345,1.23 +"United States",2012-02-01,32,321.31,54320,1.23 +"United States",2012-02-01,32,321.31,54320,1.23 +Spain,2012-02-01,66,555.42,00241,1.23 ` - df := pandas.ReadCSV(strings.NewReader(csvStr)) + df := ReadCSV(strings.NewReader(csvStr)) fmt.Println(df) - filename := "test-tutorials-w01.csv" + filename := "./testfiles/test-tutorials-w01.csv" _ = df.WriteCSV(filename) buf := new(bytes.Buffer) _ = df.WriteCSV(buf) - df = pandas.ReadCSV(filename) + df = ReadCSV(filename) fmt.Println(df) df.SetNames("a", "b", "c", "d", "e") //s1 := df.Col("d") -- Gitee From dea352f53b2f5fc2df2fdcc720a801c5dfd3392b Mon Sep 17 00:00:00 2001 From: wangfeng Date: Sun, 5 Feb 2023 10:21:50 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dexcel=E8=AF=BB=E5=86=99?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98:=201.=20=E6=97=B6=E9=97=B4=E4=B8=8D?= =?UTF-8?q?=E8=A7=84=E8=8C=83=202.=20=E6=B7=B1=E8=AF=81=E7=9A=84=E8=AF=81?= =?UTF-8?q?=E5=88=B8=E4=BB=A3=E7=A0=81=E4=B8=A2=E5=A4=B1=E5=89=8D=E7=BC=80?= =?UTF-8?q?=EF=BC=8C=E7=96=91=E4=BC=BC=E6=8C=89=E7=85=A7int=E5=A4=84?= =?UTF-8?q?=E7=90=86=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dataframe_excel.go | 14 +++++++++++--- dataframe_type.go | 18 ++++++++++++++++++ go.mod | 9 +++++++++ go.sum | 24 ++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/dataframe_excel.go b/dataframe_excel.go index 33a86f0..7365926 100644 --- a/dataframe_excel.go +++ b/dataframe_excel.go @@ -4,7 +4,9 @@ import ( "fmt" "github.com/mymmsc/gox/logger" "github.com/mymmsc/gox/util/homedir" - "github.com/tealeg/xlsx" + xlsv1 "github.com/tealeg/xlsx" + xlsv3 "github.com/tealeg/xlsx/v3" + "strings" ) // 读取excel文件 @@ -19,7 +21,7 @@ func ReadExcel(filename string, options ...LoadOption) DataFrame { return DataFrame{Err: err} } //filename := "test.xlsx" - xlFile, err := xlsx.OpenFile(filepath) + xlFile, err := xlsv1.OpenFile(filepath) if err != nil { return DataFrame{Err: err} } @@ -29,6 +31,12 @@ func ReadExcel(filename string, options ...LoadOption) DataFrame { for _, row := range sheet.Rows { col := make([]string, 0) for _, cell := range row.Cells { + //cell.SetStringFormula("%s") + if cell.IsTime() { + cell.SetFormat("yyyy-mm-dd") + } else if strings.HasPrefix(cell.Value, "0") { + cell.SetFormat("") + } text := cell.String() col = append(col, text) } @@ -47,7 +55,7 @@ func (self DataFrame) WriteExcel(filename string, options ...WriteOption) error if err != nil { return err } - xlFile := xlsx.NewFile() + xlFile := xlsv3.NewFile() sheet, err := xlFile.AddSheet("Sheet(pandas)") if err != nil { return err diff --git a/dataframe_type.go b/dataframe_type.go index 0098848..ed82a10 100644 --- a/dataframe_type.go +++ b/dataframe_type.go @@ -3,14 +3,29 @@ package pandas import ( "fmt" "strconv" + "strings" ) func findTypeByString(arr []string) (Type, error) { var hasFloats, hasInts, hasBools, hasStrings bool + var stringLengthEqual = -1 + var stringLenth = -1 for _, str := range arr { if str == "" || str == "NaN" { continue } + tLen := len(str) + if strings.HasPrefix(str, "0") { + stringLengthEqual = 0 + } + if stringLenth < 1 { + if stringLengthEqual == -1 { + stringLenth = tLen + } + } else if stringLengthEqual >= 0 && tLen != stringLenth { + stringLengthEqual += 1 + } + if _, err := strconv.Atoi(str); err == nil { hasInts = true continue @@ -25,6 +40,9 @@ func findTypeByString(arr []string) (Type, error) { } hasStrings = true } + if stringLengthEqual == 0 { + hasStrings = true + } // 类型优先级, string > bool > float > int, string 为默认类型 switch { case hasStrings: diff --git a/go.mod b/go.mod index 61ff168..dae8871 100644 --- a/go.mod +++ b/go.mod @@ -7,15 +7,24 @@ require ( github.com/huandu/go-clone v1.4.1 github.com/mymmsc/gox v1.3.1 github.com/tealeg/xlsx v1.0.5 + github.com/tealeg/xlsx/v3 v3.2.4 github.com/viterin/vek v0.4.0 gonum.org/v1/gonum v0.12.0 ) require ( github.com/chewxy/math32 v1.10.1 // indirect + github.com/frankban/quicktest v1.11.2 // indirect + github.com/google/btree v1.0.0 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa // indirect github.com/viterin/partial v1.0.0 // indirect golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.5.0 // indirect ) diff --git a/go.sum b/go.sum index 0d0b4f6..0799afd 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,11 @@ github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUw github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.11.2 h1:mjwHjStlXWibxOohM7HYieIViKyh56mmt3+6viyhDDI= +github.com/frankban/quicktest v1.11.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= @@ -11,6 +16,7 @@ github.com/huandu/go-clone v1.4.1 h1:QQYjiLadyxOvdwgZoH8f1xGkvvf4+Cm8be7fo9W2QQA github.com/huandu/go-clone v1.4.1/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -20,13 +26,24 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mymmsc/gox v1.3.1 h1:CM6bGBuf5+UK/af06Dv8U8becBlh6jyZ0RP2kEsYT84= github.com/mymmsc/gox v1.3.1/go.mod h1:A67NVxiHB3ZBcRfZVjvDfIVTUe5JaUjuy/F70ucUtdk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug= +github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa h1:2cO3RojjYl3hVTbEvJVqrMaFmORhL6O06qdW42toftk= +github.com/shabbyrobe/xmlwriter v0.0.0-20200208144257-9fca06d00ffa/go.mod h1:Yjr3bdWaVWyME1kha7X0jsz3k2DgXNa1Pj3XGyUAbx8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/tealeg/xlsx v1.0.5 h1:+f8oFmvY8Gw1iUXzPk+kz+4GpbDZPK1FhPiQRd+ypgE= github.com/tealeg/xlsx v1.0.5/go.mod h1:btRS8dz54TDnvKNosuAqxrM1QgN1udgk9O34bDCnORM= +github.com/tealeg/xlsx/v3 v3.2.4 h1:QPuk5v1xEivxoEUFmqszqINF52ppWCMejEd11ju3180= +github.com/tealeg/xlsx/v3 v3.2.4/go.mod h1:0j6U48nJBWJsvo1FmYilbGo81oRdLyYInWvjb2WAeOA= github.com/viterin/partial v1.0.0 h1:e6z0cWJ+SddpXHoLU4ikIDrsI/ZE+p+hqMsB++8IfwE= github.com/viterin/partial v1.0.0/go.mod h1:K9y+kVePpmfZN510YNHoUs+6scZ2K7BLojfI8aW2nw0= github.com/viterin/vek v0.4.0 h1:P34BWVGd3pSZFma9SE+G1pTucMGtw9p79I+Hull/+Ao= @@ -35,10 +52,17 @@ golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0 h1:17k44ji3KFYG94XS5QEFC8pyu golang.org/x/exp v0.0.0-20220907003533-145caa8ea1d0/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -- Gitee