diff --git a/config/config.toml b/config/config.toml index cbc15e0..4578f3b 100644 --- a/config/config.toml +++ b/config/config.toml @@ -12,13 +12,13 @@ name = "goods_library" host = "43.139.72.196" port = 13307 user = "webprod" -password = "J1XivNvAcR21}pA6Cysm.E289" +password = "J1XivNvAcR21}pA6Cysm.E29" name = "mall_prod" -max_idle_conns = 10 +max_idle_conns = 25 max_open_conns = 100 conn_max_lifetime = 1 # 小时 -conn_max_idle_time = 30 # 分钟 +conn_max_idle_time = 45 # 分钟 # COS配置 [cossdk] diff --git a/go.mod b/go.mod index 617347c..4802811 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module fafa-crawler -go 1.25.0 +go 1.25.4 require ( github.com/go-sql-driver/mysql v1.9.3 @@ -8,7 +8,6 @@ require ( github.com/gocolly/colly v1.2.0 github.com/gofiber/fiber/v3 v3.0.0-rc.3 github.com/google/uuid v1.6.0 - github.com/joho/godotenv v1.5.1 github.com/json-iterator/go v1.1.12 github.com/mozillazg/go-pinyin v0.21.0 github.com/spf13/viper v1.21.0 @@ -37,17 +36,17 @@ require ( github.com/gofiber/utils/v2 v2.0.0-rc.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/go-querystring v1.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.18.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mozillazg/go-httpheader v0.2.1 // indirect + github.com/mozillazg/go-httpheader v0.4.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/richardlehane/mscfb v1.0.4 // indirect diff --git a/go.sum b/go.sum index 3ec2bdf..92ddf33 100644 --- a/go.sum +++ b/go.sum @@ -51,12 +51,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -73,8 +75,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= @@ -95,15 +95,17 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA= github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= +github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= github.com/mozillazg/go-pinyin v0.21.0 h1:Wo8/NT45z7P3er/9YSLHA3/kjZzbLz5hR7i+jGeIGao= github.com/mozillazg/go-pinyin v0.21.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= diff --git a/src/services/sx_goods_service.go b/src/services/sx_goods_service.go index 185e1a4..776678f 100644 --- a/src/services/sx_goods_service.go +++ b/src/services/sx_goods_service.go @@ -310,9 +310,9 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in // 使用事务确保数据一致性 err := s.db.Transaction(func(tx *gorm.DB) error { - // 创建临时存储商品和图片的切片 + // 创建临时存储商品和对应图片的切片 var products []*models.LibraryProduct - var allProductImages []*models.LibraryProductImage + var productImages [][]*models.LibraryProductImage // 遍历批次中的每个商品 for _, bean := range batch { @@ -324,12 +324,8 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in // 添加商品到待插入列表 products = append(products, bean.Product) - // 处理商品图片 - for _, image := range bean.ImageList { - // 设置商品ID（在创建商品后会更新） - image.ProductID = bean.Product.ID - allProductImages = append(allProductImages, image) - } + // 添加商品对应的图片列表 + productImages = append(productImages, bean.ImageList) } // 如果没有需要保存的商品，直接返回 @@ -342,25 +338,28 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in return fmt.Errorf("批量保存商品失败: %w", err) } - // 更新图片中的商品ID并批量插入图片 - if len(allProductImages) > 0 { - // 根据商品的条形码或名称匹配更新图片的商品ID - for i, product := range products { - for _, image := range allProductImages { - // 这里假设图片已经正确关联到商品 - // 在实际应用中，可能需要更复杂的匹配逻辑 - if image.ProductID == 0 { - image.ProductID = product.ID - } - } - // 更新原始数据中的商品ID（如果需要） - if i < len(batch) { - batch[i].Product.ID = product.ID - } - } + // 创建一个新的图片切片，用于存储设置了正确ProductID的图片 + var imagesToInsert []*models.LibraryProductImage - // 批量插入图片（使用批次大小） - if err := tx.CreateInBatches(allProductImages, BatchSize).Error; err != nil { + // 遍历所有插入的商品及其对应的图片 + for i, product := range products { + // 为当前商品的每个图片设置正确的ProductID + for _, image := range productImages[i] { + // 创建新的图片实例以避免修改原始数据 + newImage := &models.LibraryProductImage{ + ProductID: product.ID, // 使用数据库生成的真实ID + ImageURL: image.ImageURL, + Seq: image.Seq, + IsMain: image.IsMain, + Status: image.Status, + } + imagesToInsert = append(imagesToInsert, newImage) + } + } + + // 批量插入图片（使用批次大小） + if len(imagesToInsert) > 0 { + if err := tx.CreateInBatches(imagesToInsert, BatchSize).Error; err != nil { return fmt.Errorf("批量保存商品图片失败: %w", err) } }