升级 go 版本，修改 思迅同步 数据不匹配的问题
This commit is contained in:
parent
568940d3f7
commit
a4ac7c8d9a
@ -12,13 +12,13 @@ name = "goods_library"
|
||||
host = "43.139.72.196"
|
||||
port = 13307
|
||||
user = "webprod"
|
||||
password = "J1XivNvAcR21}pA6Cysm.E289"
|
||||
password = "J1XivNvAcR21}pA6Cysm.E29"
|
||||
name = "mall_prod"
|
||||
|
||||
max_idle_conns = 10
|
||||
max_idle_conns = 25
|
||||
max_open_conns = 100
|
||||
conn_max_lifetime = 1 # 小时
|
||||
conn_max_idle_time = 30 # 分钟
|
||||
conn_max_idle_time = 45 # 分钟
|
||||
|
||||
# COS配置
|
||||
[cossdk]
|
||||
|
||||
9
go.mod
9
go.mod
@ -1,6 +1,6 @@
|
||||
module fafa-crawler
|
||||
|
||||
go 1.25.0
|
||||
go 1.25.4
|
||||
|
||||
require (
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
@ -8,7 +8,6 @@ require (
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/gofiber/fiber/v3 v3.0.0-rc.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/mozillazg/go-pinyin v0.21.0
|
||||
github.com/spf13/viper v1.21.0
|
||||
@ -37,17 +36,17 @@ require (
|
||||
github.com/gofiber/utils/v2 v2.0.0-rc.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/google/go-querystring v1.0.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/klauspost/compress v1.18.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/mapstructure v1.4.3 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.2.1 // indirect
|
||||
github.com/mozillazg/go-httpheader v0.4.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/philhofer/fwd v1.2.0 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.4 // indirect
|
||||
|
||||
12
go.sum
12
go.sum
@ -51,12 +51,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@ -73,8 +75,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
@ -95,15 +95,17 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
|
||||
github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
|
||||
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
|
||||
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
|
||||
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
|
||||
github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w=
|
||||
github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
|
||||
github.com/mozillazg/go-pinyin v0.21.0 h1:Wo8/NT45z7P3er/9YSLHA3/kjZzbLz5hR7i+jGeIGao=
|
||||
github.com/mozillazg/go-pinyin v0.21.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
|
||||
@ -310,9 +310,9 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
|
||||
|
||||
// 使用事务确保数据一致性
|
||||
err := s.db.Transaction(func(tx *gorm.DB) error {
|
||||
// 创建临时存储商品和图片的切片
|
||||
// 创建临时存储商品和对应图片的切片
|
||||
var products []*models.LibraryProduct
|
||||
var allProductImages []*models.LibraryProductImage
|
||||
var productImages [][]*models.LibraryProductImage
|
||||
|
||||
// 遍历批次中的每个商品
|
||||
for _, bean := range batch {
|
||||
@ -324,12 +324,8 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
|
||||
// 添加商品到待插入列表
|
||||
products = append(products, bean.Product)
|
||||
|
||||
// 处理商品图片
|
||||
for _, image := range bean.ImageList {
|
||||
// 设置商品ID（在创建商品后会更新）
|
||||
image.ProductID = bean.Product.ID
|
||||
allProductImages = append(allProductImages, image)
|
||||
}
|
||||
// 添加商品对应的图片列表
|
||||
productImages = append(productImages, bean.ImageList)
|
||||
}
|
||||
|
||||
// 如果没有需要保存的商品，直接返回
|
||||
@ -342,25 +338,28 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
|
||||
return fmt.Errorf("批量保存商品失败: %w", err)
|
||||
}
|
||||
|
||||
// 更新图片中的商品ID并批量插入图片
|
||||
if len(allProductImages) > 0 {
|
||||
// 根据商品的条形码或名称匹配更新图片的商品ID
|
||||
for i, product := range products {
|
||||
for _, image := range allProductImages {
|
||||
// 这里假设图片已经正确关联到商品
|
||||
// 在实际应用中，可能需要更复杂的匹配逻辑
|
||||
if image.ProductID == 0 {
|
||||
image.ProductID = product.ID
|
||||
}
|
||||
}
|
||||
// 更新原始数据中的商品ID（如果需要）
|
||||
if i < len(batch) {
|
||||
batch[i].Product.ID = product.ID
|
||||
}
|
||||
}
|
||||
// 创建一个新的图片切片，用于存储设置了正确ProductID的图片
|
||||
var imagesToInsert []*models.LibraryProductImage
|
||||
|
||||
// 批量插入图片（使用批次大小）
|
||||
if err := tx.CreateInBatches(allProductImages, BatchSize).Error; err != nil {
|
||||
// 遍历所有插入的商品及其对应的图片
|
||||
for i, product := range products {
|
||||
// 为当前商品的每个图片设置正确的ProductID
|
||||
for _, image := range productImages[i] {
|
||||
// 创建新的图片实例以避免修改原始数据
|
||||
newImage := &models.LibraryProductImage{
|
||||
ProductID: product.ID, // 使用数据库生成的真实ID
|
||||
ImageURL: image.ImageURL,
|
||||
Seq: image.Seq,
|
||||
IsMain: image.IsMain,
|
||||
Status: image.Status,
|
||||
}
|
||||
imagesToInsert = append(imagesToInsert, newImage)
|
||||
}
|
||||
}
|
||||
|
||||
// 批量插入图片（使用批次大小）
|
||||
if len(imagesToInsert) > 0 {
|
||||
if err := tx.CreateInBatches(imagesToInsert, BatchSize).Error; err != nil {
|
||||
return fmt.Errorf("批量保存商品图片失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user