backend/fafa-crawler
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

升级 go 版本，修改 思迅同步 数据不匹配的问题

Browse Source
This commit is contained in:
Jack 2025-11-25 21:27:39 +08:00
parent 568940d3f7
commit a4ac7c8d9a
4 changed files with 39 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/config.toml
View File

@ -12,13 +12,13 @@ name = "goods_library"
host = "43.139.72.196"
port = 13307
user = "webprod"
password = "J1XivNvAcR21}pA6Cysm.E289"
password = "J1XivNvAcR21}pA6Cysm.E29"
name = "mall_prod"
max_idle_conns = 10
max_idle_conns = 25
max_open_conns = 100
conn_max_lifetime = 1 # 小时
conn_max_idle_time = 30 # 分钟
conn_max_idle_time = 45 # 分钟
# COS配置
[cossdk]

9
go.mod
View File

@ -1,6 +1,6 @@
module fafa-crawler
go 1.25.0
go 1.25.4
require (
github.com/go-sql-driver/mysql v1.9.3
@ -8,7 +8,6 @@ require (
github.com/gocolly/colly v1.2.0
github.com/gofiber/fiber/v3 v3.0.0-rc.3
github.com/google/uuid v1.6.0
github.com/joho/godotenv v1.5.1
github.com/json-iterator/go v1.1.12
github.com/mozillazg/go-pinyin v0.21.0
github.com/spf13/viper v1.21.0
@ -37,17 +36,17 @@ require (
github.com/gofiber/utils/v2 v2.0.0-rc.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/kennygrant/sanitize v1.2.4 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/mozillazg/go-httpheader v0.4.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect

12
go.sum
View File

@ -51,12 +51,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -73,8 +75,6 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
@ -95,15 +95,17 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w=
github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
github.com/mozillazg/go-pinyin v0.21.0 h1:Wo8/NT45z7P3er/9YSLHA3/kjZzbLz5hR7i+jGeIGao=
github.com/mozillazg/go-pinyin v0.21.0/go.mod h1:iR4EnMMRXkfpFVV5FMi4FNB6wGq9NV6uDWbUuPhP4Yc=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=

41
src/services/sx_goods_service.go
View File

@ -310,9 +310,9 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
// 使用事务确保数据一致性
err := s.db.Transaction(func(tx *gorm.DB) error {
// 创建临时存储商品和图片的切片
// 创建临时存储商品和对应图片的切片
var products []*models.LibraryProduct
var allProductImages []*models.LibraryProductImage
var productImages [][]*models.LibraryProductImage
// 遍历批次中的每个商品
for _, bean := range batch {
@ -324,12 +324,8 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
// 添加商品到待插入列表
products = append(products, bean.Product)
// 处理商品图片
for _, image := range bean.ImageList {
// 设置商品ID（在创建商品后会更新）
image.ProductID = bean.Product.ID
allProductImages = append(allProductImages, image)
}
// 添加商品对应的图片列表
productImages = append(productImages, bean.ImageList)
}
// 如果没有需要保存的商品，直接返回
@ -342,25 +338,28 @@ func (s *SxGoodsService) saveSiXunProductBatch(batch []*beans.SiXunGoodsBean) in
return fmt.Errorf("批量保存商品失败: %w", err)
}
// 更新图片中的商品ID并批量插入图片
if len(allProductImages) > 0 {
// 根据商品的条形码或名称匹配更新图片的商品ID
// 创建一个新的图片切片，用于存储设置了正确ProductID的图片
var imagesToInsert []*models.LibraryProductImage
// 遍历所有插入的商品及其对应的图片
for i, product := range products {
for _, image := range allProductImages {
// 这里假设图片已经正确关联到商品
// 在实际应用中，可能需要更复杂的匹配逻辑
if image.ProductID == 0 {
image.ProductID = product.ID
// 为当前商品的每个图片设置正确的ProductID
for _, image := range productImages[i] {
// 创建新的图片实例以避免修改原始数据
newImage := &models.LibraryProductImage{
ProductID: product.ID, // 使用数据库生成的真实ID
ImageURL: image.ImageURL,
Seq: image.Seq,
IsMain: image.IsMain,
Status: image.Status,
}
}
// 更新原始数据中的商品ID（如果需要）
if i < len(batch) {
batch[i].Product.ID = product.ID
imagesToInsert = append(imagesToInsert, newImage)
}
}
// 批量插入图片（使用批次大小）
if err := tx.CreateInBatches(allProductImages, BatchSize).Error; err != nil {
if len(imagesToInsert) > 0 {
if err := tx.CreateInBatches(imagesToInsert, BatchSize).Error; err != nil {
return fmt.Errorf("批量保存商品图片失败: %w", err)
}
}