diff --git a/README.md b/README.md index 3b1d8d7..dd185f8 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ fafa-crawler 使用以下命令运行应用程序： ``` -go run src/main.go +go run main.go ``` 服务器将在环境变量 `PORT` 指定的端口上启动，默认为 `3000`。 diff --git a/config/config.toml b/config/config.toml index a0713cf..b94ccdc 100644 --- a/config/config.toml +++ b/config/config.toml @@ -1,15 +1,18 @@ -[mysql] +[server] +port = 3001 + +[mysql_local] host = "localhost" port = 3306 user = "root" password = "1234567890" name = "goods_library" -[mysql_prod] +[mysql] host = "43.139.72.196" port = 13307 user = "webprod" -password = "v9W2ER6KG9HVtH1mjAytRvdr" +password = "J1XivNvAcR21}pA6Cysm.E29" name = "mall_prod" max_idle_conns = 10 diff --git a/main.go b/main.go index 8d58a92..8402c8b 100644 --- a/main.go +++ b/main.go @@ -2,28 +2,23 @@ package main import ( "log" - "os" "github.com/gofiber/fiber/v3" "github.com/gofiber/fiber/v3/middleware/logger" - "github.com/joho/godotenv" + "github.com/spf13/viper" "fafa-crawler/src/controller" - // "fafa-crawler/src/services" ) func main() { + // 初始化配置 + initConfig() + app := fiber.New() // 添加日志中间件 app.Use(logger.New()) - // Load environment variables from .env file - err := godotenv.Load() - if err != nil { - log.Fatalf("Error loading .env file") - } - // Initialize controller and services ctrl := controller.Controller{} @@ -37,9 +32,22 @@ func main() { app.Post("/sixun/data/sync", ctrl.SyncSiXunGoodsData) // Start the server - port := os.Getenv("PORT") + port := viper.GetString("server.port") if port == "" { - port = "3000" + port = "3001" } log.Fatal(app.Listen(":"+port, fiber.ListenConfig{EnablePrefork: true})) } + +func initConfig() { + // 设置配置文件路径和类型 + viper.SetConfigFile("./config/config.toml") + viper.SetConfigType("toml") + + // 读取配置文件 + err := viper.ReadInConfig() + if err != nil { + log.Printf("Warning: unable to read config file: %s", err) + return + } +} diff --git a/src/controller/init.go b/src/controller/init.go index 3a50020..9de0504 100644 --- a/src/controller/init.go +++ b/src/controller/init.go @@ -8,10 +8,17 @@ import ( ) var ( - hdyColly = colly_service.NewHdyCollyService() - productService = services.NewProductService() + hdyColly *colly_service.HdyCollyService + productService *services.ProductService + sxGoodsService *services.SxGoodsService ) +func init() { + hdyColly = colly_service.NewHdyCollyService() + productService = services.NewProductService() + sxGoodsService = services.NewSxGoodsService() +} + // APIResponse 标准API响应结构体(符合国内大厂规范) // 包含状态码、消息、数据和时间戳字段 type APIResponse struct { diff --git a/src/controller/sx_goods_controller.go b/src/controller/sx_goods_controller.go index bfeafe0..1791094 100644 --- a/src/controller/sx_goods_controller.go +++ b/src/controller/sx_goods_controller.go @@ -1,13 +1,15 @@ package controller import ( - "fafa-crawler/src/services" "log" + "strings" "time" "github.com/gofiber/fiber/v3" ) +var sixunFilePath = "/Users/panjunjie/work/fafa_library/sixun/" + // SyncSiXunGoodsData 处理思迅爬取的数据，上传到小发同城图片库 func (c *Controller) SyncSiXunGoodsData(ctx fiber.Ctx) error { start := time.Now() @@ -23,16 +25,18 @@ func (c *Controller) SyncSiXunGoodsData(ctx fiber.Ctx) error { return ctx.Status(fiber.StatusBadRequest).JSON(Error(fiber.StatusBadRequest, "请提供图片源目录路径")) } - sxGoodsService := services.NewSxGoodsService() - successCnt := 0 totalSrcDataCnt := int64(0) totalFilteredDataCnt := int64(0) totalSavedDataCnt := int64(0) totalFilteredImgCnt := int64(0) + if !strings.HasSuffix(sixunFilePath, "/") { + sixunFilePath += "/" + } + for _, folder := range params.Folders { - path := processFilePath + folder + path := sixunFilePath + folder // 处理思迅商品数据 srcDataCnt, filteredDataCnt, savedDataCnt, filteredImgCnt := sxGoodsService.ProcessSiXunGoodsData(path) diff --git a/src/services/init.go b/src/services/init.go index 9d12d8b..afe7574 100644 --- a/src/services/init.go +++ b/src/services/init.go @@ -13,16 +13,31 @@ var ( once sync.Once dbGorm *gorm.DB ormQ *mapper.Query + + // 服务实例 + productService *ProductService + sxGoodsService *SxGoodsService ) func init() { - if dbGorm == nil { - dbutil.InitMySQLDB() - dbGorm = dbutil.MySQLDB - } + once.Do(func() { + // 初始化数据库连接 + if dbGorm == nil { + dbutil.InitMySQLDB() + dbGorm = dbutil.MySQLDB + } - // 初始化 gorm 的 mapper - mapper.SetDefault(dbGorm) + // 初始化 gorm 的 mapper + mapper.SetDefault(dbGorm) + ormQ = mapper.Use(dbGorm) - ormQ = mapper.Use(dbGorm) + // 确保数据库连接不为nil + if dbGorm == nil { + panic("数据库连接未初始化") + } + + // 初始化所有服务实例 + productService = &ProductService{db: dbGorm} + sxGoodsService = &SxGoodsService{db: dbGorm} + }) } diff --git a/src/services/product_service.go b/src/services/product_service.go index 43b2f86..1d2fd97 100644 --- a/src/services/product_service.go +++ b/src/services/product_service.go @@ -14,18 +14,14 @@ import ( ) var ( - productService *ProductService - filePath = "/media/images/goods_library/" + filePath = "/media/images/goods_library/" ) type ProductService struct { db *gorm.DB // Add database connection } -func NewProductService() *ProductService { // Pass database connection - once.Do(func() { - productService = &ProductService{db: dbGorm} // Initialize with database connection - }) +func NewProductService() *ProductService { return productService } diff --git a/src/services/sx_goods_service.go b/src/services/sx_goods_service.go index 9e23e0d..185e1a4 100644 --- a/src/services/sx_goods_service.go +++ b/src/services/sx_goods_service.go @@ -23,18 +23,13 @@ const ( BatchSize = 400 ) -var ( - sxGoodsService *SxGoodsService -) +var () type SxGoodsService struct { db *gorm.DB // Add database connection } -func NewSxGoodsService() *SxGoodsService { // Pass database connection - once.Do(func() { - sxGoodsService = &SxGoodsService{db: dbGorm} // Initialize with database connection - }) +func NewSxGoodsService() *SxGoodsService { return sxGoodsService } @@ -190,6 +185,7 @@ func (s *SxGoodsService) JsonDataConvBeans(path string) (srcDataCnt, targetDataC if product.Category2Nd == "" { product.Category2Nd = product.Category1St } + product.Category = product.Category2Nd // 如果Intro为空，使用Title的值 if product.Intro == "" { diff --git a/src/services/sx_service.go b/src/services/sx_service.go deleted file mode 100644 index 5e568ea..0000000 --- a/src/services/sx_service.go +++ /dev/null @@ -1 +0,0 @@ -package services