From 75e76370a208de7c03035bcf2193cebc15d2fe5a Mon Sep 17 00:00:00 2001 From: liyj <1617420630@qq.com> Date: Fri, 8 Aug 2025 17:05:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=95=86=E5=93=81=E5=90=8C=E6=AD=A5excel?= =?UTF-8?q?=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/product/ShopProductBase.java | 3 +- .../suisung/mall/common/utils/CommonUtil.java | 11 +++ .../service/ShopBaseProductBrandService.java | 2 +- .../impl/ShopBaseProductBrandServiceImpl.java | 5 +- .../controller/ShopSyncImportController.java | 6 -- .../sync/exelModel/SxCategoryModelExcel.java | 6 +- .../sync/listen/ShopBatchSubmitListener.java | 80 +++++++++++++++---- .../sync/service/SyncThirdDataService.java | 1 + .../impl/ShopSyncImportServiceImpl.java | 37 ++++++++- .../impl/SyncThirdDataServiceImpl.java | 5 -- 10 files changed, 120 insertions(+), 36 deletions(-) diff --git a/mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java b/mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java index eff57040..cad8b52b 100644 --- a/mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java +++ b/mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java @@ -172,7 +172,8 @@ public class ShopProductBase implements Serializable{ private BigDecimal shop_weight; @ApiModelProperty("是否特价商品,0否，1是") - private String is_special="0"; + @TableField(updateStrategy=NOT_EMPTY) + private String is_special; @ApiModelProperty(value = "单价") @TableField(updateStrategy=NOT_EMPTY) diff --git a/mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java b/mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java index 2fb4341e..befee276 100644 --- a/mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java +++ b/mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java @@ -18,6 +18,7 @@ import java.util.*; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; /** @@ -327,5 +328,15 @@ public class CommonUtil { return splitRatio.compareTo(BigDecimal.ZERO) >= 0 && splitRatio.compareTo(new BigDecimal(100)) <= 0; } + /** + * 对象去重工具 + * @param keyExtractor + * @return + * @param + */ + public static Predicate distinctByKey(Function keyExtractor) { + Set seen = new HashSet<>(); + return t -> seen.add(keyExtractor.apply(t)); + } } diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java b/mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java index 3dd83068..ca7475cb 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java @@ -39,7 +39,7 @@ public interface ShopBaseProductBrandService extends IBaseService selectByBrandName(String brand_name); + List selectByBrandName(String brand_name,Integer storeId); /** * 新增或更新品牌，判断品牌名一样后，不一样新增，再判断关键字段是否一样，不一样更改。 diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java b/mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java index 906ea0a9..3a8a0d33 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java @@ -185,9 +185,10 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl selectByBrandName(String brand_name) { + public List selectByBrandName(String brand_name,Integer storeId) { QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("brand_name", brand_name); + queryWrapper.eq("store_id", storeId); return list(queryWrapper); } @@ -200,7 +201,7 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl list = selectByBrandName(shopBaseProductBrand.getBrand_name()); + List list = selectByBrandName(shopBaseProductBrand.getBrand_name(),shopBaseProductBrand.getStore_id()); if (ObjectUtil.isEmpty(list)) { // 新增记录 if (saveOrUpdateBrand(shopBaseProductBrand)) { diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java index d2822635..a6c54ccb 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java @@ -59,8 +59,6 @@ public class ShopSyncImportController extends BaseControllerImpl { @ApiOperation(value = "品牌数据导入", notes = "品牌数据导入") @RequestMapping(value = "/brandImportData", method = RequestMethod.POST) public CommonResult brandImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) { - // ImportResult result = productMappingService.importData(file); - // return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result); return shopSyncImportService.importBrandData(file,storeId); } @@ -71,8 +69,6 @@ public class ShopSyncImportController extends BaseControllerImpl { @ApiOperation(value = "商品分类数据导入", notes = "分类数据导入") @RequestMapping(value = "/categoryImportData", method = RequestMethod.POST) public CommonResult categoryImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) { - // ImportResult result = productMappingService.importData(file); - // return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result); return shopSyncImportService.importCategoryData(file,storeId); } @@ -83,8 +79,6 @@ public class ShopSyncImportController extends BaseControllerImpl { @ApiOperation(value = "商品数据导入", notes = "分类数据导入") @RequestMapping(value = "/shopImportData", method = RequestMethod.POST) public CommonResult shopImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) { - // ImportResult result = productMappingService.importData(file); - // return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result); shopSyncImportService.importShopsData(file,storeId); return CommonResult.success("服务器正则处理文件，稍后查看商品列表"); } diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java index b60450df..60fa80f7 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java @@ -26,7 +26,7 @@ public class SxCategoryModelExcel { */ @ApiModelProperty(value = "产品类型") @ExcelIgnore - private String product_type=category_name; + private String product_type=this.category_name; /** * 第一级分类 当前分类的最顶层，如 生鲜->蔬菜->菜苗 如果当前分类（category_name）为菜苗，则第一级分类是”生鲜“，第二级分类是”蔬菜“，如果当前分类是蔬菜，则第一级分类是生鲜，第二级分类为空 * 第二级分类 当前分类最顶层数的第二层 @@ -39,6 +39,6 @@ public class SxCategoryModelExcel { private String second_category_name; @ApiModelProperty(value = "品牌名称") - @ExcelIgnore - private String brandName="其他品牌"; + @ExcelProperty(value = "品牌名称", index =3) + private String brandName; } \ No newline at end of file diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java index 33d44387..56cf50d9 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java @@ -1,9 +1,12 @@ package com.suisung.mall.shop.sync.listen; +import cn.hutool.core.util.ObjectUtil; import cn.hutool.json.JSONArray; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.google.gson.Gson; +import com.suisung.mall.common.utils.CommonUtil; +import com.suisung.mall.common.utils.StringUtils; import com.suisung.mall.shop.sync.exelModel.SxGoosModelExcel; import com.suisung.mall.shop.sync.service.SyncThirdDataService; import lombok.Getter; @@ -11,17 +14,19 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.math.BigDecimal; +import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; @Component @Slf4j public class ShopBatchSubmitListener extends AnalysisEventListener { // 批处理阈值 - private static final int BATCH_SIZE = 10; + private static final int BATCH_SIZE = 500; // 数据缓存 private List cachedDataList = new ArrayList<>(BATCH_SIZE); @@ -51,7 +56,8 @@ public class ShopBatchSubmitListener extends AnalysisEventListener(); this.success = new AtomicInteger(); this.fails = new AtomicInteger(); @@ -98,21 +104,61 @@ public class ShopBatchSubmitListener extends AnalysisEventListener batchCopy = new ArrayList<>(cachedDataList); + List batchCopy = new ArrayList<>(deduplicateById(cachedDataList)); + log.info("去重前：{};去重后：{}" , cachedDataList.size(), batchCopy.size()); + final int index = batchSize.get(); futures.add(executorService.submit(()->{ - try { - Gson gson=new Gson(); - String jsonShops=gson.toJson(batchCopy); - JSONArray jsonArray=new JSONArray(jsonShops); - syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps); - log.info("已提交批次: {} 条", cachedDataList.size()); - success.getAndIncrement(); - return "完成"+batchSize.get(); - } catch (Exception e) { - fails.getAndIncrement(); - return "失败："+batchSize.get()+";失败原因:"+e.getMessage(); + int i=0; + while (true){ + i++; + try { + Gson gson=new Gson(); + String jsonShops=gson.toJson(batchCopy); + JSONArray jsonArray=new JSONArray(jsonShops); + syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps); + log.info("已提交批次: {} 条", batchCopy.size()); + success.getAndIncrement(); + return "完成批次："+index; + } catch (Exception e) { + if(i<2){ + continue; + } + fails.getAndIncrement(); + return "失败批次："+index+";失败原因:"+e.getMessage(); + } } })); } + /** + * 数据处理 + * @param list + * @return + */ + private List deduplicateById(List list) { + return list.stream() + .peek(sxGoosModelExcel -> { + if(StringUtils.isNotEmpty(sxGoosModelExcel.getShop_spec())){ + sxGoosModelExcel.setProduct_spec(Collections.singletonList(sxGoosModelExcel.getShop_spec())); + } + if(null==sxGoosModelExcel.getUnit()){ + sxGoosModelExcel.setUnit(""); + } + if(ObjectUtil.isEmpty(sxGoosModelExcel.getBuy_limit())){ + sxGoosModelExcel.setBuy_limit(0); + } + if(StringUtils.isEmpty(sxGoosModelExcel.getBrand_name())){ + sxGoosModelExcel.setBrand_name("其它品牌"); + } + if(ObjectUtil.isEmpty(sxGoosModelExcel.getStock())){ + sxGoosModelExcel.setStock(BigDecimal.ZERO); + } + sxGoosModelExcel.setProduct_number(sxGoosModelExcel.getProduct_barcode()); + sxGoosModelExcel.setOriginal_price(sxGoosModelExcel.getRetail_price()); + }) + .filter(CommonUtil.distinctByKey(SxGoosModelExcel::getProduct_number)) + .collect(Collectors.toList()); + } + + } diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java index 4515d8a0..849daf30 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java @@ -197,4 +197,5 @@ public interface SyncThirdDataService { int baseSaveOrUpdateGoodsBatch(JSONArray goodsListJSON,String storeId,String isNegativeAllowed, Map brandMaps); + void syncPrimaryKey(); } diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java index e861a53d..46adfd41 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java @@ -6,14 +6,18 @@ import com.alibaba.excel.EasyExcel; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.gson.Gson; import com.suisung.mall.common.api.CommonResult; +import com.suisung.mall.common.enums.DicEnum; import com.suisung.mall.common.exception.ApiException; import com.suisung.mall.common.modules.base.ShopBaseProductBrand; import com.suisung.mall.common.modules.base.ShopBaseProductCategory; import com.suisung.mall.common.modules.sync.StoreDbConfig; import com.suisung.mall.shop.base.service.ShopBaseProductBrandService; +import com.suisung.mall.shop.base.service.ShopBaseProductCategoryService; +import com.suisung.mall.shop.number.service.ShopNumberSeqService; import com.suisung.mall.shop.sync.excleHandle.TemplateStyleHandler; import com.suisung.mall.shop.sync.exelModel.*; import com.suisung.mall.shop.sync.listen.ShopBatchSubmitListener; +import com.suisung.mall.shop.sync.service.ProductMappingService; import com.suisung.mall.shop.sync.service.ShopSyncImportService; import com.suisung.mall.shop.sync.service.StoreDbConfigService; import com.suisung.mall.shop.sync.service.SyncThirdDataService; @@ -48,7 +52,15 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService { private ShopBaseProductBrandService productBrandService; @Autowired private StoreDbConfigService storeDbConfigService; + @Autowired + private ProductMappingService productMappingService; + @Autowired + private ShopNumberSeqService shopNumberSeqService; + @Autowired + private ShopBaseProductCategoryService shopBaseProductCategoryService; + private final int limitCnt = 100; + @Override public void downloadBrandTemplate(HttpServletResponse response) { try { @@ -193,14 +205,20 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService { } // 读取商品分类Excel数据 private List readCategoryExcelData(String filePath) { - return EasyExcel.read(filePath) + List excelList= EasyExcel.read(filePath) .head(SxCategoryModelExcel.class) .sheet() .doReadSync(); + excelList.forEach(excel->{ + excel.setProduct_type(excel.getCategory_name()); + }); + return excelList; } // 读取商品Excel数据 private void readAndImportShopsExcelData(String filePath,String storeId) { + cleanCache(storeId);//清理redis数据 + initData(storeId);//初始化分类到redis缓存 Map brandMaps = productBrandService.getBrandMapByStoreId(storeId); QueryWrapper storeDbConfigQueryWrapper = new QueryWrapper<>(); storeDbConfigQueryWrapper.eq("store_id", storeId); @@ -211,5 +229,22 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService { shopBatchSubmitListener.setBrandMaps(brandMaps); shopBatchSubmitListener.setIsNegativeAllowed(isNegativeAllowed); EasyExcel.read(filePath,SxGoosModelExcel.class,shopBatchSubmitListener).sheet().doRead(); + productMappingService.syncAllProductMapping(Integer.valueOf(storeId), DicEnum.YESORNO_0.getCode()); + syncThirdDataService.syncShopImages(Integer.valueOf(storeId)); + cleanCache(storeId);//清理redis数据 + } + + private void cleanCache(String storeId){ + syncThirdDataService.syncPrimaryKey(); + shopNumberSeqService.clearKey(); + shopBaseProductCategoryService.clearCategoryCache(storeId); + shopNumberSeqService.clearKeyStoreItemSepcId(); + productBrandService.clearBrandMapByStoreId(storeId); + } + + private void initData(String storeId){ + shopBaseProductCategoryService.getCategoryListByStoreId(storeId); } } + + diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java index 1416e351..a416cff1 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java @@ -151,11 +151,6 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements @Autowired private ShopBaseProductCategoryService shopBaseProductCategoryService; - @Autowired - private ShopProductSpecItemService shopProductSpecItemService; - - @Autowired - private ShopBaseProductSpecService baseProductSpecService; @Value("#{accountBaseConfigService.getConfig('tengxun_default_dir')}") private String TENGXUN_DEFA;