商品同步excel导入

mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java

@@ -172,7 +172,8 @@ public class ShopProductBase implements Serializable{
     private BigDecimal  shop_weight;
 
     @ApiModelProperty("是否特价商品,0否，1是")
-    private String is_special="0";
+    @TableField(updateStrategy=NOT_EMPTY)
+    private String is_special;
 
     @ApiModelProperty(value = "单价")
     @TableField(updateStrategy=NOT_EMPTY)

mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java

@@ -19,6 +19,7 @@ import java.util.*;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 /**
@@ -353,5 +354,15 @@ public class CommonUtil {
         return splitRatio.compareTo(BigDecimal.ZERO) >= 0 && splitRatio.compareTo(new BigDecimal(100)) <= 0;
     }
 
+    /**
+     * 对象去重工具
+     * @param keyExtractor
+     * @return
+     * @param <T>
+     */
+    public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
+        Set<Object> seen = new HashSet<>();
+        return t -> seen.add(keyExtractor.apply(t));
+    }
 
 }

mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java

@@ -39,7 +39,7 @@ public interface ShopBaseProductBrandService extends IBaseService<ShopBaseProduc
      * @param brand_name
      * @return
      */
-    List<ShopBaseProductBrand> selectByBrandName(String brand_name);
+    List<ShopBaseProductBrand> selectByBrandName(String brand_name,Integer storeId);
 
     /**
      * 新增或更新品牌，判断品牌名一样后，不一样新增，再判断关键字段是否一样，不一样更改。

mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java

@@ -185,9 +185,10 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl<ShopBasePro
      * @return
      */
     @Override
-    public List<ShopBaseProductBrand> selectByBrandName(String brand_name) {
+    public List<ShopBaseProductBrand> selectByBrandName(String brand_name,Integer storeId) {
         QueryWrapper<ShopBaseProductBrand> queryWrapper = new QueryWrapper<>();
         queryWrapper.eq("brand_name", brand_name);
+        queryWrapper.eq("store_id", storeId);
         return list(queryWrapper);
     }
 
@@ -200,7 +201,7 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl<ShopBasePro
     @Override
     public int saveOrUpdateBrand2(ShopBaseProductBrand shopBaseProductBrand) {
 
-        List<ShopBaseProductBrand> list = selectByBrandName(shopBaseProductBrand.getBrand_name());
+        List<ShopBaseProductBrand> list = selectByBrandName(shopBaseProductBrand.getBrand_name(),shopBaseProductBrand.getStore_id());
         if (ObjectUtil.isEmpty(list)) {
             // 新增记录
             if (saveOrUpdateBrand(shopBaseProductBrand)) {

mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java

@@ -59,8 +59,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
     @ApiOperation(value = "品牌数据导入", notes = "品牌数据导入")
     @RequestMapping(value = "/brandImportData", method = RequestMethod.POST)
     public CommonResult brandImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
-        // ImportResult result = productMappingService.importData(file);
-        //  return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
         return shopSyncImportService.importBrandData(file,storeId);
     }
 
@@ -71,8 +69,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
     @ApiOperation(value = "商品分类数据导入", notes = "分类数据导入")
     @RequestMapping(value = "/categoryImportData", method = RequestMethod.POST)
     public CommonResult categoryImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
-       // ImportResult result = productMappingService.importData(file);
-      //  return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
         return shopSyncImportService.importCategoryData(file,storeId);
     }
 
@@ -83,8 +79,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
     @ApiOperation(value = "商品数据导入", notes = "分类数据导入")
     @RequestMapping(value = "/shopImportData", method = RequestMethod.POST)
     public CommonResult shopImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
-       // ImportResult result = productMappingService.importData(file);
-       // return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
          shopSyncImportService.importShopsData(file,storeId);
          return CommonResult.success("服务器正则处理文件，稍后查看商品列表");
     }

mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java

@@ -26,7 +26,7 @@ public class SxCategoryModelExcel {
      */
     @ApiModelProperty(value = "产品类型")
     @ExcelIgnore
-    private String product_type=category_name;
+    private String product_type=this.category_name;
     /**
      * 第一级分类 当前分类的最顶层，如  生鲜->蔬菜->菜苗  如果当前分类（category_name）为菜苗，则第一级分类是”生鲜“，第二级分类是”蔬菜“，如果当前分类是蔬菜，则第一级分类是生鲜，第二级分类为空
      * 第二级分类 当前分类最顶层数的第二层
@@ -39,6 +39,6 @@ public class SxCategoryModelExcel {
     private String second_category_name;
 
     @ApiModelProperty(value = "品牌名称")
-    @ExcelIgnore
+    @ExcelProperty(value = "品牌名称", index =3)
-    private String brandName="其他品牌";
+    private String brandName;
 }

mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java

@@ -1,9 +1,12 @@
 package com.suisung.mall.shop.sync.listen;
 
+import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.json.JSONArray;
 import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.event.AnalysisEventListener;
 import com.google.gson.Gson;
+import com.suisung.mall.common.utils.CommonUtil;
+import com.suisung.mall.common.utils.StringUtils;
 import com.suisung.mall.shop.sync.exelModel.SxGoosModelExcel;
 import com.suisung.mall.shop.sync.service.SyncThirdDataService;
 import lombok.Getter;
@@ -11,17 +14,19 @@ import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 
-import java.util.ArrayList;
+import java.math.BigDecimal;
-import java.util.List;
+import java.util.*;
-import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 @Component
 @Slf4j
 public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelExcel> {
     // 批处理阈值
-    private static final int BATCH_SIZE = 10;
+    private static final int BATCH_SIZE = 500;
     // 数据缓存
     private List<SxGoosModelExcel> cachedDataList = new ArrayList<>(BATCH_SIZE);
 
@@ -51,7 +56,8 @@ public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelEx
         this.syncThirdDataService = syncThirdDataService;
         // 创建线程池（根据CPU核心数优化）
         int corePoolSize = Runtime.getRuntime().availableProcessors();
-        this.executorService = Executors.newFixedThreadPool(corePoolSize);
+        log.info("核心线程数量{}" , corePoolSize);
+        this.executorService = Executors.newFixedThreadPool(6);
         this.futures = new ArrayList<>();
         this.success = new AtomicInteger();
         this.fails = new AtomicInteger();
@@ -98,21 +104,61 @@ public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelEx
 
     private void submitBatch() {
         // 复制当前批次数据（避免异步修改）
-        List<SxGoosModelExcel> batchCopy = new ArrayList<>(cachedDataList);
+        List<SxGoosModelExcel> batchCopy = new ArrayList<>(deduplicateById(cachedDataList));
+        log.info("去重前：{};去重后：{}" , cachedDataList.size(), batchCopy.size());
+        final int index = batchSize.get();
         futures.add(executorService.submit(()->{
-            try {
+            int i=0;
-                Gson gson=new Gson();
+            while (true){
-                String jsonShops=gson.toJson(batchCopy);
+                i++;
-                JSONArray jsonArray=new JSONArray(jsonShops);
+                try {
-                syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps);
+                    Gson gson=new Gson();
-                log.info("已提交批次: {} 条", cachedDataList.size());
+                    String jsonShops=gson.toJson(batchCopy);
-                success.getAndIncrement();
+                    JSONArray jsonArray=new JSONArray(jsonShops);
-                return "完成"+batchSize.get();
+                    syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps);
-            } catch (Exception e) {
+                    log.info("已提交批次: {} 条", batchCopy.size());
-                fails.getAndIncrement();
+                    success.getAndIncrement();
-                return "失败："+batchSize.get()+";失败原因:"+e.getMessage();
+                    return "完成批次："+index;
+                } catch (Exception e) {
+                    if(i<2){
+                        continue;
+                    }
+                    fails.getAndIncrement();
+                    return "失败批次："+index+";失败原因:"+e.getMessage();
+                }
             }
         }));
     }
 
+    /**
+     * 数据处理
+     * @param list
+     * @return
+     */
+    private List<SxGoosModelExcel> deduplicateById(List<SxGoosModelExcel> list) {
+        return list.stream()
+                .peek(sxGoosModelExcel -> {
+                    if(StringUtils.isNotEmpty(sxGoosModelExcel.getShop_spec())){
+                        sxGoosModelExcel.setProduct_spec(Collections.singletonList(sxGoosModelExcel.getShop_spec()));
+                    }
+                    if(null==sxGoosModelExcel.getUnit()){
+                        sxGoosModelExcel.setUnit("");
+                    }
+                    if(ObjectUtil.isEmpty(sxGoosModelExcel.getBuy_limit())){
+                        sxGoosModelExcel.setBuy_limit(0);
+                    }
+                    if(StringUtils.isEmpty(sxGoosModelExcel.getBrand_name())){
+                        sxGoosModelExcel.setBrand_name("其它品牌");
+                    }
+                    if(ObjectUtil.isEmpty(sxGoosModelExcel.getStock())){
+                        sxGoosModelExcel.setStock(BigDecimal.ZERO);
+                    }
+                    sxGoosModelExcel.setProduct_number(sxGoosModelExcel.getProduct_barcode());
+                    sxGoosModelExcel.setOriginal_price(sxGoosModelExcel.getRetail_price());
+                })
+                .filter(CommonUtil.distinctByKey(SxGoosModelExcel::getProduct_number))
+                .collect(Collectors.toList());
+    }
+
+
 }

mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java

@@ -197,4 +197,5 @@ public interface SyncThirdDataService {
     int baseSaveOrUpdateGoodsBatch(JSONArray goodsListJSON,String storeId,String isNegativeAllowed,
                                                   Map<String,Integer> brandMaps);
 
+    void syncPrimaryKey();
 }

mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java

@@ -6,14 +6,18 @@ import com.alibaba.excel.EasyExcel;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.google.gson.Gson;
 import com.suisung.mall.common.api.CommonResult;
+import com.suisung.mall.common.enums.DicEnum;
 import com.suisung.mall.common.exception.ApiException;
 import com.suisung.mall.common.modules.base.ShopBaseProductBrand;
 import com.suisung.mall.common.modules.base.ShopBaseProductCategory;
 import com.suisung.mall.common.modules.sync.StoreDbConfig;
 import com.suisung.mall.shop.base.service.ShopBaseProductBrandService;
+import com.suisung.mall.shop.base.service.ShopBaseProductCategoryService;
+import com.suisung.mall.shop.number.service.ShopNumberSeqService;
 import com.suisung.mall.shop.sync.excleHandle.TemplateStyleHandler;
 import com.suisung.mall.shop.sync.exelModel.*;
 import com.suisung.mall.shop.sync.listen.ShopBatchSubmitListener;
+import com.suisung.mall.shop.sync.service.ProductMappingService;
 import com.suisung.mall.shop.sync.service.ShopSyncImportService;
 import com.suisung.mall.shop.sync.service.StoreDbConfigService;
 import com.suisung.mall.shop.sync.service.SyncThirdDataService;
@@ -48,7 +52,15 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
     private ShopBaseProductBrandService productBrandService;
     @Autowired
     private StoreDbConfigService storeDbConfigService;
+    @Autowired
+    private ProductMappingService productMappingService;
+    @Autowired
+    private ShopNumberSeqService shopNumberSeqService;
+    @Autowired
+    private ShopBaseProductCategoryService shopBaseProductCategoryService;
+
     private final int limitCnt = 100;
+
     @Override
     public void downloadBrandTemplate(HttpServletResponse response) {
         try {
@@ -193,14 +205,20 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
     }
     // 读取商品分类Excel数据
     private List<SxCategoryModelExcel> readCategoryExcelData(String filePath) {
-        return EasyExcel.read(filePath)
+        List<SxCategoryModelExcel> excelList= EasyExcel.read(filePath)
                 .head(SxCategoryModelExcel.class)
                 .sheet()
                 .doReadSync();
+         excelList.forEach(excel->{
+            excel.setProduct_type(excel.getCategory_name());
+        });
+        return excelList;
     }
 
     // 读取商品Excel数据
     private void readAndImportShopsExcelData(String filePath,String storeId) {
+        cleanCache(storeId);//清理redis数据
+        initData(storeId);//初始化分类到redis缓存
         Map<String, Integer> brandMaps = productBrandService.getBrandMapByStoreId(storeId);
         QueryWrapper<StoreDbConfig> storeDbConfigQueryWrapper = new QueryWrapper<>();
         storeDbConfigQueryWrapper.eq("store_id", storeId);
@@ -211,5 +229,22 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
         shopBatchSubmitListener.setBrandMaps(brandMaps);
         shopBatchSubmitListener.setIsNegativeAllowed(isNegativeAllowed);
         EasyExcel.read(filePath,SxGoosModelExcel.class,shopBatchSubmitListener).sheet().doRead();
+        productMappingService.syncAllProductMapping(Integer.valueOf(storeId), DicEnum.YESORNO_0.getCode());
+        syncThirdDataService.syncShopImages(Integer.valueOf(storeId));
+        cleanCache(storeId);//清理redis数据
+    }
+
+    private void cleanCache(String storeId){
+        syncThirdDataService.syncPrimaryKey();
+        shopNumberSeqService.clearKey();
+        shopBaseProductCategoryService.clearCategoryCache(storeId);
+        shopNumberSeqService.clearKeyStoreItemSepcId();
+        productBrandService.clearBrandMapByStoreId(storeId);
+    }
+
+    private void initData(String storeId){
+        shopBaseProductCategoryService.getCategoryListByStoreId(storeId);
     }
 }
+
+

mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java

@@ -151,11 +151,6 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
     @Autowired
     private ShopBaseProductCategoryService shopBaseProductCategoryService;
 
-    @Autowired
-    private ShopProductSpecItemService shopProductSpecItemService;
-
-    @Autowired
-    private ShopBaseProductSpecService baseProductSpecService;
 
     @Value("#{accountBaseConfigService.getConfig('tengxun_default_dir')}")
     private String TENGXUN_DEFA;