backend/java-mall
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

商品同步excel导入

Browse Source
This commit is contained in:
liyj 2025-08-08 17:05:13 +08:00
parent 2d12bb9353
commit 75e76370a2
10 changed files with 120 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mall-common/src/main/java/com/suisung/mall/common/modules/product/ShopProductBase.java
View File

@ -172,7 +172,8 @@ public class ShopProductBase implements Serializable{
private BigDecimal shop_weight;
@ApiModelProperty("是否特价商品,0否，1是")
private String is_special="0";
@TableField(updateStrategy=NOT_EMPTY)
private String is_special;
@ApiModelProperty(value = "单价")
@TableField(updateStrategy=NOT_EMPTY)

11
mall-common/src/main/java/com/suisung/mall/common/utils/CommonUtil.java
View File

@ -18,6 +18,7 @@ import java.util.*;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
/**
@ -327,5 +328,15 @@ public class CommonUtil {
return splitRatio.compareTo(BigDecimal.ZERO) >= 0 && splitRatio.compareTo(new BigDecimal(100)) <= 0;
}
/**
* 对象去重工具
* @param keyExtractor
* @return
* @param <T>
*/
public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
Set<Object> seen = new HashSet<>();
return t -> seen.add(keyExtractor.apply(t));
}
}

2
mall-shop/src/main/java/com/suisung/mall/shop/base/service/ShopBaseProductBrandService.java
View File

@ -39,7 +39,7 @@ public interface ShopBaseProductBrandService extends IBaseService<ShopBaseProduc
* @param brand_name
* @return
*/
List<ShopBaseProductBrand> selectByBrandName(String brand_name);
List<ShopBaseProductBrand> selectByBrandName(String brand_name,Integer storeId);
/**
* 新增或更新品牌，判断品牌名一样后，不一样新增，再判断关键字段是否一样，不一样更改。

5
mall-shop/src/main/java/com/suisung/mall/shop/base/service/impl/ShopBaseProductBrandServiceImpl.java
View File

@ -185,9 +185,10 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl<ShopBasePro
* @return
*/
@Override
public List<ShopBaseProductBrand> selectByBrandName(String brand_name) {
public List<ShopBaseProductBrand> selectByBrandName(String brand_name,Integer storeId) {
QueryWrapper<ShopBaseProductBrand> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("brand_name", brand_name);
queryWrapper.eq("store_id", storeId);
return list(queryWrapper);
}
@ -200,7 +201,7 @@ public class ShopBaseProductBrandServiceImpl extends BaseServiceImpl<ShopBasePro
@Override
public int saveOrUpdateBrand2(ShopBaseProductBrand shopBaseProductBrand) {
List<ShopBaseProductBrand> list = selectByBrandName(shopBaseProductBrand.getBrand_name());
List<ShopBaseProductBrand> list = selectByBrandName(shopBaseProductBrand.getBrand_name(),shopBaseProductBrand.getStore_id());
if (ObjectUtil.isEmpty(list)) {
// 新增记录
if (saveOrUpdateBrand(shopBaseProductBrand)) {

6
mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/ShopSyncImportController.java
View File

@ -59,8 +59,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
@ApiOperation(value = "品牌数据导入", notes = "品牌数据导入")
@RequestMapping(value = "/brandImportData", method = RequestMethod.POST)
public CommonResult brandImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
// ImportResult result = productMappingService.importData(file);
// return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
return shopSyncImportService.importBrandData(file,storeId);
}
@ -71,8 +69,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
@ApiOperation(value = "商品分类数据导入", notes = "分类数据导入")
@RequestMapping(value = "/categoryImportData", method = RequestMethod.POST)
public CommonResult categoryImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
// ImportResult result = productMappingService.importData(file);
// return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
return shopSyncImportService.importCategoryData(file,storeId);
}
@ -83,8 +79,6 @@ public class ShopSyncImportController extends BaseControllerImpl {
@ApiOperation(value = "商品数据导入", notes = "分类数据导入")
@RequestMapping(value = "/shopImportData", method = RequestMethod.POST)
public CommonResult shopImportData(@RequestParam("file") MultipartFile file,@RequestParam("storeId")String storeId) {
// ImportResult result = productMappingService.importData(file);
// return !result.getErrorMessages().isEmpty() ?CommonResult.failed((IErrorCode) result.getErrorMessages()):CommonResult.success(result);
shopSyncImportService.importShopsData(file,storeId);
return CommonResult.success("服务器正则处理文件，稍后查看商品列表");
}

6
mall-shop/src/main/java/com/suisung/mall/shop/sync/exelModel/SxCategoryModelExcel.java
View File

@ -26,7 +26,7 @@ public class SxCategoryModelExcel {
*/
@ApiModelProperty(value = "产品类型")
@ExcelIgnore
private String product_type=category_name;
private String product_type=this.category_name;
/**
* 第一级分类 当前分类的最顶层，如 生鲜->蔬菜->菜苗 如果当前分类（category_name）为菜苗，则第一级分类是”生鲜“，第二级分类是”蔬菜“，如果当前分类是蔬菜，则第一级分类是生鲜，第二级分类为空
* 第二级分类 当前分类最顶层数的第二层
@ -39,6 +39,6 @@ public class SxCategoryModelExcel {
private String second_category_name;
@ApiModelProperty(value = "品牌名称")
@ExcelIgnore
private String brandName="其他品牌";
@ExcelProperty(value = "品牌名称", index =3)
private String brandName;
}

80
mall-shop/src/main/java/com/suisung/mall/shop/sync/listen/ShopBatchSubmitListener.java
View File

@ -1,9 +1,12 @@
package com.suisung.mall.shop.sync.listen;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONArray;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.google.gson.Gson;
import com.suisung.mall.common.utils.CommonUtil;
import com.suisung.mall.common.utils.StringUtils;
import com.suisung.mall.shop.sync.exelModel.SxGoosModelExcel;
import com.suisung.mall.shop.sync.service.SyncThirdDataService;
import lombok.Getter;
@ -11,17 +14,19 @@ import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.math.BigDecimal;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@Component
@Slf4j
public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelExcel> {
// 批处理阈值
private static final int BATCH_SIZE = 10;
private static final int BATCH_SIZE = 500;
// 数据缓存
private List<SxGoosModelExcel> cachedDataList = new ArrayList<>(BATCH_SIZE);
@ -51,7 +56,8 @@ public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelEx
this.syncThirdDataService = syncThirdDataService;
// 创建线程池（根据CPU核心数优化）
int corePoolSize = Runtime.getRuntime().availableProcessors();
this.executorService = Executors.newFixedThreadPool(corePoolSize);
log.info("核心线程数量{}" , corePoolSize);
this.executorService = Executors.newFixedThreadPool(6);
this.futures = new ArrayList<>();
this.success = new AtomicInteger();
this.fails = new AtomicInteger();
@ -98,21 +104,61 @@ public class ShopBatchSubmitListener extends AnalysisEventListener<SxGoosModelEx
private void submitBatch() {
// 复制当前批次数据（避免异步修改）
List<SxGoosModelExcel> batchCopy = new ArrayList<>(cachedDataList);
List<SxGoosModelExcel> batchCopy = new ArrayList<>(deduplicateById(cachedDataList));
log.info("去重前：{};去重后：{}" , cachedDataList.size(), batchCopy.size());
final int index = batchSize.get();
futures.add(executorService.submit(()->{
try {
Gson gson=new Gson();
String jsonShops=gson.toJson(batchCopy);
JSONArray jsonArray=new JSONArray(jsonShops);
syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps);
log.info("已提交批次: {} 条", cachedDataList.size());
success.getAndIncrement();
return "完成"+batchSize.get();
} catch (Exception e) {
fails.getAndIncrement();
return "失败："+batchSize.get()+";失败原因:"+e.getMessage();
int i=0;
while (true){
i++;
try {
Gson gson=new Gson();
String jsonShops=gson.toJson(batchCopy);
JSONArray jsonArray=new JSONArray(jsonShops);
syncThirdDataService.baseSaveOrUpdateGoodsBatch(jsonArray,storeId,isNegativeAllowed,brandMaps);
log.info("已提交批次: {} 条", batchCopy.size());
success.getAndIncrement();
return "完成批次："+index;
} catch (Exception e) {
if(i<2){
continue;
}
fails.getAndIncrement();
return "失败批次："+index+";失败原因:"+e.getMessage();
}
}
}));
}
/**
* 数据处理
* @param list
* @return
*/
private List<SxGoosModelExcel> deduplicateById(List<SxGoosModelExcel> list) {
return list.stream()
.peek(sxGoosModelExcel -> {
if(StringUtils.isNotEmpty(sxGoosModelExcel.getShop_spec())){
sxGoosModelExcel.setProduct_spec(Collections.singletonList(sxGoosModelExcel.getShop_spec()));
}
if(null==sxGoosModelExcel.getUnit()){
sxGoosModelExcel.setUnit("");
}
if(ObjectUtil.isEmpty(sxGoosModelExcel.getBuy_limit())){
sxGoosModelExcel.setBuy_limit(0);
}
if(StringUtils.isEmpty(sxGoosModelExcel.getBrand_name())){
sxGoosModelExcel.setBrand_name("其它品牌");
}
if(ObjectUtil.isEmpty(sxGoosModelExcel.getStock())){
sxGoosModelExcel.setStock(BigDecimal.ZERO);
}
sxGoosModelExcel.setProduct_number(sxGoosModelExcel.getProduct_barcode());
sxGoosModelExcel.setOriginal_price(sxGoosModelExcel.getRetail_price());
})
.filter(CommonUtil.distinctByKey(SxGoosModelExcel::getProduct_number))
.collect(Collectors.toList());
}
}

1
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncThirdDataService.java
View File

@ -197,4 +197,5 @@ public interface SyncThirdDataService {
int baseSaveOrUpdateGoodsBatch(JSONArray goodsListJSON,String storeId,String isNegativeAllowed,
Map<String,Integer> brandMaps);
void syncPrimaryKey();
}

37
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ShopSyncImportServiceImpl.java
View File

@ -6,14 +6,18 @@ import com.alibaba.excel.EasyExcel;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.gson.Gson;
import com.suisung.mall.common.api.CommonResult;
import com.suisung.mall.common.enums.DicEnum;
import com.suisung.mall.common.exception.ApiException;
import com.suisung.mall.common.modules.base.ShopBaseProductBrand;
import com.suisung.mall.common.modules.base.ShopBaseProductCategory;
import com.suisung.mall.common.modules.sync.StoreDbConfig;
import com.suisung.mall.shop.base.service.ShopBaseProductBrandService;
import com.suisung.mall.shop.base.service.ShopBaseProductCategoryService;
import com.suisung.mall.shop.number.service.ShopNumberSeqService;
import com.suisung.mall.shop.sync.excleHandle.TemplateStyleHandler;
import com.suisung.mall.shop.sync.exelModel.*;
import com.suisung.mall.shop.sync.listen.ShopBatchSubmitListener;
import com.suisung.mall.shop.sync.service.ProductMappingService;
import com.suisung.mall.shop.sync.service.ShopSyncImportService;
import com.suisung.mall.shop.sync.service.StoreDbConfigService;
import com.suisung.mall.shop.sync.service.SyncThirdDataService;
@ -48,7 +52,15 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
private ShopBaseProductBrandService productBrandService;
@Autowired
private StoreDbConfigService storeDbConfigService;
@Autowired
private ProductMappingService productMappingService;
@Autowired
private ShopNumberSeqService shopNumberSeqService;
@Autowired
private ShopBaseProductCategoryService shopBaseProductCategoryService;
private final int limitCnt = 100;
@Override
public void downloadBrandTemplate(HttpServletResponse response) {
try {
@ -193,14 +205,20 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
}
// 读取商品分类Excel数据
private List<SxCategoryModelExcel> readCategoryExcelData(String filePath) {
return EasyExcel.read(filePath)
List<SxCategoryModelExcel> excelList= EasyExcel.read(filePath)
.head(SxCategoryModelExcel.class)
.sheet()
.doReadSync();
excelList.forEach(excel->{
excel.setProduct_type(excel.getCategory_name());
});
return excelList;
}
// 读取商品Excel数据
private void readAndImportShopsExcelData(String filePath,String storeId) {
cleanCache(storeId);//清理redis数据
initData(storeId);//初始化分类到redis缓存
Map<String, Integer> brandMaps = productBrandService.getBrandMapByStoreId(storeId);
QueryWrapper<StoreDbConfig> storeDbConfigQueryWrapper = new QueryWrapper<>();
storeDbConfigQueryWrapper.eq("store_id", storeId);
@ -211,5 +229,22 @@ public class ShopSyncImportServiceImpl implements ShopSyncImportService {
shopBatchSubmitListener.setBrandMaps(brandMaps);
shopBatchSubmitListener.setIsNegativeAllowed(isNegativeAllowed);
EasyExcel.read(filePath,SxGoosModelExcel.class,shopBatchSubmitListener).sheet().doRead();
productMappingService.syncAllProductMapping(Integer.valueOf(storeId), DicEnum.YESORNO_0.getCode());
syncThirdDataService.syncShopImages(Integer.valueOf(storeId));
cleanCache(storeId);//清理redis数据
}
private void cleanCache(String storeId){
syncThirdDataService.syncPrimaryKey();
shopNumberSeqService.clearKey();
shopBaseProductCategoryService.clearCategoryCache(storeId);
shopNumberSeqService.clearKeyStoreItemSepcId();
productBrandService.clearBrandMapByStoreId(storeId);
}
private void initData(String storeId){
shopBaseProductCategoryService.getCategoryListByStoreId(storeId);
}
}

5
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java
View File

@ -151,11 +151,6 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
@Autowired
private ShopBaseProductCategoryService shopBaseProductCategoryService;
@Autowired
private ShopProductSpecItemService shopProductSpecItemService;
@Autowired
private ShopBaseProductSpecService baseProductSpecService;
@Value("#{accountBaseConfigService.getConfig('tengxun_default_dir')}")
private String TENGXUN_DEFA;