backend/java-mall
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增数据库存储文件内容，新增配置字段

Browse Source
This commit is contained in:
liyj 2025-11-10 16:20:55 +08:00
parent e3741192c4
commit 45a4166b0b
10 changed files with 230 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mall-common/src/main/java/com/suisung/mall/common/modules/sync/StoreDbConfig.java
View File

@ -128,6 +128,14 @@ public class StoreDbConfig implements Serializable {
private String shopGapTime;
@TableField(value = "sale_account",updateStrategy = FieldStrategy.NOT_EMPTY)
@NotBlank(message = "营业员账号")
@ApiModelProperty(value = "营业员账号")
private String saleAccount;
@TableField(value = "is_sync_active",updateStrategy = FieldStrategy.NOT_EMPTY)
@ApiModelProperty(value = "是否同步活动(0:否,1:是)")
private String isSyncActive;
@TableField(value = "is_sync_member",updateStrategy = FieldStrategy.NOT_EMPTY)
@ApiModelProperty(value = "是否同步会员(0:否,1:是)")
private String isSyncMember;
}

53
mall-common/src/main/java/com/suisung/mall/common/modules/sync/SyncStoreData.java Normal file
View File

@ -0,0 +1,53 @@
package com.suisung.mall.common.modules.sync;
import com.baomidou.mybatisplus.annotation.*;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.io.Serializable;
import java.util.Date;
@Data
@TableName("sycn_store_data")
@ApiModel("商品同步数据")
public class SyncStoreData implements Serializable {
@TableId(value = "sync_store_data_id", type = IdType.INPUT)
@ApiModelProperty("主键ID")
private String syncStoreDataId;
@TableField("store_id")
@ApiModelProperty("店铺ID")
private String storeId;
@TableField("content")
@ApiModelProperty("同步文本内容")
private String content;
@TableField("file_name")
@ApiModelProperty("文件名称")
private String fileName;
@TableField("folder")
@ApiModelProperty("文件路径")
private String folder;
@TableField("file_path")
@ApiModelProperty("文件全路径")
private String filePath;
@TableField("status")
@ApiModelProperty("处理状态0：未处理，1：已处理")
private String status;
@TableField("create_time")
@ApiModelProperty(value = "创建时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
@TableField("update_time")
@ApiModelProperty(value = "更新时间")
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
}

56
mall-common/src/main/java/com/suisung/mall/common/utils/TextCompressionUtil.java Normal file
View File

@ -0,0 +1,56 @@
package com.suisung.mall.common.utils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
public class TextCompressionUtil {
/**
* 压缩文本
*/
public static String compress(String text) {
if (text == null || text.length() == 0) {
return text;
}
try {
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (GZIPOutputStream gzip = new GZIPOutputStream(out)) {
gzip.write(text.getBytes(StandardCharsets.UTF_8));
}
return Base64.getEncoder().encodeToString(out.toByteArray());
} catch (IOException e) {
throw new RuntimeException("压缩失败", e);
}
}
/**
* 解压缩文本
*/
public static String decompress(String compressedText) {
if (compressedText == null || compressedText.length() == 0) {
return compressedText;
}
try {
byte[] compressedData = Base64.getDecoder().decode(compressedText);
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (ByteArrayInputStream in = new ByteArrayInputStream(compressedData);
GZIPInputStream gzip = new GZIPInputStream(in)) {
byte[] buffer = new byte[1024];
int len;
while ((len = gzip.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
}
return out.toString(String.valueOf(StandardCharsets.UTF_8));
} catch (IOException e) {
throw new RuntimeException("解压缩失败", e);
}
}
}

4
mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/StoreDbConfigController.java
View File

@ -10,12 +10,8 @@ package com.suisung.mall.shop.sync.controller;
import com.suisung.mall.common.api.CommonResult;
import com.suisung.mall.common.modules.sync.StoreDbConfig;
import com.suisung.mall.common.modules.sync.SyncApp;
import com.suisung.mall.common.modules.sync.SyncFileLog;
import com.suisung.mall.common.service.impl.BaseControllerImpl;
import com.suisung.mall.core.web.controller.BaseController;
import com.suisung.mall.shop.sync.service.StoreDbConfigService;
import com.suisung.mall.shop.sync.service.SyncFileLogService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;

12
mall-shop/src/main/java/com/suisung/mall/shop/sync/mapper/SyncStoreDataMapper.java Normal file
View File

@ -0,0 +1,12 @@
package com.suisung.mall.shop.sync.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.suisung.mall.common.modules.sync.SyncStoreData;
import org.springframework.stereotype.Repository;
@Repository
public interface SyncStoreDataMapper extends BaseMapper<SyncStoreData> {
}

11
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncStoreDataService.java Normal file
View File

@ -0,0 +1,11 @@
package com.suisung.mall.shop.sync.service;
import com.suisung.mall.common.modules.sync.SyncStoreData;
import com.suisung.mall.core.web.service.IBaseService;
public interface SyncStoreDataService extends IBaseService<SyncStoreData> {
}

15
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncStoreDataServiceImpl.java Normal file
View File

@ -0,0 +1,15 @@
package com.suisung.mall.shop.sync.service.impl;
import com.suisung.mall.common.modules.sync.SyncStoreData;
import com.suisung.mall.core.web.service.impl.BaseServiceImpl;
import com.suisung.mall.shop.sync.mapper.SyncStoreDataMapper;
import com.suisung.mall.shop.sync.service.SyncStoreDataService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@Service
@Transactional(rollbackFor = Exception.class)
@Slf4j
public class SyncStoreDataServiceImpl extends BaseServiceImpl<SyncStoreDataMapper, SyncStoreData> implements SyncStoreDataService {
}

64
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java
View File

@ -45,6 +45,7 @@ import com.suisung.mall.common.pojo.req.SyncThirdMemberReq;
import com.suisung.mall.common.pojo.res.ThirdApiRes;
import com.suisung.mall.common.utils.I18nUtil;
import com.suisung.mall.common.utils.StringUtils;
import com.suisung.mall.common.utils.TextCompressionUtil;
import com.suisung.mall.shop.base.service.ShopBaseProductBrandService;
import com.suisung.mall.shop.base.service.ShopBaseProductCategoryService;
import com.suisung.mall.shop.number.service.ShopNumberSeqService;
@ -65,13 +66,13 @@ import com.suisung.mall.shop.store.service.ShopStoreActivityBaseService;
import com.suisung.mall.shop.store.service.ShopStoreActivityItemService;
import com.suisung.mall.shop.sync.Utils.ActiveShopJsonUtils;
import com.suisung.mall.shop.sync.Utils.BigDecimalFormatter;
import com.suisung.mall.shop.sync.Utils.ThreadFileUtils;
import com.suisung.mall.shop.sync.dto.ActiveModel;
import com.suisung.mall.shop.sync.dto.ActiveShopInfo;
import com.suisung.mall.shop.sync.dto.BrandModel;
import com.suisung.mall.shop.sync.keymanage.RedisKey;
import com.suisung.mall.shop.sync.service.*;
import io.seata.spring.annotation.GlobalTransactional;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections4.ListUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -107,6 +108,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import static com.suisung.mall.common.utils.I18nUtil._;
@ -181,6 +183,9 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
@Autowired
private ProductQuantityConsumptionService productQuantityConsumptionService;
@Autowired
private SyncStoreDataService syncStoreDataService;
/**
* 批量保存商品的分类
*
@ -517,6 +522,15 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
return new ThirdApiRes().fail(500, "文件上传失败:签名验证失败");
}
logger.info("path-{},parent-{},filename-{},root-{}", path, path.getParent(), path.getFileName().toString(), path.getRoot());
SyncStoreData sxStoreData = new SyncStoreData();
sxStoreData.setSyncStoreDataId(DigestUtils.md5Hex(filePath));
sxStoreData.setStoreId(syncAppO.getStore_id());
sxStoreData.setContent(TextCompressionUtil.compress(jsonStr));
sxStoreData.setFileName(filName);
sxStoreData.setFolder(folder);
sxStoreData.setFilePath(filePath);
sxStoreData.setStatus("0");
syncStoreDataService.saveOrUpdate(sxStoreData);
// String filaPath=path.toString();
// if(filePath.contains(":")){
// filePath=filePath.substring(filePath.indexOf(":")+1);
@ -568,7 +582,7 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
});
//upLoadZipToOss(newFolders.get(0));//上传文件到cos
dowloadAndUnZip(newFolders.get(0));//读取cos文件回本地
// dowloadAndUnZip(newFolders.get(0));//读取cos文件回本地
syncPrimaryKey();
shopNumberSeqService.clearKey();
shopBaseProductCategoryService.clearCategoryCache(storeId);
@ -577,8 +591,8 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
ExecutorService executor = Executors.newFixedThreadPool(6);
List<Future<?>> futures = new ArrayList<>();
// 提交任务
AtomicInteger success = new AtomicInteger();
AtomicInteger fails = new AtomicInteger();
AtomicInteger success = new AtomicInteger(0);
AtomicInteger fails = new AtomicInteger(0);
List<String> failFolders = new ArrayList<>();
List<String> failMessage = new ArrayList<>();
shopBaseProductCategoryService.getCategoryListByStoreId(storeId);
@ -592,19 +606,32 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
// Map ShopBaseProductSpecMap = baseProductSpecService.getShopBaseProductSpecMap(Integer.valueOf(storeId));//切割商品缓存
// long seconds=System.currentTimeMillis();
// Date productSaleTime=Date.from(Instant.now().plusSeconds(seconds));
List<String> fileNames=new ArrayList<>();
for(int i=0;i<newFolders.size();i++){
String fileName = "goods_" + (i + 1) + ".txt";
fileNames.add(fileName);
}
QueryWrapper<SyncStoreData> syncStoreDataQueryWrapper = new QueryWrapper<>();
syncStoreDataQueryWrapper.eq("store_id", storeId);
syncStoreDataQueryWrapper.eq("status", 0);
syncStoreDataQueryWrapper.in("file_name",fileNames);
List<SyncStoreData> syncStoreDataList= syncStoreDataService.list(syncStoreDataQueryWrapper);
Map<String,String> syncDataMap= syncStoreDataList.stream().collect(Collectors.toMap(SyncStoreData::getSyncStoreDataId, SyncStoreData::getContent));
for (int i = 0; i < newFolders.size(); i++) {
final int taskId = i;
final String isNegativeAllowed = storeDbConfig.getIsNegativeAllowed();
//String priorityMode = storeDbConfig.getPriorityMode();
//boolean isUpdatePrice= ObjectUtil.isNotEmpty(storeDbConfig.getRefreshTime());//是否更新所有切割价格
threadNum.incrementAndGet();
Map<String, String> finalSyncDataMap = syncDataMap;
futures.add(executor.submit(() -> {
int count = 0;//失败重试机制，当失败重试一次，再次失败则记录到数据库中
while (true) {
count++;
String taskName = newFolders.get(taskId);
String fileName = "good_" + (taskId + 1) + ".txt";
JSONArray jsonArray = new ThreadFileUtils().processFolder(taskName, newFolders.get(taskId));
// String taskName = newFolders.get(taskId);
String fileName = "goods_" + (taskId + 1) + ".txt";
String sycnDataId=DigestUtils.md5Hex(newFolders.get(taskId) + fileName);
JSONArray jsonArray = getSyncDataContent(finalSyncDataMap,sycnDataId);
try {
baseSaveOrUpdateGoodsBatch(jsonArray, storeId, isNegativeAllowed, brandMaps);
success.getAndIncrement();
@ -657,6 +684,12 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
if (CollUtil.isNotEmpty(syncFileLogs)) {
syncFileLogService.saveBatch(syncFileLogs, syncFileLogs.size());
}
syncStoreDataList=syncStoreDataList.stream().peek(syncStoreData -> {
syncStoreData.setStatus("1");
}).collect(Collectors.toList());
if(!syncStoreDataList.isEmpty()){
syncStoreDataService.updateBatchById(syncStoreDataList,syncStoreDataList.size());//处理结束后更新已处理
}
//todo 定时清理文件，建议用服务器脚本
logger.info("执行成功{}个文件，失败{}个文件", success, fails);
logger.info("同步商品数据执行结束");
@ -1538,5 +1571,22 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements
return searchService.importAllProductImage(updateTime);
}
/**
* 读取数据库的content
* @return
*/
public JSONArray getSyncDataContent(Map<String,String> map,String key){
JSONArray jsonArray =null;
try {
String content=map.get(key);
content= TextCompressionUtil.decompress(content);
jsonArray = JSONUtil.parseArray(content);
// logger.info("成功处理文件: {}",key);
} catch (RuntimeException e) {
throw new RuntimeException("处理文件失败: " + e.getMessage(), e);
}
return jsonArray;
}
}

2
sql/shop/dev/20251108_dml.sql Normal file
View File

@ -0,0 +1,2 @@
ALTER table store_db_config add is_sync_active char(1) NOT NULL DEFAULT '0' COMMENT '是否同步活动(0:否,1:是)';
ALTER table store_db_config add is_sync_member char(1) NOT NULL DEFAULT '0' COMMENT '是否同步会员(0:否,1:是)';

15
sql/shop/dev/20251110_dml.sql Normal file
View File

@ -0,0 +1,15 @@
CREATE TABLE `sycn_store_data` (
`sync_store_data_id` varchar(32) NOT NULL COMMENT '主键ID',
`store_id` varchar(64) NOT NULL COMMENT '店铺ID',
`content` MEDIUMTEXT DEFAULT NULL COMMENT '同步文本内容',
`file_name` varchar(64) NOT NULL COMMENT '文件名称',
`folder` varchar(64) COMMENT '文件路径',
`file_path` varchar(64) COMMENT '文件全路径',
`status` varchar(1) NOT NULL default '0' COMMENT '处理状态0：未处理，1：已处理',
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`sync_store_data_id`),
KEY `idx_store_id_status` (`store_id`,`status`) USING BTREE,
KEY `idx_file_name` (`file_name`),
KEY `idx_folder` (`folder`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='商品同步数据 ';