diff --git a/mall-common/src/main/java/com/suisung/mall/common/modules/sync/StoreDbConfig.java b/mall-common/src/main/java/com/suisung/mall/common/modules/sync/StoreDbConfig.java index 18923329..260e536a 100644 --- a/mall-common/src/main/java/com/suisung/mall/common/modules/sync/StoreDbConfig.java +++ b/mall-common/src/main/java/com/suisung/mall/common/modules/sync/StoreDbConfig.java @@ -128,6 +128,14 @@ public class StoreDbConfig implements Serializable { private String shopGapTime; @TableField(value = "sale_account",updateStrategy = FieldStrategy.NOT_EMPTY) - @NotBlank(message = "营业员账号") + @ApiModelProperty(value = "营业员账号") private String saleAccount; + + @TableField(value = "is_sync_active",updateStrategy = FieldStrategy.NOT_EMPTY) + @ApiModelProperty(value = "是否同步活动(0:否,1:是)") + private String isSyncActive; + + @TableField(value = "is_sync_member",updateStrategy = FieldStrategy.NOT_EMPTY) + @ApiModelProperty(value = "是否同步会员(0:否,1:是)") + private String isSyncMember; } \ No newline at end of file diff --git a/mall-common/src/main/java/com/suisung/mall/common/modules/sync/SyncStoreData.java b/mall-common/src/main/java/com/suisung/mall/common/modules/sync/SyncStoreData.java new file mode 100644 index 00000000..f843069d --- /dev/null +++ b/mall-common/src/main/java/com/suisung/mall/common/modules/sync/SyncStoreData.java @@ -0,0 +1,53 @@ +package com.suisung.mall.common.modules.sync; + +import com.baomidou.mybatisplus.annotation.*; +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.util.Date; + +@Data +@TableName("sycn_store_data") +@ApiModel("商品同步数据") +public class SyncStoreData implements Serializable { + @TableId(value = "sync_store_data_id", type = IdType.INPUT) + @ApiModelProperty("主键ID") + private String syncStoreDataId; + + @TableField("store_id") + @ApiModelProperty("店铺ID") + private String storeId; + + @TableField("content") + @ApiModelProperty("同步文本内容") + private String content; + + @TableField("file_name") + @ApiModelProperty("文件名称") + private String fileName; + + @TableField("folder") + @ApiModelProperty("文件路径") + private String folder; + + @TableField("file_path") + @ApiModelProperty("文件全路径") + private String filePath; + + @TableField("status") + @ApiModelProperty("处理状态0：未处理，1：已处理") + private String status; + + @TableField("create_time") + @ApiModelProperty(value = "创建时间") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date createTime; + + @TableField("update_time") + @ApiModelProperty(value = "更新时间") + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") + private Date updateTime; +} diff --git a/mall-common/src/main/java/com/suisung/mall/common/utils/TextCompressionUtil.java b/mall-common/src/main/java/com/suisung/mall/common/utils/TextCompressionUtil.java new file mode 100644 index 00000000..908e77fe --- /dev/null +++ b/mall-common/src/main/java/com/suisung/mall/common/utils/TextCompressionUtil.java @@ -0,0 +1,56 @@ +package com.suisung.mall.common.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class TextCompressionUtil { + /** + * 压缩文本 + */ + public static String compress(String text) { + if (text == null || text.length() == 0) { + return text; + } + + try { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (GZIPOutputStream gzip = new GZIPOutputStream(out)) { + gzip.write(text.getBytes(StandardCharsets.UTF_8)); + } + return Base64.getEncoder().encodeToString(out.toByteArray()); + } catch (IOException e) { + throw new RuntimeException("压缩失败", e); + } + } + + /** + * 解压缩文本 + */ + public static String decompress(String compressedText) { + if (compressedText == null || compressedText.length() == 0) { + return compressedText; + } + + try { + byte[] compressedData = Base64.getDecoder().decode(compressedText); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ByteArrayInputStream in = new ByteArrayInputStream(compressedData); + GZIPInputStream gzip = new GZIPInputStream(in)) { + + byte[] buffer = new byte[1024]; + int len; + while ((len = gzip.read(buffer)) > 0) { + out.write(buffer, 0, len); + } + } + return out.toString(String.valueOf(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException("解压缩失败", e); + } + } +} diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/StoreDbConfigController.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/StoreDbConfigController.java index b11fbd06..a9a73ce9 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/StoreDbConfigController.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/controller/StoreDbConfigController.java @@ -10,12 +10,8 @@ package com.suisung.mall.shop.sync.controller; import com.suisung.mall.common.api.CommonResult; import com.suisung.mall.common.modules.sync.StoreDbConfig; -import com.suisung.mall.common.modules.sync.SyncApp; -import com.suisung.mall.common.modules.sync.SyncFileLog; import com.suisung.mall.common.service.impl.BaseControllerImpl; -import com.suisung.mall.core.web.controller.BaseController; import com.suisung.mall.shop.sync.service.StoreDbConfigService; -import com.suisung.mall.shop.sync.service.SyncFileLogService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.beans.factory.annotation.Autowired; diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/mapper/SyncStoreDataMapper.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/mapper/SyncStoreDataMapper.java new file mode 100644 index 00000000..2f7a2e65 --- /dev/null +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/mapper/SyncStoreDataMapper.java @@ -0,0 +1,12 @@ +package com.suisung.mall.shop.sync.mapper; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.suisung.mall.common.modules.sync.SyncStoreData; +import org.springframework.stereotype.Repository; + +@Repository +public interface SyncStoreDataMapper extends BaseMapper { + + +} diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncStoreDataService.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncStoreDataService.java new file mode 100644 index 00000000..ebd7e7ab --- /dev/null +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/SyncStoreDataService.java @@ -0,0 +1,11 @@ +package com.suisung.mall.shop.sync.service; + + +import com.suisung.mall.common.modules.sync.SyncStoreData; +import com.suisung.mall.core.web.service.IBaseService; + +public interface SyncStoreDataService extends IBaseService { + + + +} diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncStoreDataServiceImpl.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncStoreDataServiceImpl.java new file mode 100644 index 00000000..aefc521a --- /dev/null +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncStoreDataServiceImpl.java @@ -0,0 +1,15 @@ +package com.suisung.mall.shop.sync.service.impl; + +import com.suisung.mall.common.modules.sync.SyncStoreData; +import com.suisung.mall.core.web.service.impl.BaseServiceImpl; +import com.suisung.mall.shop.sync.mapper.SyncStoreDataMapper; +import com.suisung.mall.shop.sync.service.SyncStoreDataService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@Transactional(rollbackFor = Exception.class) +@Slf4j +public class SyncStoreDataServiceImpl extends BaseServiceImpl implements SyncStoreDataService { +} diff --git a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java index bb76332b..ba40f412 100644 --- a/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java +++ b/mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncThirdDataServiceImpl.java @@ -45,6 +45,7 @@ import com.suisung.mall.common.pojo.req.SyncThirdMemberReq; import com.suisung.mall.common.pojo.res.ThirdApiRes; import com.suisung.mall.common.utils.I18nUtil; import com.suisung.mall.common.utils.StringUtils; +import com.suisung.mall.common.utils.TextCompressionUtil; import com.suisung.mall.shop.base.service.ShopBaseProductBrandService; import com.suisung.mall.shop.base.service.ShopBaseProductCategoryService; import com.suisung.mall.shop.number.service.ShopNumberSeqService; @@ -65,13 +66,13 @@ import com.suisung.mall.shop.store.service.ShopStoreActivityBaseService; import com.suisung.mall.shop.store.service.ShopStoreActivityItemService; import com.suisung.mall.shop.sync.Utils.ActiveShopJsonUtils; import com.suisung.mall.shop.sync.Utils.BigDecimalFormatter; -import com.suisung.mall.shop.sync.Utils.ThreadFileUtils; import com.suisung.mall.shop.sync.dto.ActiveModel; import com.suisung.mall.shop.sync.dto.ActiveShopInfo; import com.suisung.mall.shop.sync.dto.BrandModel; import com.suisung.mall.shop.sync.keymanage.RedisKey; import com.suisung.mall.shop.sync.service.*; import io.seata.spring.annotation.GlobalTransactional; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.ListUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,6 +108,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; import static com.suisung.mall.common.utils.I18nUtil._; @@ -181,6 +183,9 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements @Autowired private ProductQuantityConsumptionService productQuantityConsumptionService; + @Autowired + private SyncStoreDataService syncStoreDataService; + /** * 批量保存商品的分类 * @@ -517,6 +522,15 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements return new ThirdApiRes().fail(500, "文件上传失败:签名验证失败"); } logger.info("path-{},parent-{},filename-{},root-{}", path, path.getParent(), path.getFileName().toString(), path.getRoot()); + SyncStoreData sxStoreData = new SyncStoreData(); + sxStoreData.setSyncStoreDataId(DigestUtils.md5Hex(filePath)); + sxStoreData.setStoreId(syncAppO.getStore_id()); + sxStoreData.setContent(TextCompressionUtil.compress(jsonStr)); + sxStoreData.setFileName(filName); + sxStoreData.setFolder(folder); + sxStoreData.setFilePath(filePath); + sxStoreData.setStatus("0"); + syncStoreDataService.saveOrUpdate(sxStoreData); // String filaPath=path.toString(); // if(filePath.contains(":")){ // filePath=filePath.substring(filePath.indexOf(":")+1); @@ -568,7 +582,7 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements }); //upLoadZipToOss(newFolders.get(0));//上传文件到cos - dowloadAndUnZip(newFolders.get(0));//读取cos文件回本地 + // dowloadAndUnZip(newFolders.get(0));//读取cos文件回本地 syncPrimaryKey(); shopNumberSeqService.clearKey(); shopBaseProductCategoryService.clearCategoryCache(storeId); @@ -577,8 +591,8 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements ExecutorService executor = Executors.newFixedThreadPool(6); List> futures = new ArrayList<>(); // 提交任务 - AtomicInteger success = new AtomicInteger(); - AtomicInteger fails = new AtomicInteger(); + AtomicInteger success = new AtomicInteger(0); + AtomicInteger fails = new AtomicInteger(0); List failFolders = new ArrayList<>(); List failMessage = new ArrayList<>(); shopBaseProductCategoryService.getCategoryListByStoreId(storeId); @@ -592,19 +606,32 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements // Map ShopBaseProductSpecMap = baseProductSpecService.getShopBaseProductSpecMap(Integer.valueOf(storeId));//切割商品缓存 // long seconds=System.currentTimeMillis(); // Date productSaleTime=Date.from(Instant.now().plusSeconds(seconds)); + List fileNames=new ArrayList<>(); + for(int i=0;i syncStoreDataQueryWrapper = new QueryWrapper<>(); + syncStoreDataQueryWrapper.eq("store_id", storeId); + syncStoreDataQueryWrapper.eq("status", 0); + syncStoreDataQueryWrapper.in("file_name",fileNames); + List syncStoreDataList= syncStoreDataService.list(syncStoreDataQueryWrapper); + Map syncDataMap= syncStoreDataList.stream().collect(Collectors.toMap(SyncStoreData::getSyncStoreDataId, SyncStoreData::getContent)); for (int i = 0; i < newFolders.size(); i++) { final int taskId = i; final String isNegativeAllowed = storeDbConfig.getIsNegativeAllowed(); //String priorityMode = storeDbConfig.getPriorityMode(); //boolean isUpdatePrice= ObjectUtil.isNotEmpty(storeDbConfig.getRefreshTime());//是否更新所有切割价格 threadNum.incrementAndGet(); + Map finalSyncDataMap = syncDataMap; futures.add(executor.submit(() -> { int count = 0;//失败重试机制，当失败重试一次，再次失败则记录到数据库中 while (true) { count++; - String taskName = newFolders.get(taskId); - String fileName = "good_" + (taskId + 1) + ".txt"; - JSONArray jsonArray = new ThreadFileUtils().processFolder(taskName, newFolders.get(taskId)); + // String taskName = newFolders.get(taskId); + String fileName = "goods_" + (taskId + 1) + ".txt"; + String sycnDataId=DigestUtils.md5Hex(newFolders.get(taskId) + fileName); + JSONArray jsonArray = getSyncDataContent(finalSyncDataMap,sycnDataId); try { baseSaveOrUpdateGoodsBatch(jsonArray, storeId, isNegativeAllowed, brandMaps); success.getAndIncrement(); @@ -657,6 +684,12 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements if (CollUtil.isNotEmpty(syncFileLogs)) { syncFileLogService.saveBatch(syncFileLogs, syncFileLogs.size()); } + syncStoreDataList=syncStoreDataList.stream().peek(syncStoreData -> { + syncStoreData.setStatus("1"); + }).collect(Collectors.toList()); + if(!syncStoreDataList.isEmpty()){ + syncStoreDataService.updateBatchById(syncStoreDataList,syncStoreDataList.size());//处理结束后更新已处理 + } //todo 定时清理文件，建议用服务器脚本 logger.info("执行成功{}个文件，失败{}个文件", success, fails); logger.info("同步商品数据执行结束"); @@ -1538,5 +1571,22 @@ public class SyncThirdDataServiceImpl extends SyncBaseThirdSxAbstract implements return searchService.importAllProductImage(updateTime); } + /** + * 读取数据库的content + * @return + */ + public JSONArray getSyncDataContent(Map map,String key){ + JSONArray jsonArray =null; + try { + String content=map.get(key); + content= TextCompressionUtil.decompress(content); + jsonArray = JSONUtil.parseArray(content); + // logger.info("成功处理文件: {}",key); + } catch (RuntimeException e) { + throw new RuntimeException("处理文件失败: " + e.getMessage(), e); + } + return jsonArray; + } + } diff --git a/sql/shop/dev/20251108_dml.sql b/sql/shop/dev/20251108_dml.sql new file mode 100644 index 00000000..3619d66e --- /dev/null +++ b/sql/shop/dev/20251108_dml.sql @@ -0,0 +1,2 @@ +ALTER table store_db_config add is_sync_active char(1) NOT NULL DEFAULT '0' COMMENT '是否同步活动(0:否,1:是)'; +ALTER table store_db_config add is_sync_member char(1) NOT NULL DEFAULT '0' COMMENT '是否同步会员(0:否,1:是)'; \ No newline at end of file diff --git a/sql/shop/dev/20251110_dml.sql b/sql/shop/dev/20251110_dml.sql new file mode 100644 index 00000000..0173abfa --- /dev/null +++ b/sql/shop/dev/20251110_dml.sql @@ -0,0 +1,15 @@ +CREATE TABLE `sycn_store_data` ( + `sync_store_data_id` varchar(32) NOT NULL COMMENT '主键ID', + `store_id` varchar(64) NOT NULL COMMENT '店铺ID', + `content` MEDIUMTEXT DEFAULT NULL COMMENT '同步文本内容', + `file_name` varchar(64) NOT NULL COMMENT '文件名称', + `folder` varchar(64) COMMENT '文件路径', + `file_path` varchar(64) COMMENT '文件全路径', + `status` varchar(1) NOT NULL default '0' COMMENT '处理状态0：未处理，1：已处理', + `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`sync_store_data_id`), + KEY `idx_store_id_status` (`store_id`,`status`) USING BTREE, + KEY `idx_file_name` (`file_name`), + KEY `idx_folder` (`folder`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='商品同步数据 '; \ No newline at end of file