backend/java-mall
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增es模糊商品图片匹配功能

Browse Source
This commit is contained in:
liyj 2025-07-18 16:14:35 +08:00
parent 54ad2d6d3c
commit 5891ecd7c0
21 changed files with 1489 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mall-common/src/main/java/com/suisung/mall/common/enums/DicEnum.java
View File

@ -31,6 +31,9 @@ public enum DicEnum {
PRIORITY_MODE_2("2", "自动优先","priorityMode","优先方式","更新时不做商品切割"),
GOODS_UN_SYNC_SX("1", "白条猪","unSyncGoodsSX","思迅非同步商品","白条猪"),
ES_SEARCH_TYPE_1("1", "高亮查询","esSearchType","es查询方式","精准查询"),
ES_SEARCH_TYPE_2("2", "模糊查询","esSearchType","es查询方式","模糊查询"),
;
;
private String code;

6
mall-common/src/main/java/com/suisung/mall/common/feignService/SearchService.java
View File

@ -2,12 +2,14 @@ package com.suisung.mall.common.feignService;
import com.suisung.mall.common.api.CommonResult;
import com.suisung.mall.common.modules.product.ShopProductIndex;
import com.suisung.mall.common.pojo.dto.ProductImageSearchDTO;
import com.suisung.mall.common.pojo.dto.ProductRecommendDTO;
import com.suisung.mall.common.pojo.dto.ProductSearchDTO;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
/**
* @program: mall-suite
@ -36,4 +38,8 @@ public interface SearchService {
@PostMapping("/esProduct/batchImport")
CommonResult batchImport(@RequestBody List<ShopProductIndex> shopProductIndexList);
@PostMapping("/esProductImage/searchProductImageList")
Map<String,List<ProductImageSearchDTO>> searchProductImageList(@RequestBody List<ProductImageSearchDTO> esProductImages,@RequestParam("esSearchType") String esSearchType);
}

53
mall-common/src/main/java/com/suisung/mall/common/pojo/dto/ProductImageSearchDTO.java Normal file
View File

@ -0,0 +1,53 @@
package com.suisung.mall.common.pojo.dto;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
@Data
@EqualsAndHashCode(callSuper = false)
@ApiModel(value = "商品图库搜索", description = "商品图库搜索DTO")
public class ProductImageSearchDTO {
@ApiModelProperty(value = "名称索引关键字(DOT)")
private String barcode;
@ApiModelProperty(value = "产品名称:店铺平台先在对用表中检索后通过id检索,检索使用")
private String productName;
@ApiModelProperty(value = "名称索引关键字(DOT)")
private String cleanName;
@ApiModelProperty(value = "关键字")
private String keywords;
@ApiModelProperty(value = "商品简称")
private String productShortName;
@ApiModelProperty(value = "品牌")
private String brand;
@ApiModelProperty(value = "规格")
private String specs;
@ApiModelProperty(value = "分类")
private String category;
@ApiModelProperty(value = "主图")
private String thumb;
@ApiModelProperty(value = "图库地址")
private String imagesUrls;
@ApiModelProperty(value = "商品图库id")
private Long imageId;
@ApiModelProperty(value = "商品id")
private Long productId;
@ApiModelProperty(value = "相似度")
private float similarity;
}

385
mall-common/src/main/java/com/suisung/mall/common/utils/JiebaUtils.java
View File

@ -9,11 +9,22 @@
package com.suisung.mall.common.utils;
import com.huaban.analysis.jieba.JiebaSegmenter;
import com.huaban.analysis.jieba.WordDictionary;
import org.springframework.stereotype.Component;
import java.util.List;
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.suisung.mall.common.utils.ProductTitleUtil.*;
/**
* 结巴分词工具类
*/
@ -21,14 +32,195 @@ import java.util.stream.Collectors;
public class JiebaUtils {
private final JiebaSegmenter segmenter = new JiebaSegmenter();
// 中文保护标记（不会被分割）
private static final String PROTECT_START = "开始";
private static final String PROTECT_END = "结束";
// 正则模式：匹配数字+单位的组合
private static final String PROTECT_UNIT = "__UNIT__";
private static final String PROTECT_SPEC = "__SPEC__";
private static final String PROTECT_REGEX = "__MIXED__";
private static final String PROTECT_DIMENSION = "__DIMENSION__";
private static final String UNIT_CHINISE = "__UNIT_CHINISE__";
private static final String UNIT_EVERY = "__UNIT_EVERY__";
// 正则模式：匹配数字+单位的组合
private static final Pattern UNIT_REGEX = Pattern.compile("\\d+[.]?\\d*[a-zA-Z]+"); // 匹配500ml/1.8L
private static final Pattern SPEC_REGEX = Pattern.compile("\\d+[a-zA-Z]+[*]\\d+"); // 匹配500L*10
private static final Pattern MIXED_REGEX = Pattern.compile("[a-zA-Z]+[-]?\\d+"); // 匹配RSCW-1949
private static final Pattern DIMENSION_REGEX = Pattern.compile("\\d+(?:\\\\.\\\\d+)?[\\\\u4e00-\\\\u9fa5a-zA-Z]+"); // 匹配维度（如2.0*2.3）
private static final Pattern UNIT_CHN_REGEX = Pattern.compile("([0-9零一二三四五六七八九十百千万亿]+)(条|个|卷)\\b");//匹配只有数字+单位的，如牛油果2个
private static final Pattern UNIT_EVERY_REGEX = Pattern.compile("([0-9零一二三四五六七八九十百千万亿]+)([个条份根盒包])(?:\\s*([0-9]+)(g|克|ml|毫升)|\\s*/([袋箱盒份]))");//匹配商品名称+数量+数量单位+重量的 如牛油果2个150克
//private static final Pattern DIMENSION_REGEX = Pattern.compile("([\\u4e00-\\u9fa5]+)(\\d+\\.?\\d*\\*\\d+\\.?\\d*(?:米)?)([\\u4e00-\\u9fa5]+)");
private static final Map<String,Pattern> PROTECT_PATTERNS = new HashMap<String,Pattern>(){{
put(PROTECT_UNIT,UNIT_REGEX);
put(PROTECT_SPEC,SPEC_REGEX);
put(PROTECT_REGEX,MIXED_REGEX);
put(PROTECT_DIMENSION,DIMENSION_REGEX);
put(UNIT_CHINISE,UNIT_CHN_REGEX);
put(UNIT_EVERY,UNIT_EVERY_REGEX);
}};
private static final Map<String,String> spectialrestoreMap = new HashMap<String,String>(){{
put("enDash","-");
put("start","\\*");
put("dot","\\.");
}};
private static void loadUserDict() {
// 方法1：通过文件加载
WordDictionary dictionary = WordDictionary.getInstance();
Path path = null;
try {
path = Paths.get(Objects.requireNonNull(JiebaUtils.class.getClassLoader()
.getResource("dic/userdict.txt")).toURI());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
if(path.toFile().exists()){
dictionary.loadUserDict(path);
}
public static void main(String[] args) {
JiebaUtils jiebaUtils = new JiebaUtils();
String text = "农行桂平";
List<String> words = jiebaUtils.segment(text);
System.out.println(words);
}
/**
* 是否为特色单位
* @param text
*/
private static boolean isSpecialUnitShop(String text){
AtomicBoolean isSpecialUnitShop = new AtomicBoolean(false);
SPECIAL_UNIT_CHN.forEach(str -> {
if(text.contains(str)){
isSpecialUnitShop.set(true);
return;
}
});
return isSpecialUnitShop.get();
}
/**
* 是否为特色单位
* @param text
*/
private static boolean isUnitShop(String text){
AtomicBoolean isUnitShop = new AtomicBoolean(false);
UNITS.forEach(str -> {
if(text.contains(str)){
isUnitShop.set(true);
return;
}
});
return isUnitShop.get();
}
/**
* 是否包好两个单位
* @param text
*/
private static boolean isSpecailAndUnit(String text){
return isSpecialUnitShop(text) && isUnitShop(text);
}
// 预处理方法：保护特定模式不被分割
private static String protectPatterns(String text) {
StringBuffer sb = new StringBuffer();
boolean special = isSpecialUnitShop(text);
boolean unit = isUnitShop(text);
if(special&&unit){
Matcher matcher = UNIT_EVERY_REGEX.matcher(text);
if(matcher.find()) {
String original = matcher.group();
// 使用中文保护标记包裹原始值
String protectedText = PROTECT_START + original + PROTECT_END;
matcher.appendReplacement(sb, Matcher.quoteReplacement(protectedText));
matcher.appendTail(sb);
}
}else if(special){
Matcher matcher = UNIT_CHN_REGEX.matcher(text);
if(matcher.find()) {
String original = matcher.group();
// 使用中文保护标记包裹原始值
String protectedText = PROTECT_START + original + PROTECT_END;
matcher.appendReplacement(sb, Matcher.quoteReplacement(protectedText));
matcher.appendTail(sb);
}
} else {
for (Map.Entry<String, Pattern> entry : PROTECT_PATTERNS.entrySet()) {
Matcher matcher = entry.getValue().matcher(text);
while (matcher.find()) {
String original = matcher.group();
// 使用中文保护标记包裹原始值
String protectedText = PROTECT_START + original + PROTECT_END;
matcher.appendReplacement(sb, Matcher.quoteReplacement(protectedText));
matcher.appendTail(sb);
break;
}
if (sb.length() > 0) {
break;
}
}
}
String result = sb.toString();
if (result.isEmpty()){
result=text;
}
if(result.contains("*")){
result = result.replaceAll(spectialrestoreMap.get("start"),"start");
}
if(result.contains("-")){
result = result.replaceAll(spectialrestoreMap.get("enDash"),"enDash");
}
if(result.contains(".")){
result = result.replaceAll(spectialrestoreMap.get("dot"),"dot");
}
return result;
}
// 后处理方法：恢复被保护的原始值
private static List<String> restorePatterns(List<String> tokens) {
List<String> result = new ArrayList<>();
StringBuilder protectedPart = new StringBuilder();
boolean inProtected = false;
for (String token : tokens) {
token=token.replaceAll("enDash",spectialrestoreMap.get("enDash"));
token=token.replaceAll("start",spectialrestoreMap.get("start"));
token=token.replaceAll("dot",spectialrestoreMap.get("dot"));
// 检查保护标记开始
if (token.contains(PROTECT_START)) {
inProtected = true;
// 只移除标记部分，保留内容
protectedPart.append(token.replace(PROTECT_START, ""));
continue;
}
// 检查保护标记结束
if (token.contains(PROTECT_END)) {
inProtected = false;
// 只移除标记部分，保留内容
protectedPart.append(token.replace(PROTECT_END, ""));
result.add(protectedPart.toString());
protectedPart.setLength(0);
continue;
}
if (inProtected) {
protectedPart.append(token);
} else {
result.add(token);
}
}
// 处理未结束的保护块
if (inProtected && protectedPart.length() > 0) {
result.add(protectedPart.toString());
}
return result;
}
/**
* 精确模式分词
*/
@ -48,4 +240,185 @@ public class JiebaUtils {
.map(token -> token.word)
.collect(Collectors.toList());
}
/**
* 搜索模式分词（更细粒度）字典分词
*/
public List<String> segmentForDicSearch(String text) {
loadUserDict();
return segmenter.process(text, JiebaSegmenter.SegMode.SEARCH)
.stream()
.map(token -> token.word)
.collect(Collectors.toList());
}
/**
* 搜索模式分词（更细粒度）字典分词+排除识别商品
*/
public static List<String> extractKeywords(String text) {
JiebaSegmenter segmenter = new JiebaSegmenter();
loadUserDict();
String protectedText = protectPatterns(text);
System.out.println("protectedText: " + protectedText);
List<String> tokens = segmenter.sentenceProcess(protectedText).stream()
//.filter(word -> word.length() > 1) // 过滤单字
// .sorted(Comparator.reverseOrder()) // 按词典词频降序
.distinct()
.collect(Collectors.toList());
return restorePatterns(tokens);
}
/**
* 传入清洗商品名称，通过分词获取商品 品牌，规格，名称，分类，关键字等数据
* @param cleanProductName
* @return
*/
public static Map<String,String> getShopDetails(String cleanProductName){
Map<String, String> fields = new HashMap<>(4);
//分词
List<String> words = extractKeywords(cleanProductName);
// resultMap.put("productShortName",productName);
// resultMap.put("brand",productName);
// resultMap.put("specs",productName);
// resultMap.put("category",productName);
// resultMap.put("keywords",productName);
List<String> remainingWords = new ArrayList<>(words.size());
for (String word : words) {
if (BRAND_LIBRARY.contains(word)) {
fields.putIfAbsent("brand", word);
} else if (CATEGORY_LIBRARY.contains(word)) {
fields.putIfAbsent("category", word);
} else if (startsWithDigit(word)) {
fields.putIfAbsent("specs", word.toLowerCase());
}else if(SPECIAL_NAME.contains(word)){
fields.putIfAbsent("productShortName", word);
}else {
remainingWords.add(word);
}
}
String specialWords = specialWordMapping(cleanProductName);
if(StringUtils.isNotEmpty(specialWords)){//加入字段清洗
List<String> specialWordsList = new ArrayList<>(Arrays.asList(specialWords.split(",")));
if(!remainingWords.isEmpty()){
specialWordsList.addAll(remainingWords);
remainingWords= specialWordsList.stream().distinct().collect(Collectors.toList());//去重
}else {
remainingWords=specialWordsList;
}
}
fields.put("keywords", StringUtils.join(remainingWords, ","));
if(StringUtils.isNotEmpty(fields.get("productShortName"))){
return fields;
}
/**
*
*/
if(words.size()>1){
int index=words.size()-1;
if(!startsWithLetterOrDigitRegex(words.get(index))){
fields.putIfAbsent("productShortName", words.get(index));
}else {
fields.putIfAbsent("productShortName", words.get(index-1));
}
}else {
fields.putIfAbsent("productShortName", words.get(0));
}
return fields;
}
/**
* 特色字段匹配
* @return
*/
private static String specialWordMapping(String text){
AtomicReference<String> specialWord = new AtomicReference<>("");
for (Map.Entry<String, String> entry : SHOP_NAME_MAPPING.entrySet()) {
String k = entry.getKey();
String val = entry.getValue();
if (text.contains(k)) {
specialWord.set(val);
return specialWord.get();
}
}
return specialWord.get();
}
/**
* 以字母或者数字开头
* @param str
* @return
*/
public static boolean startsWithLetterOrDigitRegex(String str) {
return str != null && !str.isEmpty() &&
str.matches("^[a-zA-Z\\d].*");
}
/**
* 以字母开头
* @param input
* @return
*/
public boolean startsWithLetter(String input) {
if (input == null || input.isEmpty()) {
return false;
}
return Character.isLetter(input.charAt(0));
}
/**
* 以数字开头
* @param input
* @return
*/
public static boolean startsWithDigit(String input) {
if (input == null || input.isEmpty()) {
return false;
}
return Character.isDigit(input.charAt(0));
}
/**
* 去除所有的字母和数字
* @param text
* @return
*/
public static String cleanNumberAndDigit(String text) {
return text.replaceAll("[a-zA-Z0-9]", "");
}
/**
* 替换文字
* @param text
* @return
*/
public static String repalceText(String text,String repalceTex) {
if(StringUtils.isNotEmpty(repalceTex)){
return text.replaceAll(repalceTex, "");
}
return text;
}
public static void main(String[] args) {
// JiebaUtils jiebaUtils = new JiebaUtils();
//String text = "云计算和区块链是热门技术";
// String text = "新鲜牛肉饺子500g";
//String text = "志高1.8L电热水壶";
// String text = "（单充）数据线2m";
// String text = "雅安利2.0*2.3四件套";
// String text = "RSCW-1949剃须刀";
System.out.println(cleanNumberAndDigit("日本豆腐3条"));
String text="六指鼠童袜001";
List<String> words = JiebaUtils.extractKeywords(text);
System.out.println(words);
Map<String,String> shopMap= JiebaUtils.getShopDetails(ProductTitleUtil.cleanTitle2(text));
System.out.println(shopMap);
}
}

69
mall-common/src/main/java/com/suisung/mall/common/utils/ProductTitleUtil.java
View File

@ -26,7 +26,7 @@ public class ProductTitleUtil {
"特价", "折扣", "优惠", "促销", "限时", "秒杀", "抢购", "直降", "满减",
"赠品", "包邮", "新品", "热卖", "爆款", "推荐", "精选", "特惠", "清仓",
"正品", "原装", "官方", "正版", "品牌", "优质", "好用", "新款", "老款",
"【", "】", "（", "）", "[]", "()", "「", "」", "！", "！！", "??", "?"
"【", "】", "（", "）", "[]", "()", "「", "」", "！", "！！", "??", "?","袋装","盒装","约"
)));
/**
@ -40,22 +40,26 @@ public class ProductTitleUtil {
/**
* 预编译正则表达式（提升性能）
*/
private static final Pattern NUMERIC_PATTERN = Pattern.compile(".*\\d+.*");
public static final Pattern NUMERIC_PATTERN = Pattern.compile(".*\\d+.*");
private static final Pattern UNIT_PATTERN = Pattern.compile("(\\d+\\.?\\d*)([a-zA-Z]+)");
private static final Pattern TITLE_FILTER_PATTERN = Pattern.compile("[^a-zA-Z0-9\u4e00-\u9fa5]");
private static final Pattern SYMBOL_PATTERN = Pattern.compile("[\\p{P}\\p{S}\\s]+"); // 符号和空格
// private static final Pattern SYMBOL_PATTERN = Pattern.compile("[\\p{P}\\p{S}\\s]+"); // 符号和空格
private static final Pattern SYMBOL_PATTERN = Pattern.compile("[^\\p{L}\\d.*/]|_");//去除.,*以外的符号和所有空格
/**
* 品牌词库（初始化后不可变）
*/
private static final Set<String> BRAND_LIBRARY = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("华为", "苹果", "小米", "三星", "美的", "格力", "耐克", "阿迪达斯", "海尔")
public static final Set<String> BRAND_LIBRARY = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("华为", "苹果", "小米", "三星", "美的", "格力", "耐克", "阿迪达斯", "海尔","雀巢","伊利","蒙牛","达能","乐事","多力多滋","三只松鼠","良品铺子","可口可乐",
"农夫山泉","元气森林","红牛","雅诗兰黛","欧莱雅","玉兰油","科颜氏","宝洁","汰渍","帮宝适","联合利华","Unilever","含多芬","清扬","大窑","谢村桥牌阡糯","谢村桥牌阡",
"六个核桃","大豫竹","优乐多","安慕希","纳爱斯","舒客","宜轩", "蓝月亮","海尔","美的","松下","戴森","耐克","安踏","李宁","特仑苏","纯甄","安井","三全","哇哈哈",
"龙江家园","达利园","春光","妙芙","南星","利嘉旺","卡得福","泓一","爱乡亲","思念","得力","中雪","江南点心局","德庄","六指鼠",
"依水塬","乌苏啤酒","阿尔卑斯", "瑞旗","振雷","中狗","宝视达","冷酸灵","骆驼","NIKE","PAMU","康师傅","信智利","双兔","安足莱","新博美","新博","创利")
));
/**
* 品类词库（初始化后不可变）
*/
private static final Set<String> CATEGORY_LIBRARY = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("手机", "电脑", "空调", "冰箱", "运动鞋", "T恤", "洗发水", "洗衣液")
public static final Set<String> CATEGORY_LIBRARY = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("手机", "电脑", "空调", "冰箱", "运动鞋", "T恤", "洗发水", "洗衣液","猪脚")
));
static {
@ -69,6 +73,23 @@ public class ProductTitleUtil {
UNIT_NORMAL_MAP = Collections.unmodifiableMap(map);
}
//特色字段映射
public static final Map<String,String> SHOP_NAME_MAPPING = new HashMap<String,String>(){{
put("番茄","西红柿,番茄");
put("西红柿","西红柿,番茄");
put("女","女,美女,女生,女士");
put("男","男,男生,男士");
put("猪肉包",",猪肉包,包子");
put("叉烧包","叉烧包,包子");
put("香菇青菜包","香菇青菜包,包子");
}};
//特殊商品
public static final Set<String> SPECIAL_NAME = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("水饺", "面条", "包子","卷纸","卫生纸","紫菜汤","猪肉包","叉烧包","香菇青菜包","面包")
));
static {
Map<String, Integer> map = new HashMap<>(4);
map.put("brand", 30);
@ -77,7 +98,13 @@ public class ProductTitleUtil {
map.put("attribute", 25);
FIELD_WEIGHTS = Collections.unmodifiableMap(map);
}
//特色的单位
public static final Set<String> SPECIAL_UNIT_CHN = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("个","条","份","盒","份","袋")
));
public static final Set<String> UNITS = Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("g","克","kg","千克","ml","毫升","l","升","cm","厘米","mm","毫米","份","盒","份","袋")
));
private ProductTitleUtil() {
}
@ -148,6 +175,7 @@ public class ProductTitleUtil {
return cleaned;
}
/* ---------------------------- 私有方法 ---------------------------- */
/**
@ -171,10 +199,15 @@ public class ProductTitleUtil {
* 单位归一化（优化性能）
*/
private static String normalizeUnit(String word) {
java.util.regex.Matcher matcher = UNIT_PATTERN.matcher(word);
return matcher.matches() ?
matcher.group(1) + UNIT_NORMAL_MAP.getOrDefault(matcher.group(2).toLowerCase(), matcher.group(2)) :
word;
// word=word.toLowerCase();
word=word.replaceAll("克","g");
word=word.replaceAll("毫升","ml");
word=word.replaceAll("升","l");
return word;
// java.util.regex.Matcher matcher = UNIT_PATTERN.matcher(word);
// return matcher.matches() ?
// matcher.group(1) + UNIT_NORMAL_MAP.getOrDefault(matcher.group(2).toLowerCase(), matcher.group(2)) :
// word;
}
/**
@ -200,6 +233,7 @@ public class ProductTitleUtil {
return fields;
}
/**
* 加权得分计算（优化分支预测）
*/
@ -283,12 +317,17 @@ public class ProductTitleUtil {
String[] testTitles = {
"【限时秒杀】三只松鼠开心果100g特价促销",
"华为Mate60 Pro 512G手机 官方旗舰店正品",
"Nike Air Max 运动鞋 男款 42码 新品热卖"
"Nike Air Max 运动鞋 男款 42码 新品热卖",
"香港63g烧猪肉脯",
"自然派蜜汁猪肉脯",
"三全702克芥菜猪肉水饺",
"志高GB18电热水壶",
"雅安利2.0*2.3四件套"
};
for (String title : testTitles) {
System.out.println("原始标题: " + title);
System.out.println("清洗后: " + cleanTitle(title));
System.out.println("清洗后: " + cleanTitle2(title));
System.out.println("-------------------");
}

42
mall-search/src/main/java/com/suisung/mall/search/controller/EsProductImageController.java Normal file
View File

@ -0,0 +1,42 @@
package com.suisung.mall.search.controller;
import com.suisung.mall.common.api.CommonResult;
import com.suisung.mall.common.pojo.dto.ProductImageSearchDTO;
import com.suisung.mall.search.service.EsProductImageService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
/**
* 搜索商品管理Controller
*/
@RestController
@Api(tags = "EsProductController", description = "搜索商品图库管理")
@RequestMapping("/esProductImage")
public class EsProductImageController {
@Autowired
private EsProductImageService esProductImageService;
@ApiOperation(value = "导入所有数据库中商品到ES")
@RequestMapping(value = "/importAll", method = RequestMethod.POST)
public CommonResult importAllList() {
int count = esProductImageService.importAll();
return CommonResult.success(count);
}
@RequestMapping(value = "/search", method = RequestMethod.POST)
public CommonResult search(@RequestBody ProductImageSearchDTO esProductImage) {
List<ProductImageSearchDTO> search = esProductImageService.search(esProductImage);
return CommonResult.success(search);
}
@RequestMapping(value = "/searchProductImageList", method = RequestMethod.POST)
public Map<String,List<ProductImageSearchDTO>> searchProductImageList(@RequestBody List<ProductImageSearchDTO> esProductImages,@RequestParam("esSearchType") String esSearchType) {
Map<String,List<ProductImageSearchDTO>> search = esProductImageService.searchList(esProductImages,esSearchType);
return search;
}
}

31
mall-search/src/main/java/com/suisung/mall/search/dao/EsProductImageDao.java Normal file
View File

@ -0,0 +1,31 @@
package com.suisung.mall.search.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.suisung.mall.search.domain.EsProduct;
import com.suisung.mall.search.domain.EsProductImage;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* 搜索商品管理自定义Dao
*/
@Repository
public interface EsProductImageDao extends BaseMapper<EsProductImage> {
/**
* 获取指定ID的搜索商品
*/
List<EsProductImage> getAllEsProductList(@Param("productId") Long productId);
/**
* 分页查询数据
* @param start
* @param row
* @return
*/
List<EsProductImage> getPageEsProductList(@Param("start") Integer start,@Param("row") Integer row);
Integer getPageTotal();
}

67
mall-search/src/main/java/com/suisung/mall/search/domain/EsProductImage.java Normal file
View File

@ -0,0 +1,67 @@
package com.suisung.mall.search.domain;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.*;
import java.io.Serializable;
/**
* 搜索商品的信息
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@Document(indexName = "index_product_image",shards = 3,replicas = 1)
public class EsProductImage implements Serializable {
private static final long serialVersionUID = -1L;
@Id
@Field(type = FieldType.Keyword)
private Long libId;
@Field(type = FieldType.Keyword)
@ApiModelProperty(value = "名称索引关键字(DOT)")
private String barcode;
@Field(analyzer = "product_analyzer",searchAnalyzer="product_analyzer",type = FieldType.Text)
@ApiModelProperty(value = "产品名称:店铺平台先在对用表中检索后通过id检索,检索使用")
private String productName;
@ApiModelProperty(value = "清洗名称")
@Field(analyzer = "product_analyzer",type = FieldType.Text)
private String cleanName;
@ApiModelProperty(value = "关键字")
@Field(analyzer = "product_analyzer",type = FieldType.Text)
private String keywords;
@ApiModelProperty(value = "商品简称")
@Field(analyzer = "product_analyzer",type = FieldType.Text)
private String productShortName;
@ApiModelProperty(value = "品牌")
@Field(type = FieldType.Keyword)
private String brand;
@ApiModelProperty(value = "规格")
@Field(analyzer = "product_analyzer",type = FieldType.Text)
private String specs;
@ApiModelProperty(value = "分类")
@Field(analyzer = "product_analyzer",type = FieldType.Text)
private String category;
@ApiModelProperty(value = "主图")
@Field(type = FieldType.Keyword)
private String thumb;
@ApiModelProperty(value = "附图")
@Field(type = FieldType.Keyword)
private String imagesUrls;
}

22
mall-search/src/main/java/com/suisung/mall/search/repository/EsProductImageRepository.java Normal file
View File

@ -0,0 +1,22 @@
package com.suisung.mall.search.repository;
import com.suisung.mall.search.domain.EsProduct;
import com.suisung.mall.search.domain.EsProductImage;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
/**
* 搜索商品ES操作类
*/
public interface EsProductImageRepository extends ElasticsearchRepository<EsProductImage, Long> {
/**
* 搜索查询
*
* @param productName 商品名称
* @param productNameIndex 商品索引
* @param page 分页信息
*/
Page<EsProduct> findByProductName(String productName, String productNameIndex, Pageable page);
}

19
mall-search/src/main/java/com/suisung/mall/search/service/EsProductImageService.java Normal file
View File

@ -0,0 +1,19 @@
package com.suisung.mall.search.service;
import com.suisung.mall.common.pojo.dto.ProductImageSearchDTO;
import java.util.List;
import java.util.Map;
/**
* 商品搜索管理Service
*/
public interface EsProductImageService {
/**
* 从数据库中导入所有商品到ES
*/
int importAll();
List<ProductImageSearchDTO> search(ProductImageSearchDTO productImageSearchDTO);
Map<String,List<ProductImageSearchDTO>> searchList(List<ProductImageSearchDTO> productImageSearchDTOS, String esSearchType);
}

389
mall-search/src/main/java/com/suisung/mall/search/service/impl/EsProductImageServiceImpl.java Normal file
View File

@ -0,0 +1,389 @@
package com.suisung.mall.search.service.impl;
import cn.hutool.core.util.PageUtil;
import com.alibaba.csp.sentinel.util.StringUtil;
import com.suisung.mall.common.pojo.dto.ProductImageSearchDTO;
import com.suisung.mall.common.utils.JiebaUtils;
import com.suisung.mall.common.utils.ProductTitleUtil;
import com.suisung.mall.search.dao.EsProductImageDao;
import com.suisung.mall.search.domain.EsProductImage;
import com.suisung.mall.search.repository.EsProductImageRepository;
import com.suisung.mall.search.service.EsProductImageService;
import com.suisung.mall.search.utils.CommonUtil;
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.Query;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
/**
* 商品搜索管理Service实现类
*/
@Service
public class EsProductImageServiceImpl implements EsProductImageService {
private static final Logger logger = LoggerFactory.getLogger(EsProductImageServiceImpl.class);
@Autowired
private EsProductImageDao esProductImageDao;
@Autowired
private EsProductImageRepository esProductImageRepository;
@Autowired
private ElasticsearchRestTemplate elasticsearchRestTemplate;
private static final Integer BATCH_SIZE=1000;
@Override
public int importAll() {
Integer total = esProductImageDao.getPageTotal();
if(Objects.isNull(total)||Objects.equals(total,0)){
return 0;
}
esProductImageRepository.deleteAll();
Integer pages= CommonUtil.getPagesCount(total,BATCH_SIZE);
for (int i = 1; i <= pages; i++) {
List<EsProductImage> esProductList= esProductImageDao.getPageEsProductList((i-1)*BATCH_SIZE,BATCH_SIZE);
esProductList.forEach(item->{
String cleanTitle=ProductTitleUtil.cleanTitle2(item.getProductName());
item.setCleanName(cleanTitle);//清理数据
Map<String,String> shopDetailMap= JiebaUtils.getShopDetails(cleanTitle);//分解数据
if(StringUtil.isEmpty(item.getProductShortName())){
item.setProductShortName(shopDetailMap.get("productShortName"));
}
if(StringUtil.isEmpty(item.getSpecs())){
item.setSpecs(shopDetailMap.get("specs"));
}
if(StringUtil.isEmpty(item.getBrand())){
item.setBrand(shopDetailMap.get("brand"));
}
item.setKeywords(shopDetailMap.get("keywords"));
});
saveBatchEsProduct(esProductList);
}
return total;
}
private int saveBatchEsProduct(List<EsProductImage> esProductImages){
Iterable<EsProductImage> esProductImageIterable = esProductImageRepository.saveAll(esProductImages);
Iterator<EsProductImage> iterator = esProductImageIterable.iterator();
int result = 0;
while (iterator.hasNext()) {
result++;
iterator.next();
}
return result;
}
@Override
public List<ProductImageSearchDTO> search(ProductImageSearchDTO esProductImage) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
if(StringUtil.isNotEmpty(esProductImage.getCleanName())){
boolQueryBuilder.filter(QueryBuilders.matchPhraseQuery("cleanName",esProductImage.getCleanName()));
nativeSearchQueryBuilder.withFilter(boolQueryBuilder);
}else {
List<FunctionScoreQueryBuilder.FilterFunctionBuilder> filterFunctionBuilders = new ArrayList<>();
filterFunctionBuilders.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(QueryBuilders.matchQuery("productName", esProductImage.getProductName()),
ScoreFunctionBuilders.weightFactorFunction(20)));
FunctionScoreQueryBuilder.FilterFunctionBuilder[] builders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[filterFunctionBuilders.size()];
filterFunctionBuilders.toArray(builders);
FunctionScoreQueryBuilder functionScoreQueryBuilder = QueryBuilders.functionScoreQuery(builders)
.scoreMode(FunctionScoreQuery.ScoreMode.SUM)
.setMinScore(2);
nativeSearchQueryBuilder.withQuery(functionScoreQueryBuilder);
// HighlightBuilder highlightBuilder=new HighlightBuilder();
// highlightBuilder.field("productName");
// HighlightQuery highlightQuery=new HighlightQuery(highlightBuilder);
}
// nativeSearchQueryBuilder.withQuery(QueryBuilders.termQuery("productName", esProductImage.getProductName()));
NativeSearchQuery searchQuery = nativeSearchQueryBuilder.build();
SearchHits<EsProductImage> searchHits = elasticsearchRestTemplate.search(searchQuery, EsProductImage.class);
List<SearchHit<EsProductImage>> searchHitList= searchHits.getSearchHits();
List<ProductImageSearchDTO> esProductImages=new ArrayList<>();
searchHitList.forEach(searchHit->{
EsProductImage es=searchHit.getContent();
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
BeanUtils.copyProperties(es,productImageSearchDTO);
esProductImages.add(productImageSearchDTO) ;
});
return esProductImages;
}
@Override
public Map<String,List<ProductImageSearchDTO>> searchList(List<ProductImageSearchDTO> esProductImage,String esSearchType) {
// List<String> productNameIndexList = esProductImage.stream().map(ProductImageSearchDTO::getCleanName).collect(Collectors.toList());
// if(DicEnum.ES_SEARCH_TYPE_1.getCode().equals(esSearchType)){
// return queryEsProductImageByKeyWord(productNameIndexList);
// }
return batchSearchProducts(esProductImage,1);
}
private List<ProductImageSearchDTO> queryEsProductImageByKeyWord(List<String> productNameIndexList) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
boolQueryBuilder.filter(QueryBuilders.termsQuery("cleanName", productNameIndexList));
nativeSearchQueryBuilder.withQuery(boolQueryBuilder);
SearchHits<EsProductImage> searchHits = elasticsearchRestTemplate.search(nativeSearchQueryBuilder.build(), EsProductImage.class);
List<SearchHit<EsProductImage>> searchHitList= searchHits.getSearchHits();
List<ProductImageSearchDTO> esProductImages=new ArrayList<>();
searchHitList.forEach(searchHit->{
EsProductImage esProductImage=searchHit.getContent();
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
BeanUtils.copyProperties(esProductImage,productImageSearchDTO);
esProductImages.add(productImageSearchDTO) ;
});
return esProductImages;
}
private List<ProductImageSearchDTO> queryEsProductImage(List<String> productNameIndexList) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
//List<String> productNameIndexList = esProductImage.stream().map(EsProductImage::getProductNameIndex).collect(Collectors.toList());
boolQueryBuilder.filter(QueryBuilders.termsQuery("cleanName", productNameIndexList));
nativeSearchQueryBuilder.withQuery(boolQueryBuilder);
SearchHits<EsProductImage> searchHits = elasticsearchRestTemplate.search(nativeSearchQueryBuilder.build(), EsProductImage.class);
List<SearchHit<EsProductImage>> searchHitList= searchHits.getSearchHits();
List<ProductImageSearchDTO> esProductImages=new ArrayList<>();
searchHitList.forEach(searchHit->{
EsProductImage esProductImage=searchHit.getContent();
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
BeanUtils.copyProperties(esProductImage,productImageSearchDTO);
esProductImages.add(productImageSearchDTO) ;
});
return esProductImages;
}
public List<ProductImageSearchDTO> searchSimilarProducts(String keyword, int size) {
// 1. 预处理搜索关键词（过滤无用词）
String cleanKeyword = ProductTitleUtil.cleanTitle2(keyword);
// 构建ES查询
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
// 1. 基础匹配查询（匹配productName字段）
MatchQueryBuilder matchQuery = QueryBuilders.matchQuery("productName", cleanKeyword)
.boost(1.0f); // 基础匹配权重
// 2. 模糊查询（匹配cleanName.keyword字段）
QueryBuilder fuzzyQuery = QueryBuilders.fuzzyQuery("cleanName.keyword", cleanKeyword)
.fuzziness(Fuzziness.AUTO) // 自动确定模糊度
.maxExpansions(50) // 最大扩展项数
.prefixLength(1) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.5f); // 模糊查询权重
// 3. 组合查询（布尔查询）
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
.should(matchQuery) // 基础匹配
.should(fuzzyQuery); // 模糊匹配
QueryBuilder shortNameFuzzyQuery = null;
MatchQueryBuilder brandMatchQuery =null;
QueryBuilder specsFuzzyQuery =null;
QueryBuilder keyWordsFuzzyQuery =null;
Map<String,String> shopDetailMap= JiebaUtils.getShopDetails(cleanKeyword);//分解数据
if(StringUtil.isNotEmpty(shopDetailMap.get("productShortName"))){
shortNameFuzzyQuery = QueryBuilders.fuzzyQuery("productShortName.keyword", shopDetailMap.get("productShortName"))
.fuzziness(Fuzziness.AUTO) // 自动确定模糊度
.maxExpansions(50) // 最大扩展项数
.prefixLength(1) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.5f); // 模糊查询权重
boolQuery.should(shortNameFuzzyQuery);
}
if(StringUtil.isNotEmpty(shopDetailMap.get("brand"))){
brandMatchQuery = QueryBuilders.matchQuery("brand", shopDetailMap.get("brand"))
.boost(0.5f);
boolQuery.should(brandMatchQuery);
}
if(StringUtil.isNotEmpty(shopDetailMap.get("specs"))){
specsFuzzyQuery = QueryBuilders.fuzzyQuery("specs.keyword", shopDetailMap.get("specs"))
.fuzziness(Fuzziness.AUTO) // 自动确定模糊度
.maxExpansions(50) // 最大扩展项数
.prefixLength(1) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.2f); // 模糊查询权重
boolQuery.should(specsFuzzyQuery);
}
if(StringUtil.isNotEmpty(shopDetailMap.get("keywords"))){
keyWordsFuzzyQuery = QueryBuilders.fuzzyQuery("keywords", shopDetailMap.get("keywords"))
.fuzziness(Fuzziness.ONE) // 自动确定模糊度
.maxExpansions(20) // 最大扩展项数
.prefixLength(2) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.2f); // 模糊查询权重
boolQuery.should(keyWordsFuzzyQuery);
}
// 4. 构建最终查询
nativeSearchQueryBuilder.withQuery(boolQuery);
nativeSearchQueryBuilder.withPageable(PageRequest.of(0, size));
// 5. 执行查询
SearchHits<EsProductImage> searchHits = elasticsearchRestTemplate.search(
nativeSearchQueryBuilder.build(),
EsProductImage.class
);
List<SearchHit<EsProductImage>> searchHitList= searchHits.getSearchHits();
List<ProductImageSearchDTO> esProductImages=new ArrayList<>();
searchHitList.forEach(searchHit->{
EsProductImage esProductImage=searchHit.getContent();
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
BeanUtils.copyProperties(esProductImage,productImageSearchDTO);
esProductImages.add(productImageSearchDTO) ;
});
//SearchResponse response = client.search(request, RequestOptions.DEFAULT);
//List<EsProductImage> results = parseResponse(response);
return esProductImages;
}
/**
* 模糊批量匹配
* @param esProductImage
* @param sizePerKeyword
* @return
*/
public Map<String, List<ProductImageSearchDTO>> batchSearchProducts(List<ProductImageSearchDTO> esProductImage, int sizePerKeyword) {
List<String> keywords = esProductImage.stream().map(ProductImageSearchDTO::getCleanName).collect(Collectors.toList());
// 1. 准备批量查询
List<QueryBuilder> queries = new ArrayList<>();
Map<String, String> keywordToCleanMap = new HashMap<>();
// 2. 为每个关键词构建查询
for (String keyword : keywords) {
String cleanKeyword = ProductTitleUtil.cleanTitle2(keyword);
keywordToCleanMap.put(cleanKeyword, keyword); // 保留原始关键词映射
// 构建组合查询
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
//.should(QueryBuilders.matchQuery("barcode", cleanKeyword).boost(1.0f))
.should(QueryBuilders.matchQuery("productName", JiebaUtils.cleanNumberAndDigit(cleanKeyword)).boost(1.0f))
.should(QueryBuilders.fuzzyQuery("cleanName.keyword", JiebaUtils.cleanNumberAndDigit(cleanKeyword))
.fuzziness(Fuzziness.ONE)
.maxExpansions(20)
.prefixLength(1)
.transpositions(true)
.boost(0.5f))
;
int count =2;
Map<String,String> shopDetailMap= JiebaUtils.getShopDetails(cleanKeyword);//分解数据
if(StringUtil.isNotEmpty(shopDetailMap.get("productShortName"))){
boolQuery.should( QueryBuilders.fuzzyQuery("productShortName.keyword", shopDetailMap.get("productShortName")) // 模糊查询权重
.fuzziness(Fuzziness.AUTO) // 自动确定模糊度
.maxExpansions(50) // 最大扩展项数
.prefixLength(2) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.5f));
count+=1;
}
if(StringUtil.isNotEmpty(shopDetailMap.get("brand"))){
boolQuery.should( QueryBuilders.matchQuery("brand", shopDetailMap.get("brand"))
.boost(0.5f));
count+=1;
}
if(StringUtil.isNotEmpty(shopDetailMap.get("specs"))){
boolQuery.should(QueryBuilders.fuzzyQuery("specs.keyword", shopDetailMap.get("specs")) // 模糊查询权重
.fuzziness(Fuzziness.TWO) // 自动确定模糊度
.maxExpansions(50) // 最大扩展项数
.prefixLength(1) // 必须匹配的前缀长度
.transpositions(true) // 允许字符调换
.boost(0.01f));
count+=1;
}
if(StringUtil.isNotEmpty(shopDetailMap.get("keywords"))){
String keywordsValue=shopDetailMap.get("keywords");
boolQuery.should(QueryBuilders.matchQuery("keywords", keywordsValue)
.boost(0.9f)); // 提高权重
// List<String> words=Arrays.asList(keywordsValue.split(","));
// words.forEach(word -> {
// boolQuery.should(QueryBuilders.fuzzyQuery("keywords.keyword", word)// 模糊查询权重
// .fuzziness(Fuzziness.ONE) // 自动确定模糊度
// .maxExpansions(100) // 最大扩展项数
// .prefixLength(1) // 必须匹配的前缀长度
// .transpositions(true) // 允许字符调换
// .boost(0.1f));
// });
// boolQuery.should(QueryBuilders.fuzzyQuery("keywords", keywordsValue)// 模糊查询权重
// .fuzziness(Fuzziness.ONE) // 自动确定模糊度
// .maxExpansions(100) // 最大扩展项数
// .prefixLength(1) // 必须匹配的前缀长度
// .transpositions(true) // 允许字符调换
// .boost(0.8f));
count+=1;
}
boolQuery.minimumShouldMatch("50%");
if(count>3){
boolQuery.minimumShouldMatch("70%");
}
queries.add(boolQuery);
}
// 3. 构建批量请求
NativeSearchQueryBuilder bulkQueryBuilder = new NativeSearchQueryBuilder()
.withPageable(PageRequest.of(0, sizePerKeyword));
// 4. 使用MultiSearch执行批量查询
List<Query> searchQueries = queries.stream()
.map(query -> bulkQueryBuilder.withQuery(query).build())
.collect(Collectors.toList());
IndexCoordinates indexCoordinates = IndexCoordinates.of("index_product_image");
List<SearchHits<EsProductImage>> results = elasticsearchRestTemplate.multiSearch(
searchQueries,
EsProductImage.class,
indexCoordinates
);
// 5. 整理结果（按原始关键词分组）
Map<String, List<ProductImageSearchDTO>> resultMap = new HashMap<>();
for (int i = 0; i < results.size(); i++) {
String cleanKeyword = ProductTitleUtil.cleanTitle2(keywords.get(i));
String originalKeyword = keywordToCleanMap.get(cleanKeyword);
int finalI = i;
List<ProductImageSearchDTO> products = results.get(i).getSearchHits().stream()
.filter(searchHit->originalKeyword.contains(searchHit.getContent().getProductShortName()))//过非商品
.map(searchHit->{
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
BeanUtils.copyProperties(searchHit.getContent(),productImageSearchDTO);
productImageSearchDTO.setProductId(esProductImage.get(finalI).getProductId());
productImageSearchDTO.setImageId(esProductImage.get(finalI).getImageId());
productImageSearchDTO.setSimilarity(searchHit.getScore());
return productImageSearchDTO;
})
.collect(Collectors.toList());
resultMap.put(originalKeyword, products);
}
return resultMap;
}
}

98
mall-search/src/main/java/com/suisung/mall/search/utils/CommonUtil.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Copyright (c) 2025. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
* Morbi non lorem porttitor neque feugiat blandit. Ut vitae ipsum eget quam lacinia accumsan.
* Etiam sed turpis ac ipsum condimentum fringilla. Maecenas magna.
* Proin dapibus sapien vel ante. Aliquam erat volutpat. Pellentesque sagittis ligula eget metus.
* Vestibulum commodo. Ut rhoncus gravida arcu.
*/
package com.suisung.mall.search.utils;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
public class CommonUtil {
private final static String apiUrl = "http://4ei8850868ux.vicp.fun";
public static JSONObject sendPostRequestToSiXun(String urlPath, JSONObject params) {
String resp = HttpUtil.post(apiUrl + urlPath, params.toString());
if (StrUtil.isBlank(resp)) {
return null;
}
JSONObject respObj = JSONUtil.parseObj(resp);
return respObj;
}
/**
* 根据总条数和分页大小，求页数
*
* @param total
* @param pageSize
* @return
*/
public static Integer getPagesCount(Integer total, Integer pageSize) {
if (total == null || pageSize == null || pageSize <= 0 || total <= 0) {
return 0;
}
int pagesCount = 0;
pagesCount = total / pageSize;
if (total % pageSize > 0) {
// 有余数
pagesCount++;
} else {
if (pagesCount == 0) {
pagesCount = 1;
}
}
return pagesCount;
}
/**
* 接口是否成功执行返回
*
* @param jsonObject
* @return
*/
public static Boolean isSuccess(JSONObject jsonObject) {
if (jsonObject == null) {
return false;
}
return jsonObject.get("code") != null && jsonObject.getStr("code").equals("0");
}
/**
* 接口是否成功执行返回
*
* @param jsonObject
* @return
*/
public static Boolean hasSuccessData(JSONObject jsonObject) {
if (jsonObject == null) {
return false;
}
return jsonObject.get("code") != null && jsonObject.getStr("code").equals("0") && jsonObject.get("data") != null;
}
/**
* 通过json节点表达式，获取节点json字符串，注：驼峰命名改成下划线命名
*
* @param jsonObject
* @param expression json 节点表达式比如： data.list, msg, code
* @return
*/
public static String toUnderlineJson(JSONObject jsonObject, String expression) {
return StrUtil.toUnderlineCase(jsonObject.getByPath(expression, String.class));
}
}

71
mall-search/src/main/resources/dao/EsProductImage.xml Normal file
View File

@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.suisung.mall.search.dao.EsProductImageDao">
<resultMap id="shop_product_index" type="com.suisung.mall.search.domain.EsProductImage">
<result property="libId" column="id"/>
<result property="barcode" column="barcode"/>
<result property="category" column="category"/>
<result property="productName" column="name"/>
<result property="productShortName" column="product_short_name"/>
<result property="thumb" column="thumb"/>
<result property="imagesUrls" column="merged_image_url"/>
</resultMap>
<select id="getAllEsProductList" resultMap="shop_product_index">
SELECT
lp.id,
lp.barcode,
lp.name,
lp.product_short_name,
lp.category,
lp.thumb,
(
SELECT GROUP_CONCAT(image_url SEPARATOR ',')
FROM library_product_image
WHERE product_id = lp.id
) AS merged_image_url
FROM library_product lp
<where>
<if test="productId!=null">
lp.id=#{productId}
</if>
</where>
</select>
<select id="getPageEsProductList" resultMap="shop_product_index">
SELECT
lp.id,
lp.barcode,
lp.name,
lp.product_short_name,
lp.category,
lp.thumb,
(
SELECT GROUP_CONCAT(image_url SEPARATOR ',')
FROM library_product_image
WHERE product_id = lp.id
) AS merged_image_url
FROM library_product lp
limit #{start},#{row}
</select>
<select id="getPageTotal" resultType="Integer">
select count(1)
from
(
SELECT
lp.id,
lp.barcode,
lp.name,
lp.product_short_name,
lp.category,
lp.thumb,
(
SELECT GROUP_CONCAT(image_url SEPARATOR ',')
FROM library_product_image
WHERE product_id = lp.id
) AS merged_image_url
FROM library_product lp
)temp
</select>
</mapper>

76
mall-search/src/main/resources/elasticsearch/all.json Normal file
View File

@ -0,0 +1,76 @@
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 1,
"analysis": {
"analyzer": {
"product_analyzer": {
"type": "custom",
"tokenizer": "ik_max_word",
"filter": [
"stop"
]
},
"filter_stopwords": {
"type": "stop",
"stopwords": ["特价", "折扣", "优惠", "促销", "限时", "秒杀", "抢购", "直降", "满减",
"赠品", "包邮", "新品", "热卖", "爆款", "推荐", "精选", "特惠", "清仓",
"正品", "原装", "官方", "正版", "品牌", "优质", "好用", "新款", "老款",
"【", "】", "（", "）", "[]", "()", "「", "」", "！", "！！", "??", "?"]
}
}
}
},
"mappings": {
"properties": {
"id": { "type": "keyword" },
"barcode": { "type": "keyword" },
"productName": {
"type": "text",
"analyzer": "product_analyzer",
"search_analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"keywords": {
"type": "text",
"analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"productShortName": {
"type": "text",
"analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"brand": {"type": "keyword" },
"specs": {
"type": "text",
"analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"category": {
"type": "text",
"analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"cleanName": {
"type": "text",
"analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"thumb": { "type": "keyword"},
"imagesUrls": { "type": "keyword"}
}
}
}

19
mall-search/src/main/resources/elasticsearch/mappings.json Normal file
View File

@ -0,0 +1,19 @@
{
"properties": {
"id": { "type": "keyword" },
"barcode": { "type": "keyword" },
"productName": {
"type": "text",
"analyzer": "product_analyzer",
"search_analyzer": "product_analyzer",
"fields": {
"keyword": { "type": "keyword" }
}
},
"cleanName": {
"type": "text",
"analyzer": "product_analyzer"
},
"imagesUrls": { "type": "keyword" }
}
}

23
mall-search/src/main/resources/elasticsearch/settings.json Normal file
View File

@ -0,0 +1,23 @@
{
"number_of_shards": 3,
"number_of_replicas": 1,
"analysis": {
"analyzer": {
"product_analyzer": {
"type": "custom",
"tokenizer": "ik_max_word",
"filter": [
"lowercase",
"stop",
"word_delimiter"
]
}
},
"filter": {
"filter_stopwords": {
"type": "stop",
"stopwords": ["特价", "折扣", "优惠", "促销", "限时", "秒杀"]
}
}
}
}

1
mall-shop/src/main/java/com/suisung/mall/shop/base/controller/admin/ShopBaseProductSpecController.java
View File

@ -68,6 +68,7 @@ public class ShopBaseProductSpecController {
if (CheckUtil.isNotEmpty(shopBaseProductSpec.getSpec_name())) {
queryWrapper.like("spec_name", shopBaseProductSpec.getSpec_name());
}
queryWrapper.orderByAsc("spec_order");
Page<ShopBaseProductSpec> pageList= shopBaseProductSpecService.lists(queryWrapper, pageNum, pageSize);
List<ShopBaseProductSpec> shopBaseProductSpecList= pageList.getRecords();
shopBaseProductSpecList=shopStoreBaseService.fixStoreDataShopBaseProductSpec(shopBaseProductSpecList);

2
mall-shop/src/main/java/com/suisung/mall/shop/sixun/controller/SxSyncController.java
View File

@ -89,7 +89,7 @@ public class SxSyncController {
@ApiOperation(value = "获取思迅商品新增到数据库", notes = "获取思迅商品新增到数据库")
@RequestMapping(value = "/goods/sync2", method = RequestMethod.POST)
public CommonResult syncGoods(@RequestParam(name = "storeId", defaultValue = "1") String storeId) {
if( shopProductBaseService.syncSxGoodsToShopProductBase(storeId)){
if(shopProductBaseService.syncSxGoodsToShopProductBase(storeId)){
return CommonResult.success();
}

7
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/ProductMappingServiceImpl.java
View File

@ -169,6 +169,9 @@ public class ProductMappingServiceImpl extends BaseServiceImpl<ProductMappingMap
ShopProductSpecItem shopProductSpecItem=processShopProductSpecItem(shopProductBaseList.get(i),shopProductItems.get(i).getCategory_id(),shopProductSpecItemMap,ShopBaseProductSpecMap,productMappingMap,null);
if(shopProductSpecItem!=null){
shopProductBaseList.get(i).setProduct_state_id(StateCode.PRODUCT_STATE_OFF_THE_SHELF);
ShopProductIndex shopProductIndex=new ShopProductIndex();
shopProductIndex.setProduct_id(shopProductBaseList.get(i).getProduct_id());
shopProductIndex.setProduct_state_id(StateCode.PRODUCT_STATE_OFF_THE_SHELF);
shopProductItems.get(i).setItem_enable(StateCode.PRODUCT_STATE_OFF_THE_SHELF);
shopProductItems.get(i).setItem_is_default(1);
if(shopProductSpecItem.isUpdate()){
@ -265,7 +268,7 @@ public class ProductMappingServiceImpl extends BaseServiceImpl<ProductMappingMap
}
// 使用IN查询优化（根据数据库特性可能需要分批）
QueryWrapper<ShopProductIndex> query = new QueryWrapper<>();
query.select("product_id", "product_name", "store_id");
query.select("product_id", "product_name", "store_id","product_state_id");
// 构建OR条件 (store_id=X AND product_number=Y) OR (store_id=A AND product_number=B)...
shopProductBaseList.forEach(base -> {
@ -274,7 +277,7 @@ public class ProductMappingServiceImpl extends BaseServiceImpl<ProductMappingMap
});
List<ShopProductIndex> updateShopProductIndexList= shopProductIndexService.list(query);
updateShopProductIndexList.forEach(shopProductIndex -> {
shopProductIndex.setProduct_state_id(StateCode.PRODUCT_STATE_NORMAL);
shopProductIndex.setProduct_state_id(StateCode.PRODUCT_STATE_OFF_THE_SHELF);
});
shopProductIndexService.updateBatchById(updateShopProductIndexList);
}

56
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncBaseThirdSxAbstract.java
View File

@ -61,6 +61,7 @@ import java.math.BigDecimal;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@ -96,6 +97,28 @@ public abstract class SyncBaseThirdSxAbstract{
@Autowired
private LibraryProductService libraryProductService;
@Autowired
public static final Set<String> FORBID_CATEGORY= Collections.unmodifiableSet(new HashSet<>(
Arrays.asList("香烟类","香烟","烟类", "烟")
));
/**
* 是否位禁售目录
* @param category
* @return
*/
private String getForbidCategory(String category){
AtomicReference<String> forbidName= new AtomicReference<>("");
FORBID_CATEGORY.stream().allMatch(forbidCate -> {
if(category.contains(forbidCate)){
forbidName.set(forbidCate);
return false;
}
return true;
});
return forbidName.get();
}
/**
* 对商品分类进行保存
* @param list
@ -108,6 +131,10 @@ public abstract class SyncBaseThirdSxAbstract{
int count = 0;
List<ShopBaseProductType> productTypeList = new ArrayList<>();
for (int i = 0; i < list.size(); i++) {
String categoryName=list.get(i).getCategory_name();
if(StringUtils.isNotEmpty(getForbidCategory(categoryName))){
continue;
}
list.get(i).setStore_id(storeId); // app 记录传进来
list.get(i).setData_source(2); // 思迅数据来源
list.get(i).setCategory_is_enable(1);
@ -152,25 +179,30 @@ public abstract class SyncBaseThirdSxAbstract{
Integer firstParentId = 0;
list.get(i).setCategory_parent_id(firstParentId);//设置默认值
if (StrUtil.isNotBlank(o.getStr("first_category_name"))) {
String firstCategoryName = o.getStr("first_category_name");
String forbidCategoryName=getForbidCategory(firstCategoryName);
if(StringUtils.isNotEmpty(forbidCategoryName)){
firstCategoryName=firstCategoryName.replace(forbidCategoryName,"");
}
// TODO storeId 不判断一下吗？
ShopBaseProductCategory cate = productCategoryService.getCategoryByName(o.getStr("first_category_name"));
ShopBaseProductCategory cate = productCategoryService.getCategoryByName(firstCategoryName);
if (cate != null) {
list.get(i).setCategory_parent_id(cate.getCategory_id());
} else{
// 新增一个（第一级）父类
ShopBaseProductCategory firstCate = new ShopBaseProductCategory();
firstCate.setCategory_name(o.getStr("first_category_name"));
firstCate.setCategory_name(firstCategoryName);
firstCate.setParent_id(0);
firstCate.setStore_id(storeId);
firstCate.setType_id(typeId);
firstCate.setData_source(2);
List<LibraryProductDTO> libraryProductDTOS=libraryProductService.matchLibraryProducts(null,o.getStr("first_category_name"),new ArrayList<>());
List<LibraryProductDTO> libraryProductDTOS=libraryProductService.matchLibraryProducts(null,firstCategoryName,new ArrayList<>());
if(CollectionUtil.isNotEmpty(libraryProductDTOS)){
firstCate.setCategory_image(libraryProductDTOS.get(0).getThumb());
}
if (productCategoryService.saveOrUpdate(firstCate)) {
// 当前子分类的父类id
firstParentId = firstCate.getId();
firstParentId = firstCate.getCategory_id();
list.get(i).setCategory_parent_id(firstParentId);
}
}
@ -179,25 +211,30 @@ public abstract class SyncBaseThirdSxAbstract{
// 处理（第二级）父类字段 产品分类
if (StrUtil.isNotBlank(o.getStr("second_category_name"))) {
String secondCategoryName = o.getStr("second_category_name");
String forbidCategoryName=getForbidCategory(secondCategoryName);
if(StringUtils.isNotEmpty(forbidCategoryName)){
secondCategoryName=secondCategoryName.replace(forbidCategoryName,"");
}
// TODO storeId 不判断一下吗？
ShopBaseProductCategory cate = productCategoryService.getCategoryByName(o.getStr("second_category_name"));
ShopBaseProductCategory cate = productCategoryService.getCategoryByName(secondCategoryName);
if (cate != null) {
list.get(i).setCategory_parent_id(cate.getCategory_id());
} else {
// 新增一个（第二级）父类
ShopBaseProductCategory secondCate = new ShopBaseProductCategory();
secondCate.setCategory_name(o.getStr("second_category_name"));
secondCate.setCategory_name(secondCategoryName);
secondCate.setCategory_parent_id(firstParentId);
secondCate.setStore_id(storeId);
secondCate.setType_id(typeId);
secondCate.setData_source(2);
List<LibraryProductDTO> libraryProductDTOS=libraryProductService.matchLibraryProducts(null,o.getStr("second_category_name"),new ArrayList<>());
List<LibraryProductDTO> libraryProductDTOS=libraryProductService.matchLibraryProducts(null,secondCategoryName,new ArrayList<>());
if(CollectionUtil.isNotEmpty(libraryProductDTOS)){
secondCate.setCategory_image(libraryProductDTOS.get(0).getThumb());
}
if (productCategoryService.saveOrUpdate(secondCate)) {
// 当前子分类的第二级父类id
list.get(i).setCategory_parent_id(secondCate.getId());
list.get(i).setCategory_parent_id(secondCate.getCategory_id());
}
}
}
@ -215,7 +252,6 @@ public abstract class SyncBaseThirdSxAbstract{
} else {
continue;
}
}
List<LibraryProductDTO> libraryProductDTOS=libraryProductService.matchLibraryProducts(null,list.get(i).getCategory_name(),new ArrayList<>());
if(CollectionUtil.isNotEmpty(libraryProductDTOS)){
@ -689,6 +725,8 @@ public abstract class SyncBaseThirdSxAbstract{
String cateGoryId="";
if(null!=categoryMap.get(jsonObj.get("first_category_name"))){
cateGoryId=categoryMap.get(jsonObj.get("first_category_name")).toString();
}else {
return;
}
Integer categoryId = Convert.toInt(cateGoryId, 0);
Integer storeIdInt = Convert.toInt(storeId);

94
mall-shop/src/main/java/com/suisung/mall/shop/sync/service/impl/SyncShopImageServiceImpl.java
View File

@ -9,12 +9,17 @@
package com.suisung.mall.shop.sync.service.impl;
import cn.hutool.core.collection.CollectionUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.suisung.mall.common.api.StateCode;
import com.suisung.mall.common.enums.DicEnum;
import com.suisung.mall.common.feignService.SearchService;
import com.suisung.mall.common.modules.product.ShopProductBase;
import com.suisung.mall.common.modules.product.ShopProductImage;
import com.suisung.mall.common.modules.product.ShopProductIndex;
import com.suisung.mall.common.modules.product.ShopProductItem;
import com.suisung.mall.common.modules.sync.ImageMappingDto;
import com.suisung.mall.common.pojo.dto.ProductImageSearchDTO;
import com.suisung.mall.common.utils.StringUtils;
import com.suisung.mall.shop.product.mapper.ShopProductImageMapper;
@ -31,6 +36,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@ -59,6 +65,9 @@ public class SyncShopImageServiceImpl implements SyncShopImageService {
@Autowired
private ShopProductIndexService shopProductIndexService;
@Autowired
private SearchService searchService;
@Value("${project.static_domain}")
private String staticDomain;
@ -96,10 +105,13 @@ public class SyncShopImageServiceImpl implements SyncShopImageService {
case MAPPING_PRODUCTNAME:
typename="产品名称模糊匹配";
//先把数据查到临时表再查询
shopImageMappingTempMapper.deleteShopImageMappingTemp(storeId);//匹配结束，删除临时表数据
shopImageMappingTempMapper.mergeShopImageMappingTemp(storeId);
total= shopProductImageMapper.mappingByProductNameCount(storeId);
// shopImageMappingTempMapper.deleteShopImageMappingTemp(storeId);//匹配结束，删除临时表数据
//shopImageMappingTempMapper.mergeShopImageMappingTemp(storeId);
//total= shopProductImageMapper.mappingByProductNameCount(storeId);
QueryWrapper<ShopProductImage> queryWrapper=new QueryWrapper<>();
queryWrapper.eq("store_id",storeId);
queryWrapper.eq("item_image_default","1");
total=(int)shopProductImageService.count(queryWrapper);
break;
}
if(total!=null&&total<1){
@ -112,11 +124,18 @@ public class SyncShopImageServiceImpl implements SyncShopImageService {
AtomicInteger success = new AtomicInteger();
AtomicInteger fails = new AtomicInteger();
for (int i = 1; i <= pages; i++) {
List<ImageMappingDto> imageMappingDtos = shopProductImageMapper.mappingProductImage(type,storeId,(i-1)*BATCH_SIZE,BATCH_SIZE);
final int finalI = i;
futures.add(executor.submit(() -> {
if(MAPPING_PRODUCTNAME.equals(type)){
QueryWrapper<ShopProductImage> queryWrapper=new QueryWrapper<>();
queryWrapper.eq("store_id",storeId);
queryWrapper.eq("item_image_default","1");
for (int i = 1; i <= pages; i++) {
// List<ImageMappingDto> imageMappingDtos = shopProductImageMapper.mappingProductImage(type,storeId,(i-1)*BATCH_SIZE,BATCH_SIZE);
Page<ShopProductImage> pageList= shopProductImageService.lists(queryWrapper,i,BATCH_SIZE);
List<ShopProductImage> list=pageList.getRecords();
final int finalI = i;
futures.add(executor.submit(() -> {
try {
List<ImageMappingDto> imageMappingDtos=CovertToShopProductImage(list);
syncBatchShopImage(imageMappingDtos);
success.getAndIncrement();
return "成功" + finalI;
@ -124,14 +143,31 @@ public class SyncShopImageServiceImpl implements SyncShopImageService {
fails.getAndIncrement();
return "失败"+finalI;
}
}));
}));
}
}else {
for (int i = 1; i <= pages; i++) {
List<ImageMappingDto> imageMappingDtos = shopProductImageMapper.mappingProductImage(type,storeId,(i-1)*BATCH_SIZE,BATCH_SIZE);
final int finalI = i;
futures.add(executor.submit(() -> {
try {
syncBatchShopImage(imageMappingDtos);
success.getAndIncrement();
return "图库匹配成功" + finalI;
} catch (Exception e) {
fails.getAndIncrement();
return "图库匹配失败"+finalI+":"+e.getMessage();
}
}));
}
}
// 等待所有任务完成
for (Future<?> future : futures) {
try {
log.info("任务结果: " + future.get());
log.info("图库匹配任务结果: " + future.get());
} catch (Exception e) {
log.error("任务执行异常: " + e.getMessage());
log.error("图库匹配任务执行异常: " + e.getMessage());
}
}
executor.shutdown();
@ -199,4 +235,40 @@ public class SyncShopImageServiceImpl implements SyncShopImageService {
}
}
/**
* 用es查询，转换List<ImageMappingDto>
* @param shopProductImageList
* @return
*/
private List<ImageMappingDto> CovertToShopProductImage(List<ShopProductImage> shopProductImageList){
List<ImageMappingDto> imageMappingDtos=new ArrayList<>();
List<ProductImageSearchDTO> productImageSearchDTOS=new ArrayList<>();
for (ShopProductImage shopProductImage:shopProductImageList){
if(StringUtils.isNotEmpty(shopProductImage.getProduct_name())){
ProductImageSearchDTO productImageSearchDTO=new ProductImageSearchDTO();
productImageSearchDTO.setProductName(shopProductImage.getProduct_name());
productImageSearchDTO.setCleanName(shopProductImage.getProduct_name());
productImageSearchDTO.setImageId(shopProductImage.getProduct_image_id());
productImageSearchDTO.setProductId(shopProductImage.getProduct_id());
productImageSearchDTOS.add(productImageSearchDTO);
}
}
Map<String,List<ProductImageSearchDTO>> productImageList= searchService.searchProductImageList(productImageSearchDTOS, DicEnum.ES_SEARCH_TYPE_2.getCode());
productImageList.forEach((k,v)->{
if(!v.isEmpty()){
ImageMappingDto imageMappingDto=new ImageMappingDto();
ProductImageSearchDTO productImageSearchDTO=v.get(0);
imageMappingDto.setThumb(productImageSearchDTO.getThumb());
imageMappingDto.setMergedImageUrl(productImageSearchDTO.getImagesUrls());
imageMappingDto.setImgProductName(k);
imageMappingDto.setProductId(productImageSearchDTO.getProductId());
imageMappingDto.setStoreId(shopProductImageList.get(0).getStore_id());
imageMappingDto.setProductImageId(productImageSearchDTO.getImageId());
imageMappingDtos.add(imageMappingDto);
}
});
return imageMappingDtos;
}
}