ai_dify_plat_sass/ai_dify_plat_api
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1.文本相似度加快审查速度2.忽略技术标准修改

Browse Source
This commit is contained in:
cjh
2026-06-02 17:55:00 +08:00
parent c7fa70707f
commit bc8f2acbc3
3 changed files with 271 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
ruoyi-modules/ai-intelligent-review/src/main/java/org/dromara/review/service/impl/BiddingAnalysisResultServiceImpl.java
View File

@@ -247,8 +247,13 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
SmartFilterConfig smartFilterConfig) {
try {
int totalItems1 = paragraphs1.size();
int totalItems2 = paragraphs2.size();
// 确定是否在相似度计算时忽略标点符号
boolean ignorePunctuation = smartFilterConfig != null
&& Boolean.TRUE.equals(smartFilterConfig.getIgnorePunctuationAndShortText());
List<SimilarityParagraph> preparedParagraphs1 = prepareSimilarityParagraphs(paragraphs1, ignorePunctuation);
List<SimilarityParagraph> preparedParagraphs2 = prepareSimilarityParagraphs(paragraphs2, ignorePunctuation);
int totalItems1 = preparedParagraphs1.size();
int totalItems2 = preparedParagraphs2.size();
int similarItemCount = 0;
List<Map<String, Object>> duplicateContents = new ArrayList<>();
@@ -261,42 +266,24 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
return new SimilarityResult(0L, duplicateContents);
}
// 确定是否在相似度计算时忽略标点符号
boolean ignorePunctuation = smartFilterConfig != null
&& Boolean.TRUE.equals(smartFilterConfig.getIgnorePunctuationAndShortText());
// 比较每对段落的相似度
// 缓存已查询过的pdfOssId结果
Map<Long, Long> pdfOssIdCache = new HashMap<>();
Map<Long, Long> dtlIdCache = new HashMap<>();
for (int i = 0; i < totalItems1; i++) {
String para1 = paragraphs1.get(i).get("text").toString();
String page1 = paragraphs1.get(i).get("page").toString();
String paragraphId = paragraphs1.get(i).get("paragraphId").toString();
// 直接从ossIds1列表中获取当前段落对应的ossId
Long ossId1 = (Long) paragraphs1.get(i).get("ossId");
if (para1 == null || para1.trim().isEmpty()) {
continue;
}
SimilarityParagraph left = preparedParagraphs1.get(i);
for (int j = 0; j < totalItems2; j++) {
// 如果段落2的第j个元素已经被匹配过，则跳过
if (matchedIndices2.contains(j)) {
continue;
}
String page2 = paragraphs2.get(j).get("page").toString();
String para2 = paragraphs2.get(j).get("text").toString();
String paragraphId2 = paragraphs2.get(j).get("paragraphId").toString();
// 直接从ossIds1列表中获取当前段落对应的ossId
Long ossId2 = (Long) paragraphs2.get(j).get("ossId");
if (para2 == null || para2.trim().isEmpty()) {
SimilarityParagraph right = preparedParagraphs2.get(j);
if (!canReachSimilarityThreshold(left, right, threshold)) {
continue;
}
double similarity = calculateParagraphSimilarity(para1, para2, ignorePunctuation);
double similarity = left.normalizedText.equals(right.normalizedText)
? 1.0
: calculateParagraphSimilarity(left.normalizedText, right.normalizedText);
// 如果相似度超过阈值，则认为是重复内容
if (similarity * 100 >= threshold) {
similarItemCount++;
@@ -307,22 +294,22 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
// 为每个重复段落创建包含内容和具体文件ID的Map
com.alibaba.fastjson.JSONObject duplicateItem = new com.alibaba.fastjson.JSONObject();
duplicateItem.put("smallContent", para1);
if (para1.length() > 20) {
content = para1.substring(0, 20);
duplicateItem.put("smallContent", left.originalText);
if (left.originalText.length() > 20) {
content = left.originalText.substring(0, 20);
duplicateItem.put("smallContent", content);
}
duplicateItem.put("smallContentB", para2);
if (para2.length() > 20) {
content = para2.substring(0, 20);
duplicateItem.put("smallContentB", right.originalText);
if (right.originalText.length() > 20) {
content = right.originalText.substring(0, 20);
duplicateItem.put("smallContentB", content);
}
duplicateItem.put("content", para1);
duplicateItem.put("contentB", para2);
duplicateItem.put("page2",page2);
duplicateItem.put("page1", page1);
duplicateItem.put("paragraphId2", paragraphId2);
duplicateItem.put("paragraphId1", paragraphId);
duplicateItem.put("content", left.originalText);
duplicateItem.put("contentB", right.originalText);
duplicateItem.put("page2", right.page);
duplicateItem.put("page1", left.page);
duplicateItem.put("paragraphId2", right.paragraphId);
duplicateItem.put("paragraphId1", left.paragraphId);
// 使用缓存获取pdf1
// Long pdf1 = pdfOssIdCache.get(ossId1);
@@ -337,19 +324,19 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
// pdf2 = biddingProposalDtlMapper.findPdfOssId(ossId2);
// pdfOssIdCache.put(ossId2, pdf2);
// }
Long dtlId1 = dtlIdCache.get(ossId1);
Long dtlId1 = dtlIdCache.get(left.ossId);
if (dtlId1 == null) {
dtlId1 = biddingProposalDtlMapper.findDtlIdByOssId(ossId1);
dtlIdCache.put(ossId1, dtlId1);
dtlId1 = biddingProposalDtlMapper.findDtlIdByOssId(left.ossId);
dtlIdCache.put(left.ossId, dtlId1);
}
Long dtlId2 = dtlIdCache.get(ossId2);
Long dtlId2 = dtlIdCache.get(right.ossId);
if (dtlId2 == null) {
dtlId2 = biddingProposalDtlMapper.findDtlIdByOssId(ossId2);
dtlIdCache.put(ossId2, dtlId2);
dtlId2 = biddingProposalDtlMapper.findDtlIdByOssId(right.ossId);
dtlIdCache.put(right.ossId, dtlId2);
}
//文本相似度结果，每个重复段落中已经包含了对应的具体文件ID
BiddingAnalysisResultDtl textResult = addResult(duplicateItem.toString(), 3, ossId1, ossId2, dtlId1, dtlId2);
BiddingAnalysisResultDtl textResult = addResult(duplicateItem.toString(), 3, left.ossId, right.ossId, dtlId1, dtlId2);
biddingResultList.add(textResult);
break; // 避免一个段落被多次匹配
}
@@ -509,15 +496,93 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
}
/**
* 计算两个段落的相似度(适用于中文文本)
* 使用字符级别的相似度算法,结合LCS和字符重叠度
*
* @param para1 段落1
* @param para2 段落2
* @param ignorePunctuation 是否在比较时忽略标点符号
* @return 相似度(0.0-1.0之间的值,1.0表示完全相同)
* 预处理段落，避免在两两比较时重复清洗文本和构建字符集合。
*/
private double calculateParagraphSimilarity(String para1, String para2, boolean ignorePunctuation) {
private List<SimilarityParagraph> prepareSimilarityParagraphs(List<Map<String, Object>> paragraphs, boolean ignorePunctuation) {
List<SimilarityParagraph> result = new ArrayList<>();
if (paragraphs == null || paragraphs.isEmpty()) {
return result;
}
for (Map<String, Object> paragraph : paragraphs) {
Object textObj = paragraph.get("text");
if (textObj == null) {
continue;
}
String originalText = textObj.toString().trim();
if (originalText.isEmpty()) {
continue;
}
String normalizedText = ignorePunctuation ? removePunctuation(originalText) : originalText;
normalizedText = normalizedText.trim();
if (normalizedText.isEmpty()) {
continue;
}
result.add(new SimilarityParagraph(
originalText,
normalizedText,
String.valueOf(paragraph.getOrDefault("page", "")),
String.valueOf(paragraph.getOrDefault("paragraphId", "")),
toLong(paragraph.get("ossId")),
buildCharSet(normalizedText)
));
}
return result;
}
private boolean canReachSimilarityThreshold(SimilarityParagraph left, SimilarityParagraph right, int threshold) {
if (left.normalizedText.equals(right.normalizedText)) {
return true;
}
int maxLength = Math.max(left.length, right.length);
if (maxLength == 0) {
return false;
}
double lengthUpperBound = (double) Math.min(left.length, right.length) / maxLength;
double charOverlap = calculateCharOverlapSimilarity(left.uniqueChars, right.uniqueChars);
double similarityUpperBound = lengthUpperBound * 0.7 + charOverlap * 0.3;
return similarityUpperBound * 100 >= threshold;
}
private String removePunctuation(String text) {
StringBuilder builder = new StringBuilder(text.length());
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
int type = Character.getType(c);
if (type != Character.CONNECTOR_PUNCTUATION
&& type != Character.DASH_PUNCTUATION
&& type != Character.START_PUNCTUATION
&& type != Character.END_PUNCTUATION
&& type != Character.INITIAL_QUOTE_PUNCTUATION
&& type != Character.FINAL_QUOTE_PUNCTUATION
&& type != Character.OTHER_PUNCTUATION) {
builder.append(c);
}
}
return builder.toString();
}
private Set<Character> buildCharSet(String text) {
Set<Character> chars = new HashSet<>();
for (int i = 0; i < text.length(); i++) {
char c = text.charAt(i);
if (!Character.isWhitespace(c)) {
chars.add(c);
}
}
return chars;
}
private Long toLong(Object value) {
if (value instanceof Long) {
return (Long) value;
}
if (value instanceof Number) {
return ((Number) value).longValue();
}
return value == null ? null : Long.valueOf(value.toString());
}
private double calculateParagraphSimilarity(String para1, String para2) {
if (para1 == null || para2 == null) {
return 0.0;
}
@@ -526,12 +591,6 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
String text1 = para1.trim();
String text2 = para2.trim();
// 如果配置了忽略标点符号，则在比较时临时移除标点符号（但不影响原始文本）
if (ignorePunctuation) {
text1 = text1.replaceAll("\\p{P}", "");
text2 = text2.replaceAll("\\p{P}", "");
}
// 如果任一文本为空，返回0
if (text1.isEmpty() || text2.isEmpty()) {
return 0.0;
@@ -561,6 +620,27 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
return finalSimilarity;
}
private static class SimilarityParagraph {
private final String originalText;
private final String normalizedText;
private final String page;
private final String paragraphId;
private final Long ossId;
private final int length;
private final Set<Character> uniqueChars;
private SimilarityParagraph(String originalText, String normalizedText, String page,
String paragraphId, Long ossId, Set<Character> uniqueChars) {
this.originalText = originalText;
this.normalizedText = normalizedText;
this.page = page;
this.paragraphId = paragraphId;
this.ossId = ossId;
this.length = normalizedText.length();
this.uniqueChars = uniqueChars;
}
}
/**
* 基于最长公共子序列(LCS)计算相似度
*/
@@ -603,21 +683,13 @@ public class BiddingAnalysisResultServiceImpl implements IBiddingAnalysisResultS
*/
private double calculateCharOverlapSimilarity(String text1, String text2) {
// 将文本转换为字符集合(去除空白字符和标点符号)
Set<Character> chars1 = new HashSet<>();
Set<Character> chars2 = new HashSet<>();
Set<Character> chars1 = buildCharSet(text1);
Set<Character> chars2 = buildCharSet(text2);
for (char c : text1.toCharArray()) {
if (!Character.isWhitespace(c)) {
chars1.add(c);
}
}
for (char c : text2.toCharArray()) {
if (!Character.isWhitespace(c)) {
chars2.add(c);
}
}
return calculateCharOverlapSimilarity(chars1, chars2);
}
private double calculateCharOverlapSimilarity(Set<Character> chars1, Set<Character> chars2) {
if (chars1.isEmpty() || chars2.isEmpty()) {
return 0.0;
}