admin/HaE
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Version: 4.2 Update

Browse Source 
Signed-off-by: gh0stkey <24655118+gh0stkey@users.noreply.github.com>
This commit is contained in:
gh0stkey
2025-05-06 11:24:17 +08:00
parent baa7270f46
commit 95e1cb4dc1
19 changed files with 300 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
src/main/java/hae/instances/http/utils/RegularMatcher.java
View File

@@ -8,7 +8,8 @@ import dk.brics.automaton.AutomatonMatcher;
import dk.brics.automaton.RegExp;
import dk.brics.automaton.RunAutomaton;
import hae.Config;
import hae.cache.MessageCache;
import hae.cache.DataCache;
import hae.utils.ConfigLoader;
import hae.utils.DataManager;
import hae.utils.string.HashCalculator;
import hae.utils.string.StringProcessor;
@@ -20,14 +21,17 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class RegularMatcher {
private static final Map<String, Pattern> nfaPatternCache = new ConcurrentHashMap<>();
private static final Map<String, RunAutomaton> dfaAutomatonCache = new ConcurrentHashMap<>();
private final MontoyaApi api;
private final ConfigLoader configLoader;
public RegularMatcher(MontoyaApi api) {
public RegularMatcher(MontoyaApi api, ConfigLoader configLoader) {
this.api = api;
this.configLoader = configLoader;
}
public synchronized static void putDataToGlobalMap(MontoyaApi api, String host, String name, List<String> dataList, boolean flag) {
public synchronized static void updateGlobalMatchCache(MontoyaApi api, String host, String name, List<String> dataList, boolean flag) {
// 添加到全局变量中，便于Databoard检索
if (!Objects.equals(host, "") && host != null) {
Config.globalDataMap.compute(host, (existingHost, existingMap) -> {
@@ -74,95 +78,123 @@ public class RegularMatcher {
}
}
public Map<String, Map<String, Object>> match(String host, String type, String message, String header, String body) {
// 先从缓存池里判断是否有已经匹配好的结果
String messageIndex = HashCalculator.calculateHash(message.getBytes());
Map<String, Map<String, Object>> map = MessageCache.get(messageIndex);
if (map != null) {
return map;
} else {
// 最终返回的结果
Map<String, Map<String, Object>> finalMap = new HashMap<>();
Config.globalRules.keySet().parallelStream().forEach(i -> {
for (Object[] objects : Config.globalRules.get(i)) {
// 多线程执行，一定程度上减少阻塞现象
String matchContent = "";
// 遍历获取规则
List<String> result;
Map<String, Object> tmpMap = new HashMap<>();
public Map<String, Map<String, Object>> performRegexMatching(String host, String type, String message, String header, String body) {
// 删除动态响应头再进行存储
String originalMessage = message;
String dynamicHeader = configLoader.getDynamicHeader();
boolean loaded = (Boolean) objects[0];
String name = objects[1].toString();
String f_regex = objects[2].toString();
String s_regex = objects[3].toString();
String format = objects[4].toString();
String color = objects[5].toString();
String scope = objects[6].toString();
String engine = objects[7].toString();
boolean sensitive = (Boolean) objects[8];
// 判断规则是否开启与作用域
if (loaded && (scope.contains(type) || scope.contains("any") || type.equals("any"))) {
switch (scope) {
case "any":
case "request":
case "response":
matchContent = message;
break;
case "any header":
case "request header":
case "response header":
matchContent = header;
break;
case "any body":
case "request body":
case "response body":
matchContent = body;
break;
case "request line":
case "response line":
matchContent = message.split("\\r?\\n", 2)[0];
break;
default:
break;
}
try {
result = new ArrayList<>(matchByRegex(f_regex, s_regex, matchContent, format, engine, sensitive));
} catch (Exception e) {
api.logging().logToError(String.format("[x] Error Info:\nName: %s\nRegex: %s", name, f_regex));
api.logging().logToError(e.getMessage());
continue;
}
// 去除重复内容
HashSet tmpList = new HashSet(result);
result.clear();
result.addAll(tmpList);
if (!result.isEmpty()) {
tmpMap.put("color", color);
String dataStr = String.join(Config.boundary, result);
tmpMap.put("data", dataStr);
String nameAndSize = String.format("%s (%s)", name, result.size());
finalMap.put(nameAndSize, tmpMap);
putDataToGlobalMap(api, host, name, result, true);
}
}
}
});
MessageCache.put(messageIndex, finalMap);
return finalMap;
if (!dynamicHeader.isBlank()) {
String modifiedHeader = header.replaceAll(String.format("(%s):.*?\r\n", configLoader.getDynamicHeader()), "");
message = message.replace(header, modifiedHeader);
}
String messageIndex = HashCalculator.calculateHash(message.getBytes());
// 从数据缓存中读取
Map<String, Map<String, Object>> dataCacheMap = DataCache.get(messageIndex);
// 存在则返回
if (dataCacheMap != null) {
return dataCacheMap;
}
// 最终返回的结果
String firstLine = originalMessage.split("\\r?\\n", 2)[0];
Map<String, Map<String, Object>> finalMap = applyMatchingRules(host, type, originalMessage, firstLine, header, body);
// 数据缓存写入，有可能是空值，当作匹配过的索引不再匹配
DataCache.put(messageIndex, finalMap);
return finalMap;
}
private List<String> matchByRegex(String f_regex, String s_regex, String content, String format, String engine, boolean sensitive) {
private Map<String, Map<String, Object>> applyMatchingRules(String host, String type, String message, String firstLine, String header, String body) {
Map<String, Map<String, Object>> finalMap = new HashMap<>();
Config.globalRules.keySet().parallelStream().forEach(i -> {
for (Object[] objects : Config.globalRules.get(i)) {
String matchContent = "";
// 遍历获取规则
List<String> result;
Map<String, Object> tmpMap = new HashMap<>();
boolean loaded = (Boolean) objects[0];
String name = objects[1].toString();
String f_regex = objects[2].toString();
String s_regex = objects[3].toString();
String format = objects[4].toString();
String color = objects[5].toString();
String scope = objects[6].toString();
String engine = objects[7].toString();
boolean sensitive = (Boolean) objects[8];
// 判断规则是否开启与作用域
if (loaded && (scope.contains(type) || scope.contains("any") || type.equals("any"))) {
// 在此处检查内容是否缓存，缓存则返回为空
switch (scope) {
case "any":
case "request":
case "response":
matchContent = message;
break;
case "any header":
case "request header":
case "response header":
matchContent = header;
break;
case "any body":
case "request body":
case "response body":
matchContent = body;
break;
case "request line":
case "response line":
matchContent = firstLine;
break;
default:
break;
}
// 匹配内容为空则跳出
if (matchContent.isBlank()) {
break;
}
try {
result = new ArrayList<>(executeRegexEngine(f_regex, s_regex, matchContent, format, engine, sensitive));
} catch (Exception e) {
api.logging().logToError(String.format("[x] Error Info:\nName: %s\nRegex: %s", name, f_regex));
api.logging().logToError(e.getMessage());
continue;
}
// 去除重复内容
HashSet tmpList = new HashSet(result);
result.clear();
result.addAll(tmpList);
if (!result.isEmpty()) {
tmpMap.put("color", color);
String dataStr = String.join(Config.boundary, result);
tmpMap.put("data", dataStr);
String nameAndSize = String.format("%s (%s)", name, result.size());
finalMap.put(nameAndSize, tmpMap);
updateGlobalMatchCache(api, host, name, result, true);
}
}
}
});
return finalMap;
}
private List<String> executeRegexEngine(String f_regex, String s_regex, String content, String format, String engine, boolean sensitive) {
List<String> retList = new ArrayList<>();
if ("nfa".equals(engine)) {
Matcher matcher = createPatternMatcher(f_regex, content, sensitive);
retList.addAll(extractMatches(s_regex, format, sensitive, matcher));
retList.addAll(extractRegexMatchResults(s_regex, format, sensitive, matcher));
} else {
// DFA不支持格式化输出，因此不关注format
String newContent = content;
@@ -172,44 +204,44 @@ public class RegularMatcher {
newFirstRegex = f_regex.toLowerCase();
}
AutomatonMatcher autoMatcher = createAutomatonMatcher(newFirstRegex, newContent);
retList.addAll(extractMatches(s_regex, autoMatcher, content));
retList.addAll(extractRegexMatchResults(s_regex, autoMatcher, content));
}
return retList;
}
private List<String> extractMatches(String s_regex, String format, boolean sensitive, Matcher matcher) {
private List<String> extractRegexMatchResults(String s_regex, String format, boolean sensitive, Matcher matcher) {
List<String> matches = new ArrayList<>();
if (s_regex.isEmpty()) {
matches.addAll(getFormatString(matcher, format));
matches.addAll(formatMatchResults(matcher, format));
} else {
while (matcher.find()) {
String matchContent = matcher.group(1);
if (!matchContent.isEmpty()) {
matcher = createPatternMatcher(s_regex, matchContent, sensitive);
matches.addAll(getFormatString(matcher, format));
matches.addAll(formatMatchResults(matcher, format));
}
}
}
return matches;
}
private List<String> extractMatches(String s_regex, AutomatonMatcher autoMatcher, String content) {
private List<String> extractRegexMatchResults(String s_regex, AutomatonMatcher autoMatcher, String content) {
List<String> matches = new ArrayList<>();
if (s_regex.isEmpty()) {
matches.addAll(getFormatString(autoMatcher, content));
matches.addAll(formatMatchResults(autoMatcher, content));
} else {
while (autoMatcher.find()) {
String s = autoMatcher.group();
if (!s.isEmpty()) {
autoMatcher = createAutomatonMatcher(s_regex, getSubString(content, s));
matches.addAll(getFormatString(autoMatcher, content));
autoMatcher = createAutomatonMatcher(s_regex, extractMatchedContent(content, s));
matches.addAll(formatMatchResults(autoMatcher, content));
}
}
}
return matches;
}
private List<String> getFormatString(Matcher matcher, String format) {
private List<String> formatMatchResults(Matcher matcher, String format) {
List<Integer> indexList = parseIndexesFromString(format);
List<String> stringList = new ArrayList<>();
@@ -222,20 +254,20 @@ public class RegularMatcher {
return "";
}).toArray();
stringList.add(MessageFormat.format(reorderIndex(format), params));
stringList.add(MessageFormat.format(normalizeFormatIndexes(format), params));
}
}
return stringList;
}
private List<String> getFormatString(AutomatonMatcher matcher, String content) {
private List<String> formatMatchResults(AutomatonMatcher matcher, String content) {
List<String> stringList = new ArrayList<>();
while (matcher.find()) {
String s = matcher.group(0);
if (!s.isEmpty()) {
stringList.add(getSubString(content, s));
stringList.add(extractMatchedContent(content, s));
}
}
@@ -243,14 +275,21 @@ public class RegularMatcher {
}
private Matcher createPatternMatcher(String regex, String content, boolean sensitive) {
Pattern pattern = sensitive ? Pattern.compile(regex) : Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
Pattern pattern = nfaPatternCache.computeIfAbsent(regex, k -> {
int flags = sensitive ? 0 : Pattern.CASE_INSENSITIVE;
return Pattern.compile(regex, flags);
});
return pattern.matcher(content);
}
private AutomatonMatcher createAutomatonMatcher(String regex, String content) {
RegExp regexp = new RegExp(regex);
Automaton auto = regexp.toAutomaton();
RunAutomaton runAuto = new RunAutomaton(auto, true);
RunAutomaton runAuto = dfaAutomatonCache.computeIfAbsent(regex, k -> {
RegExp regexp = new RegExp(regex);
Automaton auto = regexp.toAutomaton();
return new RunAutomaton(auto, true);
});
return runAuto.newMatcher(content);
}
@@ -269,15 +308,16 @@ public class RegularMatcher {
return indexes;
}
private String getSubString(String content, String s) {
private String extractMatchedContent(String content, String s) {
byte[] contentByte = api.utilities().byteUtils().convertFromString(content);
byte[] sByte = api.utilities().byteUtils().convertFromString(s);
int startIndex = api.utilities().byteUtils().indexOf(contentByte, sByte, false, 1, contentByte.length);
int endIndex = startIndex + s.length();
return content.substring(startIndex, endIndex);
}
private String reorderIndex(String format) {
private String normalizeFormatIndexes(String format) {
Pattern pattern = Pattern.compile("\\{(\\d+)}");
Matcher matcher = pattern.matcher(format);
int count = 0;
@@ -287,6 +327,7 @@ public class RegularMatcher {
format = format.replace(matchStr, newStr);
count++;
}
return format;
}
}