package hae.instances.http.utils; import burp.api.montoya.MontoyaApi; import burp.api.montoya.persistence.PersistedList; import burp.api.montoya.persistence.PersistedObject; import dk.brics.automaton.Automaton; import dk.brics.automaton.AutomatonMatcher; import dk.brics.automaton.RegExp; import dk.brics.automaton.RunAutomaton; import hae.Config; import hae.cache.DataCache; import hae.utils.ConfigLoader; import hae.utils.DataManager; import hae.utils.string.HashCalculator; import hae.utils.string.StringProcessor; import java.text.MessageFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; public class RegularMatcher { private static final Map nfaPatternCache = new ConcurrentHashMap<>(); private static final Map dfaAutomatonCache = new ConcurrentHashMap<>(); private static final Pattern formatIndexPattern = Pattern.compile("\\{(\\d+)}"); private final MontoyaApi api; private final ConfigLoader configLoader; public RegularMatcher(MontoyaApi api, ConfigLoader configLoader) { this.api = api; this.configLoader = configLoader; } public synchronized static void updateGlobalMatchCache(MontoyaApi api, String host, String name, List dataList, boolean flag) { // 添加到全局变量中,便于Databoard检索 if (!Objects.equals(host, "") && host != null) { Config.globalDataMap.compute(host, (existingHost, existingMap) -> { Map> gRuleMap = Optional.ofNullable(existingMap).orElse(new ConcurrentHashMap<>()); gRuleMap.merge(name, new ArrayList<>(dataList), (existingList, newList) -> { Set combinedSet = new LinkedHashSet<>(existingList); combinedSet.addAll(newList); return new ArrayList<>(combinedSet); }); if (flag) { // 数据存储在BurpSuite空间内 try { DataManager dataManager = new DataManager(api); PersistedObject persistedObject = PersistedObject.persistedObject(); gRuleMap.forEach((kName, vList) -> { PersistedList persistedList = PersistedList.persistedStringList(); persistedList.addAll(vList); persistedObject.setStringList(kName, persistedList); }); dataManager.putData("data", host, persistedObject); } catch (Exception ignored) { } } return gRuleMap; }); String[] splitHost = host.split("\\."); String onlyHost = host.split(":")[0]; String anyHost = (splitHost.length > 2 && !StringProcessor.matchHostIsIp(onlyHost)) ? StringProcessor.replaceFirstOccurrence(onlyHost, splitHost[0], "*") : ""; if (!Config.globalDataMap.containsKey(anyHost) && !anyHost.isEmpty()) { // 添加通配符Host,实际数据从查询哪里将所有数据提取 Config.globalDataMap.put(anyHost, new HashMap<>()); } if (!Config.globalDataMap.containsKey("*")) { // 添加通配符全匹配,同上 Config.globalDataMap.put("*", new HashMap<>()); } } } public Map> performRegexMatching(String host, String type, String message, String header, String body) { // 删除动态响应头再进行存储 String originalMessage = message; String dynamicHeader = configLoader.getDynamicHeader(); if (!dynamicHeader.isBlank()) { String modifiedHeader = header.replaceAll(String.format("(%s):.*?\r\n", configLoader.getDynamicHeader()), ""); message = message.replace(header, modifiedHeader); } String messageIndex = HashCalculator.calculateHash(message.getBytes()); // 从数据缓存中读取 Map> dataCacheMap = DataCache.get(messageIndex); // 存在则返回 if (dataCacheMap != null) { return dataCacheMap; } // 最终返回的结果 String firstLine = originalMessage.split("\\r?\\n", 2)[0]; Map> finalMap = applyMatchingRules(host, type, originalMessage, firstLine, header, body); // 数据缓存写入,有可能是空值,当作匹配过的索引不再匹配 DataCache.put(messageIndex, finalMap); return finalMap; } private Map> applyMatchingRules(String host, String type, String message, String firstLine, String header, String body) { Map> finalMap = new HashMap<>(); Config.globalRules.keySet().parallelStream().forEach(i -> { for (Object[] objects : Config.globalRules.get(i)) { String matchContent = ""; // 遍历获取规则 List result; Map tmpMap = new HashMap<>(); boolean loaded = (Boolean) objects[0]; String name = objects[1].toString(); String f_regex = objects[2].toString(); String s_regex = objects[3].toString(); String format = objects[4].toString(); String color = objects[5].toString(); String scope = objects[6].toString(); String engine = objects[7].toString(); boolean sensitive = (Boolean) objects[8]; // 判断规则是否开启与作用域 if (loaded && (scope.contains(type) || scope.contains("any") || type.equals("any"))) { // 在此处检查内容是否缓存,缓存则返回为空 switch (scope) { case "any": case "request": case "response": matchContent = message; break; case "any header": case "request header": case "response header": matchContent = header; break; case "any body": case "request body": case "response body": matchContent = body; break; case "request line": case "response line": matchContent = firstLine; break; default: break; } // 匹配内容为空则跳出 if (matchContent.isBlank()) { break; } try { result = new ArrayList<>(executeRegexEngine(f_regex, s_regex, matchContent, format, engine, sensitive)); } catch (Exception e) { api.logging().logToError(String.format("[x] Error Info:\nName: %s\nRegex: %s", name, f_regex)); api.logging().logToError(e.getMessage()); continue; } // 去除重复内容 HashSet tmpList = new HashSet(result); result.clear(); result.addAll(tmpList); if (!result.isEmpty()) { tmpMap.put("color", color); String dataStr = String.join(Config.boundary, result); tmpMap.put("data", dataStr); String nameAndSize = String.format("%s (%s)", name, result.size()); finalMap.put(nameAndSize, tmpMap); updateGlobalMatchCache(api, host, name, result, true); } } } }); return finalMap; } private List executeRegexEngine(String f_regex, String s_regex, String content, String format, String engine, boolean sensitive) { List retList = new ArrayList<>(); if ("nfa".equals(engine)) { Matcher matcher = createPatternMatcher(f_regex, content, sensitive); retList.addAll(extractRegexMatchResults(s_regex, format, sensitive, matcher)); } else { // DFA不支持格式化输出,因此不关注format String newContent = content; String newFirstRegex = f_regex; if (!sensitive) { newContent = content.toLowerCase(); newFirstRegex = f_regex.toLowerCase(); } AutomatonMatcher autoMatcher = createAutomatonMatcher(newFirstRegex, newContent); retList.addAll(extractRegexMatchResults(s_regex, autoMatcher, content)); } return retList; } private List extractRegexMatchResults(String s_regex, String format, boolean sensitive, Matcher matcher) { List matches = new ArrayList<>(); if (s_regex.isEmpty()) { matches.addAll(formatMatchResults(matcher, format)); } else { while (matcher.find()) { String matchContent = matcher.group(1); if (!matchContent.isEmpty()) { Matcher secondMatcher = createPatternMatcher(s_regex, matchContent, sensitive); matches.addAll(formatMatchResults(secondMatcher, format)); } } } return matches; } private List extractRegexMatchResults(String s_regex, AutomatonMatcher autoMatcher, String content) { List matches = new ArrayList<>(); if (s_regex.isEmpty()) { matches.addAll(formatMatchResults(autoMatcher, content)); } else { while (autoMatcher.find()) { String s = autoMatcher.group(); if (!s.isEmpty()) { autoMatcher = createAutomatonMatcher(s_regex, extractMatchedContent(content, s)); matches.addAll(formatMatchResults(autoMatcher, content)); } } } return matches; } private List formatMatchResults(Matcher matcher, String format) { List stringList = new ArrayList<>(); // 当format为{0}时,直接返回第一个捕获组,避免格式化开销 if ("{0}".equals(format)) { while (matcher.find()) { if (matcher.groupCount() > 0 && !matcher.group(1).isEmpty()) { stringList.add(matcher.group(1)); } } return stringList; } // 需要复杂格式化的情况 List indexList = parseIndexesFromString(format); while (matcher.find()) { if (!matcher.group(1).isEmpty()) { Object[] params = indexList.stream().map(i -> { if (!matcher.group(i + 1).isEmpty()) { return matcher.group(i + 1); } return ""; }).toArray(); stringList.add(MessageFormat.format(normalizeFormatIndexes(format), params)); } } return stringList; } private List formatMatchResults(AutomatonMatcher matcher, String content) { List stringList = new ArrayList<>(); while (matcher.find()) { String s = matcher.group(0); if (!s.isEmpty()) { stringList.add(extractMatchedContent(content, s)); } } return stringList; } private Matcher createPatternMatcher(String regex, String content, boolean sensitive) { Pattern pattern = nfaPatternCache.computeIfAbsent(regex, k -> { int flags = sensitive ? 0 : Pattern.CASE_INSENSITIVE; return Pattern.compile(regex, flags); }); return pattern.matcher(content); } private AutomatonMatcher createAutomatonMatcher(String regex, String content) { RunAutomaton runAuto = dfaAutomatonCache.computeIfAbsent(regex, k -> { RegExp regexp = new RegExp(regex); Automaton auto = regexp.toAutomaton(); return new RunAutomaton(auto, true); }); return runAuto.newMatcher(content); } private LinkedList parseIndexesFromString(String input) { LinkedList indexes = new LinkedList<>(); Matcher matcher = formatIndexPattern.matcher(input); while (matcher.find()) { String index = matcher.group(1); if (!index.isEmpty()) { indexes.add(Integer.valueOf(index)); } } return indexes; } private String extractMatchedContent(String content, String s) { byte[] contentByte = api.utilities().byteUtils().convertFromString(content); byte[] sByte = api.utilities().byteUtils().convertFromString(s); int startIndex = api.utilities().byteUtils().indexOf(contentByte, sByte, false, 1, contentByte.length); int endIndex = startIndex + s.length(); return content.substring(startIndex, endIndex); } private String normalizeFormatIndexes(String format) { Matcher matcher = formatIndexPattern.matcher(format); int count = 0; while (matcher.find()) { String newStr = String.format("{%s}", count); String matchStr = matcher.group(0); format = format.replace(matchStr, newStr); count++; } return format; } }