Merge pull request #566 from luoliwoshang/llcppg/symg/header

[wip] llcppg/symg

c/cjson/cjson.go

@@ -134,6 +134,18 @@ func (o *JSON) PrintUnformatted() *c.Char { return nil }
 // llgo:link (*JSON).PrintBuffered C.cJSON_PrintBuffered
 func (o *JSON) PrintBuffered(prebuffer c.Int, fmt c.Int) *c.Char { return nil }
 
+// llgo:link (*JSON).GetObjectItemCaseSensitive C.cJSON_GetObjectItemCaseSensitive
+func (o *JSON) GetObjectItemCaseSensitive(key *c.Char) *JSON { return nil }
+
+// llgo:link (*JSON).GetArraySize C.cJSON_GetArraySize
+func (o *JSON) GetArraySize() c.Int { return 0 }
+
+// llgo:link (*JSON).GetArrayItem C.cJSON_GetArrayItem
+func (o *JSON) GetArrayItem(index c.Int) *JSON { return nil }
+
+// llgo:link (*JSON).GetStringValue C.cJSON_GetStringValue
+func (o *JSON) GetStringValue() *c.Char { return nil }
+
 //go:linkname Free C.cJSON_free
 func Free(ptr unsafe.Pointer)
 

chore/_xtool/llcppsymg/config/config.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"errors"
+	"unsafe"
+
+	"github.com/goplus/llgo/c"
+	"github.com/goplus/llgo/c/cjson"
+	"github.com/goplus/llgo/chore/llcppg/types"
+)
+
+type Conf struct {
+	*cjson.JSON
+	*types.Config
+}
+
+func GetConf(data []byte) (Conf, error) {
+	parsedConf := cjson.ParseBytes(data)
+	if parsedConf == nil {
+		return Conf{}, errors.New("failed to parse config")
+	}
+
+	config := &types.Config{
+		Name:         GetStringItem(parsedConf, "name", ""),
+		CFlags:       GetStringItem(parsedConf, "cflags", ""),
+		Libs:         GetStringItem(parsedConf, "libs", ""),
+		Include:      GetStringArrayItem(parsedConf, "include"),
+		TrimPrefixes: GetStringArrayItem(parsedConf, "trimPrefixes"),
+	}
+
+	return Conf{
+		JSON:   parsedConf,
+		Config: config,
+	}, nil
+}
+
+func GetString(obj *cjson.JSON) (value string) {
+	str := obj.GetStringValue()
+	return unsafe.String((*byte)(unsafe.Pointer(str)), c.Strlen(str))
+}
+
+func GetStringItem(obj *cjson.JSON, key string, defval string) (value string) {
+	item := obj.GetObjectItemCaseSensitive(c.AllocaCStr(key))
+	if item == nil {
+		return defval
+	}
+	return GetString(item)
+}
+
+func GetStringArrayItem(obj *cjson.JSON, key string) (value []string) {
+	item := obj.GetObjectItemCaseSensitive(c.AllocaCStr(key))
+	if item == nil {
+		return
+	}
+	value = make([]string, item.GetArraySize())
+	for i := range value {
+		value[i] = GetString(item.GetArrayItem(c.Int(i)))
+	}
+	return
+}

chore/_xtool/llcppsymg/llcppsymg.go

@@ -17,12 +17,22 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"unsafe"
 
 	"github.com/goplus/llgo/c"
 	"github.com/goplus/llgo/c/cjson"
+	"github.com/goplus/llgo/chore/_xtool/llcppsymg/config"
+	"github.com/goplus/llgo/chore/_xtool/llcppsymg/parse"
+	"github.com/goplus/llgo/chore/llcppg/types"
+	"github.com/goplus/llgo/cpp/llvm"
+	"github.com/goplus/llgo/xtool/nm"
 )
 
 func main() {
@@ -40,14 +50,25 @@ func main() {
 	}
 	check(err)
 
-	conf := cjson.ParseBytes(data)
+	conf, err := config.GetConf(data)
-	if conf == nil {
+	check(err)
-		fmt.Fprintln(os.Stderr, "Failed to parse config file:", cfgFile)
-		os.Exit(1)
-	}
 	defer conf.Delete()
 
-	c.Printf(c.Str("%s"), conf.Print())
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to parse config file:", cfgFile)
+	}
+	symbols, err := parseDylibSymbols(conf.Libs)
+
+	check(err)
+
+	filepaths := generateHeaderFilePath(conf.CFlags, conf.Include)
+	astInfos, err := parse.ParseHeaderFile(filepaths)
+	check(err)
+
+	symbolInfo := getCommonSymbols(symbols, astInfos, conf.TrimPrefixes)
+
+	err = genSymbolTableFile(symbolInfo)
+	check(err)
 }
 
 func check(err error) {
@@ -55,3 +76,230 @@ func check(err error) {
 		panic(err)
 	}
 }
+
+func parseDylibSymbols(lib string) ([]types.CPPSymbol, error) {
+	dylibPath, err := generateDylibPath(lib)
+	if err != nil {
+		return nil, errors.New("failed to generate dylib path")
+	}
+
+	files, err := nm.New("").List(dylibPath)
+	if err != nil {
+		return nil, errors.New("failed to list symbols in dylib")
+	}
+
+	var symbols []types.CPPSymbol
+
+	for _, file := range files {
+		for _, sym := range file.Symbols {
+			demangleName := decodeSymbolName(sym.Name)
+			symbols = append(symbols, types.CPPSymbol{
+				Symbol:       sym,
+				DemangleName: demangleName,
+			})
+		}
+	}
+
+	return symbols, nil
+}
+
+func generateDylibPath(lib string) (string, error) {
+	output := lib
+	libPath := ""
+	libName := ""
+	for _, part := range strings.Fields(string(output)) {
+		if strings.HasPrefix(part, "-L") {
+			libPath = part[2:]
+		} else if strings.HasPrefix(part, "-l") {
+			libName = part[2:]
+		}
+	}
+
+	if libPath == "" || libName == "" {
+		return "", fmt.Errorf("failed to parse pkg-config output: %s", output)
+	}
+
+	dylibPath := filepath.Join(libPath, "lib"+libName+".dylib")
+	return dylibPath, nil
+}
+
+func decodeSymbolName(symbolName string) string {
+	if symbolName == "" {
+		return ""
+	}
+
+	demangled := llvm.ItaniumDemangle(symbolName, true)
+	if demangled == nil {
+		return symbolName
+	}
+	defer c.Free(unsafe.Pointer(demangled))
+
+	demangleName := c.GoString(demangled)
+	if demangleName == "" {
+		return symbolName
+	}
+
+	decodedName := strings.TrimSpace(demangleName)
+	decodedName = strings.ReplaceAll(decodedName,
+		"std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const",
+		"std::string")
+
+	return decodedName
+}
+
+func generateHeaderFilePath(cflags string, files []string) []string {
+	prefixPath := cflags
+	prefixPath = strings.TrimPrefix(prefixPath, "-I")
+	var includePaths []string
+	for _, file := range files {
+		includePaths = append(includePaths, filepath.Join(prefixPath, "/"+file))
+	}
+	return includePaths
+}
+
+func getCommonSymbols(dylibSymbols []types.CPPSymbol, astInfoList []types.ASTInformation, prefix []string) []types.SymbolInfo {
+	var commonSymbols []types.SymbolInfo
+	functionNameMap := make(map[string]int)
+
+	for _, astInfo := range astInfoList {
+		for _, dylibSym := range dylibSymbols {
+			if strings.TrimPrefix(dylibSym.Name, "_") == astInfo.Symbol {
+				cppName := generateCPPName(astInfo)
+				functionNameMap[cppName]++
+				symbolInfo := types.SymbolInfo{
+					Mangle: strings.TrimPrefix(dylibSym.Name, "_"),
+					CPP:    cppName,
+					Go:     generateMangle(astInfo, functionNameMap[cppName], prefix),
+				}
+				commonSymbols = append(commonSymbols, symbolInfo)
+				break
+			}
+		}
+	}
+
+	return commonSymbols
+}
+
+func generateCPPName(astInfo types.ASTInformation) string {
+	cppName := astInfo.Name
+	if astInfo.Class != "" {
+		cppName = astInfo.Class + "::" + astInfo.Name
+	}
+	return cppName
+}
+
+func generateMangle(astInfo types.ASTInformation, count int, prefixes []string) string {
+	astInfo.Class = removePrefix(astInfo.Class, prefixes)
+	astInfo.Name = removePrefix(astInfo.Name, prefixes)
+	res := ""
+	if astInfo.Class != "" {
+		if astInfo.Class == astInfo.Name {
+			res = "(*" + astInfo.Class + ")." + "Init"
+			if count > 1 {
+				res += "__" + strconv.Itoa(count-1)
+			}
+		} else if astInfo.Name == "~"+astInfo.Class {
+			res = "(*" + astInfo.Class + ")." + "Dispose"
+			if count > 1 {
+				res += "__" + strconv.Itoa(count-1)
+			}
+		} else {
+			res = "(*" + astInfo.Class + ")." + astInfo.Name
+			if count > 1 {
+				res += "__" + strconv.Itoa(count-1)
+			}
+		}
+	} else {
+		res = astInfo.Name
+		if count > 1 {
+			res += "__" + strconv.Itoa(count-1)
+		}
+	}
+	return res
+}
+
+func removePrefix(str string, prefixes []string) string {
+	for _, prefix := range prefixes {
+		if strings.HasPrefix(str, prefix) {
+			return strings.TrimPrefix(str, prefix)
+		}
+	}
+	return str
+}
+
+func genSymbolTableFile(symbolInfos []types.SymbolInfo) error {
+	// keep open follow code block can run successfully
+	for i := range symbolInfos {
+		println("symbol", symbolInfos[i].Go)
+	}
+
+	fileName := "llcppg.symb.json"
+	existingSymbols, err := readExistingSymbolTable(fileName)
+	if err != nil {
+		return err
+	}
+
+	for i := range symbolInfos {
+		if existingSymbol, exists := existingSymbols[symbolInfos[i].Mangle]; exists {
+			symbolInfos[i].Go = existingSymbol.Go
+		}
+	}
+
+	root := cjson.Array()
+	defer root.Delete()
+
+	for _, symbol := range symbolInfos {
+		item := cjson.Object()
+		item.SetItem(c.Str("mangle"), cjson.String(c.AllocaCStr(symbol.Mangle)))
+		item.SetItem(c.Str("c++"), cjson.String(c.AllocaCStr(symbol.CPP)))
+		item.SetItem(c.Str("go"), cjson.String(c.AllocaCStr(symbol.Go)))
+		root.AddItem(item)
+	}
+
+	cStr := root.Print()
+	if cStr == nil {
+		return errors.New("symbol table is empty")
+	}
+	defer c.Free(unsafe.Pointer(cStr))
+
+	data := unsafe.Slice((*byte)(unsafe.Pointer(cStr)), c.Strlen(cStr))
+
+	if err := os.WriteFile(fileName, data, 0644); err != nil {
+		return errors.New("failed to write symbol table file")
+	}
+	return nil
+}
+func readExistingSymbolTable(fileName string) (map[string]types.SymbolInfo, error) {
+	existingSymbols := make(map[string]types.SymbolInfo)
+
+	if _, err := os.Stat(fileName); err != nil {
+		return existingSymbols, nil
+	}
+
+	data, err := os.ReadFile(fileName)
+	if err != nil {
+		return nil, errors.New("failed to read symbol table file")
+	}
+
+	parsedJSON := cjson.ParseBytes(data)
+	if parsedJSON == nil {
+		return nil, errors.New("failed to parse JSON")
+	}
+
+	arraySize := parsedJSON.GetArraySize()
+
+	for i := 0; i < int(arraySize); i++ {
+		item := parsedJSON.GetArrayItem(c.Int(i))
+		if item == nil {
+			continue
+		}
+		symbol := types.SymbolInfo{
+			Mangle: config.GetStringItem(item, "mangle", ""),
+			CPP:    config.GetStringItem(item, "c++", ""),
+			Go:     config.GetStringItem(item, "go", ""),
+		}
+		existingSymbols[symbol.Mangle] = symbol
+	}
+
+	return existingSymbols, nil
+}

chore/_xtool/llcppsymg/parse/parse.go

@@ -0,0 +1,153 @@
+package parse
+
+import (
+	"errors"
+	"strconv"
+	"unsafe"
+
+	"github.com/goplus/llgo/c"
+	"github.com/goplus/llgo/c/clang"
+	"github.com/goplus/llgo/chore/llcppg/types"
+)
+
+type Context struct {
+	namespaceName string
+	className     string
+	astInfo       []types.ASTInformation
+	currentFile   string
+}
+
+func newContext() *Context {
+	return &Context{
+		astInfo: make([]types.ASTInformation, 0),
+	}
+}
+
+func (c *Context) setNamespaceName(name string) {
+	c.namespaceName = name
+}
+
+func (c *Context) setClassName(name string) {
+	c.className = name
+}
+
+func (c *Context) setCurrentFile(filename string) {
+	c.currentFile = filename
+}
+
+var context = newContext()
+
+func collectFuncInfo(cursor clang.Cursor) types.ASTInformation {
+
+	info := types.ASTInformation{
+		Namespace: context.namespaceName,
+		Class:     context.className,
+	}
+
+	cursorStr := cursor.String()
+	symbol := cursor.Mangling()
+
+	info.Name = c.GoString(cursorStr.CStr())
+
+	info.Symbol = c.GoString(symbol.CStr())
+	if len(info.Symbol) >= 1 {
+		if info.Symbol[0] == '_' {
+			info.Symbol = info.Symbol[1:]
+		}
+	}
+
+	defer symbol.Dispose()
+	defer cursorStr.Dispose()
+
+	if context.namespaceName != "" {
+		info.Namespace = context.namespaceName
+	}
+	if context.className != "" {
+		info.Class = context.className
+	}
+
+	typeStr := cursor.ResultType().String()
+	defer typeStr.Dispose()
+	info.ReturnType = c.GoString(typeStr.CStr())
+
+	info.Parameters = make([]types.Parameter, cursor.NumArguments())
+	for i := 0; i < int(cursor.NumArguments()); i++ {
+		argCurSor := cursor.Argument(c.Uint(i))
+		argType := argCurSor.Type().String()
+		argName := argCurSor.String()
+		info.Parameters[i] = types.Parameter{
+			Name: c.GoString(argName.CStr()),
+			Type: c.GoString(argType.CStr()),
+		}
+
+		argType.Dispose()
+		argName.Dispose()
+	}
+
+	return info
+}
+
+func visit(cursor, parent clang.Cursor, clientData c.Pointer) clang.ChildVisitResult {
+	if cursor.Kind == clang.Namespace {
+		nameStr := cursor.String()
+		context.setNamespaceName(c.GoString(nameStr.CStr()))
+		clang.VisitChildren(cursor, visit, nil)
+		context.setNamespaceName("")
+	} else if cursor.Kind == clang.ClassDecl {
+		nameStr := cursor.String()
+		context.setClassName(c.GoString(nameStr.CStr()))
+		clang.VisitChildren(cursor, visit, nil)
+		context.setClassName("")
+	} else if cursor.Kind == clang.CXXMethod || cursor.Kind == clang.FunctionDecl || cursor.Kind == clang.Constructor || cursor.Kind == clang.Destructor {
+		loc := cursor.Location()
+		var file clang.File
+		var line, column c.Uint
+
+		loc.SpellingLocation(&file, &line, &column, nil)
+		filename := file.FileName()
+
+		if c.Strcmp(filename.CStr(), c.AllocaCStr(context.currentFile)) == 0 {
+			info := collectFuncInfo(cursor)
+			info.Location = c.GoString(filename.CStr()) + ":" + strconv.Itoa(int(line)) + ":" + strconv.Itoa(int(column))
+			context.astInfo = append(context.astInfo, info)
+		}
+
+		defer filename.Dispose()
+	}
+
+	return clang.ChildVisit_Continue
+}
+
+func ParseHeaderFile(filepaths []string) ([]types.ASTInformation, error) {
+
+	index := clang.CreateIndex(0, 0)
+	args := make([]*c.Char, 3)
+	args[0] = c.Str("-x")
+	args[1] = c.Str("c++")
+	args[2] = c.Str("-std=c++11")
+	context = newContext()
+
+	for _, filename := range filepaths {
+		unit := index.ParseTranslationUnit(
+			c.AllocaCStr(filename),
+			unsafe.SliceData(args), 3,
+			nil, 0,
+			clang.TranslationUnit_None,
+		)
+
+		if unit == nil {
+			return nil, errors.New("Unable to parse translation unit for file " + filename)
+		}
+
+		cursor := unit.Cursor()
+		context.setCurrentFile(filename)
+
+		clang.VisitChildren(cursor, visit, nil)
+
+		unit.Dispose()
+	}
+
+	index.Dispose()
+
+	return context.astInfo, nil
+}

chore/llcppg/types/types.go

@@ -16,6 +16,10 @@
 
 package types
 
+import (
+	"github.com/goplus/llgo/xtool/nm"
+)
+
 // Config represents a configuration for the llcppg tool.
 type Config struct {
 	Name         string   `json:"name"`
@@ -24,3 +28,30 @@ type Config struct {
 	Include      []string `json:"include"`
 	TrimPrefixes []string `json:"trimPrefixes"`
 }
+
+type CPPSymbol struct {
+	DemangleName string
+	*nm.Symbol
+}
+
+type ASTInformation struct {
+	Namespace   string      `json:"namespace"`
+	Class       string      `json:"class"`
+	Name        string      `json:"name"`
+	BaseClasses []string    `json:"baseClasses"`
+	ReturnType  string      `json:"returnType"`
+	Location    string      `json:"location"`
+	Parameters  []Parameter `json:"parameters"`
+	Symbol      string      `json:"symbol"`
+}
+
+type Parameter struct {
+	Name string `json:"name"`
+	Type string `json:"type"`
+}
+
+type SymbolInfo struct {
+	Mangle string `json:"mangle"` // C++ Symbol
+	CPP    string `json:"c++"`    // C++ function name
+	Go     string `json:"go"`     // Go function name
+}