build: support multiple link files in a package

.gitignore

@@ -8,6 +8,7 @@
 *.so
 *.dylib
 
+test.db
 llgo_autogen.ll
 stories*.bin
 .DS_Store

_demo/sqlite/sqlite.go

@@ -6,32 +6,34 @@ import (
 )
 
 func main() {
-	db, err := sqlite.OpenV2(c.Str(":memory:"), sqlite.OpenReadWrite|sqlite.OpenMemory, nil)
-	check(err)
+	c.Remove(c.Str("test.db"))
+
+	db, err := sqlite.Open(c.Str("test.db"))
+	check(err, "sqlite: OpenV2")
 
 	err = db.Exec(c.Str("CREATE TABLE foo (id INT, name TEXT)"), nil, nil, nil)
-	check(err)
+	check(err, "sqlite: Exec CREATE TABLE")
 
-	stmt, err := db.PrepareV3("INSERT INTO foo VALUES (?, ?)", 0, nil)
-	check(err)
+	stmt, err := db.PrepareV3("INSERT INTO foo (id, name) VALUES (?, ?)", 0, nil)
+	check(err, "sqlite: PrepareV3 INSERT")
 
 	stmt.BindInt(1, 100)
-	stmt.BindText(2, c.Str("Hello World"), sqlite.Static, nil)
+	stmt.BindText(2, c.Str("Hello World"), -1, nil)
 
 	err = stmt.Step()
-	checkDone(err)
+	checkDone(err, "sqlite: Step INSERT 1")
 
 	stmt.Reset()
 	stmt.BindInt(1, 200)
-	stmt.BindText(2, c.Str("This is llgo"), sqlite.Static, nil)
+	stmt.BindText(2, c.Str("This is llgo"), -1, nil)
 
 	err = stmt.Step()
-	checkDone(err)
+	checkDone(err, "sqlite: Step INSERT 2")
 
 	stmt.Close()
 
 	stmt, err = db.PrepareV3("SELECT * FROM foo", 0, nil)
-	check(err)
+	check(err, "sqlite: PrepareV3 SELECT")
 
 	for {
 		if err = stmt.Step(); err != sqlite.HasRow {
@@ -39,21 +41,21 @@ func main() {
 		}
 		c.Printf(c.Str("==> id=%d, name=%s\n"), stmt.ColumnInt(0), stmt.ColumnText(1))
 	}
-	checkDone(err)
+	checkDone(err, "sqlite: Step done")
 
 	stmt.Close()
 	db.Close()
 }
 
-func check(err sqlite.Errno) {
+func check(err sqlite.Errno, at string) {
 	if err != sqlite.OK {
-		c.Printf(c.Str("==> Error: (%d) %s\n"), err, err.Errstr())
+		c.Printf(c.Str("==> %s Error: (%d) %s\n"), c.AllocaCStr(at), err, err.Errstr())
 		c.Exit(1)
 	}
 }
 
-func checkDone(err sqlite.Errno) {
+func checkDone(err sqlite.Errno, at string) {
 	if err != sqlite.Done {
-		check(err)
+		check(err, at)
 	}
 }

c/c.go

@@ -71,6 +71,11 @@ func GoStringData(string) *Char
 
 // -----------------------------------------------------------------------------
 
+//go:linkname Remove C.remove
+func Remove(path *Char) Int
+
+// -----------------------------------------------------------------------------
+
 //go:linkname Exit C.exit
 func Exit(Int)
 

cl/compile.go

@@ -123,7 +123,8 @@ const (
 	PkgLLGo
 	PkgNoInit   // noinit: a package that don't need to be initialized
 	PkgDeclOnly // decl: a package that only have declarations
-	PkgLinkOnly // link: a package that don't need to be compiled but need to be linked
+	PkgLinkIR   // link llvm ir (.ll)
+	// PkgLinkBitCode // link bitcode (.bc)
 )
 
 type pkgInfo struct {

cl/import.go

@@ -91,7 +91,9 @@ func PkgKindOf(pkg *types.Package) int {
 func pkgKind(v string) int {
 	switch v {
 	case "link":
-		return PkgLinkOnly
+		return PkgLinkIR
+	// case "link:bc":
+	//	return PkgLinkBitCode
 	case "decl":
 		return PkgDeclOnly
 	case "noinit":

internal/build/build.go

@@ -167,16 +167,16 @@ func buildAllPkgs(prog llssa.Program, initial []*packages.Package, mode Mode, ve
 	}
 	for _, aPkg := range pkgs {
 		pkg := aPkg.Package
-		switch cl.PkgKindOf(pkg.Types) {
+		switch kind := cl.PkgKindOf(pkg.Types); kind {
 		case cl.PkgDeclOnly:
 			// skip packages that only contain declarations
 			// and set no export file
 			pkg.ExportFile = ""
-		case cl.PkgLinkOnly:
+		case cl.PkgLinkIR: // cl.PkgLinkBitCode:
 			// skip packages that don't need to be compiled but need to be linked
 			pkgPath := pkg.PkgPath
 			if isPkgInLLGo(pkgPath) {
-				pkg.ExportFile = strings.TrimSuffix(llgoPkgLinkFile(pkgPath), ".ll")
+				pkg.ExportFile = concatPkgLinkFiles(pkgPath)
 			} else {
 				panic("todo")
 			}
@@ -205,7 +205,7 @@ func linkMainPkg(pkg *packages.Package, pkgs []*aPackage, runtimeFiles []string,
 	needRuntime := false
 	packages.Visit([]*packages.Package{pkg}, nil, func(p *packages.Package) {
 		if p.ExportFile != "" && !isRuntimePkg(p.PkgPath) { // skip packages that only contain declarations
-			args = append(args, p.ExportFile+".ll")
+			args = appendLinkFiles(args, p.ExportFile)
 			if !needRuntime {
 				needRuntime = isNeedRuntime(p)
 			}
@@ -219,8 +219,7 @@ func linkMainPkg(pkg *packages.Package, pkgs []*aPackage, runtimeFiles []string,
 				lpkg := aPkg.LPkg
 				lpkg.FuncOf(cl.RuntimeInit).MakeBody(1).Return()
 				if needLLFile(mode) {
-					file := pkg.ExportFile + ".ll"
-					os.WriteFile(file, []byte(lpkg.String()), 0644)
+					os.WriteFile(pkg.ExportFile, []byte(lpkg.String()), 0644)
 				}
 				break
 			}
@@ -265,8 +264,8 @@ func buildPkg(prog llssa.Program, aPkg *aPackage, mode Mode, verbose bool) {
 	ret, err := cl.NewPackage(prog, aPkg.SSA, pkg.Syntax)
 	check(err)
 	if needLLFile(mode) {
-		file := pkg.ExportFile + ".ll"
-		os.WriteFile(file, []byte(ret.String()), 0644)
+		pkg.ExportFile += ".ll"
+		os.WriteFile(pkg.ExportFile, []byte(ret.String()), 0644)
 	}
 	aPkg.LPkg = ret
 }
@@ -358,8 +357,9 @@ func allLinkFiles(rt []*packages.Package) (outFiles []string) {
 	packages.Visit(rt, nil, func(p *packages.Package) {
 		pkgPath := p.PkgPath
 		if isRuntimePkg(pkgPath) {
-			outFile := llgoPkgLinkFile(pkgPath)
-			outFiles = append(outFiles, outFile)
+			llgoPkgLinkFiles(pkgPath, func(linkFile string) {
+				outFiles = append(outFiles, linkFile)
+			})
 		}
 	})
 	return
@@ -393,12 +393,57 @@ func llgoRoot() string {
 	return rootDir
 }
 
-func llgoPkgLinkFile(pkgPath string) string {
-	llFile := filepath.Join(llgoRoot()+pkgPath[len(llgoModPath):], "llgo_autogen.ll")
-	if _, err := os.Stat(llFile); os.IsNotExist(err) {
-		decodeLinkFile(llFile)
+func appendLinkFiles(args []string, file string) []string {
+	if isMultiLinkFiles(file) {
+		return append(args, strings.Split(file[1:], " ")...)
+	}
+	return append(args, file)
+}
+
+func isMultiLinkFiles(ret string) bool {
+	return len(ret) > 0 && ret[0] == ' '
+}
+
+func concatPkgLinkFiles(pkgPath string) string {
+	var b strings.Builder
+	var ret string
+	var n int
+	llgoPkgLinkFiles(pkgPath, func(linkFile string) {
+		if n == 0 {
+			ret = linkFile
+		} else {
+			b.WriteByte(' ')
+			b.WriteString(linkFile)
+		}
+		n++
+	})
+	if n > 1 {
+		b.WriteByte(' ')
+		b.WriteString(ret)
+		return b.String()
+	}
+	return ret
+}
+
+func llgoPkgLinkFiles(pkgPath string, procFile func(linkFile string)) {
+	dir := llgoRoot() + pkgPath[len(llgoModPath):] + "/"
+	llFile := dir + "llgo_autogen.ll"
+	llaFile := llFile + "a"
+	zipf, err := zip.OpenReader(llaFile)
+	if err != nil {
+		procFile(llFile)
+		return
+	}
+	defer zipf.Close()
+
+	for _, f := range zipf.File {
+		procFile(dir + f.Name)
+	}
+	if _, err := os.Stat(llFile); os.IsNotExist(err) {
+		for _, f := range zipf.File {
+			decodeFile(dir+f.Name, f)
+		}
 	}
-	return llFile
 }
 
 const (
@@ -417,6 +462,18 @@ func isPkgInMod(pkgPath, modPath string) bool {
 	return false
 }
 
+/*
+func llgoPkgLinkFile(pkgPath string) string {
+	// if kind == cl.PkgLinkBitCode {
+	//	return filepath.Join(llgoRoot()+pkgPath[len(llgoModPath):], "llgo_autogen.bc")
+	// }
+	llFile := filepath.Join(llgoRoot()+pkgPath[len(llgoModPath):], "llgo_autogen.ll")
+	if _, err := os.Stat(llFile); os.IsNotExist(err) {
+		decodeLinkFile(llFile)
+	}
+	return llFile
+}
+
 // *.ll => *.lla
 func decodeLinkFile(llFile string) {
 	zipFile := llFile + "a"
@@ -435,6 +492,20 @@ func decodeLinkFile(llFile string) {
 		os.WriteFile(llFile, data, 0644)
 	}
 }
+*/
+
+func decodeFile(outFile string, zipf *zip.File) (err error) {
+	f, err := zipf.Open()
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	data, err := io.ReadAll(f)
+	if err == nil {
+		err = os.WriteFile(outFile, data, 0644)
+	}
+	return
+}
 
 func check(err error) {
 	if err != nil {

internal/llgen/llgenf.go

@@ -19,6 +19,7 @@ package llgen
 import (
 	"go/types"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -74,10 +75,11 @@ func DoFile(fileOrPkg, outFile string) {
 }
 
 func SmartDoFile(inFile string, pkgPath ...string) {
+	const autgenFile = "llgo_autogen.ll"
 	dir, _ := filepath.Split(inFile)
 	absDir, _ := filepath.Abs(dir)
 	absDir = filepath.ToSlash(absDir)
-	fname := "llgo_autogen.ll"
+	fname := autgenFile
 	if inCompilerDir(absDir) {
 		fname = "out.ll"
 	} else if inSqlite(absDir) {
@@ -90,6 +92,17 @@ func SmartDoFile(inFile string, pkgPath ...string) {
 	} else {
 		DoFile(inFile, outFile)
 	}
+	if false && fname == autgenFile {
+		genZip(absDir, "llgo_autogen.lla", autgenFile)
+	}
+}
+
+func genZip(dir string, outFile, inFile string) {
+	cmd := exec.Command("zip", outFile, inFile)
+	cmd.Dir = dir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Run()
 }
 
 func inCompilerDir(dir string) bool {

x/llvm/llvmlink/link.go

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023 The GoPlus Authors (goplus.org). All rights reserved.
+ * Copyright (c) 2024 The GoPlus Authors (goplus.org). All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,37 @@
  * limitations under the License.
  */
 
-package gocmd
+package llvmlink
+
+import (
+	"io"
+	"os"
+	"os/exec"
+)
 
 // -----------------------------------------------------------------------------
 
-type BuildConfig struct {
-	Output string
+// Cmd represents a llvm-link command.
+type Cmd struct {
+	app string
+
+	Stdout io.Writer
+	Stderr io.Writer
+}
+
+// New creates a new llvm-link command.
+func New(app string) *Cmd {
+	if app == "" {
+		app = os.Getenv("LLGO_LLVM_ROOT") + "/bin/llvm-link"
+	}
+	return &Cmd{app, os.Stdout, os.Stderr}
+}
+
+func (p *Cmd) Exec(args ...string) error {
+	cmd := exec.Command(p.app, args...)
+	cmd.Stdout = p.Stdout
+	cmd.Stderr = p.Stderr
+	return cmd.Run()
 }
 
 // -----------------------------------------------------------------------------

x/sqlite/llgo.cfg

@@ -4,7 +4,7 @@
         "cd build.dir",
         "../sqlite/configure",
         "make",
-        "clang -emit-llvm -S -o sqlite3.ll -c sqlite3.c",
-        "$LLGO_LLVM_ROOT/bin/llvm-link -o ../llgo_autogen.bc ../sqlite.ll sqlite3.ll",
+        "clang -emit-llvm -S -o ../llgo_autogen.ll -c sqlite3.c",
+        "cd ..; rm llgo_autogen.lla; zip llgo_autogen.lla llgo_autogen.ll sqlite.ll",
     ]
 }

x/sqlite/sqlite.go

@@ -209,13 +209,15 @@ func (*Stmt) BindInt(idx Int, val Int) Errno { return 0 }
 // llgo:link (*Stmt).BindInt64 C.sqlite3_bind_int64
 func (*Stmt) BindInt64(idx Int, val int64) Errno { return 0 }
 
+/*
 const (
-	Static    Int = 0  // val is a static string
-	Transient Int = -1 // val is a transient (temporary) string
+	Static    = (func(Pointer))(nil) // val is a static string
+	Transient = (func(Pointer))(-1)  // val is a transient (temporary) string
 )
+*/
 
 // llgo:link (*Stmt).BindText C.sqlite3_bind_text
-func (*Stmt) BindText(idx Int, val *Char, lenOrKind Int, destructor func(Pointer)) Errno { return 0 }
+func (*Stmt) BindText(idx Int, val *Char, nByte Int, destructor func(Pointer)) Errno { return 0 }
 
 // -----------------------------------------------------------------------------