build: separate compiler and libs

compiler/chore/_deprecated/ar/ar.go

@@ -0,0 +1,259 @@
+/*
+ * Copyright (c) 2023 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ar
+
+import (
+	"bytes"
+	"debug/elf"
+	"debug/pe"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	wasm "github.com/aykevl/go-wasm"
+	"github.com/goplus/llgo/xtool/ar"
+)
+
+// Create creates an arcive for static linking from a list of object files
+// given as a parameter. It is equivalent to the following command:
+//
+//	ar -rcs <archivePath> <objs...>
+func Create(arfile string, objs []string) error {
+	f, err := os.Create(arfile)
+	if err != nil {
+		return err
+	}
+	err = Make(f, objs)
+	f.Close()
+	if err != nil {
+		os.Remove(arfile)
+	}
+	return err
+}
+
+// Make creates an arcive for static linking from a list of object files
+// given as a parameter. It is equivalent to the following command:
+//
+//	ar -rcs <archivePath> <objs...>
+func Make(arfile io.WriteSeeker, objs []string) error {
+	// Open the archive file.
+	arwriter := ar.NewWriter(arfile)
+	err := arwriter.WriteGlobalHeader()
+	if err != nil {
+		return err
+	}
+
+	// Open all object files and read the symbols for the symbol table.
+	symbolTable := []struct {
+		name      string // symbol name
+		fileIndex int    // index into objfiles
+	}{}
+	archiveOffsets := make([]int32, len(objs))
+	for i, objpath := range objs {
+		objfile, err := os.Open(objpath)
+		if err != nil {
+			return err
+		}
+
+		// Read the symbols and add them to the symbol table.
+		if dbg, err := elf.NewFile(objfile); err == nil {
+			symbols, err := dbg.Symbols()
+			if err != nil {
+				return err
+			}
+			for _, symbol := range symbols {
+				bind := elf.ST_BIND(symbol.Info)
+				if bind != elf.STB_GLOBAL && bind != elf.STB_WEAK {
+					// Don't include local symbols (STB_LOCAL).
+					continue
+				}
+				if elf.ST_TYPE(symbol.Info) != elf.STT_FUNC && elf.ST_TYPE(symbol.Info) != elf.STT_OBJECT {
+					// Not a function.
+					continue
+				}
+				// Include in archive.
+				symbolTable = append(symbolTable, struct {
+					name      string
+					fileIndex int
+				}{symbol.Name, i})
+			}
+		} else if dbg, err := pe.NewFile(objfile); err == nil {
+			for _, symbol := range dbg.Symbols {
+				if symbol.StorageClass != 2 {
+					continue
+				}
+				if symbol.SectionNumber == 0 {
+					continue
+				}
+				symbolTable = append(symbolTable, struct {
+					name      string
+					fileIndex int
+				}{symbol.Name, i})
+			}
+		} else if dbg, err := wasm.Parse(objfile); err == nil {
+			for _, s := range dbg.Sections {
+				switch section := s.(type) {
+				case *wasm.SectionImport:
+					for _, ln := range section.Entries {
+
+						if ln.Kind != wasm.ExtKindFunction {
+							// Not a function
+							continue
+						}
+						symbolTable = append(symbolTable, struct {
+							name      string
+							fileIndex int
+						}{ln.Field, i})
+					}
+				}
+			}
+		} else {
+			return fmt.Errorf("failed to open file %s as WASM, ELF or PE/COFF: %w", objpath, err)
+		}
+
+		// Close file, to avoid issues with too many open files (especially on
+		// MacOS X).
+		objfile.Close()
+	}
+
+	// Create the symbol table buffer.
+	// For some (sparse) details on the file format:
+	// https://en.wikipedia.org/wiki/Ar_(Unix)#System_V_(or_GNU)_variant
+	buf := &bytes.Buffer{}
+	binary.Write(buf, binary.BigEndian, int32(len(symbolTable)))
+	for range symbolTable {
+		// This is a placeholder index, it will be updated after all files have
+		// been written to the archive (see the end of this function).
+		err = binary.Write(buf, binary.BigEndian, int32(0))
+		if err != nil {
+			return err
+		}
+	}
+	for _, sym := range symbolTable {
+		_, err := buf.Write([]byte(sym.name + "\x00"))
+		if err != nil {
+			return err
+		}
+	}
+	for buf.Len()%2 != 0 {
+		// The symbol table must be aligned.
+		// This appears to be required by lld.
+		buf.WriteByte(0)
+	}
+
+	// Write the symbol table.
+	err = arwriter.WriteHeader(&ar.Header{
+		Name:    "/",
+		ModTime: time.Unix(0, 0),
+		Uid:     0,
+		Gid:     0,
+		Mode:    0,
+		Size:    int64(buf.Len()),
+	})
+	if err != nil {
+		return err
+	}
+
+	// Keep track of the start of the symbol table.
+	symbolTableStart, err := arfile.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return err
+	}
+
+	// Write symbol table contents.
+	_, err = arfile.Write(buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	// Add all object files to the archive.
+	var copyBuf bytes.Buffer
+	for i, objpath := range objs {
+		objfile, err := os.Open(objpath)
+		if err != nil {
+			return err
+		}
+		defer objfile.Close()
+
+		// Store the start index, for when we'll update the symbol table with
+		// the correct file start indices.
+		offset, err := arfile.Seek(0, io.SeekCurrent)
+		if err != nil {
+			return err
+		}
+		if int64(int32(offset)) != offset {
+			return errors.New("large archives (4GB+) not supported")
+		}
+		archiveOffsets[i] = int32(offset)
+
+		// Write the file header.
+		st, err := objfile.Stat()
+		if err != nil {
+			return err
+		}
+		err = arwriter.WriteHeader(&ar.Header{
+			Name:    filepath.Base(objfile.Name()),
+			ModTime: time.Unix(0, 0),
+			Uid:     0,
+			Gid:     0,
+			Mode:    0644,
+			Size:    st.Size(),
+		})
+		if err != nil {
+			return err
+		}
+
+		// Copy the file contents into the archive.
+		// First load all contents into a buffer, then write it all in one go to
+		// the archive file. This is a bit complicated, but is necessary because
+		// io.Copy can't deal with files that are of an odd size.
+		copyBuf.Reset()
+		n, err := io.Copy(&copyBuf, objfile)
+		if err != nil {
+			return fmt.Errorf("could not copy object file into ar file: %w", err)
+		}
+		if n != st.Size() {
+			return errors.New("file modified during ar creation")
+		}
+		_, err = arwriter.Write(copyBuf.Bytes())
+		if err != nil {
+			return fmt.Errorf("could not copy object file into ar file: %w", err)
+		}
+
+		// File is not needed anymore.
+		objfile.Close()
+	}
+
+	// Create symbol indices.
+	indicesBuf := &bytes.Buffer{}
+	for _, sym := range symbolTable {
+		err = binary.Write(indicesBuf, binary.BigEndian, archiveOffsets[sym.fileIndex])
+		if err != nil {
+			return err
+		}
+	}
+
+	// Overwrite placeholder indices.
+	if _, err = arfile.Seek(symbolTableStart+4, io.SeekStart); err == nil {
+		_, err = arfile.Write(indicesBuf.Bytes())
+	}
+	return err
+}

compiler/chore/_deprecated/clang/parser/pages.go

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2022 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+// -----------------------------------------------------------------------------
+
+const pageSize = 1024 * 1024
+
+type PagedWriter struct {
+	pages []*[pageSize]byte
+	last  *[pageSize]byte
+	off   int
+}
+
+func NewPagedWriter() *PagedWriter {
+	return &PagedWriter{last: new([pageSize]byte)}
+}
+
+func (p *PagedWriter) Write(buf []byte) (written int, err error) {
+	for {
+		n := copy(p.last[p.off:], buf[written:])
+		written += n
+		if written >= len(buf) {
+			p.off += n
+			return
+		}
+		p.pages = append(p.pages, p.last)
+		p.last, p.off = new([pageSize]byte), 0
+	}
+}
+
+func (p *PagedWriter) Len() int {
+	return len(p.pages)*pageSize + p.off
+}
+
+func (p *PagedWriter) Bytes() []byte {
+	out, n := make([]byte, p.Len()), 0
+	for _, page := range p.pages {
+		n += copy(out[n:], page[:])
+	}
+	copy(out[n:], p.last[:p.off])
+	return out
+}
+
+/*
+func (p *PagedWriter) ToReader() *PagedReader {
+	return &PagedReader{src: p, curr: p.getPage(0)}
+}
+
+func (p *PagedWriter) getPage(ipage int) []byte {
+	if ipage == len(p.pages) { // last page
+		return p.last[:p.off]
+	}
+	return p.pages[ipage][:]
+}
+
+// -----------------------------------------------------------------------------
+
+type PagedReader struct {
+	src   *PagedWriter
+	curr  []byte
+	off   int
+	ipage int
+}
+
+func (p *PagedReader) WriteTo(w io.Writer) (written int64, err error) {
+	n, err := w.Write(p.curr[p.off:])
+	written = int64(n)
+	if err != nil {
+		return
+	}
+	src, ipage := p.src, p.ipage
+	for {
+		if ipage == len(src.pages) { // last page
+			p.ipage, p.off = ipage, len(p.curr)
+			return
+		}
+		ipage++
+		page := src.getPage(ipage)
+		n, err = w.Write(page)
+		written += int64(n)
+		if err != nil {
+			p.ipage, p.curr, p.off = ipage, page, n
+			return
+		}
+	}
+}
+
+func (p *PagedReader) Read(buf []byte) (nread int, err error) {
+	for {
+		n := copy(buf[nread:], p.curr[p.off:])
+		nread += n
+		p.off += n
+		if nread >= len(buf) {
+			return
+		}
+		src := p.src
+		if p.ipage == len(src.pages) { // last page
+			err = io.EOF
+			return
+		}
+		p.ipage++
+		p.curr, p.off = src.getPage(p.ipage), 0
+	}
+}
+*/
+
+// -----------------------------------------------------------------------------

compiler/chore/_deprecated/clang/parser/parse.go

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2022 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/goplus/llgo/xtool/clang/ast"
+	jsoniter "github.com/json-iterator/go"
+)
+
+type Mode uint
+
+// -----------------------------------------------------------------------------
+
+type ParseError struct {
+	Err    error
+	Stderr []byte
+}
+
+func (p *ParseError) Error() string {
+	if len(p.Stderr) > 0 {
+		return string(p.Stderr)
+	}
+	return p.Err.Error()
+}
+
+// -----------------------------------------------------------------------------
+
+type Config struct {
+	Json   *[]byte
+	Flags  []string
+	Stderr bool
+}
+
+func DumpAST(filename string, conf *Config) (result []byte, warning []byte, err error) {
+	if conf == nil {
+		conf = new(Config)
+	}
+	skiperr := strings.HasSuffix(filename, "vfprintf.c.i")
+	stdout := NewPagedWriter()
+	stderr := new(bytes.Buffer)
+	args := []string{"-Xclang", "-ast-dump=json", "-fsyntax-only", filename}
+	if len(conf.Flags) != 0 {
+		args = append(conf.Flags, args...)
+	}
+	cmd := exec.Command("clang", args...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = stdout
+	if conf.Stderr && !skiperr {
+		cmd.Stderr = os.Stderr
+	} else {
+		cmd.Stderr = stderr
+	}
+	err = cmd.Run()
+	errmsg := stderr.Bytes()
+	if err != nil && !skiperr {
+		return nil, nil, &ParseError{Err: err, Stderr: errmsg}
+	}
+	return stdout.Bytes(), errmsg, nil
+}
+
+// -----------------------------------------------------------------------------
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+func ParseFileEx(filename string, mode Mode, conf *Config) (file *ast.Node, warning []byte, err error) {
+	out, warning, err := DumpAST(filename, conf)
+	if err != nil {
+		return
+	}
+	if conf != nil && conf.Json != nil {
+		*conf.Json = out
+	}
+	file = new(ast.Node)
+	err = json.Unmarshal(out, file)
+	if err != nil {
+		err = &ParseError{Err: err}
+	}
+	return
+}
+
+func ParseFile(filename string, mode Mode) (file *ast.Node, warning []byte, err error) {
+	return ParseFileEx(filename, mode, nil)
+}
+
+// -----------------------------------------------------------------------------

compiler/chore/_deprecated/clangast/clangast.go

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2022 The GoPlus Authors (goplus.org). All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/goplus/llgo/_xtool/clang/parser"
+)
+
+var (
+	dump = flag.Bool("dump", false, "dump AST")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "Usage: clangast [-dump] source.i\n")
+	flag.PrintDefaults()
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if flag.NArg() < 1 {
+		usage()
+		return
+	}
+	var file = flag.Arg(0)
+	var err error
+	if *dump {
+		doc, _, e := parser.DumpAST(file, nil)
+		if e == nil {
+			os.Stdout.Write(doc)
+			return
+		}
+		err = e
+	} else {
+		doc, _, e := parser.ParseFile(file, 0)
+		if e == nil {
+			enc := json.NewEncoder(os.Stdout)
+			enc.SetIndent("", "  ")
+			check(enc.Encode(doc))
+			return
+		}
+		err = e
+	}
+	fmt.Fprintln(os.Stderr, err)
+	os.Exit(1)
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

compiler/chore/_deprecated/go.mod

@@ -0,0 +1,14 @@
+module github.com/goplus/llgo/_xtool
+
+go 1.20
+
+require (
+	github.com/aykevl/go-wasm v0.0.1
+	github.com/goplus/llgo v0.9.0
+	github.com/json-iterator/go v1.1.12
+)
+
+require (
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+)

compiler/chore/_deprecated/go.sum

@@ -0,0 +1,19 @@
+github.com/aykevl/go-wasm v0.0.1 h1:lPxy8l48P39W7I0tLrtCrLfZBOUq9IWZ7odGdyJP2AM=
+github.com/aykevl/go-wasm v0.0.1/go.mod h1:b4nggwg3lEkNKOU4wzhtLKz2q2sLxSHFnc98aGt6z/Y=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/goplus/llgo v0.9.0 h1:yaJzQperGUafEaHc9VlVQVskIngacoTNweEXY0GRi0Q=
+github.com/goplus/llgo v0.9.0/go.mod h1:M3UwiYdPZFyx7m2J0+6Ti1dYVA3uOO1WvSBocuE8N7M=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=