diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..f405b33 --- /dev/null +++ b/.cursorrules @@ -0,0 +1,29 @@ +You are an expert AI programming assistant specializing in building APIs with Go, using the standard library's net/http package and the new ServeMux introduced in Go 1.22. + +Always use the latest stable version of Go (1.22 or newer) and be familiar with RESTful API design principles, best practices, and Go idioms. + +- Follow the user's requirements carefully & to the letter. +- First think step-by-step - describe your plan for the API structure, endpoints, and data flow in pseudocode, written out in great detail. +- Confirm the plan, then write code! +- Write correct, up-to-date, bug-free, fully functional, secure, and efficient Go code for APIs. +- Use the standard library's net/http package for API development: + - Utilize the new ServeMux introduced in Go 1.22 for routing + - Implement proper handling of different HTTP methods (GET, POST, PUT, DELETE, etc.) + - Use method handlers with appropriate signatures (e.g., func(w http.ResponseWriter, r \*http.Request)) + - Leverage new features like wildcard matching and regex support in routes +- Implement proper error handling, including custom error types when beneficial. +- Use appropriate status codes and format JSON responses correctly. +- Implement input validation for API endpoints. +- Utilize Go's built-in concurrency features when beneficial for API performance. +- Follow RESTful API design principles and best practices. +- Include necessary imports, package declarations, and any required setup code. +- Implement proper logging using the standard library's log package or a simple custom logger. +- Consider implementing middleware for cross-cutting concerns (e.g., logging, authentication). +- Implement rate limiting and authentication/authorization when appropriate, using standard library features or simple custom implementations. +- Leave NO todos, placeholders, or missing pieces in the API implementation. +- Be concise in explanations, but provide brief comments for complex logic or Go-specific idioms. +- Always use English in comments and code. +- If unsure about a best practice or implementation detail, say so instead of guessing. +- Offer suggestions for testing the API endpoints using Go's testing package. + +Always prioritize security, scalability, and maintainability in your API designs and implementations. Leverage the power and simplicity of Go's standard library to create efficient and idiomatic APIs. diff --git a/.github/assets/python3-embed.pc b/.github/assets/python3-embed.pc new file mode 100644 index 0000000..9399f83 --- /dev/null +++ b/.github/assets/python3-embed.pc @@ -0,0 +1,12 @@ +prefix=${pcfiledir}/../.. +exec_prefix=${prefix} +libdir=${exec_prefix} +includedir=${prefix}/include + +Name: Python +Description: Embed Python into an application +Requires: +Version: 3.13 +Libs.private: +Libs: -L${libdir} -lpython313 +Cflags: -I${includedir} \ No newline at end of file diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b34578b..26665a3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -43,11 +43,24 @@ jobs: strategy: fail-fast: false matrix: - os: - - macos-latest - - ubuntu-24.04 - runs-on: ${{matrix.os}} + sys: + - {os: macos-latest, shell: bash} + - {os: ubuntu-24.04, shell: bash} + - {os: windows-latest, shell: bash} + defaults: + run: + shell: ${{ matrix.sys.shell }} + runs-on: ${{matrix.sys.os}} steps: + # - uses: msys2/setup-msys2@v2 + # if: matrix.sys.os == 'windows-latest' + # with: + # update: true + # install: >- + # curl + # git + # pkg-config + - uses: actions/checkout@v4 - name: Set up Go @@ -55,14 +68,44 @@ jobs: with: go-version: 1.23 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + update-environment: true + + - name: Generate Python pkg-config for windows (patch) + if: matrix.sys.os == 'windows-latest' + run: | + mkdir -p $PKG_CONFIG_PATH + cp .github/assets/python3-embed.pc $PKG_CONFIG_PATH/ + + - name: Install tiny-pkg-config for windows (patch) + if: matrix.sys.os == 'windows-latest' + run: | + set -x + curl -L https://github.com/cpunion/tiny-pkg-config/releases/download/v0.2.0/tiny-pkg-config_Windows_x86_64.zip -o /tmp/tiny-pkg-config.zip + unzip /tmp/tiny-pkg-config.zip -d $HOME/bin + mv $HOME/bin/tiny-pkg-config.exe $HOME/bin/pkg-config.exe + echo $PKG_CONFIG_PATH + cat $PKG_CONFIG_PATH/python3-embed.pc + pkg-config --libs python3-embed + pkg-config --cflags python3-embed + - name: Build - run: go build -v ./... + run: go install -v ./... - name: Test with coverage run: go test -v -coverprofile=coverage.txt -covermode=atomic ./... + + - name: Test gopy + run: | + gopy init $HOME/foo + cd $HOME/foo + gopy build -v . + gopy run -v . + gopy install -v . - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-24.04' uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/cmd/add.go b/cmd/add.go new file mode 100644 index 0000000..cf9e787 --- /dev/null +++ b/cmd/add.go @@ -0,0 +1,39 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// addCmd represents the add command +var addCmd = &cobra.Command{ + Use: "add", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("add called") + }, +} + +func init() { + rootCmd.AddCommand(addCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // addCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // addCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/build.go b/cmd/build.go new file mode 100644 index 0000000..902a3a5 --- /dev/null +++ b/cmd/build.go @@ -0,0 +1,37 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + "os" + + "github.com/cpunion/go-python/cmd/internal/rungo" + "github.com/spf13/cobra" +) + +// buildCmd represents the build command +var buildCmd = &cobra.Command{ + Use: "build [flags] [package]", + Short: "Build a Go package with Python environment configured", + Long: func() string { + intro := "Build compiles a Go package with the Python environment properly configured.\n\n" + help, err := rungo.GetGoCommandHelp("build") + if err != nil { + return intro + "Failed to get go help: " + err.Error() + } + return intro + help + }(), + DisableFlagParsing: true, + Run: func(cmd *cobra.Command, args []string) { + if err := rungo.RunGoCommand("build", args); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + os.Exit(1) + } + }, +} + +func init() { + rootCmd.AddCommand(buildCmd) +} diff --git a/cmd/gopy/gopy.go b/cmd/gopy/gopy.go new file mode 100644 index 0000000..1d8d9db --- /dev/null +++ b/cmd/gopy/gopy.go @@ -0,0 +1,10 @@ +/* +Copyright © 2024 NAME HERE +*/ +package main + +import "github.com/cpunion/go-python/cmd" + +func main() { + cmd.Execute() +} diff --git a/cmd/init.go b/cmd/init.go new file mode 100644 index 0000000..5174e31 --- /dev/null +++ b/cmd/init.go @@ -0,0 +1,127 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "bufio" + "fmt" + "io" + "os" + "strings" + + "github.com/cpunion/go-python/cmd/internal/create" + "github.com/cpunion/go-python/cmd/internal/install" + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +var bold = color.New(color.Bold).SprintFunc() + +// isDirEmpty checks if a directory is empty +func isDirEmpty(path string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// promptYesNo asks user for confirmation +func promptYesNo(prompt string) bool { + reader := bufio.NewReader(os.Stdin) + fmt.Printf("%s [y/N]: ", prompt) + response, err := reader.ReadString('\n') + if err != nil { + return false + } + + response = strings.ToLower(strings.TrimSpace(response)) + return response == "y" || response == "yes" +} + +// initCmd represents the init command +var initCmd = &cobra.Command{ + Use: "init [path]", + Short: "Initialize a new go-python project", + Long: `Initialize a new go-python project in the specified directory. +If no path is provided, it will initialize in the current directory. + +Example: + gopy init + gopy init my-project + gopy init --debug my-project + gopy init -v my-project`, + Run: func(cmd *cobra.Command, args []string) { + // Get project path + projectPath := "." + if len(args) > 0 { + projectPath = args[0] + } + + // Get flags + debug, _ := cmd.Flags().GetBool("debug") + verbose, _ := cmd.Flags().GetBool("verbose") + goVersion, _ := cmd.Flags().GetString("go-version") + pyVersion, _ := cmd.Flags().GetString("python-version") + pyBuildDate, _ := cmd.Flags().GetString("python-build-date") + pyFreeThreaded, _ := cmd.Flags().GetBool("python-free-threaded") + tinyPkgConfigVersion, _ := cmd.Flags().GetString("tiny-pkg-config-version") + + // Check if directory exists + if _, err := os.Stat(projectPath); err == nil { + // Directory exists, check if it's empty + empty, err := isDirEmpty(projectPath) + if err != nil { + fmt.Printf("Error checking directory: %v\n", err) + return + } + + if !empty { + if !promptYesNo(fmt.Sprintf("Directory %s is not empty. Do you want to continue?", projectPath)) { + fmt.Println("Operation cancelled") + return + } + } + } else if !os.IsNotExist(err) { + fmt.Printf("Error checking directory: %v\n", err) + return + } + + // Create project using the create package + fmt.Printf("\n%s\n", bold("Creating project...")) + if err := create.Project(projectPath, verbose); err != nil { + fmt.Printf("Error creating project: %v\n", err) + return + } + + // Install dependencies + fmt.Printf("\n%s\n", bold("Installing dependencies...")) + if err := install.Dependencies(projectPath, goVersion, tinyPkgConfigVersion, pyVersion, pyBuildDate, pyFreeThreaded, debug, verbose); err != nil { + fmt.Printf("Error installing dependencies: %v\n", err) + return + } + + fmt.Printf("\n%s\n", bold("Successfully initialized go-python project in "+projectPath)) + fmt.Println("\nNext steps:") + fmt.Println("1. cd", projectPath) + fmt.Println("2. gopy run .") + }, +} + +func init() { + rootCmd.AddCommand(initCmd) + initCmd.Flags().Bool("debug", false, "Install debug version of Python (not available on Windows)") + initCmd.Flags().BoolP("verbose", "v", false, "Enable verbose output") + initCmd.Flags().String("tiny-pkg-config-version", "v0.2.0", "tiny-pkg-config version to install") + initCmd.Flags().String("go-version", "1.23.3", "Go version to install") + initCmd.Flags().String("python-version", "3.13.0", "Python version to install") + initCmd.Flags().String("python-build-date", "20241016", "Python build date") + initCmd.Flags().Bool("python-free-threaded", false, "Install free-threaded version of Python") +} diff --git a/cmd/install.go b/cmd/install.go new file mode 100644 index 0000000..b7a8497 --- /dev/null +++ b/cmd/install.go @@ -0,0 +1,37 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + "os" + + "github.com/cpunion/go-python/cmd/internal/rungo" + "github.com/spf13/cobra" +) + +// installCmd represents the install command +var installCmd = &cobra.Command{ + Use: "install [flags] [packages]", + Short: "Install Go packages with Python environment configured", + Long: func() string { + intro := "Install compiles and installs Go packages with the Python environment properly configured.\n\n" + help, err := rungo.GetGoCommandHelp("install") + if err != nil { + return intro + "Failed to get go help: " + err.Error() + } + return intro + help + }(), + DisableFlagParsing: true, + Run: func(cmd *cobra.Command, args []string) { + if err := rungo.RunGoCommand("install", args); err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } + }, +} + +func init() { + rootCmd.AddCommand(installCmd) +} diff --git a/cmd/internal/create/create.go b/cmd/internal/create/create.go new file mode 100644 index 0000000..6f6f34b --- /dev/null +++ b/cmd/internal/create/create.go @@ -0,0 +1,152 @@ +package create + +import ( + "bufio" + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/fatih/color" +) + +//go:embed templates/* +var templates embed.FS + +var ( + green = color.New(color.FgGreen).SprintFunc() + yellow = color.New(color.FgYellow).SprintFunc() +) + +// promptOverwrite asks user whether to overwrite a file +func promptOverwrite(path string) (bool, bool) { + reader := bufio.NewReader(os.Stdin) + for { + fmt.Printf("%s %s exists. Overwrite (Yes/No/All)? [y/n/a] ", yellow("conflict"), path) + response, err := reader.ReadString('\n') + if err != nil { + return false, false + } + + response = strings.ToLower(strings.TrimSpace(response)) + switch response { + case "y", "yes": + return true, false // overwrite this file only + case "n", "no": + return false, false // skip this file + case "a", "all": + return true, true // overwrite all files + } + } +} + +// Project initializes a new go-python project in the specified directory +func Project(projectPath string, verbose bool) error { + // Create project directory + if err := os.MkdirAll(projectPath, 0755); err != nil { + return fmt.Errorf("error creating directory: %v", err) + } + + overwriteAll := false + + // Walk through template files and copy them + err := fs.WalkDir(templates, "templates", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip the templates root directory + if path == "templates" { + return nil + } + + // Get relative path from templates directory + relPath, err := filepath.Rel("templates", path) + if err != nil { + return err + } + + // Create destination path + dstPath := filepath.Join(projectPath, relPath) + + // If it's a directory, create it + if d.IsDir() { + if err := os.MkdirAll(dstPath, 0755); err != nil { + return err + } + fmt.Printf("%s\t%s/\n", green("create"), relPath) + return nil + } + + // Check if file exists + _, err = os.Stat(dstPath) + fileExists := err == nil + + if fileExists && !overwriteAll { + overwrite, all := promptOverwrite(relPath) + if !overwrite { + fmt.Printf("%s\t%s\n", yellow("skip"), relPath) + return nil + } + overwriteAll = all + } + + // Read template file + content, err := templates.ReadFile(path) + if err != nil { + return err + } + + // Write file to destination + if err := os.WriteFile(dstPath, content, 0644); err != nil { + return err + } + + // Print status with color + if fileExists { + fmt.Printf("%s\t%s\n", yellow("overwrite"), relPath) + } else { + fmt.Printf("%s\t%s\n", green("create"), relPath) + } + + return nil + }) + + if err != nil { + return fmt.Errorf("error copying template files: %v", err) + } + + // Create go.mod file + goModPath := filepath.Join(projectPath, "go.mod") + goModExists := false + if _, err := os.Stat(goModPath); err == nil { + goModExists = true + if !overwriteAll { + overwrite, _ := promptOverwrite("go.mod") + if !overwrite { + fmt.Printf("%s\tgo.mod\n", yellow("skip")) + return nil + } + } + } + + goModContent := fmt.Sprintf(`module %s + +go 1.23 +`, filepath.Base(projectPath)) + + if err := os.WriteFile(goModPath, []byte(goModContent), 0644); err != nil { + return fmt.Errorf("error writing go.mod: %v", err) + } + + // Print go.mod status + if goModExists { + fmt.Printf("%s\tgo.mod\n", yellow("overwrite")) + } else { + fmt.Printf("%s\tgo.mod\n", green("create")) + } + + return nil +} diff --git a/cmd/internal/create/create_test.go b/cmd/internal/create/create_test.go new file mode 100644 index 0000000..b56b948 --- /dev/null +++ b/cmd/internal/create/create_test.go @@ -0,0 +1,161 @@ +package create + +import ( + "os" + "path/filepath" + "testing" +) + +// setupTestDir creates a temporary directory for testing +func setupTestDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("", "gopy-test-*") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + return dir +} + +// cleanupTestDir removes the temporary test directory +func cleanupTestDir(t *testing.T, dir string) { + t.Helper() + if err := os.RemoveAll(dir); err != nil { + t.Errorf("failed to cleanup test dir: %v", err) + } +} + +func TestProject(t *testing.T) { + tests := []struct { + name string + setup func(dir string) error + wantErr bool + }{ + { + name: "create new project in empty directory", + setup: func(dir string) error { + return nil + }, + wantErr: false, + }, + { + name: "create project with existing directory", + setup: func(dir string) error { + return os.MkdirAll(dir, 0755) + }, + wantErr: false, + }, + { + name: "create project with existing files", + setup: func(dir string) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + // Create a go.mod file + return os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module test\n"), 0644) + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create test directory + testDir := setupTestDir(t) + defer cleanupTestDir(t, testDir) + + // Setup test case + if err := tt.setup(testDir); err != nil { + t.Fatalf("test setup failed: %v", err) + } + + // Run Project function + err := Project(testDir, false) + if (err != nil) != tt.wantErr { + t.Errorf("Project() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // Verify project structure + expectedFiles := []string{ + "go.mod", + "main.go", + ".gitignore", + } + + for _, file := range expectedFiles { + path := filepath.Join(testDir, file) + if _, err := os.Stat(path); os.IsNotExist(err) { + t.Errorf("expected file %s does not exist", file) + } + } + }) + } +} + +func TestPromptOverwrite(t *testing.T) { + tests := []struct { + name string + input string + wantOverwrite bool + wantOverwriteAll bool + }{ + { + name: "answer yes", + input: "y\n", + wantOverwrite: true, + wantOverwriteAll: false, + }, + { + name: "answer no", + input: "n\n", + wantOverwrite: false, + wantOverwriteAll: false, + }, + { + name: "answer all", + input: "a\n", + wantOverwrite: true, + wantOverwriteAll: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create a temporary file to simulate stdin + tmpfile, err := os.CreateTemp("", "test-input") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + // Write test input to temp file + if _, err := tmpfile.Write([]byte(tt.input)); err != nil { + t.Fatal(err) + } + if err := tmpfile.Close(); err != nil { + t.Fatal(err) + } + + // Redirect stdin to our temp file + oldStdin := os.Stdin + f, err := os.Open(tmpfile.Name()) + if err != nil { + t.Fatal(err) + } + os.Stdin = f + defer func() { + os.Stdin = oldStdin + f.Close() + }() + + // Test promptOverwrite + gotOverwrite, gotOverwriteAll := promptOverwrite("test.txt") + if gotOverwrite != tt.wantOverwrite { + t.Errorf("promptOverwrite() overwrite = %v, want %v", gotOverwrite, tt.wantOverwrite) + } + if gotOverwriteAll != tt.wantOverwriteAll { + t.Errorf("promptOverwrite() overwriteAll = %v, want %v", gotOverwriteAll, tt.wantOverwriteAll) + } + }) + } +} diff --git a/cmd/internal/create/templates/.gitignore b/cmd/internal/create/templates/.gitignore new file mode 100644 index 0000000..b47d072 --- /dev/null +++ b/cmd/internal/create/templates/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.pyc +.env +.deps/ diff --git a/cmd/internal/create/templates/main.go b/cmd/internal/create/templates/main.go new file mode 100644 index 0000000..21d7b2f --- /dev/null +++ b/cmd/internal/create/templates/main.go @@ -0,0 +1,10 @@ +package main + +import ( + . "github.com/cpunion/go-python" +) + +func main() { + Initialize() + defer Finalize() +} diff --git a/cmd/internal/install/archive.go b/cmd/internal/install/archive.go new file mode 100644 index 0000000..0d732fb --- /dev/null +++ b/cmd/internal/install/archive.go @@ -0,0 +1,349 @@ +package install + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/klauspost/compress/zstd" +) + +// getCacheDir returns the cache directory for downloaded files +func getCacheDir() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to get user home directory: %v", err) + } + cacheDir := filepath.Join(homeDir, ".gopy", "cache") + if err := os.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create cache directory: %v", err) + } + return cacheDir, nil +} + +// getFullExtension returns the full extension for a filename (e.g., ".tar.gz" for "file.tar.gz") +func getFullExtension(filename string) string { + // Handle common multi-level extensions + for _, ext := range []string{".tar.gz", ".tar.zst"} { + if strings.HasSuffix(filename, ext) { + return ext + } + } + return filepath.Ext(filename) +} + +// downloadFileWithCache downloads a file from url and returns the path to the cached file +func downloadFileWithCache(url string) (string, error) { + cacheDir, err := getCacheDir() + if err != nil { + return "", err + } + + // Use URL's last path segment as filename + urlPath := strings.Split(url, "/") + filename := urlPath[len(urlPath)-1] + + // Calculate SHA1 hash of the URL + hasher := sha1.New() + hasher.Write([]byte(url)) + urlHash := hex.EncodeToString(hasher.Sum(nil))[:8] // Use first 8 characters of hash + + // Insert hash before the file extension, handling multi-level extensions + ext := getFullExtension(filename) + baseFilename := filename[:len(filename)-len(ext)] + cachedFilename := fmt.Sprintf("%s-%s%s", baseFilename, urlHash, ext) + cachedFile := filepath.Join(cacheDir, cachedFilename) + + // Check if file exists in cache + if _, err := os.Stat(cachedFile); err == nil { + fmt.Printf("Using cached file from %s\n", cachedFile) + return cachedFile, nil + } + + fmt.Printf("Downloading from %s\n", url) + + // Create temporary file + tmpFile, err := os.CreateTemp(cacheDir, "download-*") + if err != nil { + return "", fmt.Errorf("failed to create temporary file: %v", err) + } + tmpPath := tmpFile.Name() + defer os.Remove(tmpPath) + defer tmpFile.Close() + + // Download to temporary file + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("bad status: %s", resp.Status) + } + + _, err = io.Copy(tmpFile, resp.Body) + if err != nil { + return "", fmt.Errorf("failed to write file: %v", err) + } + + // Close the file before renaming + tmpFile.Close() + + // Rename temporary file to cached file + if err := os.Rename(tmpPath, cachedFile); err != nil { + return "", fmt.Errorf("failed to move file to cache: %v", err) + } + + return cachedFile, nil +} + +func downloadAndExtract(name, version, url, dir, trimPrefix string, verbose bool) error { + if verbose { + fmt.Printf("Downloading %s %s from %s\n", name, version, url) + } + + path, err := downloadFileWithCache(url) + if err != nil { + return fmt.Errorf("error downloading %s %s: %v", name, version, err) + } + + if verbose { + fmt.Printf("Extracting %s %s into %s...\n", name, version, dir) + } + + if err = os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", dir, err) + } + + // Extract based on file extension + if strings.HasSuffix(path, ".zip") { + return extractZip(path, dir) + } else if strings.HasSuffix(path, ".tar.gz") { + return extractTarGz(path, dir) + } else if strings.HasSuffix(path, ".tar.zst") { + return extractTarZst(path, dir, trimPrefix, verbose) + } else { + return fmt.Errorf("unsupported file extension for %s %s", name, version) + } +} + +// extractZip extracts a zip file to the specified directory +func extractZip(zipFile, destDir string) error { + r, err := zip.OpenReader(zipFile) + if err != nil { + return err + } + defer r.Close() + + for _, f := range r.File { + // Skip the root "go" directory + if f.Name == "go/" || f.Name == "go" { + continue + } + + // Remove "go/" prefix from paths + destPath := filepath.Join(destDir, strings.TrimPrefix(f.Name, "go/")) + + if f.FileInfo().IsDir() { + os.MkdirAll(destPath, f.Mode()) + continue + } + + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + + destFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + + srcFile, err := f.Open() + if err != nil { + destFile.Close() + return err + } + + _, err = io.Copy(destFile, srcFile) + srcFile.Close() + destFile.Close() + if err != nil { + return err + } + } + return nil +} + +// extractTarGz extracts a tar.gz file to the specified directory +func extractTarGz(tarFile, destDir string) error { + file, err := os.Open(tarFile) + if err != nil { + return err + } + defer file.Close() + + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + // Skip the root "go" directory + if header.Name == "go/" || header.Name == "go" { + continue + } + + // Remove "go/" prefix from paths + destPath := filepath.Join(destDir, strings.TrimPrefix(header.Name, "go/")) + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(destPath, os.FileMode(header.Mode)); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } + outFile, err := os.OpenFile(destPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(outFile, tr); err != nil { + outFile.Close() + return err + } + outFile.Close() + } + } + return nil +} + +// extractTarZst extracts a tar.zst file to a destination directory +func extractTarZst(src, dst, trimPrefix string, verbose bool) error { + if verbose { + fmt.Printf("Extracting from %s to %s\n", src, dst) + } + + // Open the zstd compressed file + file, err := os.Open(src) + if err != nil { + return fmt.Errorf("error opening file: %v", err) + } + defer file.Close() + + // Create zstd decoder + decoder, err := zstd.NewReader(file) + if err != nil { + return fmt.Errorf("error creating zstd decoder: %v", err) + } + defer decoder.Close() + + // Create tar reader from the decompressed stream + tr := tar.NewReader(decoder) + + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + name := header.Name + + if trimPrefix != "" { + if !strings.HasPrefix(header.Name, trimPrefix) { + continue + } + + // Remove the trimPrefix prefix + name = strings.TrimPrefix(header.Name, trimPrefix) + if name == "" { + continue + } + } + + path := filepath.Join(dst, name) + if verbose { + fmt.Printf("Extracting: %s\n", path) + } + + switch header.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, os.FileMode(header.Mode)); err != nil { + return fmt.Errorf("error creating directory %s: %v", path, err) + } + case tar.TypeReg: + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", dir, err) + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return fmt.Errorf("error creating file %s: %v", path, err) + } + + if _, err := io.Copy(file, tr); err != nil { + file.Close() + return fmt.Errorf("error writing to file %s: %v", path, err) + } + file.Close() + case tar.TypeSymlink: + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", dir, err) + } + + // Remove existing symlink if it exists + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("error removing existing symlink %s: %v", path, err) + } + + // Create new symlink + if err := os.Symlink(header.Linkname, path); err != nil { + return fmt.Errorf("error creating symlink %s -> %s: %v", path, header.Linkname, err) + } + case tar.TypeLink: + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", dir, err) + } + + // Remove existing file if it exists + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("error removing existing file %s: %v", path, err) + } + + // Create hard link relative to the destination directory + targetPath := filepath.Join(dst, strings.TrimPrefix(header.Linkname, "python/install/")) + if err := os.Link(targetPath, path); err != nil { + return fmt.Errorf("error creating hard link %s -> %s: %v", path, targetPath, err) + } + } + } + + return nil +} diff --git a/cmd/internal/install/deps.go b/cmd/internal/install/deps.go new file mode 100644 index 0000000..36c3481 --- /dev/null +++ b/cmd/internal/install/deps.go @@ -0,0 +1,78 @@ +package install + +import ( + "fmt" + "os" + "os/exec" + "runtime" + + "github.com/cpunion/go-python/internal/env" +) + +// Dependencies installs all required dependencies for the project +func Dependencies(projectPath string, goVersion, tinyPkgConfigVersion, pyVersion, pyBuildDate string, freeThreaded, debug bool, verbose bool) error { + if err := installTinyPkgConfig(projectPath, tinyPkgConfigVersion, verbose); err != nil { + return err + } + // Only install MSYS2 on Windows + if runtime.GOOS == "windows" { + if err := installMingw(projectPath, verbose); err != nil { + return err + } + } + + if err := installGo(projectPath, goVersion, verbose); err != nil { + return err + } + env.SetBuildEnv(projectPath) + + // Install Go dependencies + if err := installGoDeps(projectPath); err != nil { + return err + } + + // Install Python environment and dependencies + if err := installPythonEnv(projectPath, pyVersion, pyBuildDate, freeThreaded, debug, verbose); err != nil { + return err + } + + if runtime.GOOS == "windows" { + pythonPath := env.GetPythonRoot(projectPath) + pkgConfigDir := env.GetPythonPkgConfigDir(projectPath) + if err := generatePkgConfig(pythonPath, pkgConfigDir); err != nil { + return err + } + } + + // Update pkg-config files + if err := updatePkgConfig(projectPath); err != nil { + return err + } + + return nil +} + +// installGoDeps installs Go dependencies +func installGoDeps(projectPath string) error { + currentDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting current directory: %v", err) + } + + if err := os.Chdir(projectPath); err != nil { + return fmt.Errorf("error changing to project directory: %v", err) + } + defer func() { + _ = os.Chdir(currentDir) + }() + + fmt.Println("Installing Go dependencies...") + getCmd := exec.Command("go", "get", "-u", "github.com/cpunion/go-python") + getCmd.Stdout = os.Stdout + getCmd.Stderr = os.Stderr + if err := getCmd.Run(); err != nil { + return fmt.Errorf("error installing dependencies: %v", err) + } + + return nil +} diff --git a/cmd/internal/install/golang.go b/cmd/internal/install/golang.go new file mode 100644 index 0000000..f13b23c --- /dev/null +++ b/cmd/internal/install/golang.go @@ -0,0 +1,58 @@ +package install + +import ( + "fmt" + "runtime" + + "github.com/cpunion/go-python/internal/env" +) + +const ( + // Go download URL format + goDownloadURL = "https://go.dev/dl/go%s.%s-%s.%s" +) + +// getGoURL returns the appropriate Go download URL for the current platform +func getGoURL(version string) string { + var os, arch, ext string + + switch runtime.GOOS { + case "windows": + os = "windows" + ext = "zip" + case "darwin": + os = "darwin" + ext = "tar.gz" + case "linux": + os = "linux" + ext = "tar.gz" + default: + return "" + } + + switch runtime.GOARCH { + case "amd64": + arch = "amd64" + case "386": + arch = "386" + case "arm64": + arch = "arm64" + default: + return "" + } + + return fmt.Sprintf(goDownloadURL, version, os, arch, ext) +} + +// installGo downloads and installs Go in the project directory +func installGo(projectPath, version string, verbose bool) error { + goDir := env.GetGoDir(projectPath) + fmt.Printf("Installing Go %s in %s\n", version, goDir) + // Get download URL + url := getGoURL(version) + if url == "" { + return fmt.Errorf("unsupported platform") + } + + return downloadAndExtract("Go", version, url, goDir, "", verbose) +} diff --git a/cmd/internal/install/mingw.go b/cmd/internal/install/mingw.go new file mode 100644 index 0000000..0fd9961 --- /dev/null +++ b/cmd/internal/install/mingw.go @@ -0,0 +1,18 @@ +package install + +import ( + "fmt" + + "github.com/cpunion/go-python/internal/env" +) + +const ( + mingwVersion = "14.2.0" + mingwURL = "https://github.com/brechtsanders/winlibs_mingw/releases/download/14.2.0posix-19.1.1-12.0.0-ucrt-r2/winlibs-x86_64-posix-seh-gcc-14.2.0-llvm-19.1.1-mingw-w64ucrt-12.0.0-r2.zip" +) + +func installMingw(projectPath string, verbose bool) error { + root := env.GetMingwDir(projectPath) + fmt.Printf("Installing mingw in %v\n", root) + return downloadAndExtract("mingw", mingwVersion, mingwURL, root, "", verbose) +} diff --git a/cmd/internal/install/python.go b/cmd/internal/install/python.go new file mode 100644 index 0000000..992658d --- /dev/null +++ b/cmd/internal/install/python.go @@ -0,0 +1,418 @@ +package install + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/cpunion/go-python/internal/env" +) + +const ( + baseURL = "https://github.com/indygreg/python-build-standalone/releases/download/%s" +) + +type pythonBuild struct { + arch string + os string + variant string + debug bool + shared bool + fullPack bool +} + +// getPythonURL returns the appropriate Python standalone URL for the current platform +func getPythonURL(version, buildDate, arch, os string, freeThreaded, debug bool) string { + // Map GOARCH to Python build architecture + archMap := map[string]string{ + "amd64": "x86_64", + "arm64": "aarch64", + "386": "i686", + } + + pythonArch, ok := archMap[arch] + if !ok { + return "" + } + + build := pythonBuild{ + arch: pythonArch, + fullPack: true, + debug: debug, + } + + switch os { + case "darwin": + build.os = "apple-darwin" + if freeThreaded { + build.variant = "freethreaded" + if build.debug { + build.variant += "+debug" + } else { + build.variant += "+pgo" + } + } else { + if build.debug { + build.variant = "debug" + } else { + build.variant = "pgo" + } + } + case "linux": + build.os = "unknown-linux-gnu" + if freeThreaded { + build.variant = "freethreaded" + if build.debug { + build.variant += "+debug" + } else { + build.variant += "+pgo" + } + } else { + if build.debug { + build.variant = "debug" + } else { + build.variant = "pgo" + } + } + case "windows": + build.os = "pc-windows-msvc" + build.shared = true + if freeThreaded { + build.variant = "freethreaded+pgo" + } else { + build.variant = "pgo" + } + default: + return "" + } + + // Construct filename + filename := fmt.Sprintf("cpython-%s+%s-%s-%s", version, buildDate, build.arch, build.os) + if build.shared { + filename += "-shared" + } + filename += "-" + build.variant + if build.fullPack { + filename += "-full" + } + filename += ".tar.zst" + + return fmt.Sprintf(baseURL, buildDate) + "/" + filename +} + +// updateMacOSDylibs updates the install names of dylib files on macOS +func updateMacOSDylibs(pythonDir string, verbose bool) error { + if runtime.GOOS != "darwin" { + return nil + } + + libDir := filepath.Join(pythonDir, "lib") + entries, err := os.ReadDir(libDir) + if err != nil { + return fmt.Errorf("failed to read lib directory: %v", err) + } + + absLibDir, err := filepath.Abs(libDir) + if err != nil { + return fmt.Errorf("failed to get absolute path: %v", err) + } + + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".dylib") { + dylibPath := filepath.Join(libDir, entry.Name()) + if verbose { + fmt.Printf("Updating install name for: %s\n", dylibPath) + } + + // Get the current install name + cmd := exec.Command("otool", "-D", dylibPath) + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to get install name for %s: %v", dylibPath, err) + } + + // Parse the output to get the current install name + lines := strings.Split(string(output), "\n") + if len(lines) < 2 { + continue + } + currentName := strings.TrimSpace(lines[1]) + if currentName == "" { + continue + } + + // Calculate new install name using absolute path + newName := filepath.Join(absLibDir, filepath.Base(currentName)) + + fmt.Printf("Updating install name for %s to %s\n", dylibPath, newName) + // Update the install name + cmd = exec.Command("install_name_tool", "-id", newName, dylibPath) + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to update install name for %s: %v", dylibPath, err) + } + } + } + return nil +} + +// generatePkgConfig generates pkg-config files for Windows +func generatePkgConfig(pythonPath, pkgConfigDir string) error { + if err := os.MkdirAll(pkgConfigDir, 0755); err != nil { + return fmt.Errorf("failed to create pkgconfig directory: %v", err) + } + + // Get Python environment + pyEnv := env.NewPythonEnv(pythonPath) + pythonBin, err := pyEnv.Python() + if err != nil { + return fmt.Errorf("failed to get Python executable: %v", err) + } + + // Get Python version and check if freethreaded + cmd := exec.Command(pythonBin, "-c", ` +import sys +import sysconfig +version = f'{sys.version_info.major}.{sys.version_info.minor}' +is_freethreaded = hasattr(sys, "gettotalrefcount") +print(f'{version}\n{is_freethreaded}') +`) + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("failed to get Python info: %v", err) + } + + info := strings.Split(strings.TrimSpace(string(output)), "\n") + if len(info) != 2 { + return fmt.Errorf("unexpected Python info output format") + } + + version := info[0] + isFreethreaded := info[1] == "True" + + // Prepare version-specific library names + versionNoPoints := strings.ReplaceAll(version, ".", "") + libSuffix := "" + if isFreethreaded { + libSuffix = "t" + } + + // Template for the pkg-config files + embedTemplate := `prefix=${pcfiledir}/../.. +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Embed Python into an application +Requires: +Version: %s +Libs.private: +Libs: -L${libdir} -lpython%s%s +Cflags: -I${includedir} +` + + normalTemplate := `prefix=${pcfiledir}/../.. +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Python library +Requires: +Version: %s +Libs.private: +Libs: -L${libdir} -lpython3%s +Cflags: -I${includedir} +` + + // Generate file pairs + filePairs := []struct { + name string + template string + embed bool + }{ + {fmt.Sprintf("python-%s%s.pc", version, libSuffix), normalTemplate, false}, + {fmt.Sprintf("python-%s%s-embed.pc", version, libSuffix), embedTemplate, true}, + {"python3" + libSuffix + ".pc", normalTemplate, false}, + {"python3" + libSuffix + "-embed.pc", embedTemplate, true}, + } + + // If freethreaded, also generate non-t versions with the same content + if isFreethreaded { + additionalPairs := []struct { + name string + template string + embed bool + }{ + {fmt.Sprintf("python-%s.pc", version), normalTemplate, false}, + {fmt.Sprintf("python-%s-embed.pc", version), embedTemplate, true}, + {"python3.pc", normalTemplate, false}, + {"python3-embed.pc", embedTemplate, true}, + } + filePairs = append(filePairs, additionalPairs...) + } + + // Write all pkg-config files + for _, pair := range filePairs { + pcPath := filepath.Join(pkgConfigDir, pair.name) + var content string + if pair.embed { + content = fmt.Sprintf(pair.template, version, versionNoPoints, libSuffix) + } else { + content = fmt.Sprintf(pair.template, version, libSuffix) + } + + if err := os.WriteFile(pcPath, []byte(content), 0644); err != nil { + return fmt.Errorf("failed to write %s: %v", pair.name, err) + } + } + + return nil +} + +// updatePkgConfig updates the prefix in pkg-config files to use absolute path +func updatePkgConfig(projectPath string) error { + pythonPath := env.GetPythonRoot(projectPath) + pkgConfigDir := env.GetPythonPkgConfigDir(projectPath) + + entries, err := os.ReadDir(pkgConfigDir) + if err != nil { + return fmt.Errorf("failed to read pkgconfig directory: %v", err) + } + + absPath, err := filepath.Abs(pythonPath) + if err != nil { + return fmt.Errorf("failed to get absolute path: %v", err) + } + + // Helper function to write a .pc file with the correct prefix + writePC := func(path string, content []byte) error { + newContent := strings.ReplaceAll(string(content), "prefix=/install", "prefix="+absPath) + return os.WriteFile(path, []byte(newContent), 0644) + } + + // Regular expressions for matching file patterns + normalPattern := regexp.MustCompile(`^python-(\d+\.\d+)t?\.pc$`) + embedPattern := regexp.MustCompile(`^python-(\d+\.\d+)t?-embed\.pc$`) + + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".pc") { + pcFile := filepath.Join(pkgConfigDir, entry.Name()) + + // Read file content + content, err := os.ReadFile(pcFile) + if err != nil { + return fmt.Errorf("failed to read %s: %v", pcFile, err) + } + + // Update original file + if err := writePC(pcFile, content); err != nil { + return fmt.Errorf("failed to update %s: %v", pcFile, err) + } + + name := entry.Name() + // Create additional copies based on patterns + copies := make(map[string]bool) + + // Handle python-X.YZt.pc and python-X.YZ.pc patterns + if matches := normalPattern.FindStringSubmatch(name); matches != nil { + if strings.Contains(name, "t.pc") { + // python-3.13t.pc -> python3.pc and python3t.pc + copies["python3t.pc"] = true + copies["python3.pc"] = true + // Also create non-t version + noT := fmt.Sprintf("python-%s.pc", matches[1]) + if err := writePC(filepath.Join(pkgConfigDir, noT), content); err != nil { + return fmt.Errorf("failed to write %s: %v", noT, err) + } + } else { + // python-3.13.pc -> python3.pc + copies["python3.pc"] = true + } + } + + // Handle python-X.YZt-embed.pc and python-X.YZ-embed.pc patterns + if matches := embedPattern.FindStringSubmatch(name); matches != nil { + if strings.Contains(name, "t-embed.pc") { + // python-3.13t-embed.pc -> python3-embed.pc and python3t-embed.pc + copies["python3t-embed.pc"] = true + copies["python3-embed.pc"] = true + // Also create non-t version + noT := fmt.Sprintf("python-%s-embed.pc", matches[1]) + if err := writePC(filepath.Join(pkgConfigDir, noT), content); err != nil { + return fmt.Errorf("failed to write %s: %v", noT, err) + } + } else { + // python-3.13-embed.pc -> python3-embed.pc + copies["python3-embed.pc"] = true + } + } + + // Write all unique copies + for copyName := range copies { + copyPath := filepath.Join(pkgConfigDir, copyName) + if err := writePC(copyPath, content); err != nil { + return fmt.Errorf("failed to write %s: %v", copyPath, err) + } + } + } + } + return nil +} + +// installPythonEnv downloads and installs Python standalone build +func installPythonEnv(projectPath string, version, buildDate string, freeThreaded, debug bool, verbose bool) error { + fmt.Printf("Installing Python %s in %s\n", version, projectPath) + pythonDir := env.GetPythonRoot(projectPath) + + // Remove existing Python directory if it exists + if err := os.RemoveAll(pythonDir); err != nil { + return fmt.Errorf("error removing existing Python directory: %v", err) + } + + // Get Python URL + url := getPythonURL(version, buildDate, runtime.GOARCH, runtime.GOOS, freeThreaded, debug) + if url == "" { + return fmt.Errorf("unsupported platform") + } + + if err := downloadAndExtract("Python", version, url, pythonDir, "python/install", verbose); err != nil { + return fmt.Errorf("error downloading and extracting Python: %v", err) + } + + // After extraction, update dylib install names on macOS + if err := updateMacOSDylibs(pythonDir, verbose); err != nil { + return fmt.Errorf("error updating dylib install names: %v", err) + } + + // Create Python environment + pyEnv := env.NewPythonEnv(pythonDir) + + if verbose { + fmt.Println("Installing Python dependencies...") + } + + if err := pyEnv.RunPip("install", "--upgrade", "pip", "setuptools", "wheel"); err != nil { + return fmt.Errorf("error upgrading pip, setuptools, whell") + } + + if err := updatePkgConfig(projectPath); err != nil { + return fmt.Errorf("error updating pkg-config: %v", err) + } + + pythonHome := env.GetPythonRoot(projectPath) + pythonPath, err := pyEnv.GetPythonPath() + if err != nil { + return fmt.Errorf("failed to get Python path: %v", err) + } + // Write environment variables to env.txt + if err := env.WriteEnvFile(projectPath, pythonHome, pythonPath); err != nil { + return fmt.Errorf("error writing environment file: %v", err) + } + + return nil +} diff --git a/cmd/internal/install/python_test.go b/cmd/internal/install/python_test.go new file mode 100644 index 0000000..9002587 --- /dev/null +++ b/cmd/internal/install/python_test.go @@ -0,0 +1,337 @@ +package install + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/cpunion/go-python/internal/env" +) + +func TestGetPythonURL(t *testing.T) { + tests := []struct { + name string + arch string + os string + freeThreaded bool + + debug bool + want string + wantErr bool + }{ + { + name: "darwin-arm64-freethreaded-debug", + arch: "arm64", + os: "darwin", + freeThreaded: true, + debug: true, + want: "cpython-3.13.0+20241016-aarch64-apple-darwin-freethreaded+debug-full.tar.zst", + }, + { + name: "darwin-amd64-freethreaded-pgo", + arch: "amd64", + os: "darwin", + freeThreaded: true, + debug: false, + want: "cpython-3.13.0+20241016-x86_64-apple-darwin-freethreaded+pgo-full.tar.zst", + }, + { + name: "darwin-amd64-debug", + arch: "amd64", + os: "darwin", + freeThreaded: false, + debug: true, + want: "cpython-3.13.0+20241016-x86_64-apple-darwin-debug-full.tar.zst", + }, + { + name: "darwin-amd64-pgo", + arch: "amd64", + os: "darwin", + freeThreaded: false, + debug: false, + want: "cpython-3.13.0+20241016-x86_64-apple-darwin-pgo-full.tar.zst", + }, + { + name: "linux-amd64-freethreaded-debug", + arch: "amd64", + os: "linux", + freeThreaded: true, + debug: true, + want: "cpython-3.13.0+20241016-x86_64-unknown-linux-gnu-freethreaded+debug-full.tar.zst", + }, + { + name: "windows-amd64-freethreaded-pgo", + arch: "amd64", + os: "windows", + freeThreaded: true, + debug: false, + want: "cpython-3.13.0+20241016-x86_64-pc-windows-msvc-shared-freethreaded+pgo-full.tar.zst", + }, + { + name: "windows-386-freethreaded-pgo", + arch: "386", + os: "windows", + freeThreaded: true, + debug: false, + want: "cpython-3.13.0+20241016-i686-pc-windows-msvc-shared-freethreaded+pgo-full.tar.zst", + }, + { + name: "unsupported-arch", + arch: "mips", + os: "linux", + freeThreaded: false, + debug: false, + want: "", + wantErr: true, + }, + { + name: "unsupported-os", + arch: "amd64", + os: "freebsd", + freeThreaded: false, + debug: false, + want: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getPythonURL("3.13.0", "20241016", tt.arch, tt.os, tt.freeThreaded, tt.debug) + + if tt.wantErr { + if got != "" { + t.Errorf("getPythonURL() = %v, want empty string for error case", got) + } + return + } + + if got == "" { + t.Errorf("getPythonURL() returned empty string, want %v", tt.want) + return + } + + // Extract filename from URL + parts := strings.Split(got, "/") + filename := parts[len(parts)-1] + + if filename != tt.want { + t.Errorf("getPythonURL() = %v, want %v", filename, tt.want) + } + }) + } +} + +func TestGetCacheDir(t *testing.T) { + // Save original home dir + origHome := os.Getenv("HOME") + defer os.Setenv("HOME", origHome) + + t.Run("valid home directory", func(t *testing.T) { + tmpDir := t.TempDir() + if runtime.GOOS == "windows" { + os.Setenv("USERPROFILE", tmpDir) + } else { + os.Setenv("HOME", tmpDir) + } + + got, err := getCacheDir() + if err != nil { + t.Errorf("getCacheDir() error = %v, want nil", err) + return + } + + want := filepath.Join(tmpDir, ".gopy", "cache") + if got != want { + t.Errorf("getCacheDir() = %v, want %v", got, want) + } + + // Verify directory was created + if _, err := os.Stat(got); os.IsNotExist(err) { + t.Errorf("getCacheDir() did not create cache directory") + } + }) +} + +func TestUpdatePkgConfig(t *testing.T) { + t.Run("freethreaded pkg-config files", func(t *testing.T) { + // Create temporary directory structure + tmpDir := t.TempDir() + pkgConfigDir := env.GetPythonPkgConfigDir(tmpDir) + if err := os.MkdirAll(pkgConfigDir, 0755); err != nil { + t.Fatal(err) + } + + // Create test .pc files with freethreaded content + testFiles := map[string]string{ + "python-3.13t.pc": `prefix=/install +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Python library +Version: 3.13 +Libs: -L${libdir} -lpython3t +Cflags: -I${includedir}`, + "python-3.13t-embed.pc": `prefix=/install +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Embed Python into an application +Version: 3.13 +Libs: -L${libdir} -lpython313t +Cflags: -I${includedir}`, + } + + for filename, content := range testFiles { + if err := os.WriteFile(filepath.Join(pkgConfigDir, filename), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + + // Test updating pkg-config files + if err := updatePkgConfig(tmpDir); err != nil { + t.Errorf("updatePkgConfig() error = %v, want nil", err) + return + } + + // Verify the generated files + expectedFiles := map[string]struct { + shouldExist bool + libName string + }{ + // Freethreaded versions + "python-3.13t.pc": {true, "-lpython3t"}, + "python3t.pc": {true, "-lpython3t"}, + "python-3.13t-embed.pc": {true, "-lpython313t"}, + "python3t-embed.pc": {true, "-lpython313t"}, + // Non-t versions (same content as freethreaded) + "python-3.13.pc": {true, "-lpython3t"}, + "python3.pc": {true, "-lpython3t"}, + "python-3.13-embed.pc": {true, "-lpython313t"}, + "python3-embed.pc": {true, "-lpython313t"}, + } + + absPath, _ := filepath.Abs(filepath.Join(tmpDir, ".deps/python")) + for filename, expected := range expectedFiles { + path := filepath.Join(pkgConfigDir, filename) + if _, err := os.Stat(path); os.IsNotExist(err) { + if expected.shouldExist { + t.Errorf("Expected file %s was not created", filename) + } + continue + } + + content, err := os.ReadFile(path) + if err != nil { + t.Errorf("Failed to read file %s: %v", filename, err) + continue + } + + // Check prefix + expectedPrefix := fmt.Sprintf("prefix=%s", absPath) + if !strings.Contains(string(content), expectedPrefix) { + t.Errorf("File %s does not contain expected prefix %s", filename, expectedPrefix) + } + + // Check library name + if !strings.Contains(string(content), expected.libName) { + t.Errorf("File %s does not contain expected library name %s", filename, expected.libName) + } + } + }) + + t.Run("non-freethreaded pkg-config files", func(t *testing.T) { + // Create temporary directory structure + tmpDir := t.TempDir() + pkgConfigDir := env.GetPythonPkgConfigDir(tmpDir) + if err := os.MkdirAll(pkgConfigDir, 0755); err != nil { + t.Fatal(err) + } + + // Create test .pc files with non-freethreaded content + testFiles := map[string]string{ + "python-3.13.pc": `prefix=/install +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Python library +Version: 3.13 +Libs: -L${libdir} -lpython3 +Cflags: -I${includedir}`, + "python-3.13-embed.pc": `prefix=/install +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: Python +Description: Embed Python into an application +Version: 3.13 +Libs: -L${libdir} -lpython313 +Cflags: -I${includedir}`, + } + + for filename, content := range testFiles { + if err := os.WriteFile(filepath.Join(pkgConfigDir, filename), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + + // Test updating pkg-config files + if err := updatePkgConfig(tmpDir); err != nil { + t.Errorf("updatePkgConfig() error = %v, want nil", err) + return + } + + // Verify the generated files + expectedFiles := map[string]struct { + shouldExist bool + libName string + }{ + "python-3.13.pc": {true, "-lpython3"}, + "python3.pc": {true, "-lpython3"}, + "python-3.13-embed.pc": {true, "-lpython313"}, + "python3-embed.pc": {true, "-lpython313"}, + } + + absPath, _ := filepath.Abs(filepath.Join(tmpDir, ".deps/python")) + for filename, expected := range expectedFiles { + path := filepath.Join(pkgConfigDir, filename) + if _, err := os.Stat(path); os.IsNotExist(err) { + if expected.shouldExist { + t.Errorf("Expected file %s was not created", filename) + } + continue + } + + content, err := os.ReadFile(path) + if err != nil { + t.Errorf("Failed to read file %s: %v", filename, err) + continue + } + + // Check prefix + expectedPrefix := fmt.Sprintf("prefix=%s", absPath) + if !strings.Contains(string(content), expectedPrefix) { + t.Errorf("File %s does not contain expected prefix %s", filename, expectedPrefix) + } + + // Check library name + if !strings.Contains(string(content), expected.libName) { + t.Errorf("File %s does not contain expected library name %s", filename, expected.libName) + } + } + }) + + t.Run("missing pkgconfig directory", func(t *testing.T) { + tmpDir := t.TempDir() + err := updatePkgConfig(tmpDir) + if err == nil { + t.Error("updatePkgConfig() error = nil, want error for missing pkgconfig directory") + } + }) +} diff --git a/cmd/internal/install/tiny_pkg_config.go b/cmd/internal/install/tiny_pkg_config.go new file mode 100644 index 0000000..d15748f --- /dev/null +++ b/cmd/internal/install/tiny_pkg_config.go @@ -0,0 +1,64 @@ +package install + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/cpunion/go-python/internal/env" +) + +const ( + tinyPkgDownloadURL = "https://github.com/cpunion/tiny-pkg-config/releases/download/%s/%s" +) + +func installTinyPkgConfig(projectPath, version string, verbose bool) error { + dir := env.GetTinyPkgConfigDir(projectPath) + // Determine OS and architecture + goos := runtime.GOOS + arch := runtime.GOARCH + + // Convert OS/arch to match release file naming + osName := strings.ToUpper(goos[:1]) + goos[1:] // "darwin" -> "Darwin", "linux" -> "Linux" + archName := arch + if arch == "amd64" { + archName = "x86_64" + } + + // Construct filename and URL + ext := ".tar.gz" + if osName == "Windows" { + ext = ".zip" + } + + filename := fmt.Sprintf("tiny-pkg-config_%s_%s%s", osName, archName, ext) + downloadURL := fmt.Sprintf(tinyPkgDownloadURL, version, filename) + + if err := downloadAndExtract("tiny-pkg-config", version, downloadURL, dir, "", verbose); err != nil { + return fmt.Errorf("download and extract tiny-pkg-config failed: %w", err) + } + + // After extraction, rename the executable + oldName := "tiny-pkg-config" + newName := "pkg-config" + if runtime.GOOS == "windows" { + oldName += ".exe" + newName += ".exe" + } + + oldPath := filepath.Join(dir, oldName) + newPath := filepath.Join(dir, newName) + + // Rename the file + if err := os.Rename(oldPath, newPath); err != nil { + return fmt.Errorf("failed to rename executable: %w", err) + } + + if verbose { + fmt.Printf("Renamed %s to %s\n", oldName, newName) + } + + return nil +} diff --git a/cmd/internal/log/log.go b/cmd/internal/log/log.go new file mode 100644 index 0000000..a089bec --- /dev/null +++ b/cmd/internal/log/log.go @@ -0,0 +1,72 @@ +package log + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var ( + logger *zap.Logger + sugar *zap.SugaredLogger +) + +// Init initializes the logger with the given verbosity level +func Init(verbose bool) { + config := zap.NewDevelopmentConfig() + if !verbose { + config.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel) + } else { + config.Level = zap.NewAtomicLevelAt(zapcore.DebugLevel) + } + + // Customize output format + config.EncoderConfig.TimeKey = "" // Remove timestamp + config.EncoderConfig.LevelKey = "" // Remove log level + config.EncoderConfig.CallerKey = "" // Remove caller + config.EncoderConfig.NameKey = "" // Remove logger name + config.EncoderConfig.StacktraceKey = "" // Remove stacktrace + config.DisableCaller = true + config.DisableStacktrace = true + + var err error + logger, err = config.Build() + if err != nil { + panic(err) + } + sugar = logger.Sugar() +} + +// Debug logs a debug message +func Debug(args ...interface{}) { + sugar.Debug(args...) +} + +// Debugf logs a formatted debug message +func Debugf(template string, args ...interface{}) { + sugar.Debugf(template, args...) +} + +// Info logs an info message +func Info(args ...interface{}) { + sugar.Info(args...) +} + +// Infof logs a formatted info message +func Infof(template string, args ...interface{}) { + sugar.Infof(template, args...) +} + +// Error logs an error message +func Error(args ...interface{}) { + sugar.Error(args...) +} + +// Errorf logs a formatted error message +func Errorf(template string, args ...interface{}) { + sugar.Errorf(template, args...) +} + +// Sync flushes any buffered log entries +func Sync() error { + return logger.Sync() +} diff --git a/cmd/internal/rungo/run.go b/cmd/internal/rungo/run.go new file mode 100644 index 0000000..2b4e540 --- /dev/null +++ b/cmd/internal/rungo/run.go @@ -0,0 +1,238 @@ +package rungo + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/cpunion/go-python/internal/env" +) + +type ListInfo struct { + Dir string `json:"Dir"` + Root string `json:"Root"` +} + +// FindPackageIndex finds the package argument index by skipping flags and their values +func FindPackageIndex(args []string) int { + for i := 0; i < len(args); i++ { + arg := args[i] + if strings.HasPrefix(arg, "-") { + // Skip known flags that take values + switch arg { + case "-o", "-p", "-asmflags", "-buildmode", "-compiler", "-gccgoflags", "-gcflags", + "-installsuffix", "-ldflags", "-mod", "-modfile", "-pkgdir", "-tags", "-toolexec": + i++ // Skip the next argument as it's the flag's value + } + continue + } + return i + } + return -1 +} + +// GetPackageDir returns the directory containing the package +func GetPackageDir(pkgPath string) (string, error) { + // Get the absolute path + absPath, err := filepath.Abs(pkgPath) + if err != nil { + return "", fmt.Errorf("error resolving path: %v", err) + } + + // If it's not a directory, get its parent directory + fi, err := os.Stat(absPath) + if err != nil { + if os.IsNotExist(err) && pkgPath == "." { + // Special case: if "." doesn't exist, use current directory + dir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("error getting working directory: %v", err) + } + absPath = dir + fi, err = os.Stat(absPath) + if err != nil { + return "", fmt.Errorf("error checking path: %v", err) + } + } else { + return "", fmt.Errorf("error checking path: %v", err) + } + } + + if !fi.IsDir() { + return filepath.Dir(absPath), nil + } + return absPath, nil +} + +// RunGoCommand executes a Go command with Python environment properly configured +func RunGoCommand(command string, args []string) error { + // Find the package argument + pkgIndex := FindPackageIndex(args) + + // TODO: don't depend on external go command + listArgs := []string{"list", "-find", "-json"} + + if pkgIndex != -1 { + pkgPath := args[pkgIndex] + listArgs = append(listArgs, pkgPath) + } + cmd := exec.Command("go", listArgs...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to get module info: %v", err) + } + var listInfo ListInfo + if err := json.NewDecoder(&out).Decode(&listInfo); err != nil { + return fmt.Errorf("failed to parse module info: %v", err) + } + projectRoot := listInfo.Root + env.SetBuildEnv(projectRoot) + + // Set up environment variables + goEnv := []string{} + + // Get PYTHONPATH and PYTHONHOME from env.txt + var pythonPath, pythonHome string + if additionalEnv, err := env.ReadEnv(projectRoot); err == nil { + for key, value := range additionalEnv { + goEnv = append(goEnv, key+"="+value) + } + pythonPath = additionalEnv["PYTHONPATH"] + pythonHome = additionalEnv["PYTHONHOME"] + } else { + fmt.Fprintf(os.Stderr, "Warning: could not load environment variables: %v\n", err) + } + + // Process args to inject Python paths via ldflags + processedArgs := ProcessArgsWithLDFlags(args, projectRoot, pythonPath, pythonHome) + + // Prepare go command with processed arguments + goArgs := append([]string{"go", command}, processedArgs...) + cmd = exec.Command(goArgs[0], goArgs[1:]...) + cmd.Env = append(goEnv, os.Environ()...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if command == "run" { + cmd.Stdin = os.Stdin + } + + // Execute the command + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + return fmt.Errorf("error executing command: %v", err) + } + + return nil +} + +// ProcessArgsWithLDFlags processes command line arguments to inject Python paths via ldflags +func ProcessArgsWithLDFlags(args []string, projectRoot, pythonPath, pythonHome string) []string { + result := make([]string, 0, len(args)) + + // Prepare the -X flags we want to add + var xFlags []string + if pythonHome != "" { + xFlags = append(xFlags, fmt.Sprintf("-X 'github.com/cpunion/go-python.ProjectRoot=%s'", projectRoot)) + } + + // Prepare rpath flag if needed + var rpathFlag string + if pythonHome != "" { + pythonLibDir := filepath.Join(pythonHome, "lib") + switch runtime.GOOS { + case "darwin", "linux": + rpathFlag = fmt.Sprintf("-extldflags '-Wl,-rpath,%s'", pythonLibDir) + case "windows": + // Windows doesn't use rpath + rpathFlag = "" + default: + // Use Linux format for other Unix-like systems + rpathFlag = fmt.Sprintf("-extldflags '-Wl,-rpath=%s'", pythonLibDir) + } + } + + // Find existing -ldflags if any + foundLDFlags := false + for i := 0; i < len(args); i++ { + arg := args[i] + if strings.HasPrefix(arg, "-ldflags=") || arg == "-ldflags" { + foundLDFlags = true + // Copy everything before this arg + result = append(result, args[:i]...) + + // Get existing flags + var existingFlags string + if strings.HasPrefix(arg, "-ldflags=") { + existingFlags = strings.TrimPrefix(arg, "-ldflags=") + } else if i+1 < len(args) { + existingFlags = args[i+1] + i++ // Skip the next arg since we've consumed it + } + + // Combine all flags + var allFlags []string + if len(xFlags) > 0 { + allFlags = append(allFlags, xFlags...) + } + if strings.TrimSpace(existingFlags) != "" { + allFlags = append(allFlags, existingFlags) + } + if rpathFlag != "" { + allFlags = append(allFlags, rpathFlag) + } + + // Add combined ldflags + result = append(result, "-ldflags") + result = append(result, strings.Join(allFlags, " ")) + + // Add remaining args + result = append(result, args[i+1:]...) + break + } + } + + // If no existing -ldflags found, add new ones at the beginning if we have any flags to add + if !foundLDFlags { + if len(xFlags) > 0 || rpathFlag != "" { + var allFlags []string + allFlags = append(allFlags, xFlags...) + if rpathFlag != "" { + allFlags = append(allFlags, rpathFlag) + } + result = append(result, "-ldflags") + result = append(result, strings.Join(allFlags, " ")) + } + result = append(result, args...) + } + + return result +} + +// GetGoCommandHelp returns the formatted help text for the specified go command +func GetGoCommandHelp(command string) (string, error) { + cmd := exec.Command("go", "help", command) + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return "", err + } + + intro := fmt.Sprintf(`The command arguments and flags are fully compatible with 'go %s'. + +Following is the help message from 'go %s': +------------------------------------------------------------------------------- + +`, command, command) + + return intro + out.String() + "\n-------------------------------------------------------------------------------", nil +} diff --git a/cmd/remove.go b/cmd/remove.go new file mode 100644 index 0000000..cb641ce --- /dev/null +++ b/cmd/remove.go @@ -0,0 +1,39 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// removeCmd represents the remove command +var removeCmd = &cobra.Command{ + Use: "remove", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("remove called") + }, +} + +func init() { + rootCmd.AddCommand(removeCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // removeCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // removeCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..856482b --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,51 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "gopy", + Short: "A tool for building Go applications with Python integration", + Long: `gopy is a command line tool that helps you build, run and manage Go applications +that integrate with Python. + +It provides commands to: +- Initialize Python environment for your Go project +- Build Go applications with Python environment properly configured +- Run Go applications with Python runtime support +- Install Go packages with Python dependencies +- Add or remove Python packages to/from your project + +Use "gopy help [command]" for more information about a command.`, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) { }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-python.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/run.go b/cmd/run.go new file mode 100644 index 0000000..b28895f --- /dev/null +++ b/cmd/run.go @@ -0,0 +1,37 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + "os" + + "github.com/cpunion/go-python/cmd/internal/rungo" + "github.com/spf13/cobra" +) + +// runCmd represents the run command +var runCmd = &cobra.Command{ + Use: "run [flags] [package] [arguments...]", + Short: "Run a Go package with Python environment configured", + Long: func() string { + intro := "Run executes a Go package with the Python environment properly configured.\n\n" + help, err := rungo.GetGoCommandHelp("run") + if err != nil { + return intro + "Failed to get go help: " + err.Error() + } + return intro + help + }(), + DisableFlagParsing: true, + Run: func(cmd *cobra.Command, args []string) { + if err := rungo.RunGoCommand("run", args); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + os.Exit(1) + } + }, +} + +func init() { + rootCmd.AddCommand(runCmd) +} diff --git a/convert.go b/convert.go index 6d98f95..5993185 100644 --- a/convert.go +++ b/convert.go @@ -24,17 +24,25 @@ func From(from any) Object { case int64: return newObject(C.PyLong_FromLongLong(C.longlong(v))) case int: - return newObject(C.PyLong_FromLong(C.long(v))) + if unsafe.Sizeof(v) == unsafe.Sizeof(int64(0)) { + return newObject(C.PyLong_FromLongLong(C.longlong(v))) + } else { + return newObject(C.PyLong_FromLong(C.long(v))) + } case uint8: - return newObject(C.PyLong_FromLong(C.long(v))) + return newObject(C.PyLong_FromUnsignedLong(C.ulong(v))) case uint16: - return newObject(C.PyLong_FromLong(C.long(v))) + return newObject(C.PyLong_FromUnsignedLong(C.ulong(v))) case uint32: - return newObject(C.PyLong_FromLong(C.long(v))) + return newObject(C.PyLong_FromUnsignedLong(C.ulong(v))) case uint64: return newObject(C.PyLong_FromUnsignedLongLong(C.ulonglong(v))) case uint: - return newObject(C.PyLong_FromUnsignedLong(C.ulong(v))) + if unsafe.Sizeof(v) == unsafe.Sizeof(uint64(0)) { + return newObject(C.PyLong_FromUnsignedLongLong(C.ulonglong(v))) + } else { + return newObject(C.PyLong_FromUnsignedLong(C.ulong(v))) + } case float64: return newObject(C.PyFloat_FromDouble(C.double(v))) case float32: diff --git a/dict.go b/dict.go index 606e77f..f29f513 100644 --- a/dict.go +++ b/dict.go @@ -101,7 +101,7 @@ func (d Dict) Items() func(func(Object, Object) bool) { C.pyCriticalSection_Begin(&cs, obj) return func(fn func(Object, Object) bool) { defer C.pyCriticalSection_End(&cs) - var pos C.long + var pos C.Py_ssize_t var key, value *C.PyObject for C.PyDict_Next(obj, &pos, &key, &value) == 1 { if !fn(newObject(key), newObject(value)) { diff --git a/extension_test.go b/extension_test.go index edb4bf5..033ab27 100644 --- a/extension_test.go +++ b/extension_test.go @@ -93,20 +93,20 @@ assert obj.int32_field == 2147483647 assert obj.int64_field == 9223372036854775807 assert obj.int_field == 1234567890 assert obj.uint8_field == 255 -assert obj.uint16_field == 65535 -assert obj.uint32_field == 4294967295 -assert obj.uint64_field == 18446744073709551615 -assert obj.uint_field == 4294967295 -assert abs(obj.float32_field - 3.14) < 0.0001 -assert abs(obj.float64_field - 3.14159265359) < 0.0000001 -assert abs(obj.complex64_field - (1.5 + 2.5j)) < 0.0001 -assert abs(obj.complex128_field - (3.14 + 2.718j)) < 0.0000001 +assert obj.uint16_field == 65535, f"Expected 65535, got {obj.uint16_field}" +assert obj.uint32_field == 4294967295, f"Expected 4294967295, got {obj.uint32_field}" +assert obj.uint64_field == 18446744073709551615, f"Expected 18446744073709551615, got {obj.uint64_field}" +assert obj.uint_field == 4294967295, f"Expected 4294967295, got {obj.uint_field}" +assert abs(obj.float32_field - 3.14) < 0.0001, f"Expected 3.14, got {obj.float32_field}" +assert abs(obj.float64_field - 3.14159265359) < 0.0000001, f"Expected 3.14159265359, got {obj.float64_field}" +assert abs(obj.complex64_field - (1.5 + 2.5j)) < 0.0001, f"Expected (1.5 + 2.5j), got {obj.complex64_field}" +assert abs(obj.complex128_field - (3.14 + 2.718j)) < 0.0000001, f"Expected (3.14 + 2.718j), got {obj.complex128_field}" # verify non-C-compatible types -assert obj.string_field == "test string" -assert obj.slice_field == [1, 2, 3] -assert obj.map_field["key"] == 42 -assert obj.struct_field.x == 100 +assert obj.string_field == "test string", f"Expected 'test string', got {obj.string_field}" +assert obj.slice_field == [1, 2, 3], f"Expected [1, 2, 3], got {obj.slice_field}" +assert obj.map_field["key"] == 42, f"Expected 42, got {obj.map_field['key']}" +assert obj.struct_field.x == 100, f"Expected 100, got {obj.struct_field.x}" ` err := RunString(code) diff --git a/go.mod b/go.mod index 3873b45..0753726 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,21 @@ module github.com/cpunion/go-python -go 1.20 +go 1.21 + +toolchain go1.23.2 + +require ( + github.com/fatih/color v1.18.0 + github.com/klauspost/compress v1.17.11 + github.com/spf13/cobra v1.8.1 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/sys v0.25.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f5b484e --- /dev/null +++ b/go.sum @@ -0,0 +1,36 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/inject.go b/inject.go new file mode 100644 index 0000000..b35b2a1 --- /dev/null +++ b/inject.go @@ -0,0 +1,25 @@ +package gp + +import ( + "fmt" + "os" + + "github.com/cpunion/go-python/internal/env" +) + +var ProjectRoot string + +func init() { + if ProjectRoot == "" { + fmt.Fprintf(os.Stderr, "ProjectRoot is not set\n") + return + } + envs, err := env.ReadEnv(ProjectRoot) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to read env: %s\n", err) + return + } + for key, value := range envs { + os.Setenv(key, value) + } +} diff --git a/internal/env/env.go b/internal/env/env.go new file mode 100644 index 0000000..c9bc188 --- /dev/null +++ b/internal/env/env.go @@ -0,0 +1,179 @@ +package env + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +const ( + // depsDir is the directory for all dependencies + depsDir = ".deps" + // pyDir is the directory name for Python installation + pyDir = "python" + // goDir is the directory name for Go installation + goDir = "go" + // mingwDir is the directory name for Mingw installation + mingwDir = "mingw" + mingwRoot = mingwDir + "/mingw64" + + tinyPkgConfigDir = "tiny-pkg-config" +) + +func GetDepsDir(projectPath string) string { + return filepath.Join(projectPath, depsDir) +} + +func GetGoDir(projectPath string) string { + return filepath.Join(GetDepsDir(projectPath), goDir) +} + +// GetPythonRoot returns the Python installation root path relative to project path +func GetPythonRoot(projectPath string) string { + return filepath.Join(projectPath, depsDir, pyDir) +} + +// GetPythonBinDir returns the Python binary directory path relative to project path +func GetPythonBinDir(projectPath string) string { + return filepath.Join(GetPythonRoot(projectPath), "bin") +} + +// GetPythonLibDir returns the Python library directory path relative to project path +func GetPythonLibDir(projectPath string) string { + return filepath.Join(GetPythonRoot(projectPath), "lib") +} + +// GetPythonPkgConfigDir returns the pkg-config directory path relative to project path +func GetPythonPkgConfigDir(projectPath string) string { + return filepath.Join(GetPythonLibDir(projectPath), "pkgconfig") +} + +// GetGoRoot returns the Go installation root path relative to project path +func GetGoRoot(projectPath string) string { + return filepath.Join(projectPath, depsDir, goDir) +} + +// GetGoPath returns the Go path relative to project path +func GetGoPath(projectPath string) string { + return filepath.Join(GetGoRoot(projectPath), "packages") +} + +// GetGoBinDir returns the Go binary directory path relative to project path +func GetGoBinDir(projectPath string) string { + return filepath.Join(GetGoRoot(projectPath), "bin") +} + +// GetGoCacheDir returns the Go cache directory path relative to project path +func GetGoCacheDir(projectPath string) string { + return filepath.Join(GetGoRoot(projectPath), "go-build") +} + +func GetMingwDir(projectPath string) string { + return filepath.Join(projectPath, depsDir, mingwDir) +} + +func GetMingwRoot(projectPath string) string { + return filepath.Join(projectPath, depsDir, mingwRoot) +} + +func GetTinyPkgConfigDir(projectPath string) string { + return filepath.Join(projectPath, depsDir, tinyPkgConfigDir) +} + +func GetEnvConfigPath(projectPath string) string { + return filepath.Join(GetDepsDir(projectPath), "env.txt") +} + +func SetBuildEnv(projectPath string) { + absPath, err := filepath.Abs(projectPath) + if err != nil { + panic(err) + } + path := os.Getenv("PATH") + path = GetGoBinDir(absPath) + pathSeparator() + path + if runtime.GOOS == "windows" { + path = GetMingwRoot(absPath) + pathSeparator() + path + path = GetTinyPkgConfigDir(absPath) + pathSeparator() + path + } + os.Setenv("PATH", path) + os.Setenv("GOPATH", GetGoPath(absPath)) + os.Setenv("GOROOT", GetGoRoot(absPath)) + os.Setenv("GOCACHE", GetGoCacheDir(absPath)) +} + +func pathSeparator() string { + if runtime.GOOS == "windows" { + return ";" + } + return ":" +} + +// WriteEnvFile writes environment variables to .python/env.txt +func WriteEnvFile(projectPath, pythonHome, pythonPath string) error { + // Prepare environment variables + envVars := []string{ + fmt.Sprintf("PKG_CONFIG_PATH=%s", filepath.Join(pythonHome, "lib", "pkgconfig")), + fmt.Sprintf("PYTHONPATH=%s", strings.TrimSpace(pythonPath)), + fmt.Sprintf("PYTHONHOME=%s", pythonHome), + } + + // Write to env.txt + envFile := GetEnvConfigPath(projectPath) + if err := os.WriteFile(envFile, []byte(strings.Join(envVars, "\n")), 0644); err != nil { + return fmt.Errorf("failed to write env file: %v", err) + } + + return nil +} + +// ReadEnvFile loads environment variables from .python/env.txt in the given directory +func ReadEnvFile(projectDir string) (map[string]string, error) { + envFile := GetEnvConfigPath(projectDir) + content, err := os.ReadFile(envFile) + if err != nil { + return nil, fmt.Errorf("failed to read env file: %v", err) + } + envs := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(content)), "\n") { + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + envs[parts[0]] = parts[1] + } + } + return envs, nil +} + +func GeneratePythonEnv(pythonHome, pythonPath string) map[string]string { + path := os.Getenv("PATH") + if runtime.GOOS == "windows" { + path = filepath.Join(pythonHome) + ";" + path + } else { + path = filepath.Join(pythonHome, "bin") + ":" + path + } + return map[string]string{ + "PYTHONHOME": pythonHome, + "PYTHONPATH": pythonPath, + "PATH": path, + } +} + +func ReadEnv(projectDir string) (map[string]string, error) { + envs, err := ReadEnvFile(projectDir) + if err != nil { + return nil, err + } + pythonHome, ok := envs["PYTHONHOME"] + if !ok { + return nil, fmt.Errorf("PYTHONHOME is not set in env.txt") + } + pythonPath, ok := envs["PYTHONPATH"] + if !ok { + return nil, fmt.Errorf("PYTHONPATH is not set in env.txt") + } + for k, v := range GeneratePythonEnv(pythonHome, pythonPath) { + envs[k] = v + } + return envs, nil +} diff --git a/internal/env/env_test.go b/internal/env/env_test.go new file mode 100644 index 0000000..006fcb4 --- /dev/null +++ b/internal/env/env_test.go @@ -0,0 +1,120 @@ +package env + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestLoadEnvFile(t *testing.T) { + t.Run("valid env file", func(t *testing.T) { + // Create temporary directory structure + projectDir := t.TempDir() + pythonDir := GetPythonRoot(projectDir) + if err := os.MkdirAll(pythonDir, 0755); err != nil { + t.Fatal(err) + } + + // Create test env.txt file + envContent := map[string]string{ + "PKG_CONFIG_PATH": "/test/lib/pkgconfig", + "PYTHONPATH": "/test/lib/python3.9", + "PYTHONHOME": "/test", + } + lines := []string{} + for key, value := range envContent { + lines = append(lines, fmt.Sprintf("%s=%s", key, value)) + } + envFile := GetEnvConfigPath(projectDir) + if err := os.WriteFile(envFile, []byte(strings.Join(lines, "\n")), 0644); err != nil { + t.Fatal(err) + } + + // Test loading the env file + got, err := ReadEnvFile(projectDir) + if err != nil { + t.Errorf("LoadEnvFile() error = %v, want nil", err) + return + } + + if !reflect.DeepEqual(got, envContent) { + t.Errorf("LoadEnvFile() = %v, want %v", got, envContent) + } + }) + + t.Run("missing env file", func(t *testing.T) { + tmpDir := t.TempDir() + _, err := ReadEnvFile(tmpDir) + if err == nil { + t.Error("LoadEnvFile() error = nil, want error for missing env file") + } + }) +} + +func TestWriteEnvFile(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + + t.Run("write env file", func(t *testing.T) { + // Create temporary directory structure + projectDir := t.TempDir() + pythonDir := GetPythonRoot(projectDir) + binDir := GetPythonBinDir(projectDir) + if err := os.MkdirAll(binDir, 0755); err != nil { + t.Fatal(err) + } + + // Create mock Python executable + var pythonPath string + if runtime.GOOS == "windows" { + pythonPath = "/mock/path1;/mock/path2" + } else { + pythonPath = "/mock/path1:/mock/path2" + } + + // Test writing env file + if err := WriteEnvFile(projectDir, pythonDir, pythonPath); err != nil { + t.Errorf("writeEnvFile() error = %v, want nil", err) + return + } + + // Verify the env file was created + envFile := GetEnvConfigPath(projectDir) + if _, err := os.Stat(envFile); os.IsNotExist(err) { + t.Error("writeEnvFile() did not create env.txt") + return + } + + // Read and verify content + content, err := os.ReadFile(envFile) + if err != nil { + t.Errorf("Failed to read env.txt: %v", err) + return + } + + // Get expected path separator + pathSep := ":" + if runtime.GOOS == "windows" { + pathSep = ";" + } + + // Verify the content contains expected environment variables + envContent := string(content) + expectedVars := []string{ + fmt.Sprintf("PKG_CONFIG_PATH=%s", filepath.Join(pythonDir, "lib", "pkgconfig")), + fmt.Sprintf("PYTHONPATH=/mock/path1%s/mock/path2", pathSep), + fmt.Sprintf("PYTHONHOME=%s", pythonDir), + } + fmt.Printf("envContent:\n%v\n", envContent) + for _, v := range expectedVars { + if !strings.Contains(envContent, v) { + t.Errorf("env.txt missing expected variable %s", v) + } + } + }) +} diff --git a/internal/env/pyenv.go b/internal/env/pyenv.go new file mode 100644 index 0000000..09acc57 --- /dev/null +++ b/internal/env/pyenv.go @@ -0,0 +1,87 @@ +package env + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +// PythonEnv represents a Python environment +type PythonEnv struct { + Root string // Root directory of the Python installation +} + +// NewPythonEnv creates a new Python environment instance +func NewPythonEnv(pythonHome string) *PythonEnv { + return &PythonEnv{ + Root: pythonHome, + } +} + +// Python returns the path to the Python executable +func (e *PythonEnv) Python() (string, error) { + binDir := e.Root + if runtime.GOOS != "windows" { + binDir = filepath.Join(e.Root, "bin") + } + entries, err := os.ReadDir(binDir) + if err != nil { + return "", fmt.Errorf("failed to read bin directory: %v", err) + } + + // Single pattern to match all variants, prioritizing 't' versions + var pattern *regexp.Regexp + if runtime.GOOS == "windows" { + pattern = regexp.MustCompile(`^python3?[\d.]*t?(?:\.exe)?$`) + } else { + pattern = regexp.MustCompile(`^python3?[\d.]*t?$`) + } + + for _, entry := range entries { + if !entry.IsDir() && pattern.MatchString(entry.Name()) { + return filepath.Join(binDir, entry.Name()), nil + } + } + + return "", fmt.Errorf("python executable not found in %s", e.Root) +} + +// RunPip executes pip with the given arguments +func (e *PythonEnv) RunPip(args ...string) error { + return e.RunPythonWithOutput(nil, append([]string{"-m", "pip"}, args...)...) +} + +// RunPython executes python with the given arguments +func (e *PythonEnv) RunPython(args ...string) (string, error) { + var buf strings.Builder + err := e.RunPythonWithOutput(&buf, args...) + if err != nil { + return "", err + } + return strings.TrimSpace(buf.String()), nil +} + +func (e *PythonEnv) RunPythonWithOutput(writer io.Writer, args ...string) error { + pythonPath, err := e.Python() + if err != nil { + return err + } + + cmd := exec.Command(pythonPath, args...) + if writer != nil { + cmd.Stdout = io.MultiWriter(writer, os.Stdout) + } else { + cmd.Stdout = os.Stdout + } + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (e *PythonEnv) GetPythonPath() (string, error) { + return e.RunPython("-c", `import sys; print(':'.join(sys.path))`) +} diff --git a/unicode.go b/unicode.go index 0e7e4b7..39cb739 100644 --- a/unicode.go +++ b/unicode.go @@ -16,12 +16,12 @@ func newStr(obj *cPyObject) Str { func MakeStr(s string) Str { ptr := (*Char)(unsafe.Pointer(unsafe.StringData(s))) - length := C.long(len(s)) + length := C.Py_ssize_t(len(s)) return newStr(C.PyUnicode_FromStringAndSize(ptr, length)) } func (s Str) String() string { - var l C.long + var l C.Py_ssize_t buf := C.PyUnicode_AsUTF8AndSize(s.obj, &l) return GoStringN((*Char)(buf), int(l)) } @@ -31,7 +31,7 @@ func (s Str) Len() int { } func (s Str) ByteLen() int { - var l C.long + var l C.Py_ssize_t _ = C.PyUnicode_AsUTF8AndSize(s.obj, &l) return int(l) }