*: update server

examples/server/server.py

@@ -0,0 +1,7 @@
+import os
+
+from npiai.app import GitHub
+
+os.environ.setdefault("NPIAI_TOOL_SERVER_MODE", "true")
+instance = GitHub(access_token="123")
+instance.server()

npiai/app/github/npi.yml

@@ -2,12 +2,13 @@ main: app.py
 class: GitHub
 env:
   - name: GITHUB_ACCESS_TOKEN
-    value: 'GITHUB_ACCESS_TOKEN'
+    value: 'YOUR_GITHUB_ACCESS_TOKEN'
+    secret: true
 dependencies:
   - name: python
     version: "^3.10"
   - name: npiai
-    version: "0.1.1"
+    version: "^0.1.2"
   - name: openai
     version: "^1.30.3"
   - name: pygithub

npiai/core/app/app.py

@@ -7,6 +7,7 @@
 import re
 import signal
 import sys
+import time
 from dataclasses import asdict
 from typing import Dict, List, Optional, Any
 import logging
@@ -150,6 +151,8 @@ async def server(self):
         """Start the server"""
         self.start()
         if not bool(os.environ.get("NPIAI_TOOL_SERVER_MODE")):
+            print("Server mode is disabled, if you want to run the server, set env NPIAI_TOOL_SERVER_MODE=true")
+            print("Exiting...")
             return
 
         fapp = FastAPI()
@@ -175,11 +178,20 @@ async def root(full_path: str, request: Request):
                 raise HTTPException(status_code=500, detail="Internal Server Error")
 
         def signal_handler(sig, frame):
-            print(f"Signal {sig} received, shutting down.")
-            asyncio.create_task(self.end())
+            print(f"Signal {sig} received, shutting down...")
+            try:
+                loop = asyncio.get_running_loop()
+            except RuntimeError:  # 'RuntimeError: There is no current event loop...'
+                loop = None
+
+            tsk = loop.create_task(self.end())
+            # while not tsk.done():
+            #     time.sleep(1)
+            print("Shutdown complete")
             sys.exit(0)
 
         signal.signal(signal.SIGINT, signal_handler)
+        signal.signal(signal.SIGKILL, signal_handler)
         signal.signal(signal.SIGTERM, signal_handler)
 
         uvicorn.run(fapp, host="0.0.0.0", port=18000)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "npiai"
-version = "0.1.1"
+version = "0.1.2"
 description = ""
 authors = ["wenfeng <[email protected]>"]
 readme = "README.md"

server/api/errors.go

@@ -8,9 +8,11 @@ import (
 )
 
 var (
-	ErrParseBody      = newErrorMessage(http.StatusBadRequest, 40001, "parse body error")
-	ErrInvalidRequest = newErrorMessage(http.StatusBadRequest, 40002, "invalid request")
-	ErrObjectID       = newErrorMessage(http.StatusBadRequest, 40003, "invalid object id")
+	ErrParseBody        = newErrorMessage(http.StatusBadRequest, 40001, "parse body error")
+	ErrInvalidRequest   = newErrorMessage(http.StatusBadRequest, 40002, "invalid request")
+	ErrObjectID         = newErrorMessage(http.StatusBadRequest, 40003, "invalid object id")
+	ErrBalanceNotEnough = newErrorMessage(http.StatusBadRequest, 40004, "out of usage")
+	ErrInvitationExists = newErrorMessage(http.StatusBadRequest, 40005, "invitation already exists")
 
 	ErrUnauthorized = newErrorMessage(http.StatusUnauthorized, 40101, "unauthorized")
 
@@ -20,6 +22,8 @@ var (
 	ErrResourceNotFound = newErrorMessage(http.StatusNotFound, 40402, "resource not found")
 	DocumentNotFound    = newErrorMessage(http.StatusNotFound, 40403, "data not found in database")
 
+	ErrCannotLeave = newErrorMessage(http.StatusConflict, 40901, "cannot leave the org")
+
 	ErrInternal      = newErrorMessage(http.StatusInternalServerError, 50001, "internal error")
 	ErrUnknown       = newErrorMessage(http.StatusInternalServerError, 50002, "unknown error")
 	ErrResourceRetry = newErrorMessage(http.StatusInternalServerError, 50003, "resource retry")

server/api/request.go

@@ -15,3 +15,31 @@ type ToolRequest struct {
 	GitHubBranch string   `json:"github_branch"`
 	RepoDir      string   `json:"repo_dir"`
 }
+
+type UpdateUserProfile struct {
+	Nickname string `json:"nickname,omitempty"`
+	Picture  string `json:"picture,omitempty"`
+}
+
+type UpdateOrgProfile struct {
+	Name    string `json:"name,omitempty"`
+	Picture string `json:"picture,omitempty"`
+}
+
+type MoveAfterForm struct {
+	AfterPageID  string `json:"after_page_id" binding:"required"`
+	ParentPageID string `json:"parent_page_id" binding:"required"`
+}
+
+type InviteNewMemberForm struct {
+	Email string `json:"email" binding:"required,email"`
+	Role  string `json:"role" binding:"required,oneof=owner admin writer reader"`
+}
+
+type UpdateMembershipForm struct {
+	Role string `json:"role" binding:"required"`
+}
+
+type DeleteMembersForm struct {
+	UserIDs []string `json:"user_ids" binding:"required"`
+}

server/config/config.go

@@ -1,13 +1,9 @@
 package config
 
 import (
-	"github.com/npi-ai/npi/server/db"
 	"github.com/npi-ai/npi/server/reconcile"
 )
 
 type ServerConfig struct {
-	Port    int                            `yaml:"port"`
-	MongoDB db.MongoConfig                 `yaml:"mongodb"`
-	S3      db.S3Config                    `yaml:"s3"`
-	Tools   reconcile.ToolReconcilerConfig `yaml:"tools"`
+	Tools reconcile.ToolReconcilerConfig `yaml:"tools"`
 }

server/db/s3.go

@@ -4,16 +4,14 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
-	"os"
-	"strings"
-	"sync"
-
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/npi-ai/npi/server/constant"
+	"io"
+	"os"
+	"strings"
 )
 
 type S3 struct {
@@ -26,33 +24,44 @@ type S3Config struct {
 	AK        string `yaml:"ak"`
 	SK        string `yaml:"sk"`
 	Region    string `yaml:"region"`
+	Bucket    string `yaml:"bucket"`
+	Endpoint  string `yaml:"endpoint"`
 }
 
-var (
-	s3Svc *S3
-	once  sync.Once
-)
-
-func InitS3(cfg S3Config) *S3 {
-	once.Do(func() {
-		s3Svc = &S3{}
-		optFns := []func(*config.LoadOptions) error{
-			config.WithRegion(cfg.Region),
-		}
-		if cfg.AK != "" && cfg.SK != "" {
-			optFns = append(optFns, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AK, cfg.SK, "")))
-		}
-		s3Cfg, err := config.LoadDefaultConfig(context.TODO(), optFns...)
-		if err != nil {
-			panic(fmt.Sprintf("failed to init s3: %s", err))
-		}
-
-		s3Svc.cli = s3.NewFromConfig(s3Cfg)
+func GetCloudflareR2(cfg S3Config) *S3 {
+	//https://fef624caaf39cd90f34029a5643c7d0f.r2.cloudflarestorage.com
+	r2Resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		return aws.Endpoint{
+			SigningRegion: "auto",
+			URL:           fmt.Sprintf("https://%s.r2.cloudflarestorage.com", cfg.AccountID),
+		}, nil
 	})
-	return s3Svc
+
+	dfCfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithEndpointResolverWithOptions(r2Resolver),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AK, cfg.SK, "")),
+		config.WithRegion("auto"),
+	)
+	if err != nil {
+		panic(fmt.Sprintf("failed to init cloudflare r2: %s", err))
+	}
+	return &S3{cli: s3.NewFromConfig(dfCfg)}
 }
 
-func GetS3() *S3 {
+func GetAWSS3(cfg S3Config) *S3 {
+	s3Svc := &S3{}
+	optFns := []func(*config.LoadOptions) error{
+		config.WithRegion(cfg.Region),
+	}
+	if cfg.AK != "" && cfg.SK != "" {
+		optFns = append(optFns, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(cfg.AK, cfg.SK, "")))
+	}
+	s3Cfg, err := config.LoadDefaultConfig(context.TODO(), optFns...)
+	if err != nil {
+		panic(fmt.Sprintf("failed to init s3: %s", err))
+	}
+
+	s3Svc.cli = s3.NewFromConfig(s3Cfg)
 	return s3Svc
 }
 

server/go.mod

@@ -5,6 +5,7 @@ go 1.22.0
 toolchain go1.22.4
 
 require (
+	github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b
 	github.com/aws/aws-sdk-go-v2 v1.27.2
 	github.com/aws/aws-sdk-go-v2/config v1.27.18
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.18
@@ -14,6 +15,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20
 	github.com/pb33f/libopenapi v0.16.8
 	github.com/rs/zerolog v1.33.0
+	github.com/vincent-petithory/dataurl v1.0.0
 	go.mongodb.org/mongo-driver v1.15.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.30.2

server/go.sum

@@ -1,3 +1,8 @@
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
+github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
 github.com/aws/aws-sdk-go-v2 v1.27.2 h1:pLsTXqX93rimAOZG2FIYraDQstZaaGVVN4tNw65v0h8=
 github.com/aws/aws-sdk-go-v2 v1.27.2/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
@@ -219,6 +224,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI=
+github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
 github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk=
 github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
@@ -282,6 +289,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -311,6 +319,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
 golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
@@ -350,6 +359,7 @@ gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
 k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
 k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=

server/reconcile/tool.go

@@ -38,16 +38,19 @@ type ToolReconcilerConfig struct {
 	ValidateScript string `yaml:"validate_script"`
 	KubeConfig     string `yaml:"kubeconfig"`
 	Namespace      string `yaml:"namespace"`
+	AccessEndpoint string `yaml:"access_endpoint"`
 }
 
 func NewToolReconciler(cfg ToolReconcilerConfig) Reconciler[model.ToolInstance] {
 	return &toolReconciler{
 		coll: db.GetCollection(db.CollToolInstances),
 		builderSvc: service.NewBuilderService(
 			cfg.DockerRegistry,
-			cfg.S3Bucket,
 			cfg.Workdir,
 			cfg.ValidateScript,
+			db.S3Config{
+				Bucket: cfg.S3Bucket,
+			},
 		),
 		deploySvc: service.NewDeploymentService(cfg.KubeConfig, cfg.Namespace),
 	}
@@ -164,6 +167,7 @@ func (tc *toolReconciler) BuildingHandler(ctx context.Context, toolInstance mode
 	return db.ConvertError(err)
 }
 
+// DeployingHandler TODO add env
 func (tc *toolReconciler) DeployingHandler(ctx context.Context, toolInstance model.ToolInstance) error {
 	name := utils.GenerateRandomString(12, false, false, false)
 	if err := tc.deploySvc.CreateDeployment(ctx, name, toolInstance.Image, toolInstance.GetRuntimeEnv()); err != nil {
@@ -195,6 +199,7 @@ func (tc *toolReconciler) DeployingHandler(ctx context.Context, toolInstance mod
 		bson.M{
 			"$set": bson.M{
 				"service_url": svcUrl,
+				"deploy_name": name,
 				"updated_at":  time.Now(),
 			},
 		})