Merge pull request #63 from cybertec-postgresql/pgbackrest-clone

Pgbackrest clone

charts/postgres-operator/crds/postgresqls.yaml

@@ -133,6 +133,39 @@ spec:
                   uid:
                     format: uuid
                     type: string
+                  pgbackrest:
+                    type: object
+                    properties:
+                      configuration:
+                        type: object
+                        properties:
+                          secret:
+                            type: string
+                      options:
+                        type: object
+                        additionalProperties:
+                          type: string
+                      repo:
+                        type: object
+                        properties:
+                          storage:
+                            type: string
+                            enum:
+                              - "s3"
+                              - "gcs"
+                              - "azure"
+                              - "pvc"
+                          resource:
+                            type: string
+                          endpoint:
+                            type: string
+                          region:
+                            type: string
+                        required:
+                          - storage
+                          - resource
+                    required:
+                      - repo
               connectionPooler:
                 type: object
                 properties:

charts/postgres-operator/values.yaml

@@ -440,9 +440,11 @@ configMultisite:
   # Must be unique for each site.
   site: ""
   # IP address or hostname of shared etcd cluster used for multicluster operation.
-  etcd_host: ""
-  etcd_user: ""
-  etcd_password: ""
+  etcd:
+    hosts:
+    user: ""
+    password: ""
+    protocol: http
   # Timeout for cross site failover, and timeout for demoting to read only when accessing shared etcd cluster fails.
   # There should be adequate safety margin between the two to allow for demotion to take place.
   #ttl: 90

manifests/postgresql.crd.yaml

@@ -131,6 +131,39 @@ spec:
                   uid:
                     format: uuid
                     type: string
+                  pgbackrest:
+                    type: object
+                    properties:
+                      configuration:
+                        type: object
+                        properties:
+                          secret:
+                            type: string
+                      options:
+                        type: object
+                        additionalProperties:
+                          type: string
+                      repo:
+                        type: object
+                        properties:
+                          storage:
+                            type: string
+                            enum:
+                              - "s3"
+                              - "gcs"
+                              - "azure"
+                              - "pvc"
+                          resource:
+                            type: string
+                          endpoint:
+                            type: string
+                          region:
+                            type: string
+                        required:
+                          - storage
+                          - resource
+                    required:
+                      - repo
               connectionPooler:
                 type: object
                 properties:

pkg/apis/cpo.opensource.cybertec.at/v1/crds.go

@@ -213,6 +213,68 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
 								Type:   "string",
 								Format: "uuid",
 							},
+							"pgbackrest": {
+								Type: "object",
+								Properties: map[string]apiextv1.JSONSchemaProps{
+									"configuration": {
+										Type: "object",
+										Properties: map[string]apiextv1.JSONSchemaProps{
+											"secret": {
+												Type: "string",
+											},
+										},
+									},
+									"options": {
+										Type: "object",
+										AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
+											Schema: &apiextv1.JSONSchemaProps{
+												Type:                   "string",
+												XPreserveUnknownFields: util.True(),
+											},
+										},
+									},
+									"repo": {
+										Type: "object",
+										Properties: map[string]apiextv1.JSONSchemaProps{
+											"storage": {
+												Type: "string",
+												Enum: []apiextv1.JSON{
+													{
+														Raw: []byte(`"s3"`),
+													},
+													{
+														Raw: []byte(`"gcs"`),
+													},
+													{
+														Raw: []byte(`"azure"`),
+													},
+													{
+														Raw: []byte(`"pvc"`),
+													},
+												},
+											},
+											"resource": {
+												Type: "string",
+											},
+											"endpoint": {
+												Type: "string",
+											},
+											"region": {
+												Type: "string",
+											},
+											"account": {
+												Type: "string",
+											},
+											"key": {
+												Type: "string",
+											},
+											"keyType": {
+												Type: "string",
+											},
+										},
+									},
+								},
+							},
 						},
 					},
 					"connectionPooler": {

pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go

@@ -203,14 +203,15 @@ type TLSDescription struct {
 
 // CloneDescription describes which cluster the new should clone and up to which point in time
 type CloneDescription struct {
-	ClusterName       string `json:"cluster,omitempty"`
-	UID               string `json:"uid,omitempty"`
-	EndTimestamp      string `json:"timestamp,omitempty"`
-	S3WalPath         string `json:"s3_wal_path,omitempty"`
-	S3Endpoint        string `json:"s3_endpoint,omitempty"`
-	S3AccessKeyId     string `json:"s3_access_key_id,omitempty"`
-	S3SecretAccessKey string `json:"s3_secret_access_key,omitempty"`
-	S3ForcePathStyle  *bool  `json:"s3_force_path_style,omitempty" defaults:"false"`
+	ClusterName       string           `json:"cluster,omitempty"`
+	UID               string           `json:"uid,omitempty"`
+	EndTimestamp      string           `json:"timestamp,omitempty"`
+	S3WalPath         string           `json:"s3_wal_path,omitempty"`
+	S3Endpoint        string           `json:"s3_endpoint,omitempty"`
+	S3AccessKeyId     string           `json:"s3_access_key_id,omitempty"`
+	S3SecretAccessKey string           `json:"s3_secret_access_key,omitempty"`
+	S3ForcePathStyle  *bool            `json:"s3_force_path_style,omitempty" defaults:"false"`
+	Pgbackrest        *PgbackrestClone `json:"pgbackrest,omitempty"`
 }
 
 // Sidecar defines a container to be run in the same pod as the Postgres container.
@@ -283,6 +284,12 @@ type Pgbackrest struct {
 	Resources     *Resources        `json:"resources,omitempty"`
 }
 
+type PgbackrestClone struct {
+	Repo          Repo              `json:"repo"`
+	Options       map[string]string `json:"options"`
+	Configuration Configuration     `json:"configuration"`
+}
+
 type Repo struct {
 	Name     string            `json:"name"`
 	Storage  string            `json:"storage"`

pkg/cluster/cluster.go

@@ -366,6 +366,12 @@ func (c *Cluster) Create() (err error) {
 		c.logger.Info("a monitoring secret was successfully created")
 	}
 
+	if specHasPgbackrestClone(&c.Postgresql.Spec) {
+		if err := c.createPgbackrestCloneConfig(); err != nil {
+			return fmt.Errorf("could not create pgbackrest clone config: %v", err)
+		}
+	}
+
 	if c.multisiteEnabled() {
 		c.logger.Infof("waiting for load balancer IP to be assigned")
 		c.waitForPrimaryLoadBalancerIp()
@@ -1067,6 +1073,28 @@ func (c *Cluster) Update(oldSpec, newSpec *cpov1.Postgresql) error {
 		}
 	}()
 
+	// Clone configmap for pgbackrest
+	func() {
+		if specHasPgbackrestClone(&oldSpec.Spec) {
+			if specHasPgbackrestClone(&newSpec.Spec) {
+				// TODO: if we know cluster state and it has been initialized, then should ignore this
+				if err := c.updatePgbackrestCloneConfig(); err != nil {
+					c.logger.Warningf("could not update pgbackrest clone config: %v", err)
+					updateFailed = true
+				}
+			} else {
+				if err := c.deletePgbackrestCloneConfig(); err != nil {
+					c.logger.Warningf("could not delete pgbackrest clone config: %v", err)
+				}
+			}
+		} else if specHasPgbackrestClone(&newSpec.Spec) {
+			c.logger.Warningf("Can't add a clone specification after cluster has been initialized")
+			updateFailed = true
+		} else {
+			// TODO: try to delete just in case?
+		}
+	}()
+
 	// Statefulset
 	func() {
 		oldSs, err := c.generateStatefulSet(&oldSpec.Spec)
@@ -1225,6 +1253,10 @@ func specHasPgbackrestPVCRepo(newSpec *cpov1.PostgresSpec) bool {
 	return false
 }
 
+func specHasPgbackrestClone(newSpec *cpov1.PostgresSpec) bool {
+	return newSpec.Clone != nil && newSpec.Clone.Pgbackrest != nil
+}
+
 func syncResources(a, b *v1.ResourceRequirements) bool {
 	for _, res := range []v1.ResourceName{
 		v1.ResourceCPU,
@@ -1291,6 +1323,10 @@ func (c *Cluster) Delete() {
 		c.logger.Warningf("could not delete pod disruption budget: %v", err)
 	}
 
+	if err := c.deletePgbackrestCloneConfig(); err != nil {
+		c.logger.Warningf("could not delete pgbackrest clone config: %v", err)
+	}
+
 	for _, role := range []PostgresRole{Master, Replica, ClusterPods} {
 
 		if !c.patroniKubernetesUseConfigMaps() {