Skip to content

Commit

Permalink
use separate crd controller
Browse files Browse the repository at this point in the history
  • Loading branch information
lionelvillard committed Nov 20, 2019
1 parent 3f68a69 commit 6a5af58
Show file tree
Hide file tree
Showing 8 changed files with 312 additions and 92 deletions.
21 changes: 12 additions & 9 deletions cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"knative.dev/pkg/signals"

"github.com/lionelvillard/knative-functions-controller/pkg/dynamic"
"github.com/lionelvillard/knative-functions-controller/pkg/reconciler/crds"
"github.com/lionelvillard/knative-functions-controller/pkg/reconciler/functions"
)

Expand All @@ -47,6 +48,8 @@ var (
func main() {
flag.Parse()

// Load functions CRDs.

cfg, err := sharedmain.GetConfig(*masterURL, *kubeconfig)
if err != nil {
log.Fatal("Error building kubeconfig", err)
Expand All @@ -67,9 +70,11 @@ func main() {
log.Fatalf("Error processing environment: %v", err)
}

ctx := signals.NewContext()
// Create a controller per function CRD.

controllers := make([]injection.ControllerConstructor, len(defs.Items)+1)
controllers[0] = crds.NewController

controllers := make([]injection.ControllerConstructor, len(defs.Items))
names := make([]string, len(defs.Items))
for i, crd := range defs.Items {
gvr := schema.GroupVersionResource{
Expand All @@ -79,21 +84,19 @@ func main() {
}
names[i] = crd.Name

log.Printf("injecting %v", gvr)

injection.Default.RegisterInformer(dynamic.WithInformer(gvr))

controllers[i] = functions.NewController(gvr)
controllers[i+1] = functions.NewController(gvr)
}

// Watch for any CRD changes
ctx := signals.NewContext()

f := externalversions.NewSharedInformerFactory(clientset, time.Hour)
crdInformer := f.Apiextensions().V1beta1().CustomResourceDefinitions().Informer()
crdInformer.AddEventHandler(newHandler(names))
go crdInformer.Run(ctx.Done())

if len(controllers) > 0 {
sharedmain.MainWithConfig(ctx, "controller", cfg, controllers...)
}
sharedmain.MainWithConfig(ctx, "controller", cfg, controllers...)

<-ctx.Done()
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/apis/duck/v1alpha1/function.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ import (
"knative.dev/pkg/kmeta"
)

const (
ConfigMapAnnotation = "functions.knative.dev/configmap-version"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

Expand Down
1 change: 0 additions & 1 deletion pkg/dynamic/dynamic.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ type Key struct {

func WithInformer(gvr schema.GroupVersionResource) func(ctx context.Context) (context.Context, controller.Informer) {
return func(ctx context.Context) (context.Context, controller.Informer) {

f := factory.Get(ctx)
inf := f.ForResource(gvr)
return context.WithValue(ctx, Key{gvr: gvr}, inf), inf.Informer()
Expand Down
81 changes: 81 additions & 0 deletions pkg/reconciler/crds/controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package crds

import (
"context"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
apiextensionsclient "knative.dev/pkg/client/injection/apiextensions/client"
crdinformers "knative.dev/pkg/client/injection/apiextensions/informers/apiextensions/v1beta1/customresourcedefinition"
kubeclient "knative.dev/pkg/client/injection/kube/client"
"knative.dev/pkg/configmap"
"knative.dev/pkg/controller"
"knative.dev/pkg/logging"
servingclient "knative.dev/serving/pkg/client/injection/client"
serviceinformer "knative.dev/serving/pkg/client/injection/informers/serving/v1beta1/service"
)

const (
controllerAgentName = "crd-controller"
)

// NewController returns a new CRD reconcile controller.
func NewController(ctx context.Context, cmw configmap.Watcher) *controller.Impl {
logger := logging.FromContext(ctx)

crdInformer := crdinformers.Get(ctx)
serviceInformer := serviceinformer.Get(ctx)

r := &Reconciler{
kubeClient: kubeclient.Get(ctx),
crdClient: apiextensionsclient.Get(ctx),
crdLister: crdInformer.Lister(),
servingClient: servingclient.Get(ctx),
serviceLister: serviceInformer.Lister(),
Recorder: record.NewBroadcaster().NewRecorder(
scheme.Scheme, corev1.EventSource{Component: controllerAgentName}),
}
impl := controller.NewImpl(r, logger, "crd")

logger.Info("Setting up event handlers")

crdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{
FilterFunc: func(obj interface{}) bool {
if object, ok := obj.(metav1.Object); ok {
if labels := object.GetLabels(); labels != nil {
if v, ok := labels["functions.knative.dev/crd"]; ok && v == "true" {
// for _, name := range names {
// if name == object.GetName() {
// return false
// }
// }
return true
}
}
}
return false
},
Handler: controller.HandleAll(impl.Enqueue),
})

return impl
}
181 changes: 181 additions & 0 deletions pkg/reconciler/crds/crds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
/*
Copyright 2019 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package crds

import (
"context"
"errors"
"fmt"

"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"

apiextclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/api/equality"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
"knative.dev/pkg/controller"
"knative.dev/pkg/logging"
"knative.dev/pkg/system"
servingv1beta1 "knative.dev/serving/pkg/apis/serving/v1beta1"
servingclient "knative.dev/serving/pkg/client/clientset/versioned"
servingv1beta1listers "knative.dev/serving/pkg/client/listers/serving/v1beta1"

"github.com/lionelvillard/knative-functions-controller/pkg/reconciler/crds/resources"
)

// Reconciler implements controller.Reconciler for dynamic resources.
type Reconciler struct {
// KubeClient allows us to talk to the k8s for core APIs
kubeClient kubernetes.Interface

//
crdClient apiextclientset.Interface

// servingClient allows us to talk to the serving APIs
servingClient servingclient.Interface

// serviceLister index properties about Knative services
serviceLister servingv1beta1listers.ServiceLister

// crdLister index properties about CRDs
crdLister apiextensionsv1beta1.CustomResourceDefinitionLister

// Recorder is an event recorder for recording Event resources to the
// Kubernetes API.
Recorder record.EventRecorder
}

// Check that our Reconciler implements controller.Reconciler
var _ controller.Reconciler = (*Reconciler)(nil)

// Reconcile implements controller.Reconciler
func (r *Reconciler) Reconcile(ctx context.Context, name string) error {
logger := logging.FromContext(ctx)

crd, err := r.crdLister.Get(name)
if apierrs.IsNotFound(err) {
// The resource may no longer exist, in which case we stop processing.
logger.Errorf("resource %q no longer exists", name)
return nil
} else if err != nil {
return err
}

return r.reconcile(ctx, crd)
}

func (r *Reconciler) reconcile(ctx context.Context, crd *apiextv1beta1.CustomResourceDefinition) error {
if crd.GetDeletionTimestamp() != nil {
// Check for a DeletionTimestamp. If present, elide the normal reconcile logic.
// When a controller needs finalizer handling, it would go here.
return nil
}

functionName := crd.Spec.Names.Plural

// Make sure the function service/configmaps exists
cm, err := r.reconcileConfig(ctx, functionName)
if err != nil {
return err
}

_, err = r.reconcileService(ctx, functionName, cm)
if err != nil {

return err
}

return nil
}

func (r *Reconciler) reconcileConfig(ctx context.Context, functionName string) (*corev1.ConfigMap, error) {
logger := logging.FromContext(ctx)

cmname := fmt.Sprintf("config-function-%s", functionName)

cm, err := r.kubeClient.CoreV1().ConfigMaps(system.Namespace()).Get(cmname, metav1.GetOptions{})
if err != nil {
if apierrs.IsNotFound(err) {
cm, err = resources.MakeConfigMap(system.Namespace(), cmname)
if err != nil {
logger.Error("Failed to create the function configmap", zap.Error(err))
return nil, err
}
cm, err = r.kubeClient.CoreV1().ConfigMaps(system.Namespace()).Create(cm)
if err != nil {
logger.Error("Failed to create the function configmap", zap.Error(err))
return nil, err
}
} else {
logger.Error("Unable to get the function configmap", zap.Error(err))
return nil, err
}
}

return cm, nil
}

func (r *Reconciler) reconcileService(ctx context.Context, functionName string, cm *corev1.ConfigMap) (*servingv1beta1.Service, error) {
logger := logging.FromContext(ctx)

crd, err := r.crdLister.Get(functionName + ".functions.knative.dev")
if err != nil {
logger.Error("Failed to get function Custom Resource Definition", zap.Error(err))
return nil, fmt.Errorf("Failed to get function Custom Resource Definition: %v", err)
}

image, ok := crd.Annotations["functions.knative.dev/image"]
if !ok {
logger.Error("Missing functions.knative.dev/image annotation on function CRD", zap.Any("functionName", functionName))
return nil, errors.New("Missing functions.knative.dev/image annotation on function CRD")
}

expected := resources.MakeKnativeService(functionName, cm.ResourceVersion, image)

// Update service annotation with config map UUID.
service, err := r.serviceLister.Services("knative-functions").Get(functionName)
if err != nil {
if apierrs.IsNotFound(err) {

ksvc, err := r.servingClient.ServingV1beta1().Services("knative-functions").Create(expected)
if err != nil {
logger.Error("Failed to create the function service", zap.Error(err))
return nil, fmt.Errorf("Failed to create the function service: %v", err)
}
return ksvc, nil
}

logger.Error("Unable to get the function service", zap.Error(err))
return nil, err
} else if !equality.Semantic.DeepEqual(expected.Spec, service.Spec) {
service = service.DeepCopy()
service.Spec = expected.Spec

service, err = r.servingClient.ServingV1beta1().Services("knative-functions").Update(service)
if err != nil {
logger.Error("Failed to update the function service", zap.Error(err))
return nil, fmt.Errorf("Failed to update the function service: %v", err)
}
}

return service, nil
}
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,9 @@ package resources
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

servingv1beta1 "knative.dev/serving/pkg/apis/serving/v1beta1"
)

const (
ConfigMapAnnotation = "functions.knative.dev/cm-resourceVersion"
duckv1alpha1 "github.com/lionelvillard/knative-functions-controller/pkg/apis/duck/v1alpha1"
)

// MakeKnativeService create a knative service
Expand All @@ -37,15 +34,12 @@ func MakeKnativeService(functionName string, version, image string) *servingv1be
ObjectMeta: metav1.ObjectMeta{
Name: functionName,
Namespace: "knative-functions",
Labels: map[string]string{
FunctionRoleLabel: FunctionRole,
},
},
Spec: servingv1beta1.ServiceSpec{
ConfigurationSpec: servingv1beta1.ConfigurationSpec{
Template: servingv1beta1.RevisionTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{ConfigMapAnnotation: version},
Annotations: map[string]string{duckv1alpha1.ConfigMapAnnotation: version},
},
Spec: servingv1beta1.RevisionSpec{
PodSpec: corev1.PodSpec{
Expand Down
Loading

0 comments on commit 6a5af58

Please sign in to comment.