try better helm

.travis.yml

@@ -1,44 +1,55 @@
 sudo: required
+
+language: go
+
+go:
+  - 1.11.x
+
 services:
-- docker
-language: python
-python:
-- '2.7'
-- '3.6'
-install:
-- echo ok
-- sudo apt-get install golang-1.10-go
-- curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-- echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
-- sudo apt-get update
-- sudo apt-get install -y kubectl
-- /usr/lib/go-1.10/bin/go version
-- export GOROOT=/usr/lib/go-1.10
-- /usr/lib/go-1.10/bin/go get sigs.k8s.io/kind
-- curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | sudo bash
-- kind create cluster
-- export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
-- docker build -t yadage/crdctrl:latest -f images/crdctrl/Dockerfile images/crdctrl
-- kind load docker-image yadage/crdctrl:latest
-script:
-- echo ok
-- kubectl create clusterrolebinding permissive-binding --clusterrole=cluster-admin --user=admin --user=kubelet --group=system:serviceaccounts
-- helm init --service-account default --wait --upgrade
-- sleep 30
-- kubectl create namespace yadage
-- cd helm; helm install metacontroller; cd -
-- cd helm; helm install yadage --set crdctrl_image=yadage/crdctrl:latest; cd -
-- sleep 20
-- kubectl get pods -n yadage
-- kubectl create -f crd/pvc.yml -f crd/pv.yml
-- kubectl create -f examples/wflow_mg.yml
-- sleep 60
-- kubectl get pods
-- kubectl get pods|grep myflow-yadage|awk '{print $1}'|xargs kubectl logs -f
-- docker exec kind-control-plane cat /mnt/workdir/delphes/output.lhco
+  - docker
 
 jobs:
   include:
+    - stage: test
+      install:
+      - echo ok
+      - export GO111MODULE="on"
+      - go version
+      - go get -u sigs.k8s.io/kind
+      - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+      - echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
+      - sudo apt-get update
+      - sudo apt-get install -y kubectl
+      - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | sudo bash
+      - kind create cluster
+      - export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+      - docker build -t yadage/crdctrl:latest -f images/crdctrl/Dockerfile images/crdctrl
+      - kind load docker-image yadage/crdctrl:latest
+      script:
+      - echo ok
+      - kubectl create clusterrolebinding permissive-binding --clusterrole=cluster-admin --user=admin --user=kubelet --group=system:serviceaccounts
+      - helm init --service-account default --wait --upgrade
+      - sleep 10
+      - kubectl create namespace yadage
+      - cd helm; helm install metacontroller --namespace metacontroller; cd -
+      - >
+        cd helm;
+        helm install yadage 
+        --set crdctrl_image=yadage/crdctrl:latest 
+        --set storage.crate=true
+        --set storage.node_name=kind-control-plane
+        --namespace yadage;
+        cd -
+      - sleep 10
+      - kubectl get pods -n yadage
+      # - curl https://raw.githubusercontent.com/lukasheinrich/yadage-workflows/master/phenochain/madgraph_delphes.yml
+      # - docker run --rm -it alpine sh -c 'apk add curl;curl https://raw.githubusercontent.com/lukasheinrich/yadage-workflows/master/phenochain/madgraph_delphes.yml'
+      # - kubectl run -it  hello --image alpine -- sh -c 'apk add curl;curl https://raw.githubusercontent.com/lukasheinrich/yadage-workflows/master/phenochain/madgraph_delphes.yml'
+      # - kubectl create -f examples/wflow_mg.yml
+      # - sleep 60
+      # - kubectl get pods
+      # - kubectl get pods|grep myflow-yadage|awk '{print $1}'|xargs kubectl logs -f
+      # - docker exec kind-control-plane cat /mnt/workdir/delphes/output.lhco
     - stage: helm
       install:
         - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | sudo bash

README.md

@@ -1,3 +1,34 @@
 Kubernetes CRD for yadage workflows
 
 [![Build Status](https://travis-ci.com/yadage/yadage-crd.svg?branch=master)](https://travis-ci.com/yadage/yadage-crd)
+
+
+on a checkout
+
+```
+helm install metacontroller --namespace metacontroller
+helm install \
+--namespace yadage \
+-f ../helm_values/ci.yml \
+--set secrets.username=XXXX \
+--set secrets.password=YYYY \
+--set imageCredentials.username=ZZZZ \
+--set imageCredentials.password=AAAA \
+yadage
+```
+
+from the release
+
+```
+helm repo add yadage https://yadage.github.io/yadage-crd
+helm repo update
+helm install yadage/metacontroller --namespace metacontroller
+helm install \
+--namespace yadage \
+-f ../helm_values/ci.yml \
+--set secrets.username=XXXX \
+--set secrets.password=YYYY \
+--set imageCredentials.username=ZZZZ \
+--set imageCredentials.password=AAAA \
+yadage/yadage
+```

helm/metacontroller/templates/metacontroller-rbac.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: metacontroller
-  namespace: metacontroller
+  namespace: {{ .Release.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -23,7 +23,7 @@ metadata:
 subjects:
 - kind: ServiceAccount
   name: metacontroller
-  namespace: metacontroller
+  namespace: {{ .Release.Namespace }}
 roleRef:
   kind: ClusterRole
   name: metacontroller

helm/metacontroller/templates/metacontroller.yaml

@@ -49,7 +49,7 @@ metadata:
   labels:
     app.kubernetes.io/name: metacontroller
   name: metacontroller
-  namespace: metacontroller
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: 1
   selector:
@@ -68,6 +68,6 @@ spec:
         command: ["/usr/bin/metacontroller"]
         args:
         - --logtostderr
-        - -v=4
+        - -v={{.Values.verbosity}}
         - --discovery-interval=20s
   volumeClaimTemplates: []

helm/metacontroller/values.yaml

@@ -4,6 +4,8 @@
 
 replicaCount: 1
 
+verbosity: 4
+
 image:
   repository: nginx
   tag: stable

helm/yadage/templates/filebrowser.yml

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: filebrowser
-  # namespace: yadage
+  namespace: {{ .Values.job.namespace }}
 spec:
   selector:
     matchLabels:
@@ -33,7 +33,7 @@ spec:
         - -c
         - |
           pip install sauth
-          sauth {{ .Values.filebrowser_user }} {{ .Values.filebrowser_password }} 0.0.0.0 80
+          sauth {{ .Values.filebrowser.user }} {{ .Values.filebrowser.password }} 0.0.0.0 80
         ports:
         - containerPort: 80
         volumeMounts:
@@ -42,13 +42,13 @@ spec:
       volumes:
       - name: data
         persistentVolumeClaim:
-          claimName: {{ .Values.claim_name }}
+          claimName: {{ .Values.storage.claim_name }}
 ---
 kind: Service
 apiVersion: v1
 metadata:
   name: filebrowser
-  # namespace: yadage
+  namespace: {{ .Values.job.namespace }} # comment: this must be the namespace of the job since the PV of the jobs must be accessible here
 spec:
   type: NodePort
   selector:
@@ -63,6 +63,7 @@ kind: Service
 apiVersion: v1
 metadata:
   name: fileaccess
+  namespace: {{ .Values.job.namespace }} # comment: this must be the namespace of the job since the PV of the jobs must be accessible here
 spec:
   selector:
     app: filebrowser

helm/yadage/templates/hook.yml

@@ -8,7 +8,7 @@ apiVersion: apps/v1beta1
 kind: Deployment
 metadata:
   name: wflow-controller
-  namespace: yadage
+  namespace: {{ .Release.Namespace }}
 spec:
   replicas: 1
   selector:
@@ -21,27 +21,31 @@ spec:
     spec:
       containers:
       - name: controller
-        image: {{ .Values.crdctrl_image }}
-        imagePullPolicy: IfNotPresent
+        image: {{ .Values.crdctrl.image }}
+        imagePullPolicy: {{ .Values.crdctrl.pull_policy }}
         env:
         - name: YADKUBE_IMAGE
-          value: {{ .Values.yadage_image }}
+          value: {{ .Values.yadage.image }}
         - name: YADKUBE_IMAGE_POLICY
-          value: IfNotPresent
+          value: {{ .Values.yadage.pull_policy }}
         - name: YADKUBE_CLAIM
-          value: {{ .Values.claim_name }}
+          value: {{ .Values.storage.claim_name }}
         - name: YADKUBE_PRIVATE_TOKEN
           value: {{ .Values.private_token }}
+        - name: YADKUBE_CPU_REQUEST
+          value: {{ .Values.engine_cpu_request }}
         - name: YADKUBE_AUTH_SECRET
-          value: hepauth
+          value: {{ .Values.job.experiment_creds }}
         - name: YADKUBE_REGCRED_SECRET
-          value: hepimgcred
+          value: {{ .Values.job.registry_creds }}
+        - name: YADKUBE_RESOURCE_OPTS
+          value: '{{ .Values.job.resource_opts | toJson }}'
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: wflow-controller
-  namespace: yadage
+  namespace: {{ .Release.Namespace }}
 spec:
   selector:
     app: wflow-controller

helm/yadage/templates/job_role.yml

@@ -0,0 +1,26 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: yadage
+  namespace: {{ .Values.job.namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Values.job.svcaccount }}
+    namespace: {{ .Values.job.namespace }}
+roleRef:
+  kind: Role
+  name: yadage-engine
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: yadage-engine
+  namespace: {{ .Values.job.namespace }}
+rules:
+- apiGroups: [""]
+  resources: ["pods", "configmaps"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: ["batch", "extensions"]
+  resources: ["jobs"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]

helm/yadage/templates/secrets.yml

@@ -0,0 +1,21 @@
+apiVersion: v1
+data: {"getkrb.sh": {{(printf "echo '%s'|kinit %[email protected]" .Values.secrets.password .Values.secrets.username)|b64enc}} }
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.job.experiment_creds }}
+  namespace: {{ .Values.job.namespace }}
+---
+
+{{- define "imagePullSecret" }}
+{{- printf "{\"auths\": {\"%s\": {\"auth\": \"%s\"}}}" .Values.imageCredentials.registry (printf "%s:%s" .Values.imageCredentials.username .Values.imageCredentials.password | b64enc) | b64enc }}
+{{- end }}
+
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/dockerconfigjson
+metadata:
+  name: {{ .Values.job.registry_creds }}
+  namespace: {{ .Values.job.namespace }}
+data:
+  .dockerconfigjson: {{ template "imagePullSecret" . }}

helm/yadage/templates/storage.yml

@@ -0,0 +1,55 @@
+{{ if (eq .Values.storage.create true) and (eq .Values.storage.type "local") }}
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: local-pv
+  namespace: {{ .Values.job.namespace }}
+spec:
+  capacity:
+    storage: {{ .Values.storage.size }}
+  accessModes:
+  - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: {{ .Values.storage.sclass_name }}
+  local:
+    path: {{ .Values.storage.path }}
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - {{ .Values.storage.node_name }}
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.storage.claim_name }}
+  namespace: {{ .Values.job.namespace }}
+spec:
+  accessModes:
+  - ReadWriteMany
+  storageClassName: local-storage
+  resources:
+    requests:
+      storage: {{ .Values.storage.size }}
+
+{{ else if (eq .Values.storage.create true) and (eq .Values.storage.type "cern") }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ .Values.storage.claim_name }}
+  namespace: {{ .Values.job.namespace }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10G
+  storageClassName: {{ .Values.storage.sclass_name }}
+
+{{ else }}
+
+{{ end }}