Kubernetes-Operator：扩展Kubernetes API Resource与Custom Controller (下)

export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac)
export OS=$(uname | awk '{print tolower($0)}')
export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v1.16.0
curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH}
chmod +x operator-sdk_${OS}_${ARCH} && sudo mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk

# 创建项目
mkdir -p sidecar-operator && cd sidecar-operator
# 初始化项目，这里的domain指定了后续注册CRD对象的Group域名，init初始化后会生成代码框架、Makefile工具文件、拉取依赖代码库
operator-sdk init --domain chinalhr.github.io --repo github.com/ChinaLHR/sidecar-operator
 
# 查看项目结构
tree -L 2
.
├── Dockerfile
├── LICENSE
├── Makefile
├── PROJECT
├── README.md
├── config
│   ├── default
│   ├── manager
│   ├── manifests
│   ├── prometheus
│   ├── rbac
│   └── scorecard
├── go.mod
├── go.sum
├── hack
│   └── boilerplate.go.txt
└── main.go

operator-sdk create api --group apps --version v1alpha1 --kind SidecarSet --resource --controller

/*
Copyright 2022.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

// SidecarSetSpec defines the desired state of SidecarSet
type SidecarSetSpec struct {
	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
	// Important: Run "make" to regenerate code after modifying this file

	//query over pods that should be injected sidecar
	Selector   *metav1.LabelSelector `json:"selector,omitempty"`
	Containers []SidecarContainer    `json:"containers,omitempty"`
}

type SidecarContainer struct {
	v1.Container `json:",inline"`
}

// SidecarSetStatus defines the observed state of SidecarSet
type SidecarSetStatus struct {
	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
	// Important: Run "make" to regenerate code after modifying this file

	//the number of Pods whose labels are matched with this SidecarSet's selector
	MatchedPods int32 `json:"matchedPods"`
	//the number of matched Pods that are injected with the latest SidecarSet's
	UpdatedPods int32 `json:"updatedPods"`
	//the number of matched Pods that have a ready condition
	ReadyPods int32 `json:"readyPods"`
}

// +genclient
// +genclient:nonNamespaced
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:printcolumn:name="MATCHED",type="integer",JSONPath=".status.matchedPods",description="The number of pods matched."
// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedPods",description="The number of pods matched and updated."
// +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyPods",description="the number of matched Pods that have a ready condition."

// SidecarSet is the Schema for the sidecarsets API
type SidecarSet struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   SidecarSetSpec   `json:"spec,omitempty"`
	Status SidecarSetStatus `json:"status,omitempty"`
}

//+kubebuilder:object:root=true

// SidecarSetList contains a list of SidecarSet
type SidecarSetList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []SidecarSet `json:"items"`
}

func init() {
	//将类型注册到API组中
	SchemeBuilder.Register(&SidecarSet{}, &SidecarSetList{})
}

make generate

// SetupWithManager sets up the controller with the Manager.
func (r *SidecarSetReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).	//控制器构建器
		For(&appsv1alpha1.SidecarSet{}).		//监控的主资源,对于每个SidecarSet类型的Add/Update/Delete事件，将发送给协调循环（Reconcile）
		Complete(r)								
}

func (r *SidecarSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	_ = log.FromContext(ctx)
	// TODO(user): your logic here
	return ctrl.Result{}, nil
}

# --programmatic-validation参数创建Validating admission webhook所需要的资源
# --defaulting 参数创建Mutating admission webhook所需要的资源
operator-sdk create webhook --group apps --version v1alpha1 --kind SidecarSet --programmatic-validation

func (r *SidecarSet) ValidateCreate() error {
	sidecarsetlog.Info("validate create", "name", r.Name)
	// TODO(user): fill in your validation logic upon object creation.
	return nil
}

func (r *SidecarSet) ValidateUpdate(old runtime.Object) error {
	sidecarsetlog.Info("validate update", "name", r.Name)
	// TODO(user): fill in your validation logic upon object update.
	return nil
}

func (r *SidecarSet) ValidateDelete() error {
	sidecarsetlog.Info("validate delete", "name", r.Name)
	// TODO(user): fill in your validation logic upon object deletion.
	return nil
}

//+kubebuilder:webhook:path=/mutate-sidecar-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mpod.kb.io,admissionReviewVersions=v1,sideEffects=None

type podHandler struct {
	Client  client.Client
	Decoder *admission.Decoder
}

var (
	SidecarEnvKey = "IS_INJECTED"
	SidecarAnnotationHashKey = "chinalhr.github.io/sidecar-hash"
)

func (r *SidecarSet) Register(mgr ctrl.Manager) error {
	mgr.GetWebhookServer().Register("/mutate-sidecar-pod", &webhook.Admission{Handler: &podHandler{Client: mgr.GetClient()}})
	return nil
}

func (handler *podHandler) Handle(ctx context.Context, request admission.Request) admission.Response {
	log := log.FromContext(ctx)
	pod := &v1.Pod{}
	err := handler.Decoder.Decode(request, pod)
	copy := pod.DeepCopy()

	log.V(0).Info("begin to process sidecarSet webhook")
	if err != nil {
		return admission.Errored(http.StatusInternalServerError, err)
	}

	log.V(0).Info("sidecarSet webhook hand pod %v", "pod", pod.Name)
	err = handler.mutatingPodFunc(ctx, copy)
	if err != nil {
		return admission.Errored(http.StatusInternalServerError, err)
	}

	marshaledPod, err := json.Marshal(copy)
	return admission.PatchResponseFromRaw(request.Object.Raw, marshaledPod)
}

func (handler *podHandler) mutatingPodFunc(ctx context.Context, pod *v1.Pod) error {
	//1. get sidecarSet resource list
	sidecarSets := &SidecarSetList{}
	if err := handler.Client.List(ctx, sidecarSets); err != nil {
		return err
	}

	var sidecarContainers []v1.Container
	sidecarSetHash := make(map[string]string)

	matchNothing := true
	for _, sidecarSet := range sidecarSets.Items {
		needInject, err := PodSidecarSetMatch(pod, sidecarSet)
		if err != nil {
			return err
		}

		if !needInject {
			continue
		}

		matchNothing = false
		sidecarSetHash[sidecarSet.Name] = sidecarSet.Annotations[SidecarAnnotationHashKey]

		for i := range sidecarSet.Spec.Containers {
			sidecarContainer := &sidecarSet.Spec.Containers[i]
			sidecarContainer.Env = append(sidecarContainer.Env, v1.EnvVar{Name: SidecarEnvKey, Value: "true"})
			sidecarContainers = append(sidecarContainers, sidecarContainer.Container)
		}
	}

	if matchNothing {
		return nil
	}

	pod.Spec.Containers = append(pod.Spec.Containers, sidecarContainers...)
	if pod.Annotations == nil {
		pod.Annotations = make(map[string]string)
	}
	if len(sidecarSetHash) != 0 {
		encodedStr, err := json.Marshal(sidecarSetHash)
		if err != nil {
			return err
		}
		pod.Annotations[SidecarAnnotationHashKey] = string(encodedStr)
	}

	return nil
}

func PodSidecarSetMatch(pod *v1.Pod, sidecarSet SidecarSet) (bool, error) {
	selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.Selector)
	if err != nil {
		return false, err
	}

	if !selector.Empty() && selector.Matches(labels.Set(pod.Labels)) {
		return true, nil
	}
	return false, nil
}

// InjectDecoder injects the decoder.
func (handler *podHandler) InjectDecoder(d *admission.Decoder) error {
	handler.Decoder = d
	return nil
}

// log is for logging in this package.
var sidecarsetlog = logf.Log.WithName("sidecarset-resource")

func (r *SidecarSet) SetupWebhookWithManager(mgr ctrl.Manager) error {
	return ctrl.NewWebhookManagedBy(mgr).
		For(r).
		Complete()
}

//+kubebuilder:webhook:path=/validate-apps-chinalhr-github-io-v1alpha1-sidecarset,mutating=false,failurePolicy=fail,sideEffects=None,groups=apps.chinalhr.github.io,resources=sidecarsets,verbs=create;update,versions=v1alpha1,name=vsidecarset.kb.io,admissionReviewVersions=v1

var _ webhook.Validator = &SidecarSet{}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *SidecarSet) ValidateCreate() error {
	sidecarsetlog.Info("validate create", "name", r.Name)
	return r.validateSpec()
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (r *SidecarSet) ValidateUpdate(old runtime.Object) error {
	sidecarsetlog.Info("validate update", "name", r.Name)
	return r.validateSpec()
}

func (r *SidecarSet) validateSpec() *apierrors.StatusError {
	spec := r.Spec
	var allErrs field.ErrorList

	if spec.Selector == nil {
		allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("selector"), "no selector defined for sidecarset"))
	} else {
		if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 {
			allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("selector"), spec.Selector, "empty selector is not valid for sidecarset."))
		}
	}
	if len(allErrs) == 0 {
		return nil
	}

	return apierrors.NewInvalid(
		schema.GroupKind{Group: "apps.chinalhr.github.io", Kind: "SidecarSet"},
		r.Name, allErrs)
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
func (r *SidecarSet) ValidateDelete() error {
	sidecarsetlog.Info("validate delete", "name", r.Name)

	// TODO(user): fill in your validation logic upon object deletion.
	return nil
}

//+kubebuilder:rbac:groups=apps.chinalhr.github.io,resources=sidecarsets,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps.chinalhr.github.io,resources=sidecarsets/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=apps.chinalhr.github.io,resources=sidecarsets/finalizers,verbs=update
//+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=pods/status,verbs=get

func (r *SidecarSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	log := log.FromContext(ctx)
	//next reconcile
	reconcileResult := ctrl.Result{RequeueAfter: time.Second * 5, Requeue: true}

	//fetch the SidecarSet instance
	sidecarSet := &appsv1alpha1.SidecarSet{}
	err := r.Get(context.TODO(), req.NamespacedName, sidecarSet)
	if err != nil {
		if errors.IsNotFound(err) {
			log.Error(err, "process sidecarSet error : object not found")
			return reconcileResult, nil
		}
		log.Error(err, "process sidecarSet error : reading object error")
		return reconcile.Result{}, err
	}

	log.V(0).Info("begin to process sidecarSet [%s]", sidecarSet.Name)
	//list matched pod
	selector, err := metav1.LabelSelectorAsSelector(sidecarSet.Spec.Selector)
	if err != nil {
		log.Error(err, "process sidecarSet error")
		return reconcile.Result{}, err
	}

	listOpts := &client.ListOptions{LabelSelector: selector}

	matchedPods := &v1.PodList{}
	if err := r.Client.List(context.TODO(), matchedPods, listOpts); err != nil {
		log.Error(err, "list matched pods error")
		return reconcile.Result{}, err
	}
	log.V(0).Info("process sidecarSet matchedPods", "pods", matchedPods)

	// ignore inactive pods
	var filteredPods []*v1.Pod
	for i := range matchedPods.Items {
		pod := &matchedPods.Items[i]
		if IsPodActive(pod) {
			filteredPods = append(filteredPods, pod)
		}
	}

	//calculateSidecarSetStatus
	status, err := calculateSidecarSetStatus(sidecarSet, filteredPods)
	if err != nil {
		log.Error(err, "calculateSidecarSetStatus error")
		return reconcile.Result{}, err
	}

	//updateSidecarSetStatus
	err = r.updateSidecarSetStatus(log, sidecarSet, status)
	if err != nil {
		log.Error(err, "updateSidecarSetStatus error")
		return reconcile.Result{}, err
	}
	return reconcileResult, nil
}

# 使用spec/status字段与标记和CRD validation标记定义API，以及新增了其他资源的Webhook，需要使用make manifests生成/更新CRD与webhook的 manifests
make manifests

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.1/cert-manager.yaml

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: mutating-webhook-configuration
  annotations:
    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  name: validating-webhook-configuration
  annotations:
    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

make docker-build docker-push IMG=ccr.ccs.tencentyun.com/open-operator/sidecar-operator:1.0.0
make deploy IMG=ccr.ccs.tencentyun.com/open-operator/sidecar-operator:1.0.0