istio 源码 – Citadel 源码分析 (原创)



From: Istio 安全

源码位于 security,编译后名称为 citadel。



FROM scratch

# obtained from debian ca-certs deb using
ADD ca-certificates.tgz /
# All containers need a /tmp directory
ADD istio_ca /usr/local/bin/istio_ca

ENTRYPOINT [ "/usr/local/bin/istio_ca", "--self-signed-ca" ]


# kubectl exec -ti istio-citadel-55cdfdd57c-bh7dk -n istio-system -- /usr/local/bin/istio_ca version
Version: 1.0.5
GitRevision: c1707e45e71c75d74bf3a5dec8c7086f32f32fad
User: root@6f6ea1061f2b
GolangVersion: go1.10.4
BuildStatus: Clean


# kubectl exec -ti istio-citadel-55cdfdd57c-bh7dk -n istio-system -- /usr/local/bin/istio_ca --help
Istio Certificate Authority (CA)

  istio_ca [flags]
  istio_ca [command]

Available Commands:
  help        Help about any command
  probe       Check the liveness or readiness of a locally-running server
  version     Prints out build version information

      --append-dns-names                           Append DNS names to the certificates for webhook services. (default true)
      --cert-chain string                          Path to the certificate chain file
      --citadel-storage-namespace string           Namespace where the Citadel pod is running. Will not be used if explicit file or other storage mechanism is specified. (default "istio-system")
      --custom-dns-names string                    The list of account.namespace:customdns names, separated by comma.
      --enable-profiling                           Enabling profiling when monitoring Citadel.
      --grpc-host-identities string                The list of hostnames for istio ca server, separated by comma. (default "istio-ca,istio-citadel")
      --grpc-hostname string                       DEPRECATED, use --grpc-host-identites. (default "istio-ca")
      --grpc-port int                              The port number for Citadel GRPC server. If unspecified, Citadel will not serve GRPC requests. (default 8060)
  -h, --help                                       help for istio_ca
      --key-size int                               Size of generated private key (default 2048)
      --kube-config string                         Specifies path to kubeconfig file. This must be specified when not running inside a Kubernetes pod.
      --listened-namespace string                  Select a namespace for the CA to listen to. If unspecified, Citadel tries to use the ${NAMESPACE} environment variable. If neither is set, Citadel listens to all namespaces.
      --liveness-probe-interval duration           Interval of updating file for the liveness probe.
      --liveness-probe-path string                 Path to the file for the liveness probe.
      --log_as_json                                Whether to format output as JSON or in plain console-friendly format
      --log_caller string                          Comma-separated list of scopes for which to include caller information, scopes can be any of [default, model]
      --log_output_level string                    Comma-separated minimum per-scope logging level of messages to output, in the form of <scope>:<level>,<scope>:<level>,... where scope can be one of [default, model] and level can be one of [debug, info, warn, error, none] (default "default:info")
      --log_rotate string                          The path for the optional rotating log file
      --log_rotate_max_age int                     The maximum age in days of a log file beyond which the file is rotated (0 indicates no limit) (default 30)
      --log_rotate_max_backups int                 The maximum number of log file backups to keep before older files are deleted (0 indicates no limit) (default 1000)
      --log_rotate_max_size int                    The maximum size in megabytes of a log file beyond which the file is rotated (default 104857600)
      --log_stacktrace_level string                Comma-separated minimum per-scope logging level at which stack traces are captured, in the form of <scope>:<level>,<scope:level>,... where scope can be one of [default, model] and level can be one of [debug, info, warn, error, none] (default "default:none")
      --log_target stringArray                     The set of paths where to output the log. This can be any path as well as the special values stdout and stderr (default [stdout])
      --max-workload-cert-ttl duration             The max TTL of issued workload certificates (default 2160h0m0s)
      --monitoring-port int                        The port number for monitoring Citadel. If unspecified, Citadel will disable monitoring. (default 9093)
      --org string                                 Organization for the cert
      --probe-check-interval duration              Interval of checking the liveness of the CA. (default 30s)
      --requested-ca-cert-ttl duration             The requested TTL for the workload (default 8760h0m0s)
      --root-cert string                           Path to the root certificate file
      --self-signed-ca                             Indicates whether to use auto-generated self-signed CA certificate. When set to true, the '--signing-cert' and '--signing-key' options are ignored.
      --self-signed-ca-cert-ttl duration           The TTL of self-signed CA root certificate (default 8760h0m0s)
      --self-signed-ca-org string                  The issuer organization used in self-signed CA certificate (default to k8s.cluster.local) (default "k8s.cluster.local")
      --sign-ca-certs                              Whether Citadel signs certificates for other CAs
      --signing-cert string                        Path to the CA signing certificate file
      --signing-key string                         Path to the CA signing key file
      --upstream-ca-address string                 The IP:port address of the upstream CA. When set, the CA will rely on the upstream Citadel to provision its own certificate.
      --workload-cert-grace-period-ratio float32   The workload certificate rotation grace period, as a ratio of the workload certificate TTL. (default 0.5)
      --workload-cert-min-grace-period duration    The minimum workload certificate rotation grace period. (default 10m0s)
      --workload-cert-ttl duration                 The TTL of issued workload certificates (default 2160h0m0s)

Use "istio_ca [command] --help" for more information about a command.


    - --append-dns-names=true
    - --grpc-port=8060
    - --grpc-hostname=citadel
    - --citadel-storage-namespace=istio-system
    - --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system
    - --self-signed-ca=true

可以在其运行的 node 节点上通过命令查看

$ /usr/local/bin/istio_ca --self-signed-ca --append-dns-names=true --grpc-port=8060 --grpc-hostname=citadel --citadel-storage-namespace=istio-system --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system --self-signed-ca=true

istio-citadel 启动的 yaml 文件

# kubectl get pod istio-citadel-55cdfdd57c-bh7dk -n istio-system -o yaml
apiVersion: v1
kind: Pod
  annotations: "" "false"
  creationTimestamp: 2019-01-15T08:24:24Z
  generateName: istio-citadel-55cdfdd57c-
    istio: citadel
    pod-template-hash: 55cdfdd57c
  name: istio-citadel-55cdfdd57c-bh7dk
  namespace: istio-system
  - apiVersion: apps/v1
    blockOwnerDeletion: true
    controller: true
    kind: ReplicaSet
    name: istio-citadel-55cdfdd57c
    uid: f6db1f80-189e-11e9-ab53-00163e0c1552
  resourceVersion: "16685125"
  selfLink: /api/v1/namespaces/istio-system/pods/istio-citadel-55cdfdd57c-bh7dk
  uid: f710ae31-189e-11e9-ab53-00163e0c1552
      - preference:
          - key:
            operator: In
            - amd64
        weight: 2
      - preference:
          - key:
            operator: In
            - ppc64le
        weight: 2
      - preference:
          - key:
            operator: In
            - s390x
        weight: 2
        - matchExpressions:
          - key:
            operator: In
            - amd64
            - ppc64le
            - s390x
  - args:
    - --append-dns-names=true
    - --grpc-port=8060
    - --grpc-hostname=citadel
    - --citadel-storage-namespace=istio-system
    - --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system
    - --self-signed-ca=true
    imagePullPolicy: IfNotPresent
    name: citadel
        cpu: 10m
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    - mountPath: /var/run/secrets/
      name: istio-citadel-service-account-token-gdxfk
      readOnly: true
  dnsPolicy: ClusterFirst
  nodeName: node02
  priority: 0
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: istio-citadel-service-account
  serviceAccountName: istio-citadel-service-account
  terminationGracePeriodSeconds: 30
  - effect: NoExecute
    operator: Exists
    tolerationSeconds: 10
  - effect: NoExecute
    operator: Exists
    tolerationSeconds: 10
  - name: istio-citadel-service-account-token-gdxfk
      defaultMode: 420
      secretName: istio-citadel-service-account-token-gdxfk




// /usr/local/bin/istio_ca 
//  --self-signed-ca 
//  --append-dns-names=true 
//  --grpc-port=8060 
//  --grpc-hostname=citadel 
//  --citadel-storage-namespace=istio-system 
//  --custom-dns-names=istio-pilot-service-account.istio-system:istio-pilot.istio-system,
//     istio-ingressgateway-service-account.istio-system:istio-ingressgateway.istio-system //  --self-signed-ca=true    

rootCmd = &cobra.Command{
        Use:   "istio_ca",
        Short: "Istio Certificate Authority (CA).",
        Args:  cobra.ExactArgs(0),
        Run: func(cmd *cobra.Command, args []string) {

runCA 的主函数流程如下:

func runCA() {
    // --listened-namespace 设置 CA 监控的 namespace,如果没有指定会从 ${NAMESPACE}  环境变量中获取,如果都没有设置,Citadel 则会监听全部的 namespace.

    if value, exists := os.LookupEnv(cmd.ListenedNamespaceKey); exists {
        // When -namespace is not set, try to read the namespace from environment variable.
        if opts.listenedNamespace == "" {
            opts.listenedNamespace = value
        // Use environment variable for istioCaStorageNamespace if it exists
        opts.istioCaStorageNamespace = value

    // 验证命令行

    var webhooks map[string]controller.DNSNameEntry

    // 如果设置了添加 DNS 名字的后缀
    if opts.appendDNSNames {
        webhooks = make(map[string]controller.DNSNameEntry)
        // ServiceAccount/DNS pair for generating DNS names in certificates.
        // TODO: move it to a configmap later when we have more services to support.
        webhookServiceAccounts = []string{

        webhookServiceNames = []string{
        for i, svcAccount := range webhookServiceAccounts { 
            // istio-sidecar-injector-service-account
            // istio-galley-service-account
            webhooks[svcAccount] = controller.DNSNameEntry{
                ServiceName: webhookServiceNames[i],
                Namespace:   opts.istioCaStorageNamespace, 
                // opts.istioCaStorageNamespace 运行的 namespace,默认为  istio-system

        // ...

    // 创建连接到集群中的 client
    cs := createClientset()

    // 返回 ca.IstioCA,用于管理证书链和签新的证书
    ca := createCA(cs.CoreV1())

    // For workloads in K8s, we apply the configured workload cert TTL.
    // 1. 创建 NewSecretController 来完成对于 API Server 中的 ServiceAccount 和 Secret 的创建
    sc, err := controller.NewSecretController(ca,
        opts.workloadCertGracePeriodRatio, opts.workloadCertMinGracePeriod, opts.dualUse,
        cs.CoreV1(), opts.signCACerts, opts.listenedNamespace, webhooks)
    if err != nil {
        fatalf("Failed to create secret controller: %v", err)

    stopCh := make(chan struct{})
    // !!! 运行 NewSecretController

    // 2. 如果设置了 grpcPort,则启动相关 server
    if opts.grpcPort > 0 {
        // ...
        ch := make(chan struct{})
        // monitor service objects with "" and
        // "" annotations
        // 2.1 NewServiceController
        serviceController := kube.NewServiceController(cs.CoreV1(), opts.listenedNamespace, reg)

        // ServiceController

        // 2.2 NewServiceAccountController
        // monitor service account objects for istio mesh expansion
        serviceAccountController := kube.NewServiceAccountController(cs.CoreV1(), opts.listenedNamespace, reg)

        // The CA API uses cert with the max workload cert TTL.
        hostnames := append(strings.Split(opts.grpcHosts, ","), fqdn())
        caServer, startErr := caserver.New(ca, opts.maxWorkloadCertTTL, opts.signCACerts, hostnames, opts.grpcPort, spiffe.GetTrustDomain())
        if startErr != nil {
            fatalf("Failed to create istio ca server: %v", startErr)
        if serverErr := caServer.Run(); serverErr != nil {
            // stop the registry-related controllers
            ch <- struct{}{}

            log.Warnf("Failed to start GRPC server with error: %v", serverErr)

    monitorErrCh := make(chan error)

    // 3. Start the monitoring server.
    if opts.monitoringPort > 0 {
        monitor, mErr := monitoring.NewMonitor(opts.monitoringPort, opts.enableProfiling)
        if mErr != nil {
            fatalf("Unable to setup monitoring: %v", mErr)
        go monitor.Start(monitorErrCh)
        log.Info("Citadel monitor has started.")
        defer monitor.Close()

    log.Info("Citadel has started")

    rotatorErrCh := make(chan error)
    // Start CA client if the upstream CA address is specified.
    if len(opts.cAClientConfig.CAAddress) != 0 {
        config := &opts.cAClientConfig
        config.Env = "onprem"
        config.Platform = "vm"
        config.ForCA = true
        config.CertFile = opts.signingCertFile
        config.KeyFile = opts.signingKeyFile
        config.CertChainFile = opts.certChainFile
        config.RootCertFile = opts.rootCertFile
        config.CSRGracePeriodPercentage = cmd.DefaultCSRGracePeriodPercentage
        config.CSRMaxRetries = cmd.DefaultCSRMaxRetries
        config.CSRInitialRetrialInterval = cmd.DefaultCSRInitialRetrialInterval
        rotator, creationErr := caclient.NewKeyCertBundleRotator(config, ca.GetCAKeyCertBundle())
        if creationErr != nil {
            fatalf("Failed to create key cert bundle rotator: %v", creationErr)

        // 4. rotator 启动
        go rotator.Start(rotatorErrCh)
        log.Info("Key cert bundle rotator has started.")
        defer rotator.Stop()

    // Blocking until receives error.
    for {
        select {
        case <-monitorErrCh:
            fatalf("Monitoring server error: %v", err)
        case <-rotatorErrCh:
            fatalf("Key cert bundle rotator error: %v", err)


SecretController 内部会创建两个 Controller:

  1. ServiceAccount 的监听,如果设置了 listened-namespace,则监听该 namespace 下,否则是全部;
  2. Secret 的监听,namespace 同上,但是 Controller 只会监听自己创建的类型,即:type:””

实现的主要功能是为 ServiceAccount 创建对应的 Secret,Secret 中设置了相关的证书,在对应的 Pod 启动的时候进行加载;

// NewSecretController returns a pointer to a newly constructed SecretController instance.
func NewSecretController(ca ca.CertificateAuthority, certTTL time.Duration,
    gracePeriodRatio float32, minGracePeriod time.Duration, dualUse bool,
    core corev1.CoreV1Interface, forCA bool, namespace string, dnsNames map[string]DNSNameEntry) (*SecretController, error) {


    c := &SecretController{
        ca:               ca,
        certTTL:          certTTL,
        gracePeriodRatio: gracePeriodRatio,
        minGracePeriod:   minGracePeriod,
        dualUse:          dualUse,
        core:             core,
        forCA:            forCA,
        dnsNames:         dnsNames,
        monitoring:       newMonitoringMetrics(),

    // 监听特定 namespace 下的 ServiceAccount
    c.saStore, c.saController = cache.NewInformer(saLW, &v1.ServiceAccount{}, time.Minute, rehf)

    istioSecretSelector := fields.SelectorFromSet(map[string]string{"type": IstioSecretType}).String()

    // 监听 type:”” 的 secret 
    c.scrtStore, c.scrtController =
        cache.NewInformer(scrtLW, &v1.Secret{}, secretResyncPeriod, cache.ResourceEventHandlerFuncs{
            DeleteFunc: c.scrtDeleted,
            UpdateFunc: c.scrtUpdated,

    // ...

// Run starts the SecretController until a value is sent to stopCh.
func (sc *SecretController) Run(stopCh chan struct{}) {
    go sc.scrtController.Run(stopCh)

    // saAdded calls upsertSecret to update and insert secret
    // it throws error if the secret cache is not synchronized, but the secret exists in the system
    cache.WaitForCacheSync(stopCh, sc.scrtController.HasSynced)

    go sc.saController.Run(stopCh)

gRPC Server 启动

如果设置了 gRPC 相关的参数,则会启动相关的服务,同上也会启动两个 Controller 和 一个 gRPC Server,Controller 监听的 namespace 由 listened-namespace 设置,同上:

  1. NewServiceController:用于监听添加了注解 的 Service 对象;从注解中解出来对应的用户名对应的 Reg 注册表的映射关系中,当前 key 和 value 都是相同 c.reg.AddMapping(svcAcct, svcAcct)

    // KubeServiceAccountsOnVMAnnotation is to specify the K8s service accounts that are allowed to run
    // this service on the VMs
    KubeServiceAccountsOnVMAnnotation = ""
    // CanonicalServiceAccountsAnnotation is to specify the non-Kubernetes service accounts that
    // are allowed to run this service.
    CanonicalServiceAccountsAnnotation = ""


    // ServiceController monitors the service definition changes in a namespace. If a
    // new service is added with "" or
    // "" annotations enabled,
    // the corresponding service account will be added to the identity registry
    // for whitelisting.
    type ServiceController struct {
    core corev1.CoreV1Interface
    // identity registry object
    reg registry.Registry
    // controller for service objects
    controller cache.Controller
  2. NewServiceAccountController: 监听 ServiceAccount 对象;对于获取到 sa 信息,生成相对应的 SpiffeID 保存到 Reg 注册表的映射关系中,当前 key 和 value 都是相同 c.reg.DeleteMapping(id, id)


    // ServiceAccountController monitors service account definition changes in a namespace.
    // For each service account object, its SpiffeID is added to identity registry for
    // whitelisting purpose.
    type ServiceAccountController struct {
    core corev1.CoreV1Interface
    // identity registry object
    reg registry.Registry
    // controller for service objects
    controller cache.Controller
  3. IstioCAServiceServer:主要提供证书的生成和验证功能;

    // CreateCertificate handles an incoming certificate signing request (CSR). It does
    // authentication and authorization. Upon validated, signs a certificate that:
    // the SAN is the identity of the caller in authentication result.
    // the subject public key is the public key in the CSR.
    // the validity duration is the ValidityDuration in request, or default value if the given duration is invalid.
    // it is signed by the CA signing key.
    func (s *Server) CreateCertificate(ctx context.Context, request *pb.IstioCertificateRequest) (
    *pb.IstioCertificateResponse, error) {
    // 根据请求生成对应的证书
    _, _, certChainBytes, rootCertBytes :=
    cert, signErr :=
        []byte(request.Csr), caller.Identities, time.Duration(request.ValidityDuration)*time.Second, false)
    respCertChain := []string{string(cert)}
    respCertChain = append(respCertChain, string(rootCertBytes))
    response := &pb.IstioCertificateResponse{
        CertChain: respCertChain,
    log.Debug("CSR successfully signed.")
    return response, nil
    // HandleCSR handles an incoming certificate signing request (CSR). It does
    // proper validation (e.g. authentication) and upon validated, signs the CSR
    // and returns the resulting certificate. If not approved, reason for refusal
    // to sign is returned as part of the response object.
    // [TODO](myidpt): Deprecate this function.
    func (s *Server) HandleCSR(ctx context.Context, request *pb.CsrRequest) (*pb.CsrResponse, error) {
    csr, err := util.ParsePemEncodedCSR(request.CsrPem)
    _, err = util.ExtractIDs(csr.Extensions)
    // TODO: Call authorizer.
    _, _, certChainBytes, _ :=
    cert, signErr :=, []string{}, time.Duration(request.RequestedTtlMinutes)*time.Minute, s.forCA)
    response := &pb.CsrResponse{
        IsApproved: true,
        SignedCert: cert,
        CertChain:  certChainBytes,
    log.Debug("CSR successfully signed.")
    return response, nil


Monitor 服务主要用于对外输出检查,为 HttpServer, 主要提供 /metrics/version ,如果启用了 enableProfiling 还会启用 /debug/pprof/ 相关的路径;当 Monitor 启动以后, Citadel 则任务已经启动成功,打印以下信息:log.Info("Citadel has started")

CA client

如果指定了 upstream CA Server,还会启动一个 CA Client, 创建一个 rotator go routine,定期用于证书的轮转替换;

// Start periodically rotates the KeyCertBundle by interacting with the upstream CA.
// It is a blocking function that should run as a go routine. Thread safe.
func (c *KeyCertBundleRotator) Start(errCh chan<- error) {
    if !c.stopped {
        errCh <- fmt.Errorf("rotator already started")
    c.stopped = false

    // Make sure we mark rotator stopped after this method finishes.
    defer func() {
        c.stopped = true

    for {
        certBytes, _, _, _ := c.keycert.GetAllPem()
        if len(certBytes) != 0 {
            waitTime, ttlErr := c.certUtil.GetWaitTime(certBytes, time.Now())
            if ttlErr != nil {
                log.Errorf("Error getting TTL from cert: %v. Rotate immediately.", ttlErr)
            } else {
                timer := time.NewTimer(waitTime)
                log.Infof("Will rotate key and cert in %v.", waitTime)
                select {
                case <-c.stopCh:
                case <-timer.C:
                    // Continue in the loop.
        co, coErr := c.keycert.CertOptions()
        if coErr != nil {
            err := fmt.Errorf("failed to extact CertOptions from bundle: %v, abort auto rotation", coErr)
            errCh <- err
        certBytes, certChainBytes, privKeyBytes, rErr := c.retriever.Retrieve(co)
        if rErr != nil {
            err := fmt.Errorf("error retrieving the key and cert: %v, abort auto rotation", rErr)
            errCh <- err
        _, _, _, rootCertBytes := c.keycert.GetAllPem()
        if vErr := c.keycert.VerifyAndSetAll(certBytes, privKeyBytes, certChainBytes, rootCertBytes); vErr != nil {
            err := fmt.Errorf("cannot verify the retrieved key and cert: %v, abort auto rotation", vErr)
            errCh <- err
        log.Infof("Successfully retrieved new key and certs.")


电子邮件地址不会被公开。 必填项已用*标注