Ingress介绍

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127

// IngressSpec describes the Ingress the user wishes to exist.
type IngressSpec struct {
    // IngressClassName is the name of the IngressClass cluster resource. The
    // associated IngressClass defines which controller will implement the
    // resource. This replaces the deprecated `kubernetes.io/ingress.class`
    // annotation. For backwards compatibility, when that annotation is set, it
    // must be given precedence over this field. The controller may emit a
    // warning if the field and annotation have different values.
    // Implementations of this API should ignore Ingresses without a class
    // specified. An IngressClass resource may be marked as default, which can
    // be used to set a default value for this field. For more information,
    // refer to the IngressClass documentation.
    // +optional
    IngressClassName *string `json:"ingressClassName,omitempty" protobuf:"bytes,4,opt,name=ingressClassName"`

    // A default backend capable of servicing requests that don't match any
    // rule. At least one of 'backend' or 'rules' must be specified. This field
    // is optional to allow the loadbalancer controller or defaulting logic to
    // specify a global default.
    // +optional
    Backend *IngressBackend `json:"backend,omitempty" protobuf:"bytes,1,opt,name=backend"`

    // TLS configuration. Currently the Ingress only supports a single TLS
    // port, 443. If multiple members of this list specify different hosts, they
    // will be multiplexed on the same port according to the hostname specified
    // through the SNI TLS extension, if the ingress controller fulfilling the
    // ingress supports SNI.
    // +optional
    TLS []IngressTLS `json:"tls,omitempty" protobuf:"bytes,2,rep,name=tls"`

    // A list of host rules used to configure the Ingress. If unspecified, or
    // no rule matches, all traffic is sent to the default backend.
    // +optional
    Rules []IngressRule `json:"rules,omitempty" protobuf:"bytes,3,rep,name=rules"`
    // TODO: Add the ability to specify load-balancer IP through claims
}

// IngressTLS describes the transport layer security associated with an Ingress.
type IngressTLS struct {
    // Hosts are a list of hosts included in the TLS certificate. The values in
    // this list must match the name/s used in the tlsSecret. Defaults to the
    // wildcard host setting for the loadbalancer controller fulfilling this
    // Ingress, if left unspecified.
    // +optional
    Hosts []string `json:"hosts,omitempty" protobuf:"bytes,1,rep,name=hosts"`
    // SecretName is the name of the secret used to terminate TLS traffic on
    // port 443. Field is left optional to allow TLS routing based on SNI
    // hostname alone. If the SNI host in a listener conflicts with the "Host"
    // header field used by an IngressRule, the SNI host is used for termination
    // and value of the Host header is used for routing.
    // +optional
    SecretName string `json:"secretName,omitempty" protobuf:"bytes,2,opt,name=secretName"`
    // TODO: Consider specifying different modes of termination, protocols etc.
}

// IngressStatus describe the current state of the Ingress.
type IngressStatus struct {
    // LoadBalancer contains the current status of the load-balancer.
    // +optional
    LoadBalancer v1.LoadBalancerStatus `json:"loadBalancer,omitempty" protobuf:"bytes,1,opt,name=loadBalancer"`
}

// IngressRule represents the rules mapping the paths under a specified host to
// the related backend services. Incoming requests are first evaluated for a host
// match, then routed to the backend associated with the matching IngressRuleValue.
type IngressRule struct {
    // Host is the fully qualified domain name of a network host, as defined by RFC 3986.
    // Note the following deviations from the "host" part of the
    // URI as defined in RFC 3986:
    // 1. IPs are not allowed. Currently an IngressRuleValue can only apply to
    //    the IP in the Spec of the parent Ingress.
    // 2. The `:` delimiter is not respected because ports are not allowed.
    //    Currently the port of an Ingress is implicitly :80 for http and
    //    :443 for https.
    // Both these may change in the future.
    // Incoming requests are matched against the host before the
    // IngressRuleValue. If the host is unspecified, the Ingress routes all
    // traffic based on the specified IngressRuleValue.
    //
    // Host can be "precise" which is a domain name without the terminating dot of
    // a network host (e.g. "foo.bar.com") or "wildcard", which is a domain name
    // prefixed with a single wildcard label (e.g. "*.foo.com").
    // The wildcard character '*' must appear by itself as the first DNS label and
    // matches only a single label. You cannot have a wildcard label by itself (e.g. Host == "*").
    // Requests will be matched against the Host field in the following way:
    // 1. If Host is precise, the request matches this rule if the http host header is equal to Host.
    // 2. If Host is a wildcard, then the request matches this rule if the http host header
    // is to equal to the suffix (removing the first label) of the wildcard rule.
    // +optional
    Host string `json:"host,omitempty" protobuf:"bytes,1,opt,name=host"`
    // IngressRuleValue represents a rule to route requests for this IngressRule.
    // If unspecified, the rule defaults to a http catch-all. Whether that sends
    // just traffic matching the host to the default backend or all traffic to the
    // default backend, is left to the controller fulfilling the Ingress. Http is
    // currently the only supported IngressRuleValue.
    // +optional
    IngressRuleValue `json:",inline,omitempty" protobuf:"bytes,2,opt,name=ingressRuleValue"`
}

// IngressRuleValue represents a rule to apply against incoming requests. If the
// rule is satisfied, the request is routed to the specified backend. Currently
// mixing different types of rules in a single Ingress is disallowed, so exactly
// one of the following must be set.
type IngressRuleValue struct {
    //TODO:
    // 1. Consider renaming this resource and the associated rules so they
    // aren't tied to Ingress. They can be used to route intra-cluster traffic.
    // 2. Consider adding fields for ingress-type specific global options
    // usable by a loadbalancer, like http keep-alive.

    // +optional
    HTTP *HTTPIngressRuleValue `json:"http,omitempty" protobuf:"bytes,1,opt,name=http"`
}

// HTTPIngressRuleValue is a list of http selectors pointing to backends.
// In the example: http://<host>/<path>?<searchpart> -> backend where
// where parts of the url correspond to RFC 3986, this resource will be used
// to match against everything after the last '/' and before the first '?'
// or '#'.
type HTTPIngressRuleValue struct {
    // A collection of paths that map requests to backends.
    Paths []HTTPIngressPath `json:"paths" protobuf:"bytes,1,rep,name=paths"`
    // TODO: Consider adding fields for ingress-type specific global
    // options usable by a loadbalancer, like http keep-alive.
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

    store.informers.Ingress = infFactory.Networking().V1beta1().Ingresses().Informer()
    store.listers.Ingress.Store = store.informers.Ingress.GetStore()

    store.informers.Endpoint = infFactory.Core().V1().Endpoints().Informer()
    store.listers.Endpoint.Store = store.informers.Endpoint.GetStore()

    store.informers.Secret = infFactorySecrets.Core().V1().Secrets().Informer()
    store.listers.Secret.Store = store.informers.Endpoint.GetStore()

    store.informers.ConfigMap = infFactoryConfigmaps.Core().V1().ConfigMaps().Informer()
    store.listers.ConfigMap.Store = store.informers.ConfigMap.GetStore()

    store.informers.Service = infFactory.Core().V1().Services().Informer()
    store.listers.Service.Store = store.informers.Service.GetStore()

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179

// NewNGINXController creates a new NGINX Ingress controller.
func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXController {
    eventBroadcaster := record.NewBroadcaster()
    eventBroadcaster.StartLogging(klog.Infof)
    eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{
        Interface: config.Client.CoreV1().Events(config.Namespace),
    })

    h, err := dns.GetSystemNameServers()
    if err != nil {
        klog.Warningf("Error reading system nameservers: %v", err)
    }

    n := &NGINXController{
        isIPV6Enabled: ing_net.IsIPv6Enabled(),

        resolver:        h,
        cfg:             config,
        syncRateLimiter: flowcontrol.NewTokenBucketRateLimiter(config.SyncRateLimit, 1),

        recorder: eventBroadcaster.NewRecorder(scheme.Scheme, apiv1.EventSource{
            Component: "nginx-ingress-controller",
        }),

        stopCh:   make(chan struct{}),
        updateCh: channels.NewRingChannel(1024),

        ngxErrCh: make(chan error),

        stopLock: &sync.Mutex{},

        runningConfig: new(ingress.Configuration),

        Proxy: &TCPProxy{},

        metricCollector: mc,

        command: NewNginxCommand(),
    }

    if n.cfg.ValidationWebhook != "" {
        n.validationWebhookServer = &http.Server{
            Addr:      config.ValidationWebhook,
            Handler:   adm_controller.NewAdmissionControllerServer(&adm_controller.IngressAdmission{Checker: n}),
            TLSConfig: ssl.NewTLSListener(n.cfg.ValidationWebhookCertPath, n.cfg.ValidationWebhookKeyPath).TLSConfig(),
            // disable http/2
            // https://github.com/kubernetes/kubernetes/issues/80313
            // https://github.com/kubernetes/ingress-nginx/issues/6323#issuecomment-737239159
            TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
        }
    }

    n.store = store.New(
        config.Namespace,
        config.ConfigMapName,
        config.TCPConfigMapName,
        config.UDPConfigMapName,
        config.DefaultSSLCertificate,
        config.ResyncPeriod,
        config.Client,
        n.updateCh,
        config.DisableCatchAll)

    n.syncQueue = task.NewTaskQueue(n.syncIngress)

    if config.UpdateStatus {
        n.syncStatus = status.NewStatusSyncer(status.Config{
            Client:                 config.Client,
            PublishService:         config.PublishService,
            PublishStatusAddress:   config.PublishStatusAddress,
            IngressLister:          n.store,
            UpdateStatusOnShutdown: config.UpdateStatusOnShutdown,
            UseNodeInternalIP:      config.UseNodeInternalIP,
        })
    } else {
        klog.Warning("Update of Ingress status is disabled (flag --update-status)")
    }

    onTemplateChange := func() {
        template, err := ngx_template.NewTemplate(nginx.TemplatePath)
        if err != nil {
            // this error is different from the rest because it must be clear why nginx is not working
            klog.ErrorS(err, "Error loading new template")
            return
        }

        n.t = template
        klog.InfoS("New NGINX configuration template loaded")
        n.syncQueue.EnqueueTask(task.GetDummyObject("template-change"))
    }

    ngxTpl, err := ngx_template.NewTemplate(nginx.TemplatePath)
    if err != nil {
        klog.Fatalf("Invalid NGINX configuration template: %v", err)
    }

    n.t = ngxTpl

    _, err = watch.NewFileWatcher(nginx.TemplatePath, onTemplateChange)
    if err != nil {
        klog.Fatalf("Error creating file watcher for %v: %v", nginx.TemplatePath, err)
    }

    filesToWatch := []string{}
    err = filepath.Walk("/etc/nginx/geoip/", func(path string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        }

        if info.IsDir() {
            return nil
        }

        filesToWatch = append(filesToWatch, path)
        return nil
    })

    if err != nil {
        klog.Fatalf("Error creating file watchers: %v", err)
    }

    for _, f := range filesToWatch {
        _, err = watch.NewFileWatcher(f, func() {
            klog.InfoS("File changed detected. Reloading NGINX", "path", f)
            n.syncQueue.EnqueueTask(task.GetDummyObject("file-change"))
        })
        if err != nil {
            klog.Fatalf("Error creating file watcher for %v: %v", f, err)
        }
    }

    return n
}

// NGINXController describes a NGINX Ingress controller.
type NGINXController struct {
    cfg *Configuration

    recorder record.EventRecorder

    syncQueue *task.Queue

    syncStatus status.Syncer

    syncRateLimiter flowcontrol.RateLimiter

    // stopLock is used to enforce that only a single call to Stop send at
    // a given time. We allow stopping through an HTTP endpoint and
    // allowing concurrent stoppers leads to stack traces.
    stopLock *sync.Mutex

    stopCh   chan struct{}
    updateCh *channels.RingChannel

    // ngxErrCh is used to detect errors with the NGINX processes
    ngxErrCh chan error

    // runningConfig contains the running configuration in the Backend
    runningConfig *ingress.Configuration

    t ngx_template.TemplateWriter

    resolver []net.IP

    isIPV6Enabled bool

    isShuttingDown bool

    Proxy *TCPProxy

    store store.Storer

    metricCollector metric.Collector

    validationWebhookServer *http.Server

    command NginxExecTester
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112

// Start starts a new NGINX master process running in the foreground.
func (n *NGINXController) Start() {
    klog.InfoS("Starting NGINX Ingress controller")

    // k8s informer run
    n.store.Run(n.stopCh)

    // we need to use the defined ingress class to allow multiple leaders
    // in order to update information about ingress status
    electionID := fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.DefaultClass)
    if class.IngressClass != "" {
        electionID = fmt.Sprintf("%v-%v", n.cfg.ElectionID, class.IngressClass)
    }

    // 选主 启动 n.syncStatus.Run(stopCh)
    setupLeaderElection(&leaderElectionConfig{
        Client:     n.cfg.Client,
        ElectionID: electionID,
        OnStartedLeading: func(stopCh chan struct{}) {
            if n.syncStatus != nil {
                go n.syncStatus.Run(stopCh)
            }

            n.metricCollector.OnStartedLeading(electionID)
            // manually update SSL expiration metrics
            // (to not wait for a reload)
            n.metricCollector.SetSSLExpireTime(n.runningConfig.Servers)
        },
        OnStoppedLeading: func() {
            n.metricCollector.OnStoppedLeading(electionID)
        },
    })

    cmd := n.command.ExecCommand()

    // put NGINX in another process group to prevent it
    // to receive signals meant for the controller
    cmd.SysProcAttr = &syscall.SysProcAttr{
        Setpgid: true,
        Pgid:    0,
    }

    // SSLPassthrough true 启动本地tcp 代理，不使用nginx
    if n.cfg.EnableSSLPassthrough {
        n.setupSSLProxy()
    }

    klog.InfoS("Starting NGINX process")
    n.start(cmd)

    go n.syncQueue.Run(time.Second, n.stopCh)
    // force initial sync
    // 起始事件 "initial-sync" 入队
    n.syncQueue.EnqueueTask(task.GetDummyObject("initial-sync"))

    // In case of error the temporal configuration file will
    // be available up to five minutes after the error
    go func() {
        for {
            time.Sleep(5 * time.Minute)
            err := cleanTempNginxCfg()
            if err != nil {
                klog.ErrorS(err, "Unexpected error removing temporal configuration files")
            }
        }
    }()

    if n.validationWebhookServer != nil {
        klog.InfoS("Starting validation webhook", "address", n.validationWebhookServer.Addr,
            "certPath", n.cfg.ValidationWebhookCertPath, "keyPath", n.cfg.ValidationWebhookKeyPath)
        go func() {
            klog.ErrorS(n.validationWebhookServer.ListenAndServeTLS("", ""), "Error listening for TLS connections")
        }()
    }

    for {
        select {
        case err := <-n.ngxErrCh:
            if n.isShuttingDown {
                return
            }

            // if the nginx master process dies, the workers continue to process requests
            // until the failure of the configured livenessProbe and restart of the pod.
            if process.IsRespawnIfRequired(err) {
                return
            }

        case event := <-n.updateCh.Out():
            if n.isShuttingDown {
                break
            }

            if evt, ok := event.(store.Event); ok {
                klog.V(3).InfoS("Event received", "type", evt.Type, "object", evt.Obj)
                if evt.Type == store.ConfigurationEvent {
                    // TODO: is this necessary? Consider removing this special case
                    n.syncQueue.EnqueueTask(task.GetDummyObject("configmap-change"))
                    continue
                }

                n.syncQueue.EnqueueSkippableTask(evt.Obj)
            } else {
                klog.Warningf("Unexpected event type received %T", event)
            }
        case <-n.stopCh:
            return
        }
    }
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

// syncIngress collects all the pieces required to assemble the NGINX
// configuration file and passes the resulting data structures to the backend
// (OnUpdate) when a reload is deemed necessary.
func (n *NGINXController) syncIngress(interface{}) error {
    n.syncRateLimiter.Accept()

    if n.syncQueue.IsShuttingDown() {
        return nil
    }

    ings := n.store.ListIngresses()
    hosts, servers, pcfg := n.getConfiguration(ings)

    n.metricCollector.SetSSLExpireTime(servers)

    if n.runningConfig.Equal(pcfg) {
        klog.V(3).Infof("No configuration change detected, skipping backend reload")
        return nil
    }

    n.metricCollector.SetHosts(hosts)

    if !n.IsDynamicConfigurationEnough(pcfg) {
        klog.InfoS("Configuration changes detected, backend reload required")

        hash, _ := hashstructure.Hash(pcfg, &hashstructure.HashOptions{
            TagName: "json",
        })

        pcfg.ConfigurationChecksum = fmt.Sprintf("%v", hash)

        err := n.OnUpdate(*pcfg)
        if err != nil {
            n.metricCollector.IncReloadErrorCount()
            n.metricCollector.ConfigSuccess(hash, false)
            klog.Errorf("Unexpected failure reloading the backend:\n%v", err)
            n.recorder.Eventf(k8s.IngressPodDetails, apiv1.EventTypeWarning, "RELOAD", fmt.Sprintf("Error reloading NGINX: %v", err))
            return err
        }

        klog.InfoS("Backend successfully reloaded")
        n.metricCollector.ConfigSuccess(hash, true)
        n.metricCollector.IncReloadCount()

        n.recorder.Eventf(k8s.IngressPodDetails, apiv1.EventTypeNormal, "RELOAD", "NGINX reload triggered due to a change in configuration")
    }

    isFirstSync := n.runningConfig.Equal(&ingress.Configuration{})
    if isFirstSync {
        // For the initial sync it always takes some time for NGINX to start listening
        // For large configurations it might take a while so we loop and back off
        klog.InfoS("Initial sync, sleeping for 1 second")
        time.Sleep(1 * time.Second)
    }

    retry := wait.Backoff{
        Steps:    15,
        Duration: 1 * time.Second,
        Factor:   0.8,
        Jitter:   0.1,
    }

    err := wait.ExponentialBackoff(retry, func() (bool, error) {
        err := n.configureDynamically(pcfg)
        if err == nil {
            klog.V(2).Infof("Dynamic reconfiguration succeeded.")
            return true, nil
        }

        klog.Warningf("Dynamic reconfiguration failed: %v", err)
        return false, err
    })
    if err != nil {
        klog.Errorf("Unexpected failure reconfiguring NGINX:\n%v", err)
        return err
    }

    ri := getRemovedIngresses(n.runningConfig, pcfg)
    re := getRemovedHosts(n.runningConfig, pcfg)
    n.metricCollector.RemoveMetrics(ri, re)

    n.runningConfig = pcfg

    return nil
}




// OnUpdate is called by the synchronization loop whenever configuration
// changes were detected. The received backend Configuration is merged with the
// configuration ConfigMap before generating the final configuration file.
// Returns nil in case the backend was successfully reloaded.
func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) error {
    cfg := n.store.GetBackendConfiguration()
    cfg.Resolver = n.resolver

    content, err := n.generateTemplate(cfg, ingressCfg)
    if err != nil {
        return err
    }

    err = createOpentracingCfg(cfg)
    if err != nil {
        return err
    }

    err = n.testTemplate(content)
    if err != nil {
        return err
    }

    if klog.V(2).Enabled() {
        src, _ := ioutil.ReadFile(cfgPath)
        if !bytes.Equal(src, content) {
            tmpfile, err := ioutil.TempFile("", "new-nginx-cfg")
            if err != nil {
                return err
            }
            defer tmpfile.Close()
            err = ioutil.WriteFile(tmpfile.Name(), content, file.ReadWriteByUser)
            if err != nil {
                return err
            }

            diffOutput, err := exec.Command("diff", "-I", "'# Configuration.*'", "-u", cfgPath, tmpfile.Name()).CombinedOutput()
            if err != nil {
                if exitError, ok := err.(*exec.ExitError); ok {
                    ws := exitError.Sys().(syscall.WaitStatus)
                    if ws.ExitStatus() == 2 {
                        klog.Warningf("Failed to executing diff command: %v", err)
                    }
                }
            }

            klog.InfoS("NGINX configuration change", "diff", string(diffOutput))

            // we do not defer the deletion of temp files in order
            // to keep them around for inspection in case of error
            os.Remove(tmpfile.Name())
        }
    }

    err = ioutil.WriteFile(cfgPath, content, file.ReadWriteByUser)
    if err != nil {
        return err
    }

    o, err := n.command.ExecCommand("-s", "reload").CombinedOutput()
    if err != nil {
        return fmt.Errorf("%v\n%v", err, string(o))
    }

    return nil
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175

// Server describes a website
type Server struct {
    // Hostname returns the FQDN of the server
    Hostname string `json:"hostname"`
    // SSLPassthrough indicates if the TLS termination is realized in
    // the server or in the remote endpoint
    SSLPassthrough bool `json:"sslPassthrough"`
    // SSLCert describes the certificate that will be used on the server
    SSLCert *SSLCert `json:"sslCert"`
    // Locations list of URIs configured in the server.
    Locations []*Location `json:"locations,omitempty"`
    // Aliases return the alias of the server name
    Aliases []string `json:"aliases,omitempty"`
    // RedirectFromToWWW returns if a redirect to/from prefix www is required
    RedirectFromToWWW bool `json:"redirectFromToWWW,omitempty"`
    // CertificateAuth indicates the this server requires mutual authentication
    // +optional
    CertificateAuth authtls.Config `json:"certificateAuth"`
    // ProxySSL indicates the this server uses client certificate to access backends
    // +optional
    ProxySSL proxyssl.Config `json:"proxySSL"`
    // ServerSnippet returns the snippet of server
    // +optional
    ServerSnippet string `json:"serverSnippet"`
    // SSLCiphers returns list of ciphers to be enabled
    SSLCiphers string `json:"sslCiphers,omitempty"`
    // SSLPreferServerCiphers indicates that server ciphers should be preferred
    // over client ciphers when using the SSLv3 and TLS protocols.
    SSLPreferServerCiphers string `json:"sslPreferServerCiphers,omitempty"`
    // AuthTLSError contains the reason why the access to a server should be denied
    AuthTLSError string `json:"authTLSError,omitempty"`
}

// Location describes an URI inside a server.
// Also contains additional information about annotations in the Ingress.
//
// In some cases when more than one annotations is defined a particular order in the execution
// is required.
// The chain in the execution order of annotations should be:
// - Whitelist
// - RateLimit
// - BasicDigestAuth
// - ExternalAuth
// - Redirect
type Location struct {
    // Path is an extended POSIX regex as defined by IEEE Std 1003.1,
    // (i.e this follows the egrep/unix syntax, not the perl syntax)
    // matched against the path of an incoming request. Currently it can
    // contain characters disallowed from the conventional "path"
    // part of a URL as defined by RFC 3986. Paths must begin with
    // a '/'. If unspecified, the path defaults to a catch all sending
    // traffic to the backend.
    Path string `json:"path"`
    // PathType represents the type of path referred to by a HTTPIngressPath.
    PathType *networking.PathType `json:"pathType"`
    // IsDefBackend indicates if service specified in the Ingress
    // contains active endpoints or not. Returning true means the location
    // uses the default backend.
    IsDefBackend bool `json:"isDefBackend"`
    // Ingress returns the ingress from which this location was generated
    Ingress *Ingress `json:"ingress"`
    // IngressPath original path defined in the ingress rule
    IngressPath string `json:"ingressPath"`
    // Backend describes the name of the backend to use.
    Backend string `json:"backend"`
    // Service describes the referenced services from the ingress
    Service *apiv1.Service `json:"-"`
    // Port describes to which port from the service
    Port intstr.IntOrString `json:"port"`
    // Overwrite the Host header passed into the backend. Defaults to
    // vhost of the incoming request.
    // +optional
    UpstreamVhost string `json:"upstream-vhost"`
    // BasicDigestAuth returns authentication configuration for
    // an Ingress rule.
    // +optional
    BasicDigestAuth auth.Config `json:"basicDigestAuth,omitempty"`
    // Denied returns an error when this location cannot not be allowed
    // Requesting a denied location should return HTTP code 403.
    Denied *string `json:"denied,omitempty"`
    // CorsConfig returns the Cors Configuration for the ingress rule
    // +optional
    CorsConfig cors.Config `json:"corsConfig,omitempty"`
    // ExternalAuth indicates the access to this location requires
    // authentication using an external provider
    // +optional
    ExternalAuth authreq.Config `json:"externalAuth,omitempty"`
    // EnableGlobalAuth indicates if the access to this location requires
    // authentication using an external provider defined in controller's config
    EnableGlobalAuth bool `json:"enableGlobalAuth"`
    // HTTP2PushPreload allows to configure the HTTP2 Push Preload from backend
    // original location.
    // +optional
    HTTP2PushPreload bool `json:"http2PushPreload,omitempty"`
    // RateLimit describes a limit in the number of connections per IP
    // address or connections per second.
    // The Redirect annotation precedes RateLimit
    // +optional
    RateLimit ratelimit.Config `json:"rateLimit,omitempty"`
    // GlobalRateLimit similar to RateLimit
    // but this is applied globally across multiple replicas.
    // +optional
    GlobalRateLimit globalratelimit.Config `json:"globalRateLimit,omitempty"`
    // Redirect describes a temporal o permanent redirection this location.
    // +optional
    Redirect redirect.Config `json:"redirect,omitempty"`
    // Rewrite describes the redirection this location.
    // +optional
    Rewrite rewrite.Config `json:"rewrite,omitempty"`
    // Whitelist indicates only connections from certain client
    // addresses or networks are allowed.
    // +optional
    Whitelist ipwhitelist.SourceRange `json:"whitelist,omitempty"`
    // Proxy contains information about timeouts and buffer sizes
    // to be used in connections against endpoints
    // +optional
    Proxy proxy.Config `json:"proxy,omitempty"`
    // ProxySSL contains information about SSL configuration parameters
    // to be used in connections against endpoints
    // +optional
    ProxySSL proxyssl.Config `json:"proxySSL,omitempty"`
    // UsePortInRedirects indicates if redirects must specify the port
    // +optional
    UsePortInRedirects bool `json:"usePortInRedirects"`
    // ConfigurationSnippet contains additional configuration for the backend
    // to be considered in the configuration of the location
    ConfigurationSnippet string `json:"configurationSnippet"`
    // Connection contains connection header to override the default Connection header
    // to the request.
    // +optional
    Connection connection.Config `json:"connection"`
    // ClientBodyBufferSize allows for the configuration of the client body
    // buffer size for a specific location.
    // +optional
    ClientBodyBufferSize string `json:"clientBodyBufferSize,omitempty"`
    // DefaultBackend allows the use of a custom default backend for this location.
    // +optional
    DefaultBackend *apiv1.Service `json:"-"`
    // DefaultBackendUpstreamName is the upstream-formatted string for the name of
    // this location's custom default backend
    DefaultBackendUpstreamName string `json:"defaultBackendUpstreamName,omitempty"`
    // XForwardedPrefix allows to add a header X-Forwarded-Prefix to the request with the
    // original location.
    // +optional
    XForwardedPrefix string `json:"xForwardedPrefix,omitempty"`
    // Logs allows to enable or disable the nginx logs
    // By default access logs are enabled and rewrite logs are disabled
    Logs log.Config `json:"logs,omitempty"`
    // InfluxDB allows to monitor the incoming request by sending them to an influxdb database
    // +optional
    InfluxDB influxdb.Config `json:"influxDB,omitempty"`
    // BackendProtocol indicates which protocol should be used to communicate with the service
    // By default this is HTTP
    BackendProtocol string `json:"backend-protocol"`
    // FastCGI allows the ingress to act as a FastCGI client for a given location.
    // +optional
    FastCGI fastcgi.Config `json:"fastcgi,omitempty"`
    // CustomHTTPErrors specifies the error codes that should be intercepted.
    // +optional
    CustomHTTPErrors []int `json:"custom-http-errors"`
    // ModSecurity allows to enable and configure modsecurity
    // +optional
    ModSecurity modsecurity.Config `json:"modsecurity"`
    // Satisfy dictates allow access if any or all is set
    Satisfy string `json:"satisfy"`
    // Mirror allows you to mirror traffic to a "test" backend
    // +optional
    Mirror mirror.Config `json:"mirror,omitempty"`
    // Opentracing allows the global opentracing setting to be overridden for a location
    // +optional
    Opentracing opentracing.Config `json:"opentracing"`
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: example-ingress
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /$1
    kubernetes.io/ingress.class: "nginx"
    # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    # nginx.ingress.kubernetes.io/ssl-passthrough: "true"
spec:
  rules:
    - host: hello-world.info
      http:
        paths:
          - path: /web1
            pathType: Prefix
            backend:
              service:
                name: web-1-svc
                port:
                  number: 8080
          - path: /web2
            pathType: Prefix
            backend:
              service:
                name: web-2-svc
                port:
                  number: 8080
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ivision-ingress
  namespace: aistation
  annotations:
    # 这里不要使用rewrite-target的通配符/$1 或 $2，需根据实际业务场景配置
    # nginx.ingress.kubernetes.io/rewrite-target: /$1
    # 说明：对于不同的业务需要不同的rewrite-target处理（/ 或 /$1），则分开不同的ingress配置
    # 这里标识3个路由都rewrite到了/路径下
    nginx.ingress.kubernetes.io/rewrite-target: /
    # 在多ingress-controller中，指定使用nginx-ingress，需与nginx-ingress-controller的参数一致（默认值：nginx）
    kubernetes.io/ingress.class: "nginx"
    # 如后端服务为https，则backend-protocol: "HTTPS"
    # 该配置实际上是根据loc.BackendProtocol = anns.BackendProtocol，再进行buildProxyPass操作，
    # proxy_pass 配置方式: [proxy_pass http://upstream名称, proxy_pass https://upstream名称, proxy_pass grpc://upstream名称, ...] 
    # 在nginx.conf中，配置 proxy_pass https://upstream_balancer
    # 如果http，则配置proxy_pass http://upstream_balancer
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
    # ssl-passthrough 会本地启动tcp代理服务，绕过了nginx
    # nginx.ingress.kubernetes.io/ssl-passthrough: "true"
    # nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
spec:
  rules:
    # 采用域名
    # - host: test.example.com
    #   http:
    #     paths:
    #       - path: /
    #         pathType: Prefix
    #         backend:
    #           service:
    #             name: ivision
    #             port:
    #               number: 8443

    # 可使用ip方式，匹配路径全路由
    - http:
        paths:
          - path: /vvv
            pathType: Prefix
            backend:
              service:
                name: web-svc
                port:
                  number: 8443
          - path: /
            pathType: Prefix
            backend:
              service:
                name: web-svc
                port:
                  number: 8443
    # 可以放到另个ingress配置中              
    - http:
        paths:
          - path: /doc
            pathType: Prefix
            backend:
              service:
                name: doc-svc
                port:
                  number: 8443

1

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v0.46.0/deploy/static/provider/baremetal/deploy.yaml

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx

---
# Source: ingress-nginx/templates/controller-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx
  namespace: ingress-nginx
automountServiceAccountToken: true
---
# Source: ingress-nginx/templates/controller-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx-controller
  namespace: ingress-nginx
data:
---
# Source: ingress-nginx/templates/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
  name: ingress-nginx
rules:
  - apiGroups:
      - ''
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ''
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ''
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ''
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - extensions
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingresses/status
    verbs:
      - update
  - apiGroups:
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingressclasses
    verbs:
      - get
      - list
      - watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
  name: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ingress-nginx
subjects:
  - kind: ServiceAccount
    name: ingress-nginx
    namespace: ingress-nginx
---
# Source: ingress-nginx/templates/controller-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx
  namespace: ingress-nginx
rules:
  - apiGroups:
      - ''
    resources:
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ''
    resources:
      - configmaps
      - pods
      - secrets
      - endpoints
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ''
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - extensions
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingresses/status
    verbs:
      - update
  - apiGroups:
      - networking.k8s.io   # k8s 1.14+
    resources:
      - ingressclasses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ''
    resources:
      - configmaps
    resourceNames:
      - ingress-controller-leader-nginx
    verbs:
      - get
      - update
  - apiGroups:
      - ''
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ''
    resources:
      - events
    verbs:
      - create
      - patch
---
# Source: ingress-nginx/templates/controller-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: ingress-nginx
subjects:
  - kind: ServiceAccount
    name: ingress-nginx
    namespace: ingress-nginx
---
# Source: ingress-nginx/templates/controller-service-webhook.yaml
apiVersion: v1
kind: Service
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx-controller-admission
  namespace: ingress-nginx
spec:
  type: ClusterIP
  ports:
    - name: https-webhook
      port: 443
      targetPort: webhook
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/component: controller
---
# Source: ingress-nginx/templates/controller-service.yaml
apiVersion: v1
kind: Service
metadata:
  annotations:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: http
    - name: https
      port: 443
      protocol: TCP
      targetPort: https
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/component: controller
---
# Source: ingress-nginx/templates/controller-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: controller
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/instance: ingress-nginx
      app.kubernetes.io/component: controller
  revisionHistoryLimit: 10
  minReadySeconds: 0
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/component: controller
    spec:
      dnsPolicy: ClusterFirst
      containers:
        - name: controller
          image: k8s.gcr.io/ingress-nginx/controller:v0.46.0@sha256:52f0058bed0a17ab0fb35628ba97e8d52b5d32299fbc03cc0f6c7b9ff036b61a
          imagePullPolicy: IfNotPresent
          lifecycle:
            preStop:
              exec:
                command:
                  - /wait-shutdown
          args:
            - /nginx-ingress-controller
            - --election-id=ingress-controller-leader
            - --ingress-class=nginx
            - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
            - --validating-webhook=:8443
            - --validating-webhook-certificate=/usr/local/certificates/cert
            - --validating-webhook-key=/usr/local/certificates/key
          securityContext:
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            runAsUser: 101
            allowPrivilegeEscalation: true
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: LD_PRELOAD
              value: /usr/local/lib/libmimalloc.so
          livenessProbe:
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            timeoutSeconds: 1
            successThreshold: 1
            failureThreshold: 5
          readinessProbe:
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            timeoutSeconds: 1
            successThreshold: 1
            failureThreshold: 3
          ports:
            - name: http
              containerPort: 80
              protocol: TCP
            - name: https
              containerPort: 443
              protocol: TCP
            - name: webhook
              containerPort: 8443
              protocol: TCP
          volumeMounts:
            - name: webhook-cert
              mountPath: /usr/local/certificates/
              readOnly: true
          resources:
            requests:
              cpu: 100m
              memory: 90Mi
      nodeSelector:
        kubernetes.io/os: linux
      serviceAccountName: ingress-nginx
      terminationGracePeriodSeconds: 300
      volumes:
        - name: webhook-cert
          secret:
            secretName: ingress-nginx-admission
---
# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
# before changing this value, check the required kubernetes version
# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  name: ingress-nginx-admission
webhooks:
  - name: validate.nginx.ingress.kubernetes.io
    matchPolicy: Equivalent
    rules:
      - apiGroups:
          - networking.k8s.io
        apiVersions:
          - v1beta1
        operations:
          - CREATE
          - UPDATE
        resources:
          - ingresses
    failurePolicy: Fail
    sideEffects: None
    admissionReviewVersions:
      - v1
      - v1beta1
    clientConfig:
      service:
        namespace: ingress-nginx
        name: ingress-nginx-controller-admission
        path: /networking/v1beta1/ingresses
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: ingress-nginx-admission
  annotations:
    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  namespace: ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: ingress-nginx-admission
  annotations:
    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
rules:
  - apiGroups:
      - admissionregistration.k8s.io
    resources:
      - validatingwebhookconfigurations
    verbs:
      - get
      - update
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: ingress-nginx-admission
  annotations:
    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ingress-nginx-admission
subjects:
  - kind: ServiceAccount
    name: ingress-nginx-admission
    namespace: ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ingress-nginx-admission
  annotations:
    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  namespace: ingress-nginx
rules:
  - apiGroups:
      - ''
    resources:
      - secrets
    verbs:
      - get
      - create
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ingress-nginx-admission
  annotations:
    helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: ingress-nginx-admission
subjects:
  - kind: ServiceAccount
    name: ingress-nginx-admission
    namespace: ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: ingress-nginx-admission-create
  annotations:
    helm.sh/hook: pre-install,pre-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  namespace: ingress-nginx
spec:
  template:
    metadata:
      name: ingress-nginx-admission-create
      labels:
        helm.sh/chart: ingress-nginx-3.30.0
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/version: 0.46.0
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: admission-webhook
    spec:
      containers:
        - name: create
          image: docker.io/jettech/kube-webhook-certgen:v1.5.1
          imagePullPolicy: IfNotPresent
          args:
            - create
            - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
            - --namespace=$(POD_NAMESPACE)
            - --secret-name=ingress-nginx-admission
          env:
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
      restartPolicy: OnFailure
      serviceAccountName: ingress-nginx-admission
      securityContext:
        runAsNonRoot: true
        runAsUser: 2000
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: ingress-nginx-admission-patch
  annotations:
    helm.sh/hook: post-install,post-upgrade
    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
  labels:
    helm.sh/chart: ingress-nginx-3.30.0
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/version: 0.46.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: admission-webhook
  namespace: ingress-nginx
spec:
  template:
    metadata:
      name: ingress-nginx-admission-patch
      labels:
        helm.sh/chart: ingress-nginx-3.30.0
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/version: 0.46.0
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: admission-webhook
    spec:
      containers:
        - name: patch
          image: docker.io/jettech/kube-webhook-certgen:v1.5.1
          imagePullPolicy: IfNotPresent
          args:
            - patch
            - --webhook-name=ingress-nginx-admission
            - --namespace=$(POD_NAMESPACE)
            - --patch-mutating=false
            - --secret-name=ingress-nginx-admission
            - --patch-failure-policy=Fail
          env:
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
      restartPolicy: OnFailure
      serviceAccountName: ingress-nginx-admission
      securityContext:
        runAsNonRoot: true
        runAsUser: 2000