diff --git a/cmd/kube-apiserver/app/config.go b/cmd/kube-apiserver/app/config.go index 67e20223cf66f..ead606bc00948 100644 --- a/cmd/kube-apiserver/app/config.go +++ b/cmd/kube-apiserver/app/config.go @@ -20,6 +20,7 @@ import ( apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/informerfactoryhack" "k8s.io/apiserver/pkg/util/webhook" aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" @@ -77,7 +78,7 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { Options: opts, } - genericConfig, _, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( + genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( opts.CompletedOptions, []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme}, controlplane.DefaultAPIResourceConfigSource(), @@ -99,13 +100,13 @@ func NewConfig(opts options.CompletedOptions) (*Config, error) { return nil, err } - apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, kubeAPIs.ControlPlane.VersionedInformers, pluginInitializer, opts.CompletedOptions, opts.MasterCount, conversionFactory) + apiExtensions, err := controlplaneapiserver.CreateAPIExtensionsConfig(*kubeAPIs.ControlPlane.Generic, informerfactoryhack.Wrap(versionedInformers), pluginInitializer, opts.CompletedOptions, opts.MasterCount, conversionFactory) if err != nil { return nil, err } c.ApiExtensions = apiExtensions - aggregator, err := controlplaneapiserver.CreateAggregatorConfig(*kubeAPIs.ControlPlane.Generic, opts.CompletedOptions, kubeAPIs.ControlPlane.VersionedInformers, serviceResolver, kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Extra.PeerProxy, pluginInitializer) + aggregator, err := controlplaneapiserver.CreateAggregatorConfig(*kubeAPIs.ControlPlane.Generic, opts.CompletedOptions, informerfactoryhack.Wrap(versionedInformers), serviceResolver, kubeAPIs.ControlPlane.ProxyTransport, kubeAPIs.ControlPlane.Extra.PeerProxy, pluginInitializer) if err != nil { return nil, err } diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index bdd697c22315a..0a3c51e462b8f 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -26,6 +26,7 @@ import ( "net/url" "os" + kcpinformers "github.com/kcp-dev/client-go/informers" "github.com/spf13/cobra" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -34,6 +35,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/admission" genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" + "k8s.io/apiserver/pkg/informerfactoryhack" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/egressselector" serverstorage "k8s.io/apiserver/pkg/server/storage" @@ -202,7 +204,7 @@ func CreateProxyTransport() *http.Transport { func CreateKubeAPIServerConfig( opts options.CompletedOptions, genericConfig *genericapiserver.Config, - versionedInformers clientgoinformers.SharedInformerFactory, + versionedInformers kcpinformers.SharedInformerFactory, storageFactory *serverstorage.DefaultStorageFactory, ) ( *controlplane.Config, @@ -225,7 +227,7 @@ func CreateKubeAPIServerConfig( return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err) } - serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) + serviceResolver := buildServiceResolver(opts.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, informerfactoryhack.Wrap(versionedInformers)) controlplaneConfig, admissionInitializers, err := controlplaneapiserver.CreateConfig(opts.CompletedOptions, genericConfig, versionedInformers, storageFactory, serviceResolver, kubeInitializers) if err != nil { return nil, nil, nil, err diff --git a/pkg/api/genericcontrolplanescheme/scheme.go b/pkg/api/genericcontrolplanescheme/scheme.go deleted file mode 100644 index 41cbdd58f4fba..0000000000000 --- a/pkg/api/genericcontrolplanescheme/scheme.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package genericcontrolplanescheme - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" -) - -var ( - // Scheme is the default instance of runtime.Scheme to which control plane types - // in the Kubernetes API are already registered. - // NOTE: If you are copying this file to start a new api group, STOP! This Scheme - // is special and should appear ONLY in the api group, unless you really know what - // you're doing. - Scheme = runtime.NewScheme() - - // Codecs provides access to encoding and decoding for the scheme - Codecs = serializer.NewCodecFactory(Scheme) - - // ParameterCodec handles versioning of objects that are converted to query parameters. - ParameterCodec = runtime.NewParameterCodec(Scheme) -) diff --git a/pkg/apis/core/install/genericcontrolplane/install.go b/pkg/apis/core/install/genericcontrolplane/install.go deleted file mode 100644 index a33dd9593ffba..0000000000000 --- a/pkg/apis/core/install/genericcontrolplane/install.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package install installs the v1 monolithic api, making it available as an -// option to all of the API encoding/decoding machinery. -package genericcontrolplane - -import ( - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/kubernetes/pkg/apis/core" - v1 "k8s.io/kubernetes/pkg/apis/core/v1" -) - -// Install registers the API group and adds types to a scheme -func Install(scheme *runtime.Scheme) { - utilruntime.Must(core.AddToGenericControlPlaneScheme(scheme)) - utilruntime.Must(v1.AddToControlPlaneScheme(scheme)) - utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion)) -} diff --git a/pkg/controlplane/apiserver/apis.go b/pkg/controlplane/apiserver/apis.go index f1f7878698ae2..0e8d3dad306e2 100644 --- a/pkg/controlplane/apiserver/apis.go +++ b/pkg/controlplane/apiserver/apis.go @@ -19,6 +19,7 @@ package apiserver import ( "fmt" + "k8s.io/apiserver/pkg/informerfactoryhack" "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" @@ -70,7 +71,7 @@ func (c *CompletedConfig) DefaultStorageProviders() ([]RESTStorageProvider, erro ExtendExpiration: c.Extra.ExtendExpiration, ServiceAccountMaxExpiration: c.Extra.ServiceAccountMaxExpiration, APIAudiences: c.Generic.Authentication.APIAudiences, - Informers: c.Extra.VersionedInformers, + Informers: informerfactoryhack.Wrap(c.Extra.VersionedInformers), }, apiserverinternalrest.StorageProvider{}, authenticationrest.RESTStorageProvider{Authenticator: c.Generic.Authentication.Authenticator, APIAudiences: c.Generic.Authentication.APIAudiences}, diff --git a/pkg/controlplane/apiserver/completion.go b/pkg/controlplane/apiserver/completion.go index 41937bbdf0639..c23e9dee5712c 100644 --- a/pkg/controlplane/apiserver/completion.go +++ b/pkg/controlplane/apiserver/completion.go @@ -20,6 +20,7 @@ import ( "time" "k8s.io/apiserver/pkg/endpoints/discovery" + "k8s.io/apiserver/pkg/informerfactoryhack" genericapiserver "k8s.io/apiserver/pkg/server" ) @@ -44,7 +45,7 @@ type CompletedConfig struct { func (c *Config) Complete() CompletedConfig { cfg := completedConfig{ - c.Generic.Complete(c.VersionedInformers), + c.Generic.Complete(informerfactoryhack.Wrap(c.VersionedInformers)), &c.Extra, } diff --git a/pkg/controlplane/apiserver/config.go b/pkg/controlplane/apiserver/config.go index 7d295e0b2e593..f2627091b24a8 100644 --- a/pkg/controlplane/apiserver/config.go +++ b/pkg/controlplane/apiserver/config.go @@ -35,6 +35,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/discovery/aggregated" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" genericfeatures "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/informerfactoryhack" "k8s.io/apiserver/pkg/reconcilers" peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericapiserver "k8s.io/apiserver/pkg/server" @@ -69,6 +70,10 @@ import ( "k8s.io/kubernetes/pkg/serviceaccount" ) +// LocalAdminCluster is the default logical cluster that kube-apiserver's +// objects, e.g. the RBAC bootstrap policy land in. +var LocalAdminCluster = logicalcluster.Name("system:admin") + // Config defines configuration for the master type Config struct { Generic *genericapiserver.Config @@ -112,7 +117,7 @@ type Extra struct { SystemNamespaces []string - VersionedInformers clientgoinformers.SharedInformerFactory + VersionedInformers kcpinformers.SharedInformerFactory } // BuildGenericConfig takes the generic controlplane apiserver options and produces @@ -125,8 +130,7 @@ func BuildGenericConfig( getOpenAPIDefinitions func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition, ) ( genericConfig *genericapiserver.Config, - versionedClusterInformers kcpinformers.SharedInformerFactory, - versionedInformers clientgoinformers.SharedInformerFactory, + versionedInformers kcpinformers.SharedInformerFactory, storageFactory *serverstorage.DefaultStorageFactory, lastErr error, ) { @@ -204,15 +208,14 @@ func BuildGenericConfig( lastErr = fmt.Errorf("failed to create cluster clientset: %v", err) return } - versionedInformers = clientgoinformers.NewSharedInformerFactory(clusterClient.Cluster(logicalcluster.NewPath("system:admin")), 10*time.Minute) - versionedClusterInformers = kcpinformers.NewSharedInformerFactory(clusterClient, 10*time.Minute) + versionedInformers = kcpinformers.NewSharedInformerFactory(clusterClient, 10*time.Minute) // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present - if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clusterClient, versionedClusterInformers); lastErr != nil { + if lastErr = s.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clusterClient, versionedInformers); lastErr != nil { return } - genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers) + genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, informerfactoryhack.Wrap(versionedInformers)) if err != nil { lastErr = fmt.Errorf("invalid authorization config: %v", err) return @@ -227,7 +230,7 @@ func BuildGenericConfig( } if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) && s.GenericServerRunOptions.EnablePriorityAndFairness { - genericConfig.FlowControl, lastErr = BuildPriorityAndFairness(s, clusterClient.Cluster(logicalcluster.NewPath("system:admin")), versionedInformers) + genericConfig.FlowControl, lastErr = BuildPriorityAndFairness(s, clusterClient.Cluster(logicalcluster.NewPath("system:admin")), informerfactoryhack.Wrap(versionedInformers)) } if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) { @@ -270,7 +273,7 @@ func BuildPriorityAndFairness(s controlplaneapiserver.CompletedOptions, extclien func CreateConfig( opts controlplaneapiserver.CompletedOptions, genericConfig *genericapiserver.Config, - versionedInformers clientgoinformers.SharedInformerFactory, + versionedInformers kcpinformers.SharedInformerFactory, storageFactory *serverstorage.DefaultStorageFactory, serviceResolver aggregatorapiserver.ServiceResolver, additionalInitializers []admission.PluginInitializer, @@ -311,7 +314,7 @@ func CreateConfig( } // build peer proxy config only if peer ca file exists if opts.PeerCAFile != "" { - config.PeerProxy, err = BuildPeerProxy(versionedInformers, genericConfig.StorageVersionManager, opts.ProxyClientCertFile, + config.PeerProxy, err = BuildPeerProxy(informerfactoryhack.Wrap(versionedInformers), genericConfig.StorageVersionManager, opts.ProxyClientCertFile, opts.ProxyClientKeyFile, opts.PeerCAFile, opts.PeerAdvertiseAddress, genericConfig.APIServerID, config.Extra.PeerEndpointLeaseReconciler, config.Generic.Serializer) if err != nil { return nil, nil, err @@ -339,7 +342,7 @@ func CreateConfig( // setup admission genericAdmissionConfig := controlplaneadmission.Config{ - ExternalInformers: versionedInformers, + ExternalInformers: informerfactoryhack.Wrap(versionedInformers), LoopbackClientConfig: genericConfig.LoopbackClientConfig, } genericInitializers, admissionPostStartHook, err := genericAdmissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider) @@ -356,7 +359,7 @@ func CreateConfig( } err = opts.Admission.ApplyTo( genericConfig, - versionedInformers, + informerfactoryhack.Wrap(versionedInformers), clientgoExternalClient, dynamicExternalClient, utilfeature.DefaultFeatureGate, diff --git a/pkg/genericcontrolplane/aggregator/aggregator.go b/pkg/controlplane/apiserver/miniaggregator/aggregator.go similarity index 83% rename from pkg/genericcontrolplane/aggregator/aggregator.go rename to pkg/controlplane/apiserver/miniaggregator/aggregator.go index e07f9f38b6e19..d162b5076b636 100644 --- a/pkg/genericcontrolplane/aggregator/aggregator.go +++ b/pkg/controlplane/apiserver/miniaggregator/aggregator.go @@ -16,27 +16,29 @@ limitations under the License. // Package aggregator contains a server that aggregates content from a generic control // plane server, apiextensions server, and CustomResourceDefinitions. -package aggregator +package miniaggregator import ( "fmt" "net/http" "github.com/emicklei/go-restful/v3" + apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator" "k8s.io/kube-openapi/pkg/handler" - "k8s.io/kubernetes/pkg/genericcontrolplane/apis" + + controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver" ) var ( @@ -85,16 +87,16 @@ type CompletedMiniAggregatorConfig struct { *completedMiniAggregatorConfig } -// MiniAggregatorServer sits in front of the GenericControlPlane and -// CustomResourceDefinitions servers and aggregates them. +// MiniAggregatorServer sits in front of the Apis and +// ApiExtensions servers and aggregates them. type MiniAggregatorServer struct { // GenericAPIServer is the aggregator's server. GenericAPIServer *genericapiserver.GenericAPIServer - // GenericControlPlane is the server for the minimal control plane. It serves + // Apis is the server for the minimal control plane. It serves // APIs such as core v1, certificates.k8s.io, RBAC, etc. - GenericControlPlane *apis.GenericControlPlane - // CustomResourceDefinitions is the server for API extensions. - CustomResourceDefinitions *apiextensionsapiserver.CustomResourceDefinitions + Apis *controlplaneapiserver.Server + // ApiExtensions is the server for API extensions. + ApiExtensions *apiextensionsapiserver.CustomResourceDefinitions } // Complete fills in any fields not set that are required to have valid data. @@ -116,7 +118,7 @@ func (cfg *MiniAggregatorConfig) Complete() CompletedMiniAggregatorConfig { // New creates a new MiniAggregatorServer. func (c completedMiniAggregatorConfig) New( delegationTarget genericapiserver.DelegationTarget, - genericControlPlane *apis.GenericControlPlane, + apis *controlplaneapiserver.Server, crds *apiextensionsapiserver.CustomResourceDefinitions, ) (*MiniAggregatorServer, error) { genericServer, err := c.GenericConfig.New("mini-aggregator", delegationTarget) @@ -125,9 +127,9 @@ func (c completedMiniAggregatorConfig) New( } s := &MiniAggregatorServer{ - GenericAPIServer: genericServer, - GenericControlPlane: genericControlPlane, - CustomResourceDefinitions: crds, + GenericAPIServer: genericServer, + Apis: apis, + ApiExtensions: crds, } // Have to do this as a filter because of how the APIServerHandler.Director serves requests. @@ -148,19 +150,19 @@ func (s *MiniAggregatorServer) filterAPIsRequest(req *restful.Request, resp *res } // Discovery for things like core, authentication, authorization, certificates, ... - gcpGroups, err := s.GenericControlPlane.GenericAPIServer.DiscoveryGroupManager.Groups(req.Request.Context(), req.Request) + gcpGroups, err := s.Apis.GenericAPIServer.DiscoveryGroupManager.Groups(req.Request.Context(), req.Request) if err != nil { http.Error(resp.ResponseWriter, fmt.Sprintf("error retrieving generic control plane discovery groups: %v", err), http.StatusInternalServerError) } // Discovery for the apiextensions group itself - apiextensionsGroups, err := s.CustomResourceDefinitions.GenericAPIServer.DiscoveryGroupManager.Groups(req.Request.Context(), req.Request) + apiextensionsGroups, err := s.ApiExtensions.GenericAPIServer.DiscoveryGroupManager.Groups(req.Request.Context(), req.Request) if err != nil { http.Error(resp.ResponseWriter, fmt.Sprintf("error retrieving apiextensions discovery groups: %v", err), http.StatusInternalServerError) } // Discovery for all the groups contributed by CRDs - crdGroups, err := s.CustomResourceDefinitions.DiscoveryGroupLister.Groups(req.Request.Context(), req.Request) + crdGroups, err := s.ApiExtensions.DiscoveryGroupLister.Groups(req.Request.Context(), req.Request) if err != nil { http.Error(resp.ResponseWriter, fmt.Sprintf("error retrieving custom resource discovery groups: %v", err), http.StatusInternalServerError) } @@ -191,29 +193,24 @@ func (s *MiniAggregatorServer) serveOpenAPI(w http.ResponseWriter, req *http.Req } // Can't use withCluster here because the GenericControlPlane doesn't have APIs coming from multiple logical clusters at this time. - controlPlaneSpec, _, _, err := downloader.Download(s.GenericControlPlane.GenericAPIServer.Handler.Director, "") + controlPlaneSpec, _, _, err := downloader.Download(s.Apis.GenericAPIServer.Handler.Director, "") // Use withCluster here because each logical cluster can have a distinct set of APIs coming from its CRDs. - crdSpecs, _, _, err := downloader.Download(withCluster(s.CustomResourceDefinitions.GenericAPIServer.Handler.Director), "") + crdSpecs, _, _, err := downloader.Download(withCluster(s.ApiExtensions.GenericAPIServer.Handler.Director), "") // TODO(ncdc): merging on the fly is expensive. We may need to optimize this (e.g. caching). mergedSpecs, err := builder.MergeSpecs(controlPlaneSpec, crdSpecs) - - openAPIVersionedService, err := handler.NewOpenAPIService(mergedSpecs) if err != nil { - klog.Errorf("Error while building OpenAPI schema: %v", err) + utilruntime.HandleError(err) } - handler := &singlePathHandler{} + h := &singlePathHandler{} // In order to reuse the kube-openapi API as much as possible, we // register the OpenAPI service in the singlePathHandler - err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", handler) - if err != nil { - klog.Errorf("Error while building OpenAPI schema: %v", err) - } + handler.NewOpenAPIService(mergedSpecs).RegisterOpenAPIVersionedService("/openapi/v2", h) - handler.ServeHTTP(w, req) + h.ServeHTTP(w, req) } // singlePathHandler is a dummy PathHandler that mainly allows grabbing a http.Handler diff --git a/pkg/controlplane/apiserver/server.go b/pkg/controlplane/apiserver/server.go index 362fba5c4261b..675e0ea342c8d 100644 --- a/pkg/controlplane/apiserver/server.go +++ b/pkg/controlplane/apiserver/server.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" apiserverfeatures "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/informerfactoryhack" peerreconcilers "k8s.io/apiserver/pkg/reconcilers" genericregistry "k8s.io/apiserver/pkg/registry/generic" genericapiserver "k8s.io/apiserver/pkg/server" @@ -132,7 +133,7 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele APIResourceConfigSource: c.APIResourceConfigSource, RESTOptionsGetter: c.Generic.RESTOptionsGetter, ClusterAuthenticationInfo: c.ClusterAuthenticationInfo, - VersionedInformers: c.VersionedInformers, + VersionedInformers: informerfactoryhack.Wrap(c.VersionedInformers), } client := kubernetes.NewForConfigOrDie(s.GenericAPIServer.LoopbackClientConfig) diff --git a/pkg/controlplane/instance.go b/pkg/controlplane/instance.go index 81b02962d3498..c1e80c519a2be 100644 --- a/pkg/controlplane/instance.go +++ b/pkg/controlplane/instance.go @@ -55,6 +55,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/endpoints/discovery" + "k8s.io/apiserver/pkg/informerfactoryhack" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/client-go/kubernetes" @@ -336,7 +337,7 @@ func (c CompletedConfig) New(delegationTarget genericapiserver.DelegationTarget) ExtendExpiration: c.ControlPlane.Extra.ExtendExpiration, ServiceAccountMaxExpiration: c.ControlPlane.Extra.ServiceAccountMaxExpiration, APIAudiences: c.ControlPlane.Generic.Authentication.APIAudiences, - Informers: c.ControlPlane.Extra.VersionedInformers, + Informers: informerfactoryhack.Wrap(c.ControlPlane.Extra.VersionedInformers), }, Proxy: corerest.ProxyConfig{ Transport: c.ControlPlane.Extra.ProxyTransport, @@ -402,7 +403,7 @@ func (c CompletedConfig) New(delegationTarget genericapiserver.DelegationTarget) ServicePort: c.ExtraConfig.APIServerServicePort, PublicServicePort: publicServicePort, KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort, - }, client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Services()) + }, client, c.ControlPlane.Extra.VersionedInformers.Core().V1().Services().Cluster(controlplaneapiserver.LocalAdminCluster)) s.ControlPlane.GenericAPIServer.AddPostStartHookOrDie("bootstrap-controller", func(hookContext genericapiserver.PostStartHookContext) error { kubernetesServiceCtrl.Start(hookContext.StopCh) return nil diff --git a/pkg/genericcontrolplane/admission/config.go b/pkg/genericcontrolplane/admission/config.go deleted file mode 100644 index 64f04cd0c83d0..0000000000000 --- a/pkg/genericcontrolplane/admission/config.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "time" - - utilwait "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/apiserver/pkg/admission" - genericapiserver "k8s.io/apiserver/pkg/server" - cacheddiscovery "k8s.io/client-go/discovery/cached/memory" - externalinformers "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - quotainstall "k8s.io/kubernetes/pkg/quota/v1/install" -) - -// Config holds the configuration needed to for initialize the admission plugins -type Config struct { - CloudConfigFile string - LoopbackClientConfig *rest.Config - ExternalInformers externalinformers.SharedInformerFactory -} - -// New sets up the plugins and admission start hooks needed for admission -func (c *Config) New() ([]admission.PluginInitializer, genericapiserver.PostStartHookFunc, error) { - clientset, err := kubernetes.NewForConfig(c.LoopbackClientConfig) - if err != nil { - return nil, nil, err - } - - discoveryClient := cacheddiscovery.NewMemCacheClient(clientset.Discovery()) - discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient) - kubePluginInitializer := NewPluginInitializer( - discoveryRESTMapper, - quotainstall.NewQuotaConfigurationForAdmission(), - ) - - admissionPostStartHook := func(context genericapiserver.PostStartHookContext) error { - discoveryRESTMapper.Reset() - go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, context.StopCh) - return nil - } - - return []admission.PluginInitializer{kubePluginInitializer}, admissionPostStartHook, nil -} diff --git a/pkg/genericcontrolplane/admission/initializer.go b/pkg/genericcontrolplane/admission/initializer.go deleted file mode 100644 index 15a94a2b1da96..0000000000000 --- a/pkg/genericcontrolplane/admission/initializer.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/admission/initializer" - quota "k8s.io/apiserver/pkg/quota/v1" -) - -// WantsRESTMapper defines a function which sets RESTMapper for admission plugins that need it. -type WantsRESTMapper interface { - SetRESTMapper(meta.RESTMapper) -} - -// PluginInitializer is used for initialization of the Kubernetes specific admission plugins. -type PluginInitializer struct { - restMapper meta.RESTMapper - quotaConfiguration quota.Configuration -} - -var _ admission.PluginInitializer = &PluginInitializer{} - -// NewPluginInitializer constructs new instance of PluginInitializer -// TODO: switch these parameters to use the builder pattern or just make them -// all public, this construction method is pointless boilerplate. -func NewPluginInitializer( - restMapper meta.RESTMapper, - quotaConfiguration quota.Configuration, -) *PluginInitializer { - return &PluginInitializer{ - restMapper: restMapper, - quotaConfiguration: quotaConfiguration, - } -} - -// Initialize checks the initialization interfaces implemented by each plugin -// and provide the appropriate initialization data -func (i *PluginInitializer) Initialize(plugin admission.Interface) { - if wants, ok := plugin.(WantsRESTMapper); ok { - wants.SetRESTMapper(i.restMapper) - } - - if wants, ok := plugin.(initializer.WantsQuotaConfiguration); ok { - wants.SetQuotaConfiguration(i.quotaConfiguration) - } -} diff --git a/pkg/genericcontrolplane/apiextensions.go b/pkg/genericcontrolplane/apiextensions.go deleted file mode 100644 index bbe3b20ddc635..0000000000000 --- a/pkg/genericcontrolplane/apiextensions.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package genericcontrolplane - -import ( - kcpdynamic "github.com/kcp-dev/client-go/dynamic" - kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" - - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion" - apiextensionsoptions "k8s.io/apiextensions-apiserver/pkg/cmd/server/options" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/clientsethack" - "k8s.io/apiserver/pkg/dynamichack" - "k8s.io/apiserver/pkg/features" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/util/feature" - utilfeature "k8s.io/apiserver/pkg/util/feature" - kubeexternalinformers "k8s.io/client-go/informers" - - "k8s.io/kubernetes/pkg/genericcontrolplane/options" -) - -func CreateAPIExtensionsConfig( - kubeAPIServerConfig genericapiserver.Config, - externalInformers kubeexternalinformers.SharedInformerFactory, - pluginInitializers []admission.PluginInitializer, - commandOptions options.CompletedServerRunOptions, -) (*apiextensionsapiserver.Config, error) { - // make a shallow copy to let us twiddle a few things - // most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions - genericConfig := kubeAPIServerConfig - genericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - genericConfig.RESTOptionsGetter = nil - - client, err := kcpkubernetesclientset.NewForConfig(genericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - dynamicClient, err := kcpdynamic.NewForConfig(genericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - // override genericConfig.AdmissionControl with apiextensions' scheme, - // because apiextensions apiserver should use its own scheme to convert resources. - err = commandOptions.Admission.ApplyTo( - &genericConfig, - externalInformers, - clientsethack.Wrap(client), - dynamichack.Wrap(dynamicClient), - feature.DefaultFeatureGate, - pluginInitializers...) - if err != nil { - return nil, err - } - - // copy the etcd options so we don't mutate originals. - etcdOptions := *commandOptions.Etcd - etcdOptions.StorageConfig.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) - // this is where the true decodable levels come from. - etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion) - // prefer the more compact serialization (v1beta1) for storage until http://issue.k8s.io/82292 is resolved for objects whose v1 serialization is too big but whose v1beta1 serialization can be stored - etcdOptions.StorageConfig.EncodeVersioner = runtime.NewMultiGroupVersioner(v1beta1.SchemeGroupVersion, schema.GroupKind{Group: v1beta1.GroupName}) - etcdOptions.SkipHealthEndpoints = true // avoid double wiring of health checks - if err := etcdOptions.ApplyTo(&genericConfig); err != nil { - return nil, err - } - - // override MergedResourceConfig with apiextensions defaults and registry - if err := commandOptions.APIEnablement.ApplyTo( - &genericConfig, - apiextensionsapiserver.DefaultAPIResourceConfigSource(), - apiextensionsapiserver.Scheme); err != nil { - return nil, err - } - - crdRESTOptionsGetter, err := apiextensionsoptions.NewCRDRESTOptionsGetter(etcdOptions) - if err != nil { - return nil, err - } - - conversionFactory, err := conversion.NewCRConverterFactory(serviceResolver, authResolverWrapper) - if err != nil { - return nil, err - } - - apiextensionsConfig := &apiextensionsapiserver.Config{ - GenericConfig: &genericapiserver.RecommendedConfig{ - Config: genericConfig, - SharedInformerFactory: externalInformers, - }, - ExtraConfig: apiextensionsapiserver.ExtraConfig{ - CRDRESTOptionsGetter: crdRESTOptionsGetter, - MasterCount: 1, // pass that correctly - ConversionFactory: conversionFactory, - }, - } - - // we need to clear the poststarthooks so we don't add them multiple times to all the servers (that fails) - apiextensionsConfig.GenericConfig.PostStartHooks = map[string]genericapiserver.PostStartHookConfigEntry{} - - return apiextensionsConfig, nil -} diff --git a/pkg/genericcontrolplane/apis/apis.go b/pkg/genericcontrolplane/apis/apis.go deleted file mode 100644 index 906846a9a1ade..0000000000000 --- a/pkg/genericcontrolplane/apis/apis.go +++ /dev/null @@ -1,501 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apis - -import ( - "context" - "fmt" - "net/http" - "os" - "time" - - kcpkubernetesinformers "github.com/kcp-dev/client-go/informers" - kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - admissionregistrationv1alpha1 "k8s.io/api/admissionregistration/v1alpha1" - apiserverinternalv1alpha1 "k8s.io/api/apiserverinternal/v1alpha1" - authenticationv1 "k8s.io/api/authentication/v1" - authenticationv1alpha1 "k8s.io/api/authentication/v1alpha1" - authorizationapiv1 "k8s.io/api/authorization/v1" - certificatesapiv1 "k8s.io/api/certificates/v1" - coordinationapiv1 "k8s.io/api/coordination/v1" - apiv1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - discoveryv1beta1 "k8s.io/api/discovery/v1beta1" - eventsv1 "k8s.io/api/events/v1" - eventsv1beta1 "k8s.io/api/events/v1beta1" - flowcontrolv1alpha1 "k8s.io/api/flowcontrol/v1alpha1" - flowcontrolv1beta1 "k8s.io/api/flowcontrol/v1beta1" - flowcontrolv1beta2 "k8s.io/api/flowcontrol/v1beta2" - flowcontrolv1beta3 "k8s.io/api/flowcontrol/v1beta3" - rbacv1 "k8s.io/api/rbac/v1" - resourcev1alpha2 "k8s.io/api/resource/v1alpha2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/apiserver/pkg/endpoints/discovery" - apiserverfeatures "k8s.io/apiserver/pkg/features" - "k8s.io/apiserver/pkg/registry/generic" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/dynamiccertificates" - serverstorage "k8s.io/apiserver/pkg/server/storage" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/component-helpers/apimachinery/lease" - "k8s.io/klog/v2" - "k8s.io/kubernetes/pkg/controlplane/controller/apiserverleasegc" - "k8s.io/kubernetes/pkg/controlplane/controller/clusterauthenticationtrust" - "k8s.io/kubernetes/pkg/controlplane/controller/legacytokentracking" - "k8s.io/kubernetes/pkg/genericcontrolplane/localadmin" - - // RESTStorage installers - admissionregistrationrest "k8s.io/kubernetes/pkg/registry/admissionregistration/rest" - apiserverinternalrest "k8s.io/kubernetes/pkg/registry/apiserverinternal/rest" - authenticationrest "k8s.io/kubernetes/pkg/registry/authentication/rest" - authorizationrest "k8s.io/kubernetes/pkg/registry/authorization/rest" - certificatesrest "k8s.io/kubernetes/pkg/registry/certificates/rest" - coordinationrest "k8s.io/kubernetes/pkg/registry/coordination/rest" - corerest "k8s.io/kubernetes/pkg/registry/core/rest/genericcontrolplane" - eventsrest "k8s.io/kubernetes/pkg/registry/events/rest" - rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest" - "k8s.io/kubernetes/pkg/routes" - "k8s.io/kubernetes/pkg/serviceaccount" - "k8s.io/utils/clock" -) - -const ( - // DefaultEndpointReconcilerInterval is the default amount of time for how often the endpoints for - // the kubernetes Service are reconciled. - DefaultEndpointReconcilerInterval = 10 * time.Second - // DefaultEndpointReconcilerTTL is the default TTL timeout for the storage layer - DefaultEndpointReconcilerTTL = 15 * time.Second - // IdentityLeaseComponentLabelKey is used to apply a component label to identity lease objects, indicating: - // 1. the lease is an identity lease (different from leader election leases) - // 2. which component owns this lease - IdentityLeaseComponentLabelKey = "k8s.io/component" - // KubeAPIServer defines variable used internally when referring to kube-apiserver component - KubeAPIServer = "kube-apiserver" - // KubeAPIServerIdentityLeaseLabelSelector selects kube-apiserver identity leases - KubeAPIServerIdentityLeaseLabelSelector = IdentityLeaseComponentLabelKey + "=" + KubeAPIServer -) - -var ( - // IdentityLeaseGCPeriod is the interval which the lease GC controller checks for expired leases - // IdentityLeaseGCPeriod is exposed so integration tests can tune this value. - IdentityLeaseGCPeriod = 3600 * time.Second - // IdentityLeaseDurationSeconds is the duration of kube-apiserver lease in seconds - // IdentityLeaseDurationSeconds is exposed so integration tests can tune this value. - IdentityLeaseDurationSeconds = 3600 - // IdentityLeaseRenewIntervalSeconds is the interval of kube-apiserver renewing its lease in seconds - // IdentityLeaseRenewIntervalSeconds is exposed so integration tests can tune this value. - IdentityLeaseRenewIntervalPeriod = 10 * time.Second -) - -// ExtraConfig defines extra configuration for the master -type ExtraConfig struct { - ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo - - APIResourceConfigSource serverstorage.APIResourceConfigSource - StorageFactory serverstorage.StorageFactory - EventTTL time.Duration - - EnableLogsSupport bool - ProxyTransport *http.Transport - - // Port for the apiserver service. - APIServerServicePort int - - ServiceAccountIssuer serviceaccount.TokenGenerator - ServiceAccountMaxExpiration time.Duration - ExtendExpiration bool - - // ServiceAccountIssuerDiscovery - ServiceAccountIssuerURL string - ServiceAccountJWKSURI string - ServiceAccountPublicKeys []interface{} - - VersionedInformers kcpkubernetesinformers.SharedInformerFactory -} - -// Config defines configuration for the master -type Config struct { - GenericConfig *genericapiserver.Config - ExtraConfig ExtraConfig -} - -type completedConfig struct { - GenericConfig genericapiserver.CompletedConfig - ExtraConfig *ExtraConfig -} - -// CompletedConfig embeds a private pointer that cannot be instantiated outside of this package -type CompletedConfig struct { - *completedConfig -} - -// GenericControlPlane contains state for a Kubernetes cluster master/api server. -type GenericControlPlane struct { - GenericAPIServer *genericapiserver.GenericAPIServer - - ClusterAuthenticationInfo clusterauthenticationtrust.ClusterAuthenticationInfo -} - -// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. -func (c *Config) Complete() CompletedConfig { - cfg := completedConfig{ - c.GenericConfig.Complete(nil), // TODO: when we have users of this, we will need a shared informer factory super-interface hack on both ends so as to not have to refactor the world - &c.ExtraConfig, - } - - discoveryAddresses := discovery.DefaultAddresses{DefaultAddress: cfg.GenericConfig.ExternalAddress} - cfg.GenericConfig.DiscoveryAddresses = discoveryAddresses - - return CompletedConfig{&cfg} -} - -// New returns a new instance of GenericControlPlane from the given config. -// Certain config fields will be set to a default value if unset. -// Certain config fields must be specified, including: -// -// KubeletClientConfig -func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*GenericControlPlane, error) { - s, err := c.GenericConfig.New("kube-control-plane", delegationTarget) - if err != nil { - return nil, err - } - - if c.ExtraConfig.EnableLogsSupport { - routes.Logs{}.Install(s.Handler.GoRestfulContainer) - } - - m := &GenericControlPlane{ - GenericAPIServer: s, - ClusterAuthenticationInfo: c.ExtraConfig.ClusterAuthenticationInfo, - } - - // install legacy rest storage - if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter); err != nil { - return nil, err - } - - clientset, err := kcpkubernetesclientset.NewForConfig(c.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - - // TODO: update to a version that caches success but will recheck on failure, unlike memcache discovery - discoveryClientForAdmissionRegistration := clientset.Discovery() - - // The order here is preserved in discovery. - // If resources with identical names exist in more than one of these groups (e.g. "deployments.apps"" and "deployments.extensions"), - // the order of this list determines which group an unqualified resource name (e.g. "deployments") should prefer. - // This priority order is used for local discovery, but it ends up aggregated in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go - // with specific priorities. - // TODO: describe the priority all the way down in the RESTStorageProviders and plumb it back through the various discovery - // handlers that we have. - restStorageProviders := []RESTStorageProvider{ - apiserverinternalrest.StorageProvider{}, - authenticationrest.RESTStorageProvider{Authenticator: c.GenericConfig.Authentication.Authenticator, APIAudiences: c.GenericConfig.Authentication.APIAudiences}, - authorizationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, RuleResolver: c.GenericConfig.RuleResolver}, - certificatesrest.RESTStorageProvider{}, - coordinationrest.RESTStorageProvider{}, - rbacrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer}, - admissionregistrationrest.RESTStorageProvider{Authorizer: c.GenericConfig.Authorization.Authorizer, DiscoveryClient: discoveryClientForAdmissionRegistration}, - eventsrest.RESTStorageProvider{TTL: c.ExtraConfig.EventTTL}, - } - if err := m.InstallAPIs(c.ExtraConfig.APIResourceConfigSource, c.GenericConfig.RESTOptionsGetter, restStorageProviders...); err != nil { - return nil, err - } - - localAdminCluster := localadmin.Cluster - - // Have to create this outside the post-start hook to ensure the informer is started correctly - rootConfigMapsInformer := c.ExtraConfig.VersionedInformers.Core().V1().ConfigMaps().Cluster(localAdminCluster) - // Register the informer for starting when the factory is started in a different post-start hook - _ = rootConfigMapsInformer.Informer() - - m.GenericAPIServer.AddPostStartHookOrDie("start-cluster-authentication-info-controller", func(hookContext genericapiserver.PostStartHookContext) error { - kubeClient, err := kcpkubernetesclientset.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - return err - } - - controller := clusterauthenticationtrust.NewClusterAuthenticationTrustController(m.ClusterAuthenticationInfo, kubeClient.Cluster(localAdminCluster.Path())) - - // generate a context from stopCh. This is to avoid modifying files which are relying on apiserver - // TODO: See if we can pass ctx to the current method - ctx, cancel := context.WithCancel(context.Background()) - go func() { - select { - case <-hookContext.StopCh: - cancel() // stopCh closed, so cancel our context - case <-ctx.Done(): - } - }() - - // prime values and start listeners - if m.ClusterAuthenticationInfo.ClientCA != nil { - m.ClusterAuthenticationInfo.ClientCA.AddListener(controller) - if controller, ok := m.ClusterAuthenticationInfo.ClientCA.(dynamiccertificates.ControllerRunner); ok { - // runonce to be sure that we have a value. - if err := controller.RunOnce(ctx); err != nil { - runtime.HandleError(err) - } - go controller.Run(ctx, 1) - } - } - if m.ClusterAuthenticationInfo.RequestHeaderCA != nil { - m.ClusterAuthenticationInfo.RequestHeaderCA.AddListener(controller) - if controller, ok := m.ClusterAuthenticationInfo.RequestHeaderCA.(dynamiccertificates.ControllerRunner); ok { - // runonce to be sure that we have a value. - if err := controller.RunOnce(ctx); err != nil { - runtime.HandleError(err) - } - go controller.Run(ctx, 1) - } - } - - go controller.Run(ctx, 1) - return nil - }) - - if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.APIServerIdentity) { - m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error { - kubeClient, err := kcpkubernetesclientset.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - return err - } - - leaseName := m.GenericAPIServer.APIServerID - holderIdentity := m.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID()) - - controller := lease.NewController( - clock.RealClock{}, - kubeClient.Cluster(localAdminCluster.Path()), - holderIdentity, - int32(IdentityLeaseDurationSeconds), - nil, - IdentityLeaseRenewIntervalPeriod, - leaseName, - metav1.NamespaceSystem, - labelAPIServerHeartbeat) - go controller.Run(hookContext.StopCh) - return nil - }) - m.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error { - kubeClient, err := kcpkubernetesclientset.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - return err - } - go apiserverleasegc.NewAPIServerLeaseGC( - kubeClient.Cluster(localAdminCluster.Path()), - IdentityLeaseGCPeriod, - metav1.NamespaceSystem, - KubeAPIServerIdentityLeaseLabelSelector, - ).Run(hookContext.StopCh) - return nil - }) - } - - m.GenericAPIServer.AddPostStartHookOrDie("start-legacy-token-tracking-controller", func(hookContext genericapiserver.PostStartHookContext) error { - kubeClient, err := kcpkubernetesclientset.NewForConfig(hookContext.LoopbackClientConfig) - if err != nil { - return err - } - go legacytokentracking.NewController(kubeClient.Cluster(localAdminCluster.Path())).Run(hookContext.StopCh) - return nil - }) - - return m, nil -} - -func labelAPIServerHeartbeat(lease *coordinationapiv1.Lease) error { - if lease.Labels == nil { - lease.Labels = map[string]string{} - } - // This label indicates that kube-apiserver owns this identity lease object - lease.Labels[IdentityLeaseComponentLabelKey] = KubeAPIServer - - hostname, err := os.Hostname() - if err != nil { - return err - } - - // convenience label to easily map a lease object to a specific apiserver - lease.Labels[apiv1.LabelHostname] = hostname - return nil -} - -// InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled. -func (m *GenericControlPlane) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter) error { - legacyRESTStorageProvider := corerest.LegacyRESTStorageProvider{ - StorageFactory: c.ExtraConfig.StorageFactory, - EventTTL: c.ExtraConfig.EventTTL, - ServiceAccountIssuer: c.ExtraConfig.ServiceAccountIssuer, - ExtendExpiration: c.ExtraConfig.ExtendExpiration, - ServiceAccountMaxExpiration: c.ExtraConfig.ServiceAccountMaxExpiration, - APIAudiences: c.GenericConfig.Authentication.APIAudiences, - } - _, apiGroupInfo, err := legacyRESTStorageProvider.NewLegacyRESTStorage(c.ExtraConfig.APIResourceConfigSource, restOptionsGetter) - if err != nil { - return fmt.Errorf("error building core storage: %v", err) - } - if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { // if all core storage is disabled, return. - return nil - } - - if err := m.GenericAPIServer.InstallLegacyAPIGroup(genericapiserver.DefaultLegacyAPIPrefix, &apiGroupInfo); err != nil { - return fmt.Errorf("error in registering group versions: %v", err) - } - return nil -} - -// RESTStorageProvider is a factory type for REST storage. -type RESTStorageProvider interface { - GroupName() string - NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, error) -} - -// InstallAPIs will install the APIs for the restStorageProviders if they are enabled. -func (m *GenericControlPlane) InstallAPIs(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter, restStorageProviders ...RESTStorageProvider) error { - apiGroupsInfo := []*genericapiserver.APIGroupInfo{} - - // used later in the loop to filter the served resource by those that have expired. - resourceExpirationEvaluator, err := genericapiserver.NewResourceExpirationEvaluator(*m.GenericAPIServer.Version) - if err != nil { - return err - } - - for _, restStorageBuilder := range restStorageProviders { - groupName := restStorageBuilder.GroupName() - apiGroupInfo, err := restStorageBuilder.NewRESTStorage(apiResourceConfigSource, restOptionsGetter) - if err != nil { - return fmt.Errorf("problem initializing API group %q : %v", groupName, err) - } - if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { - // If we have no storage for any resource configured, this API group is effectively disabled. - // This can happen when an entire API group, version, or development-stage (alpha, beta, GA) is disabled. - klog.Infof("API group %q is not enabled, skipping.", groupName) - continue - } - - // Remove resources that serving kinds that are removed. - // We do this here so that we don't accidentally serve versions without resources or openapi information that for kinds we don't serve. - // This is a spot above the construction of individual storage handlers so that no sig accidentally forgets to check. - resourceExpirationEvaluator.RemoveDeletedKinds(groupName, apiGroupInfo.Scheme, apiGroupInfo.VersionedResourcesStorageMap) - if len(apiGroupInfo.VersionedResourcesStorageMap) == 0 { - klog.V(1).Infof("Removing API group %v because it is time to stop serving it because it has no versions per APILifecycle.", groupName) - continue - } - - klog.V(1).Infof("Enabling API group %q.", groupName) - - if postHookProvider, ok := restStorageBuilder.(genericapiserver.PostStartHookProvider); ok { - name, hook, err := postHookProvider.PostStartHook() - if err != nil { - klog.Fatalf("Error building PostStartHook: %v", err) - } - m.GenericAPIServer.AddPostStartHookOrDie(name, hook) - } - - apiGroupsInfo = append(apiGroupsInfo, &apiGroupInfo) - } - - if err := m.GenericAPIServer.InstallAPIGroups(apiGroupsInfo...); err != nil { - return fmt.Errorf("error in registering group versions: %v", err) - } - return nil -} - -var ( - // stableAPIGroupVersionsEnabledByDefault is a list of our stable versions. - stableAPIGroupVersionsEnabledByDefault = []schema.GroupVersion{ - admissionregistrationv1.SchemeGroupVersion, - apiv1.SchemeGroupVersion, - //appsv1.SchemeGroupVersion, - authenticationv1.SchemeGroupVersion, - authorizationapiv1.SchemeGroupVersion, - //autoscalingapiv1.SchemeGroupVersion, - //autoscalingapiv2.SchemeGroupVersion, - //batchapiv1.SchemeGroupVersion, - certificatesapiv1.SchemeGroupVersion, - coordinationapiv1.SchemeGroupVersion, - discoveryv1.SchemeGroupVersion, - eventsv1.SchemeGroupVersion, - //networkingapiv1.SchemeGroupVersion, - //nodev1.SchemeGroupVersion, - //policyapiv1.SchemeGroupVersion, - rbacv1.SchemeGroupVersion, - //storageapiv1.SchemeGroupVersion, - //schedulingapiv1.SchemeGroupVersion, - } - - // legacyBetaEnabledByDefaultResources is the list of beta resources we enable. You may only add to this list - // if your resource is already enabled by default in a beta level we still serve AND there is no stable API for it. - // see https://github.com/kubernetes/enhancements/tree/master/keps/sig-architecture/3136-beta-apis-off-by-default - // for more details. - legacyBetaEnabledByDefaultResources = []schema.GroupVersionResource{ - //autoscalingapiv2beta2.SchemeGroupVersion.WithResource("horizontalpodautoscalers"), // remove in 1.26 - //storageapiv1beta1.SchemeGroupVersion.WithResource("csistoragecapacities"), // remove in 1.27 - flowcontrolv1beta1.SchemeGroupVersion.WithResource("flowschemas"), // remove in 1.26 - flowcontrolv1beta1.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.26 - flowcontrolv1beta2.SchemeGroupVersion.WithResource("flowschemas"), // remove in 1.29 - flowcontrolv1beta2.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // remove in 1.29 - flowcontrolv1beta3.SchemeGroupVersion.WithResource("flowschemas"), // deprecate in 1.29, remove in 1.32 - flowcontrolv1beta3.SchemeGroupVersion.WithResource("prioritylevelconfigurations"), // deprecate in 1.29, remove in 1.32 - } - // betaAPIGroupVersionsDisabledByDefault is for all future beta groupVersions. - betaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{ - //autoscalingapiv2beta1.SchemeGroupVersion, - //autoscalingapiv2beta2.SchemeGroupVersion, - //batchapiv1beta1.SchemeGroupVersion, - discoveryv1beta1.SchemeGroupVersion, - eventsv1beta1.SchemeGroupVersion, - //nodev1beta1.SchemeGroupVersion, // remove in 1.26 - //policyapiv1beta1.SchemeGroupVersion, - //storageapiv1beta1.SchemeGroupVersion, - flowcontrolv1beta1.SchemeGroupVersion, - flowcontrolv1beta2.SchemeGroupVersion, - flowcontrolv1beta3.SchemeGroupVersion, - } - - // alphaAPIGroupVersionsDisabledByDefault holds the alpha APIs we have. They are always disabled by default. - alphaAPIGroupVersionsDisabledByDefault = []schema.GroupVersion{ - admissionregistrationv1alpha1.SchemeGroupVersion, - apiserverinternalv1alpha1.SchemeGroupVersion, - authenticationv1alpha1.SchemeGroupVersion, - resourcev1alpha2.SchemeGroupVersion, - //networkingapiv1alpha1.SchemeGroupVersion, - //storageapiv1alpha1.SchemeGroupVersion, - flowcontrolv1alpha1.SchemeGroupVersion, - } -) - -// DefaultAPIResourceConfigSource returns default configuration for an APIResource. -func DefaultAPIResourceConfigSource() *serverstorage.ResourceConfig { - ret := serverstorage.NewResourceConfig() - // NOTE: GroupVersions listed here will be enabled by default. Don't put alpha or beta versions in the list. - ret.EnableVersions(stableAPIGroupVersionsEnabledByDefault...) - // disable alpha and beta versions explicitly so we have a full list of what's possible to serve - ret.DisableVersions(betaAPIGroupVersionsDisabledByDefault...) - ret.DisableVersions(alphaAPIGroupVersionsDisabledByDefault...) - - // enable the legacy beta resources that were present before stopped serving new beta APIs by default. - ret.EnableResources(legacyBetaEnabledByDefaultResources...) - - return ret -} diff --git a/pkg/genericcontrolplane/apis/import_known_versions.go b/pkg/genericcontrolplane/apis/import_known_versions.go deleted file mode 100644 index c781b149df3f7..0000000000000 --- a/pkg/genericcontrolplane/apis/import_known_versions.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apis - -import ( - _ "k8s.io/kubernetes/pkg/genericcontrolplane/apis/install" -) diff --git a/pkg/genericcontrolplane/apis/install/install.go b/pkg/genericcontrolplane/apis/install/install.go deleted file mode 100644 index cee0169d2fbc6..0000000000000 --- a/pkg/genericcontrolplane/apis/install/install.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package install - -import ( - "k8s.io/kubernetes/pkg/api/genericcontrolplanescheme" - admissionregistrationinstall "k8s.io/kubernetes/pkg/apis/admissionregistration/install" - authenticationinstall "k8s.io/kubernetes/pkg/apis/authentication/install" - authorizationinstall "k8s.io/kubernetes/pkg/apis/authorization/install" - certificatesinstall "k8s.io/kubernetes/pkg/apis/certificates/install" - coordinationinstall "k8s.io/kubernetes/pkg/apis/coordination/install" - genericcontrolplaneinstall "k8s.io/kubernetes/pkg/apis/core/install/genericcontrolplane" - eventsinstall "k8s.io/kubernetes/pkg/apis/events/install" - rbacinstall "k8s.io/kubernetes/pkg/apis/rbac/install" -) - -func init() { - genericcontrolplaneinstall.Install(genericcontrolplanescheme.Scheme) - authenticationinstall.Install(genericcontrolplanescheme.Scheme) - authorizationinstall.Install(genericcontrolplanescheme.Scheme) - certificatesinstall.Install(genericcontrolplanescheme.Scheme) - coordinationinstall.Install(genericcontrolplanescheme.Scheme) - rbacinstall.Install(genericcontrolplanescheme.Scheme) - eventsinstall.Install(genericcontrolplanescheme.Scheme) - admissionregistrationinstall.Install(genericcontrolplanescheme.Scheme) -} diff --git a/pkg/genericcontrolplane/localadmin/cluster.go b/pkg/genericcontrolplane/localadmin/cluster.go deleted file mode 100644 index c50e63d610c5f..0000000000000 --- a/pkg/genericcontrolplane/localadmin/cluster.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package localadmin - -import ( - "github.com/kcp-dev/logicalcluster/v3" -) - -var Cluster = logicalcluster.Name("system:admin") diff --git a/pkg/genericcontrolplane/options/flags.go b/pkg/genericcontrolplane/options/flags.go deleted file mode 100644 index eed41cdc47d85..0000000000000 --- a/pkg/genericcontrolplane/options/flags.go +++ /dev/null @@ -1,67 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - cliflag "k8s.io/component-base/cli/flag" - logsapi "k8s.io/component-base/logs/api/v1" -) - -// Flags returns flags for a specific APIServer by section name -func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) { - s.GenericServerRunOptions.AddUniversalFlags(fss.FlagSet("generic")) - s.Etcd.AddFlags(fss.FlagSet("etcd")) - s.SecureServing.AddFlags(fss.FlagSet("secure serving")) - s.Audit.AddFlags(fss.FlagSet("auditing")) - s.Features.AddFlags(fss.FlagSet("features")) - s.Authentication.AddFlags(fss.FlagSet("authentication")) - - s.APIEnablement.AddFlags(fss.FlagSet("API enablement")) - s.EgressSelector.AddFlags(fss.FlagSet("egress selector")) - s.Admission.AddFlags(fss.FlagSet("admission")) - - s.Metrics.AddFlags(fss.FlagSet("metrics")) - logsapi.AddFlags(s.Logs, fss.FlagSet("logs")) - s.Traces.AddFlags(fss.FlagSet("traces")) - - fs := fss.FlagSet("misc") - fs.DurationVar(&s.EventTTL, "event-ttl", s.EventTTL, - "Amount of time to retain events.") - - fs.BoolVar(&s.EnableLogsHandler, "enable-logs-handler", s.EnableLogsHandler, - "If true, install a /logs handler for the apiserver logs.") - fs.MarkDeprecated("enable-logs-handler", "This flag will be removed in v1.19") //nolint:golint,errcheck - - fs.Int64Var(&s.MaxConnectionBytesPerSec, "max-connection-bytes-per-sec", s.MaxConnectionBytesPerSec, ""+ - "If non-zero, throttle each user connection to this number of bytes/sec. "+ - "Currently only applies to long-running requests.") - - fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+ - "Client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins. It is expected that this "+ - "cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+ - "That CA is published in the 'extension-apiserver-authentication' configmap in "+ - "the kube-system namespace. Components receiving calls from kube-aggregator should "+ - "use that CA to perform their half of the mutual TLS verification.") - fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+ - "Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+ - "when it must call out during a request. This includes proxying requests to a user "+ - "api-server and calling out to webhook admission plugins.") - - return fss -} diff --git a/pkg/genericcontrolplane/options/options.go b/pkg/genericcontrolplane/options/options.go deleted file mode 100644 index c8c1f51d3afc0..0000000000000 --- a/pkg/genericcontrolplane/options/options.go +++ /dev/null @@ -1,198 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "os" - "strings" - "time" - - "k8s.io/apiextensions-apiserver/pkg/apiserver/conversion" - genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/apiserver/pkg/storage/storagebackend" - "k8s.io/client-go/util/keyutil" - "k8s.io/component-base/logs" - "k8s.io/component-base/metrics" - "k8s.io/klog/v2" - - kubeauthenticator "k8s.io/kubernetes/pkg/kubeapiserver/authenticator" - kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options" - "k8s.io/kubernetes/pkg/serviceaccount" -) - -// InsecurePortFlags are dummy flags, they are kept only for compatibility and will be removed in v1.24. -// TODO: remove these flags in v1.24. -var InsecurePortFlags = []string{"insecure-port", "port"} - -// ServerRunOptions runs a kubernetes api server. -type ServerRunOptions struct { - GenericServerRunOptions *genericoptions.ServerRunOptions - Etcd *genericoptions.EtcdOptions - SecureServing *genericoptions.SecureServingOptionsWithLoopback - Audit *genericoptions.AuditOptions - Features *genericoptions.FeatureOptions - Admission *genericoptions.AdmissionOptions - Authentication *kubeoptions.BuiltInAuthenticationOptions - APIEnablement *genericoptions.APIEnablementOptions - EgressSelector *genericoptions.EgressSelectorOptions - Metrics *metrics.Options - Logs *logs.Options - Traces *genericoptions.TracingOptions - - EnableLogsHandler bool - EventTTL time.Duration - MaxConnectionBytesPerSec int64 - - ProxyClientCertFile string - ProxyClientKeyFile string - - EnableAggregatorRouting bool - - ServiceAccountSigningKeyFile string - ServiceAccountIssuer serviceaccount.TokenGenerator - ServiceAccountTokenMaxExpiration time.Duration - - ShowHiddenMetricsForVersion string - - ConversionFactory conversion.Factory -} - -// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked. -type completedServerRunOptions struct { - ServerRunOptions -} - -type CompletedServerRunOptions struct { - // Embed a private pointer that cannot be instantiated outside of this package. - *completedServerRunOptions -} - -// NewServerRunOptions creates a new ServerRunOptions object with default parameters -func NewServerRunOptions() *ServerRunOptions { - s := ServerRunOptions{ - GenericServerRunOptions: genericoptions.NewServerRunOptions(), - Etcd: genericoptions.NewEtcdOptions(storagebackend.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), - SecureServing: kubeoptions.NewSecureServingOptions().WithLoopback(), - Audit: genericoptions.NewAuditOptions(), - Features: genericoptions.NewFeatureOptions(), - Admission: genericoptions.NewAdmissionOptions(), - Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(), - APIEnablement: genericoptions.NewAPIEnablementOptions(), - EgressSelector: genericoptions.NewEgressSelectorOptions(), - Metrics: metrics.NewOptions(), - Logs: logs.NewOptions(), - Traces: genericoptions.NewTracingOptions(), - - EnableLogsHandler: true, - EventTTL: 1 * time.Hour, - } - - // disable the watch cache - s.Etcd.EnableWatchCache = false - - // Overwrite the default for storage data format. - s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf" - - return &s -} - -// Complete defaults missing field values. It mutates the receiver. -func (o *ServerRunOptions) Complete() (CompletedServerRunOptions, error) { - if err := o.GenericServerRunOptions.DefaultAdvertiseAddress(o.SecureServing.SecureServingOptions); err != nil { - return CompletedServerRunOptions{}, err - } - - if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts(o.GenericServerRunOptions.AdvertiseAddress.String(), nil, nil); err != nil { - return CompletedServerRunOptions{}, fmt.Errorf("error creating self-signed certificates: %v", err) - } - - if len(o.GenericServerRunOptions.ExternalHost) == 0 { - if len(o.GenericServerRunOptions.AdvertiseAddress) > 0 { - o.GenericServerRunOptions.ExternalHost = o.GenericServerRunOptions.AdvertiseAddress.String() - } else { - if hostname, err := os.Hostname(); err == nil { - o.GenericServerRunOptions.ExternalHost = hostname - } else { - return CompletedServerRunOptions{}, fmt.Errorf("error finding host name: %v", err) - } - } - klog.Infof("external host was not specified, using %v", o.GenericServerRunOptions.ExternalHost) - } - - // Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling - // TokenRequest functionality. This defaulting was convenient, but messed up - // a lot of people when they rotated their serving cert with no idea it was - // connected to their service account keys. We are taking this opportunity to - // remove this problematic defaulting. - if o.ServiceAccountSigningKeyFile == "" { - // Default to the private server key for service account token signing - if len(o.Authentication.ServiceAccounts.KeyFiles) == 0 && o.SecureServing.ServerCert.CertKey.KeyFile != "" { - if kubeauthenticator.IsValidServiceAccountKeyFile(o.SecureServing.ServerCert.CertKey.KeyFile) { - o.Authentication.ServiceAccounts.KeyFiles = []string{o.SecureServing.ServerCert.CertKey.KeyFile} - } else { - klog.Warning("No TLS key provided, service account token authentication disabled") - } - } - } - - if o.ServiceAccountSigningKeyFile != "" && len(o.Authentication.ServiceAccounts.Issuers) != 0 && o.Authentication.ServiceAccounts.Issuers[0] != "" { - sk, err := keyutil.PrivateKeyFromFile(o.ServiceAccountSigningKeyFile) - if err != nil { - return CompletedServerRunOptions{}, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err) - } - if o.Authentication.ServiceAccounts.MaxExpiration != 0 { - lowBound := time.Hour - upBound := time.Duration(1<<32) * time.Second - if o.Authentication.ServiceAccounts.MaxExpiration < lowBound || - o.Authentication.ServiceAccounts.MaxExpiration > upBound { - return CompletedServerRunOptions{}, fmt.Errorf("the service-account-max-token-expiration must be between 1 hour and 2^32 seconds") - } - if o.Authentication.ServiceAccounts.ExtendExpiration { - if o.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.WarnOnlyBoundTokenExpirationSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, in order to correctly trigger safe transition logic, service-account-max-token-expiration must be set longer than %d seconds (currently %s)", serviceaccount.WarnOnlyBoundTokenExpirationSeconds, o.Authentication.ServiceAccounts.MaxExpiration) - } - if o.Authentication.ServiceAccounts.MaxExpiration < serviceaccount.ExpirationExtensionSeconds*time.Second { - klog.Warningf("service-account-extend-token-expiration is true, enabling tokens valid up to %d seconds, which is longer than service-account-max-token-expiration set to %s seconds", serviceaccount.ExpirationExtensionSeconds, o.Authentication.ServiceAccounts.MaxExpiration) - } - } - } - - o.ServiceAccountIssuer, err = serviceaccount.JWTTokenGenerator(o.Authentication.ServiceAccounts.Issuers[0], sk) - if err != nil { - return CompletedServerRunOptions{}, fmt.Errorf("failed to build token generator: %v", err) - } - o.ServiceAccountTokenMaxExpiration = o.Authentication.ServiceAccounts.MaxExpiration - } - - for key, value := range o.APIEnablement.RuntimeConfig { - if key == "v1" || strings.HasPrefix(key, "v1/") || - key == "api/v1" || strings.HasPrefix(key, "api/v1/") { - delete(o.APIEnablement.RuntimeConfig, key) - o.APIEnablement.RuntimeConfig["/v1"] = value - } - if key == "api/legacy" { - delete(o.APIEnablement.RuntimeConfig, key) - } - } - - return CompletedServerRunOptions{ - &completedServerRunOptions{ - ServerRunOptions: *o, - }, - }, nil -} diff --git a/pkg/genericcontrolplane/options/validation.go b/pkg/genericcontrolplane/options/validation.go deleted file mode 100644 index a02c4b9f6adba..0000000000000 --- a/pkg/genericcontrolplane/options/validation.go +++ /dev/null @@ -1,37 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - - "k8s.io/kubernetes/pkg/api/genericcontrolplanescheme" -) - -// Validate checks Options and return a slice of found errs. -func (s *CompletedServerRunOptions) Validate() []error { - var errs []error - errs = append(errs, s.Etcd.Validate()...) - errs = append(errs, s.SecureServing.Validate()...) - errs = append(errs, s.Authentication.Validate()...) - errs = append(errs, s.Audit.Validate()...) - errs = append(errs, s.Admission.Validate()...) - errs = append(errs, s.APIEnablement.Validate(genericcontrolplanescheme.Scheme, apiextensionsapiserver.Scheme)...) - errs = append(errs, s.Metrics.Validate()...) - - return errs -} diff --git a/pkg/genericcontrolplane/server.go b/pkg/genericcontrolplane/server.go deleted file mode 100644 index bcbe26b2fa9e3..0000000000000 --- a/pkg/genericcontrolplane/server.go +++ /dev/null @@ -1,280 +0,0 @@ -/* -Copyright 2014 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package app does all of the work necessary to create a Kubernetes -// APIServer by binding together the API, master and APIServer infrastructure. -// It can be configured and called directly or via the hyperkube framework. -package genericcontrolplane - -import ( - "errors" - "fmt" - "net/url" - "time" - - "github.com/kcp-dev/client-go/dynamic" - kcpkubernetesinformers "github.com/kcp-dev/client-go/informers" - kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" - oteltrace "go.opentelemetry.io/otel/trace" - extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/clientsethack" - "k8s.io/apiserver/pkg/dynamichack" - genericapifilters "k8s.io/apiserver/pkg/endpoints/filters" - openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" - genericfeatures "k8s.io/apiserver/pkg/features" - "k8s.io/apiserver/pkg/informerfactoryhack" - genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/filters" - serverstorage "k8s.io/apiserver/pkg/server/storage" - utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/apiserver/pkg/util/notfoundhandler" - "k8s.io/apiserver/pkg/util/openapi" - "k8s.io/client-go/util/keyutil" - _ "k8s.io/component-base/metrics/prometheus/workqueue" - "k8s.io/component-base/version" - aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme" - "k8s.io/kubernetes/pkg/api/genericcontrolplanescheme" - "k8s.io/kubernetes/pkg/api/legacyscheme" - generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi" - "k8s.io/kubernetes/pkg/genericcontrolplane/aggregator" - "k8s.io/kubernetes/pkg/genericcontrolplane/apis" - "k8s.io/kubernetes/pkg/genericcontrolplane/localadmin" - "k8s.io/kubernetes/pkg/genericcontrolplane/options" - "k8s.io/kubernetes/pkg/kubeapiserver" - "k8s.io/kubernetes/pkg/serviceaccount" -) - -var LocalAdminCluster = localadmin.Cluster - -type ServerChain struct { - CustomResourceDefinitions *extensionsapiserver.CustomResourceDefinitions - GenericControlPlane *apis.GenericControlPlane - MiniAggregator *aggregator.MiniAggregatorServer -} - -// CreateServerChain creates the apiservers connected via delegation. -func CreateServerChain(miniAggregatorConfig aggregator.CompletedMiniAggregatorConfig, apisConfig apis.CompletedConfig, apiExtensionConfig extensionsapiserver.CompletedConfig) (*ServerChain, error) { - notFoundHandler := notfoundhandler.New(apisConfig.GenericConfig.Serializer, genericapifilters.NoMuxAndDiscoveryIncompleteKey) - apiExtensionsServer, err := apiExtensionConfig.New(genericapiserver.NewEmptyDelegateWithCustomHandler(notFoundHandler)) - if err != nil { - return nil, fmt.Errorf("create api extensions: %v", err) - } - - kubeAPIServer, err := apisConfig.New(apiExtensionsServer.GenericAPIServer) - if err != nil { - return nil, err - } - - miniAggregatorServer, err := miniAggregatorConfig.New(kubeAPIServer.GenericAPIServer, kubeAPIServer, apiExtensionsServer) - if err != nil { - return nil, err - } - - return &ServerChain{ - CustomResourceDefinitions: apiExtensionsServer, - GenericControlPlane: kubeAPIServer, - MiniAggregator: miniAggregatorServer, - }, nil -} - -// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them -func CreateKubeAPIServerConfig( - genericConfig *genericapiserver.Config, - o options.CompletedServerRunOptions, - versionedInformers kcpkubernetesinformers.SharedInformerFactory, - additionalPluginInitializers []admission.PluginInitializer, - storageFactory *serverstorage.DefaultStorageFactory, -) ( - *apis.Config, - error, -) { - o.Metrics.Apply() - serviceaccount.RegisterMetrics() - - // Load the public keys. - var pubKeys []interface{} - for _, f := range o.Authentication.ServiceAccounts.KeyFiles { - keys, err := keyutil.PublicKeysFromFile(f) - if err != nil { - return nil, fmt.Errorf("failed to parse key file %q: %v", f, err) - } - pubKeys = append(pubKeys, keys...) - } - - config := &apis.Config{ - GenericConfig: genericConfig, - ExtraConfig: apis.ExtraConfig{ - APIResourceConfigSource: storageFactory.APIResourceConfigSource, - StorageFactory: storageFactory, - EventTTL: o.EventTTL, - EnableLogsSupport: o.EnableLogsHandler, - - VersionedInformers: versionedInformers, - - ServiceAccountIssuer: o.ServiceAccountIssuer, - ServiceAccountMaxExpiration: o.ServiceAccountTokenMaxExpiration, - ExtendExpiration: o.Authentication.ServiceAccounts.ExtendExpiration, - ServiceAccountIssuerURL: o.Authentication.ServiceAccounts.Issuers[0], // panic? - ServiceAccountJWKSURI: o.Authentication.ServiceAccounts.JWKSURI, - ServiceAccountPublicKeys: pubKeys, - }, - } - - clientCAProvider, err := o.Authentication.ClientCert.GetClientCAContentProvider() - if err != nil { - return nil, err - } - config.ExtraConfig.ClusterAuthenticationInfo.ClientCA = clientCAProvider - - requestHeaderConfig, err := o.Authentication.RequestHeader.ToAuthenticationRequestHeaderConfig() - if err != nil { - return nil, err - } - if requestHeaderConfig != nil { - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderCA = requestHeaderConfig.CAContentProvider - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderAllowedNames = requestHeaderConfig.AllowedClientNames - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderExtraHeaderPrefixes = requestHeaderConfig.ExtraHeaderPrefixes - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderGroupHeaders = requestHeaderConfig.GroupHeaders - config.ExtraConfig.ClusterAuthenticationInfo.RequestHeaderUsernameHeaders = requestHeaderConfig.UsernameHeaders - } - - client, err := kcpkubernetesclientset.NewForConfig(config.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - dynamicClient, err := dynamic.NewForConfig(config.GenericConfig.LoopbackClientConfig) - if err != nil { - return nil, err - } - if err := o.ServerRunOptions.Admission.ApplyTo( - config.GenericConfig, - informerfactoryhack.Wrap(config.ExtraConfig.VersionedInformers), - clientsethack.Wrap(client), - dynamichack.Wrap(dynamicClient), - utilfeature.DefaultFeatureGate, - additionalPluginInitializers...); err != nil { - return nil, err - } - - return config, nil -} - -// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it -func BuildGenericConfig( - o options.CompletedServerRunOptions, -) ( - genericConfig *genericapiserver.Config, - storageFactory *serverstorage.DefaultStorageFactory, - versionedInformers kcpkubernetesinformers.SharedInformerFactory, - clientgoExternalClient kcpkubernetesclientset.ClusterInterface, - lastErr error, -) { - genericConfig = genericapiserver.NewConfig(genericcontrolplanescheme.Codecs) - genericConfig.MergedResourceConfig = apis.DefaultAPIResourceConfigSource() - - if lastErr = o.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil { - return - } - - if lastErr = o.SecureServing.ApplyTo(&genericConfig.SecureServing, &genericConfig.LoopbackClientConfig); lastErr != nil { - return - } - if lastErr = o.Features.ApplyTo(genericConfig); lastErr != nil { - return - } - if lastErr = o.APIEnablement.ApplyTo(genericConfig, apis.DefaultAPIResourceConfigSource(), genericcontrolplanescheme.Scheme); lastErr != nil { - return - } - if lastErr = o.EgressSelector.ApplyTo(genericConfig); lastErr != nil { - return - } - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { - if lastErr = o.Traces.ApplyTo(genericConfig.EgressSelector, genericConfig); lastErr != nil { - return - } - } - - // wrap the definitions to revert any changes from disabled features - getOpenAPIDefinitions := openapi.GetOpenAPIDefinitionsWithoutDisabledFeatures(generatedopenapi.GetOpenAPIDefinitions) - genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(getOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, extensionsapiserver.Scheme)) - genericConfig.OpenAPIConfig.Info.Title = "Kubernetes" - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.OpenAPIV3) { - genericConfig.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config(getOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme)) - genericConfig.OpenAPIV3Config.Info.Title = "Kubernetes" - } - - genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( - sets.NewString("watch", "proxy"), - sets.NewString("attach", "exec", "proxy", "log", "portforward"), - ) - - kubeVersion := version.Get() - genericConfig.Version = &kubeVersion - - if genericConfig.EgressSelector != nil { - o.Etcd.StorageConfig.Transport.EgressLookup = genericConfig.EgressSelector.Lookup - } - if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) { - o.Etcd.StorageConfig.Transport.TracerProvider = genericConfig.TracerProvider - } else { - o.Etcd.StorageConfig.Transport.TracerProvider = oteltrace.NewNoopTracerProvider() - } - if lastErr = o.Etcd.Complete(genericConfig.StorageObjectCountTracker, genericConfig.DrainedNotify(), genericConfig.AddPostStartHook); lastErr != nil { - return - } - - storageFactoryConfig := kubeapiserver.NewStorageFactoryConfig(genericcontrolplanescheme.Scheme, genericcontrolplanescheme.Codecs) - storageFactoryConfig.APIResourceConfig = genericConfig.MergedResourceConfig - storageFactory, lastErr = storageFactoryConfig.Complete(o.Etcd).New() - if lastErr != nil { - return - } - if lastErr = o.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil { - return - } - - // Disable compression for self-communication, since we are going to be - // on a fast local network - genericConfig.LoopbackClientConfig.DisableCompression = true - - // Accept both protobuf and application/json for loopback client. - genericConfig.LoopbackClientConfig.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json" - - kubeClientConfig := genericConfig.LoopbackClientConfig - clientgoExternalClient, err := kcpkubernetesclientset.NewForConfig(kubeClientConfig) - if err != nil { - lastErr = fmt.Errorf("failed to create real external clientset: %v", err) - return - } - versionedInformers = kcpkubernetesinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute) - - // Authentication.ApplyTo requires already applied OpenAPIConfig and EgressSelector if present - if lastErr = o.Authentication.ApplyTo(&genericConfig.Authentication, genericConfig.SecureServing, genericConfig.EgressSelector, genericConfig.OpenAPIConfig, genericConfig.OpenAPIV3Config, clientgoExternalClient, versionedInformers); lastErr != nil { - return - } - - return -} - -// unimplementedServiceResolver is a webhook.ServiceResolver that always returns an error. -type unimplementedServiceResolver struct{} - -// ResolveEndpoint always returns an error that this is not yet supported. -func (r *unimplementedServiceResolver) ResolveEndpoint(namespace string, name string, port int32) (*url.URL, error) { - return nil, errors.New("webhook admission and conversions are not yet supported in kcp") -} diff --git a/staging/src/k8s.io/apiserver/pkg/clientsethack/adapter.go b/staging/src/k8s.io/apiserver/pkg/clientsethack/adapter.go index 4e14716e55f1b..0ca2e5d29b4bf 100644 --- a/staging/src/k8s.io/apiserver/pkg/clientsethack/adapter.go +++ b/staging/src/k8s.io/apiserver/pkg/clientsethack/adapter.go @@ -20,21 +20,18 @@ package clientsethack import ( kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" - admissionregistrationv1alpha1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1" - "k8s.io/client-go/kubernetes/typed/authentication/v1alpha1" - flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3" - networkingv1alpha1 "k8s.io/client-go/kubernetes/typed/networking/v1alpha1" - resourcev1alpha2 "k8s.io/client-go/kubernetes/typed/resource/v1alpha2" "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes" admissionregistrationv1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1" + admissionregistrationv1alpha1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1alpha1" admissionregistrationv1beta1 "k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1" internalv1alpha1 "k8s.io/client-go/kubernetes/typed/apiserverinternal/v1alpha1" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" appsv1beta1 "k8s.io/client-go/kubernetes/typed/apps/v1beta1" appsv1beta2 "k8s.io/client-go/kubernetes/typed/apps/v1beta2" authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + "k8s.io/client-go/kubernetes/typed/authentication/v1alpha1" authenticationv1beta1 "k8s.io/client-go/kubernetes/typed/authentication/v1beta1" authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" authorizationv1beta1 "k8s.io/client-go/kubernetes/typed/authorization/v1beta1" @@ -45,6 +42,7 @@ import ( batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" batchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" certificatesv1 "k8s.io/client-go/kubernetes/typed/certificates/v1" + certificatesv1alpha1 "k8s.io/client-go/kubernetes/typed/certificates/v1alpha1" certificatesv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" coordinationv1 "k8s.io/client-go/kubernetes/typed/coordination/v1" coordinationv1beta1 "k8s.io/client-go/kubernetes/typed/coordination/v1beta1" @@ -57,7 +55,9 @@ import ( flowcontrolv1alpha1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1" flowcontrolv1beta1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1" flowcontrolv1beta2 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2" + flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta3" networkingv1 "k8s.io/client-go/kubernetes/typed/networking/v1" + networkingv1alpha1 "k8s.io/client-go/kubernetes/typed/networking/v1alpha1" networkingv1beta1 "k8s.io/client-go/kubernetes/typed/networking/v1beta1" nodev1 "k8s.io/client-go/kubernetes/typed/node/v1" nodev1alpha1 "k8s.io/client-go/kubernetes/typed/node/v1alpha1" @@ -67,6 +67,7 @@ import ( rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1" rbacv1alpha1 "k8s.io/client-go/kubernetes/typed/rbac/v1alpha1" rbacv1beta1 "k8s.io/client-go/kubernetes/typed/rbac/v1beta1" + resourcev1alpha2 "k8s.io/client-go/kubernetes/typed/resource/v1alpha2" schedulingv1 "k8s.io/client-go/kubernetes/typed/scheduling/v1" schedulingv1alpha1 "k8s.io/client-go/kubernetes/typed/scheduling/v1alpha1" schedulingv1beta1 "k8s.io/client-go/kubernetes/typed/scheduling/v1beta1" @@ -187,6 +188,10 @@ func (h *hack) CertificatesV1() certificatesv1.CertificatesV1Interface { panic("programmer error: using a cluster-unaware clientset, need to cast this to use the cluster-aware one!") } +func (h *hack) CertificatesV1alpha1() certificatesv1alpha1.CertificatesV1alpha1Interface { + panic("programmer error: using a cluster-unaware clientset, need to cast this to use the cluster-aware one!") +} + func (h *hack) CertificatesV1beta1() certificatesv1beta1.CertificatesV1beta1Interface { panic("programmer error: using a cluster-unaware clientset, need to cast this to use the cluster-aware one!") } diff --git a/test/integration/framework/test_server.go b/test/integration/framework/test_server.go index bece9c68fbf8a..39ca11eff26d6 100644 --- a/test/integration/framework/test_server.go +++ b/test/integration/framework/test_server.go @@ -166,7 +166,7 @@ func StartTestServer(ctx context.Context, t testing.TB, setup TestServerSetup) ( t.Fatalf("failed to validate ServerRunOptions: %v", utilerrors.NewAggregate(errs)) } - genericConfig, _, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( + genericConfig, versionedInformers, storageFactory, err := controlplaneapiserver.BuildGenericConfig( completedOptions.CompletedOptions, []*runtime.Scheme{legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme}, controlplane.DefaultAPIResourceConfigSource(),