diff --git a/config/jobs/kubernetes/kops/build_jobs.py b/config/jobs/kubernetes/kops/build_jobs.py index b78f5d435e55..e8d8f9808ddb 100644 --- a/config/jobs/kubernetes/kops/build_jobs.py +++ b/config/jobs/kubernetes/kops/build_jobs.py @@ -90,6 +90,7 @@ def build_test(cloud='aws', runs_per_day=0, scenario=None, env=None, + instance_groups_overrides=None, kubernetes_feature_gates=None, build_cluster=None, cluster_name=None, @@ -114,6 +115,9 @@ def build_test(cloud='aws', if extra_flags is None: extra_flags = [] + if instance_groups_overrides is None: + instance_groups_overrides = [] + if cloud == 'aws': kops_image = distro_images[distro] kops_ssh_user = distros_ssh_user[distro] @@ -234,6 +238,7 @@ def build_test(cloud='aws', test_args=test_args, cluster_name=cluster_name, storage_e2e_cred=storage_e2e_cred, + instance_groups_overrides=instance_groups_overrides, ) spec = { @@ -318,7 +323,8 @@ def presubmit_test(branch='master', cluster_name=None, use_preset_for_account_creds=None, alert_email=None, - alert_num_failures=None): + alert_num_failures=None, + instance_groups_overrides=None): # pylint: disable=too-many-statements,too-many-branches,too-many-arguments if cloud == 'aws': if distro == "channels": @@ -355,6 +361,8 @@ def presubmit_test(branch='master', if extra_flags is None: extra_flags = [] + if instance_groups_overrides is None: + instance_groups_overrides = [] # TODO: Uncomment when dynamic discovery buckets are available # if (irsa and cloud == "aws" and scenario is None and # terraform_version is None and name != "pull-kops-aws-distro-al2023"): @@ -414,6 +422,7 @@ def presubmit_test(branch='master', build_cluster=build_cluster, test_args=test_args, cluster_name=cluster_name, + instance_groups_overrides=instance_groups_overrides, ) spec = { @@ -2043,6 +2052,31 @@ def generate_presubmits_e2e(): always_run=True, extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 ), + presubmit_test( + cloud='gce', + k8s_version='stable', + kops_channel='alpha', + name='pull-kops-e2e-k8s-gce-cilium-hyperdisk', + networking='cilium', + tab_name='e2e-gce-cilium-hyperdisk', + always_run=False, + extra_flags=[ + "--zones=us-east1-b", + "--node-size=c4-standard-4", + "--master-size=c4-standard-2", + "--set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000", + "--set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000", + "--set spec.etcdClusters[*].etcdMembers[*].volumeSize=60", + "--set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced", + "--gce-service-account=default" + ], + instance_groups_overrides=[ + "spec.rootVolume.type=hyperdisk-balanced", + "spec.rootVolume.size=60", + "spec.rootVolume.iops=10000", + "spec.rootVolume.throughput=1000", + ] + ), presubmit_test( cloud='gce', k8s_version='stable', diff --git a/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml b/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml index a0fe08529acf..567816a5ab5a 100644 --- a/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml +++ b/config/jobs/kubernetes/kops/kops-presubmits-e2e.yaml @@ -1,5 +1,5 @@ # Test jobs generated by build_jobs.py (do not manually edit) -# 41 jobs +# 42 jobs presubmits: kubernetes/kops: @@ -403,6 +403,80 @@ presubmits: testgrid-days-of-results: '90' testgrid-tab-name: e2e-gce-cilium +# {"cloud": "gce", "distro": "u2404", "extra_flags": "--zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default", "k8s_version": "stable", "kops_channel": "alpha", "networking": "cilium"} + - name: pull-kops-e2e-k8s-gce-cilium-hyperdisk + cluster: k8s-infra-prow-build + branches: + - master + always_run: false + optional: false + skip_report: false + labels: + preset-k8s-ssh: "true" + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kops + spec: + serviceAccountName: k8s-kops-test + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + imagePullPolicy: Always + command: + - runner.sh + args: + - bash + - -c + - | + make test-e2e-install + kubetest2 kops \ + -v 2 \ + --up --build --down \ + --cloud-provider=gce \ + --admin-access=0.0.0.0/0 \ + --create-args="--channel=alpha --networking=cilium --zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default" \ + --kubernetes-version=https://dl.k8s.io/release/stable.txt \ + --kops-binary-path=/home/prow/go/src/k8s.io/kops/.build/dist/linux/amd64/kops \ + --control-plane-instance-group-overrides=spec.rootVolume.type=hyperdisk-balanced \ + --node-instance-group-overrides=spec.rootVolume.type=hyperdisk-balanced \ + --control-plane-instance-group-overrides=spec.rootVolume.size=60 \ + --node-instance-group-overrides=spec.rootVolume.size=60 \ + --control-plane-instance-group-overrides=spec.rootVolume.iops=10000 \ + --node-instance-group-overrides=spec.rootVolume.iops=10000 \ + --control-plane-instance-group-overrides=spec.rootVolume.throughput=1000 \ + --node-instance-group-overrides=spec.rootVolume.throughput=1000 \ + --test=kops \ + -- \ + --test-args="-test.timeout=60m" \ + --test-package-marker=stable.txt \ + --parallel=25 + securityContext: + privileged: true + env: + - name: KUBE_SSH_KEY_PATH + value: /etc/ssh-key-secret/ssh-private + - name: KUBE_SSH_USER + value: prow + - name: GOPATH + value: /home/prow/go + resources: + limits: + cpu: "4" + memory: 6Gi + requests: + cpu: "4" + memory: 6Gi + annotations: + test.kops.k8s.io/cloud: gce + test.kops.k8s.io/distro: u2404 + test.kops.k8s.io/extra_flags: --zones=us-east1-b --node-size=c4-standard-4 --master-size=c4-standard-2 --set spec.etcdClusters[*].etcdMembers[*].volumeIOPS=10000 --set spec.etcdClusters[*].etcdMembers[*].volumeThroughput=1000 --set spec.etcdClusters[*].etcdMembers[*].volumeSize=60 --set spec.etcdClusters[*].etcdMembers[*].volumeType=hyperdisk-balanced --gce-service-account=default + test.kops.k8s.io/k8s_version: stable + test.kops.k8s.io/kops_channel: alpha + test.kops.k8s.io/networking: cilium + testgrid-dashboards: kops-presubmits, presubmits-kops, sig-cluster-lifecycle-kops + testgrid-days-of-results: '90' + testgrid-tab-name: e2e-gce-cilium-hyperdisk + # {"cloud": "gce", "distro": "u2404", "extra_flags": "--gce-service-account=default", "k8s_version": "stable", "kops_channel": "alpha", "networking": "cilium-etcd"} - name: pull-kops-e2e-k8s-gce-cilium-etcd cluster: k8s-infra-prow-build diff --git a/config/jobs/kubernetes/kops/kops-presubmits.yaml b/config/jobs/kubernetes/kops/kops-presubmits.yaml index 60a1434c1871..6d1f903713ed 100644 --- a/config/jobs/kubernetes/kops/kops-presubmits.yaml +++ b/config/jobs/kubernetes/kops/kops-presubmits.yaml @@ -12,7 +12,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -40,7 +40,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -73,7 +73,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master imagePullPolicy: Always command: - runner.sh @@ -126,7 +126,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master imagePullPolicy: Always command: - runner.sh @@ -179,7 +179,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master imagePullPolicy: Always command: - runner.sh @@ -226,7 +226,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -255,7 +255,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -282,7 +282,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -311,7 +311,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -338,7 +338,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -367,7 +367,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -395,7 +395,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -424,7 +424,7 @@ presubmits: path_alias: k8s.io/kops spec: containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master command: - runner.sh args: @@ -464,7 +464,7 @@ presubmits: spec: serviceAccountName: k8s-kops-test containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master imagePullPolicy: Always command: - runner.sh @@ -508,7 +508,7 @@ presubmits: spec: serviceAccountName: azure containers: - - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:v20251021-e2c2c9806f-master + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master imagePullPolicy: Always command: - runner.sh @@ -544,3 +544,69 @@ presubmits: annotations: testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits testgrid-tab-name: e2e-azure + - name: pull-kops-e2e-baremetal + branches: + - master + always_run: false + labels: + preset-dind-enabled: "true" + cluster: k8s-infra-prow-build + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kops + spec: + nodeSelector: + cloud.google.com/machine-family: c4 # this node type supports nested virtualization + containers: + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master + imagePullPolicy: Always + command: + - runner.sh + args: + - tests/e2e/scenarios/bare-metal/run-test + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + limits: + cpu: "7" + memory: "25Gi" + requests: + cpu: "7" + memory: "25Gi" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits + - name: pull-kops-e2e-baremetal-ipv6 + branches: + - master + always_run: false + labels: + preset-dind-enabled: "true" + cluster: k8s-infra-prow-build + decorate: true + decoration_config: + timeout: 90m + path_alias: k8s.io/kops + spec: + nodeSelector: + cloud.google.com/machine-family: c4 # this node type supports nested virtualization + containers: + - image: us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20251021-e2c2c9806f-master + imagePullPolicy: Always + command: + - runner.sh + args: + - tests/e2e/scenarios/bare-metal/scenario-ipv6 + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + limits: + cpu: "7" + memory: "25Gi" + requests: + cpu: "7" + memory: "25Gi" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kops, presubmits-kops, kops-presubmits diff --git a/config/jobs/kubernetes/kops/templates/presubmit.yaml.jinja b/config/jobs/kubernetes/kops/templates/presubmit.yaml.jinja index b92f67d6f5eb..e03c721ac042 100644 --- a/config/jobs/kubernetes/kops/templates/presubmit.yaml.jinja +++ b/config/jobs/kubernetes/kops/templates/presubmit.yaml.jinja @@ -70,6 +70,10 @@ {%- if template_path %} --template-path={{template_path}} \ {%- endif %} + {%- for flag in instance_groups_overrides %} + --control-plane-instance-group-overrides={{flag}} \ + --node-instance-group-overrides={{flag}} \ + {%- endfor %} --test=kops \ -- \ {%- if test_args %}