diff --git a/deploy/Pulumi.gcpStaging.yaml b/deploy/Pulumi.gcpStaging.yaml index 9bd7e1e36..96e65c1c7 100644 --- a/deploy/Pulumi.gcpStaging.yaml +++ b/deploy/Pulumi.gcpStaging.yaml @@ -1,7 +1,6 @@ config: - mcp-registry:gatewayStagingHandoff: gateway + mcp-registry:gatewayStagingStandalone: "true" mcp-registry:gatewayStagingPublicIP: 35.222.36.75 - mcp-registry:gatewayCanary: "true" mcp-registry:securityPatch202611: "true" mcp-registry:publicReadCompression: "true" mcp-registry:publicReadLogSampling: "true" diff --git a/deploy/README.md b/deploy/README.md index 2b0ac923c..bf11b1818 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1143,30 +1143,20 @@ The patched releases address [CloudNativePG GHSA-x8c2-3p4r-v9r6](https://github. [ingress-nginx CVE-2026-4342](https://discuss.kubernetes.io/t/security-advisory-cve-2026-4342-ingress-nginx-comment-based-nginx-configuration-injection/34349), and [Grafana CVE-2026-27880](https://grafana.com/security/security-advisories/cve-2026-27880/). -#### ingress-nginx replacement plan - -1. Evaluate a maintained Gateway API implementation in staging (start with Envoy - Gateway), on a separate load balancer and test hostname. Keep existing traffic - on ingress-nginx during qualification. Select the release against the cluster's - Kubernetes version and documented support policy. -2. Map all existing ingress behavior: ACME/cert-manager certificates, HTTPS - redirects, real client IP and rate limits, OAuth callbacks, health checks, - body/time limits, metrics, HPA and node placement. NGINX snippets are not portable: - explicitly implement public listing cache keys/TTL, public versus authenticated route isolation, invalidation behavior, gzip negotiation and `Vary` handling (including credential/cookie compression bypass); confirm that - publishing and other authenticated routes are never cached. -3. Run the existing traffic tests against both controllers, including publishing - in staging, cache isolation, burst/rollout capacity and certificate renewal. - Compare error rates, tail latency, cache hits, CPU and memory under equivalent load. -4. After parity is demonstrated, use a separate reviewed production rollout with - a low DNS TTL and a reversible traffic cutover. Retain the old load balancer - during observation; only then remove its controller, admission webhook and RBAC. - +#### ingress-nginx replacement + +Staging uses OSS NGINX Gateway Fabric with Gateway API, Vector, cert-manager, +VictoriaMetrics and Grafana. Its public Service/IP and trusted certificates are +preserved; ingress-nginx and the duplicate canary are retired. The existing +Gateway nodes are reused and the normal release flow is unchanged. + +See [standalone staging qualification](gateway-canary/standalone.md) for feature, +load, recovery, certificate and monitoring gates. There is no running legacy +fallback in staging; recovery uses a reviewed Git revision and full-stack saved +plan. The historical [canary comparison](gateway-canary/validation.md) and +[handoff](gateway-canary/handoff-validation.md) remain as evidence of earlier phases. + +Production still uses the patched ingress-nginx release. Its migration requires +a separate reviewed plan after final staging qualification and observation, +including production capacity, real alert delivery and an explicit recovery path. The interim patch does not restore upstream maintenance for the retired project. -The controller migration remains a separate rollout, not part of this version bump. - -### Parallel Gateway Fabric canary - -The staging-only [Gateway Fabric canary](gateway-canary/README.md) runs alongside -ingress-nginx using an explicit test IP. It preserves the normal release flow -and does not change production or public DNS. See its qualification record -before considering a traffic cutover. diff --git a/deploy/gateway-canary/README.md b/deploy/gateway-canary/README.md index a2b6a5afe..7173f1063 100644 --- a/deploy/gateway-canary/README.md +++ b/deploy/gateway-canary/README.md @@ -1,9 +1,14 @@ -# Staging Gateway Fabric canary +# Staging Gateway Fabric -This opt-in Pulumi deployment runs alongside ingress-nginx. `gatewayCanary` is -rejected outside GCP staging, and the cloud resources also require project -`mcp-registry-staging`. Neither release workflow nor production configuration is -changed. Normal staging pushes continue to deploy the current main image. +Staging now selects `gatewayStagingStandalone: "true"`. Its public hostnames use +OSS Gateway Fabric, with no running ingress-nginx or duplicate canary. See +[standalone.md](standalone.md) for the final topology, qualification gates and +Git-based recovery procedure. Production retains its existing ingress. Normal +staging pushes continue to deploy the current main image. + +The `gateway-canary` paths, metric prefixes and node-pool names are retained to +preserve monitoring history and infrastructure identity. Earlier comparison and +handoff reports below describe historical configurations, not the current one. The canary uses OSS NGF 2.8.1 / Gateway API 1.6.3 / Vector 0.61.0. Controller, data-plane, Vector and fixture images are pinned by digest. NGINX Plus and product @@ -14,39 +19,24 @@ current data sources while mappings are qualified. The full event stream is processed locally; GCP retains 10% of fast successful listing logs after metric extraction. -## Isolation and cost - -A separate tainted pool starts with two e2-standard-2 workers, autoscaling at most -two (node upgrades can temporarily surge). Two NGINX replicas can scale to -four. Each eligible node runs one single-threaded Vector collector requesting -500m CPU; proxy replicas request 500m. The controller requests 100m; the temporary fixture requests 100m when enabled. These are canary starting budgets, not approved production -sizing. The original staging worker/database pool, ingress, IP and storage stay -in place. The canary adds node/disks, one regional external IP and one L4 load -balancer while enabled. Account for this overlap when interpreting cost. +## Capacity and temporary fixtures -## No DNS changes +The existing two e2-standard-2 Gateway nodes remain capped at two; no additional +nodes are required by this migration. Two proxy replicas can scale to four. +Each Gateway node runs one single-threaded Vector collector requesting 500m CPU; +each proxy requests 500m and the controller requests 100m. The worker and database +pools remain unchanged. Removing the duplicate canary releases its separate IP +and L4 load balancer. Node upgrades may temporarily surge under the pool's existing +upgrade policy; that is separate from workload autoscaling. -DNS access is unavailable for this experiment. Use the exported `gatewayCanaryIP` -with the existing hostnames. For example: +The original public IP and Service are preserved. Public certificates are managed +by Pulumi/cert-manager with native Gateway HTTP01 renewal. Internal server/client +TLS certificates are also managed by cert-manager; the old bootstrap Job is gone. -```sh -curl --resolve staging.registry.modelcontextprotocol.io:443:CANARY_IP \ - https://staging.registry.modelcontextprotocol.io/v0.1/health -python3.13 gateway-canary/check.py --ip CANARY_IP --fixture-ca /tmp/fixture.crt \ - --output /tmp/canary-contract.json -``` - -TLS references the existing staging and Grafana Secrets through narrowly named -ReferenceGrants. The original cert-manager Ingress solver continues renewing -these certificates. This canary **does not validate independent Gateway HTTP01 -renewal or an end-to-end browser OAuth callback through the new IP**. Those remain -cutover gates; do not retire ingress-nginx or its solver based on these results. - -`gatewayCanaryFixtures: "true"` temporarily enables a fixture and load generator. -`gatewayCanaryComparison: "true"` additionally enables a private ingress-nginx -baseline and caps the gateway at two replicas for a matched comparison. Neither -flag is enabled in the retained canary. Turning them off removes the unthrottled -fixture listener, workload, test certificate, generator and comparison resources. +`gatewayCanaryFixtures: "true"` temporarily enables an isolated backend and load +generator on the existing worker, with a test listener on the public Gateway. +Remove this flag after qualification. `gatewayCanaryComparison` is rejected in +standalone mode; historical ingress comparisons are recorded in the reports. `fixture.gateway-canary.invalid` is a reserved test hostname, without DNS. The fixture has its own cert-manager self-signed test certificate. Supply its public @@ -60,14 +50,10 @@ the unthrottled fixture route as the application route. ## Resource ownership `pkg/k8s/gatewaycanary/assets/controller.json` is the reviewed upstream Helm -render using `ngf-values.json`, including the certificate-generation hook. -Pulumi owns individual resources so saved plans contain concrete identities; -there is no custom NGINX template or controller fork. The certificate job stays -in state after completion rather than generating perpetual TTL replacements. -This is the upstream bootstrap path for inter-plane TLS. Before production, -manage these internal certificates through the existing cert-manager installation, -as recommended by the [upstream certificate guide](https://docs.nginx.com/nginx-gateway-fabric/install/secure-certificates/). -That is separate from public listener certificate renewal. +render using `ngf-values.json`. Standalone deployment replaces its bootstrap +certificate hook with cert-manager resources. Pulumi owns concrete resources; +there is no custom NGINX template or controller fork. Public Certificates are +protected, have no legacy Ingress owner, and retain their existing trusted Secrets. `crds.json` contains Gateway API's standard installation plus the pinned NGF chart CRDs. `routes.json` is the canary's native Gateway/NGF policy configuration. @@ -85,20 +71,13 @@ NGF and Gateway API are Apache-2.0; NGINX OSS is BSD-2-Clause and Vector is MPL- These core licenses do not constitute a complete transitive image/SBOM audit. `assets-provenance.json` records the chart digest and vendored asset hashes. -## Rollback and release flow - -The normal staging hostname continues to reach ingress-nginx throughout. To stop -using the canary, stop the explicit-IP probes; no DNS rollback is needed. To remove -its infrastructure, set `gatewayCanary: "false"`, preview and apply through the -normal staging workflow. Review deletion of canary CRDs carefully if anything -else begins using Gateway API; they are cluster-scoped. Do not remove shared CRDs -while another controller relies on them. Original application, ingress, databases -and volumes must remain in the removal preview. +## Recovery and release flow -No image pin, release-wrapper or workflow dispatch mode is introduced. For local -qualification only, `REGISTRY_STAGING_IMAGE` can select the already built release -digest without persisting a staging configuration pin. A later normal main push -resumes normal image selection. +There is no live legacy fallback. Use the reviewed Git recovery procedure in +[standalone.md](standalone.md), preserving the protected public IP and certificates. +No image pin or workflow dispatch mode is introduced. A normal main push builds +and deploys its exact image digest. Temporary local qualification may use +`REGISTRY_STAGING_IMAGE` without persisting a staging pin. ## Monitoring follow-up @@ -118,46 +97,47 @@ must each have separate evidence or be marked pending. ## Repeating qualification -Use only the staging context. Enable `gatewayCanaryComparison` for matched tests, -preview/apply, and wait for both two-replica controllers to become ready. Build -`load.go` for Linux amd64 and copy it to `/work/loadgen` in `load-generator`. -Export **only** `fixture-tls`'s `tls.crt` field to a local file, and place that -public certificate at `/work/fixture.crt` in the generator. Do not export its key. -The generator validates this certificate and the fixture hostname. +Use only the staging context. Temporarily enable `gatewayCanaryFixtures`, build +`deploy/infra`, preview the full stack and apply its saved plan. Build `load.go` +for Linux amd64 and copy it to `/work/loadgen` in the `registry-gateway-canary` +namespace's `load-generator` Pod. Export **only** `ingress-nginx/fixture-tls`'s +`tls.crt` to a local file and `/work/fixture.crt` in the generator. -From `deploy/`, run each arm sequentially with identical arguments: +From `deploy/`, run scenarios sequentially: ```sh -python3 gateway-canary/measure.py --arm baseline --rate 1000 --seconds 120 \ - --epoch matched-run --output /tmp/baseline-result +export REGISTRY_GATEWAY_TARGET=public +python3.13 gateway-canary/check.py --ip 35.222.36.75 --fixture-ca /tmp/fixture.crt \ + --output /tmp/gateway-contract.json python3 gateway-canary/measure.py --arm ngf --rate 1000 --seconds 120 \ - --epoch matched-run --output /tmp/gateway-result + --epoch qualification --output /tmp/gateway-load +python3 gateway-canary/autoscale.py --output /tmp/gateway-autoscale +python3 gateway-canary/resilience.py --scenario proxy --output /tmp/gateway-proxy ``` -Use `--miss-percent 20 --rate 250` for the mixed test. Do not run other fixture -probes during measurement. The driver discovers the internal Service addresses; -GKE's Service range is outside RFC1918, so it explicitly permits those discovered -addresses while retaining TLS verification. Results include offered/achieved -load, errors, cache outcomes, CPU/memory samples and metric reconciliation. -This fixed payload is a controlled comparison, not a production traffic replay. - -`limits.py` runs 1,920 bounded real-canary health requests to verify source-IP -limiting; give it `--ip`, `--fixture-ca` and `--output`. `rollout.py --output DIR` -replaces one canary proxy during a fixture load. `collector_restart.py --output -FILE` checks a **quiescent** same-node collector restart, then rerun a short -`measure.py` check. Collector counters reset on restart; persisted file positions -prevent historical replay but do not guarantee exactly-once delivery across a -crash. Raw counter totals from deleted exporter instances must not be added to -current exporter totals; use rates/increases for time-series dashboards. - -For autoscaling, disable `gatewayCanaryComparison` and enable only -`gatewayCanaryFixtures`; this removes the temporary baseline and restores the -2–4 replica HPA while retaining the fixture. Wait for the cooldown to return to -two replicas, then run `python3 gateway-canary/autoscale.py --output DIR`. This -uses 256 load workers, 1,000 requests/second and a four-minute window, recording -HPA, readiness, node-count and metric evidence. Afterwards remove both test flags, -preview/apply, and delete the orphaned self-signed `fixture-tls` Secret if -cert-manager's certificate owner-reference option is disabled. The live staging -and Grafana certificate Secrets must remain untouched. - -Public staging handoff and rollback: [handoff.md](handoff.md). +Use `--miss-percent 20 --rate 250` for mixed traffic. Do not run other fixture +probes during measurement. The driver discovers the public Gateway's internal +Service IP and retains hostname/TLS verification. This measures the active data +plane with a controlled fixture, not a production traffic replay or internet +capacity benchmark. Public HTTPS probes run separately during every scenario. + +`limits.py` verifies real source-IP limiting; run it with two proxy replicas. +`resilience.py` also supports controller, rollout and node scenarios. Rollout +tests default to ten minutes to cover both the update and restoration with +graceful draining. The node scenario defaults to five minutes; it drains one +Gateway node, verifies at least a minute of overlap with the generator’s actual +traffic interval, and always attempts to uncordon it. The rollout +scenario patches the managed NginxProxy temporarily, then restores it. + +`collector_restart.py` checks a quiescent same-node restart, followed by a new +load measurement. Checkpoints prevent historical replay, but there is no +exactly-once crash guarantee. Deleted exporter counters must not be added to +live exporter totals; dashboards use rates/increases. + +Remove the fixture flag, preview/apply, then remove the orphaned test TLS Secret +only after confirming its Certificate and test listener are gone. Preserve both +public trusted certificates. Validate the final full-stack preview and normal +GitOps deployment before beginning the final observation period. + +Historical reports: [comparison](validation.md), [monitoring](monitoring-validation.md), +[public handoff](handoff.md). diff --git a/deploy/gateway-canary/autoscale.py b/deploy/gateway-canary/autoscale.py index afd385a8e..26580bb86 100644 --- a/deploy/gateway-canary/autoscale.py +++ b/deploy/gateway-canary/autoscale.py @@ -2,17 +2,19 @@ """Exercise the staging canary HPA from two to four replicas under fixture load.""" import argparse, json, subprocess, time from pathlib import Path +from measure import gateway_target a = argparse.ArgumentParser() a.add_argument('--output', type=Path, required=True) args = a.parse_args() args.output.mkdir(parents=True, exist_ok=True) K = ['kubectl', '--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging'] -NS = 'registry-gateway-canary' -NAME = 'registry-canary-registry-gateway-canary' +NS, NAME = gateway_target() def get(*args): return json.loads(subprocess.check_output(K + list(args) + ['-o', 'json'])) h = get('-n', NS, 'get', 'hpa', NAME) +node_names = sorted(n['metadata']['name'] for n in get('get', 'nodes')['items']) +assert len(node_names) == 4, 'Qualification requires the original four-node cluster' assert h['spec']['maxReplicas'] == 4 assert h['status'].get('currentReplicas') == 2, 'wait for the cooldown to return to two replicas' assert not any((p['metadata']['name'].startswith('registry-canary-baseline-') for p in get('-n', NS, 'get', 'pods')['items'])) @@ -23,10 +25,12 @@ def get(*args): h = get('-n', NS, 'get', 'hpa', NAME) d = get('-n', NS, 'get', 'deployment', NAME) nodes = get('get', 'nodes', '-l', 'cloud.google.com/gke-nodepool=registry-gateway-canary')['items'] - row = {'at': time.time(), 'hpa': h['status'], 'deployment': d['status'], 'nodes': len(nodes)} + cluster_nodes = sorted(n['metadata']['name'] for n in get('get', 'nodes')['items']) + row = {'cluster_nodes': cluster_nodes, 'at': time.time(), 'hpa': h['status'], 'deployment': d['status'], 'nodes': len(nodes)} rows.append(row) print(json.dumps({'current': h['status'].get('currentReplicas'), 'desired': h['status'].get('desiredReplicas'), 'ready': d['status'].get('readyReplicas'), 'nodes': len(nodes)}), flush=True) time.sleep(10) (args.output / 'operation.json').write_text(json.dumps({'exit': p.returncode, 'samples': rows}, indent=2) + '\n') + assert all(r['cluster_nodes'] == node_names for r in rows), 'Cluster nodes changed during scaling qualification' assert p.returncode == 0, p.returncode assert max((r['deployment'].get('readyReplicas', 0) for r in rows)) >= 4, 'did not reach four ready replicas' diff --git a/deploy/gateway-canary/certificate_rotation.py b/deploy/gateway-canary/certificate_rotation.py new file mode 100644 index 000000000..e09aa1220 --- /dev/null +++ b/deploy/gateway-canary/certificate_rotation.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Force renewal and verify Gateway mTLS credentials propagate, without exposing keys.""" +import argparse +import base64 +import hashlib +import json +import subprocess +import ssl +import time +from pathlib import Path +from measure import K, NS, get +from monitoring_staging import IMAGE + + +def run(*args): + return subprocess.run(K + list(args), check=True, capture_output=True, text=True, timeout=60) + + +def fingerprint(namespace, name): + public = run('-n', namespace, 'get', 'secret', name, '-o', 'jsonpath={.data.tls\\.crt}').stdout + pem = base64.b64decode(public).decode().split('-----END CERTIFICATE-----')[0] + '-----END CERTIFICATE-----' + return hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() + + +def controller_processes(controller): + selector = ','.join(k + '=' + v for k, v in controller['spec']['selector']['matchLabels'].items()) + pods = get('-n', NS, 'get', 'pods', '-l', selector)['items'] + assert len(pods) == controller['spec']['replicas'], 'Controller is rolling; wait before renewal' + result = {} + for pod in pods: + assert not pod['metadata'].get('deletionTimestamp'), 'Controller is terminating' + statuses = pod['status'].get('containerStatuses', []) + assert statuses and all(c.get('ready') for c in statuses), 'Controller is not ready' + result[pod['metadata']['uid']] = { + c['name']: {'restart_count': c['restartCount'], 'container_id': c['containerID']} + for c in statuses} + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cmctl', required=True) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args() + controller = get('-n', NS, 'get', 'deployment', 'registry-gateway-canary') + generation = controller['metadata']['generation'] + processes_before = controller_processes(controller) + started = time.time() + report = {'started': started, 'renewals': []} + for name in ['gateway-server-tls', 'gateway-agent-tls']: + cert = get('-n', NS, 'get', 'certificate', name) + previous_revision = cert['status']['revision'] + before = fingerprint(NS, name) + subprocess.run([args.cmctl, 'renew', name, '-n', NS, '--context', K[1].split('=', 1)[1]], capture_output=True, check=True, timeout=30) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + cert = get('-n', NS, 'get', 'certificate', name) + conditions = {c['type']: c['status'] for c in cert.get('status', {}).get('conditions', [])} + if cert.get('status', {}).get('revision', 0) > previous_revision and conditions.get('Ready') == 'True' and conditions.get('Issuing') != 'True': + break + time.sleep(2) + else: + raise RuntimeError(name + ' renewal timeout') + after = fingerprint(NS, name) + assert before != after + report['renewals'].append({'name': name, 'revision': cert['status']['revision'], 'certificate_changed': True}) + print(name + ' renewed', flush=True) + # The controller duplicates the agent Secret into each Gateway namespace. + deadline = time.monotonic() + 180 + deployment = get('-n', 'ingress-nginx', 'get', 'deployment', 'registry-edge-registry-gateway-canary') + copied_secret = next(v['secret']['secretName'] for v in deployment['spec']['template']['spec']['volumes'] if v['name'] == 'nginx-agent-tls') + assert copied_secret.endswith('-gateway-agent-tls') + while fingerprint('ingress-nginx', copied_secret) != fingerprint(NS, 'gateway-agent-tls'): + assert time.monotonic() < deadline, 'Agent Secret did not propagate' + time.sleep(3) + assert get('-n', NS, 'get', 'deployment', 'registry-gateway-canary')['metadata']['generation'] == generation + # A fresh mTLS handshake verifies the running server loaded the renewed leaf + # and accepts the renewed client certificate. Credentials stay mounted in-cluster. + probe = 'gateway-certificate-rotation-proof' + expected = fingerprint(NS, 'gateway-server-tls') + service = get('-n', NS, 'get', 'service', 'registry-gateway-canary') + port = next(p['port'] for p in service['spec']['ports'] if p['name'] == 'agent-grpc') + script = f"""import ssl,socket,hashlib,json,time +c=ssl.create_default_context(cafile="/tls/ca.crt") +c.minimum_version=ssl.TLSVersion.TLSv1_3 +c.load_cert_chain("/tls/tls.crt","/tls/tls.key") +c.set_alpn_protocols(["h2"]) +for attempt in range(60): + with c.wrap_socket(socket.create_connection(("registry-gateway-canary",{port}),timeout=15),server_hostname="registry-gateway-canary.registry-gateway-canary.svc") as s: + s.sendall(bytes([80,82,73,32,42,32,72,84,84,80,47,50,46,48,13,10,13,10,83,77,13,10,13,10,0,0,0,4,0,0,0,0,0])) + frame=b"" + while len(frame)<9: + chunk=s.recv(9-len(frame)) + if not chunk: raise RuntimeError("EOF before HTTP/2 frame header") + frame+=chunk + assert frame[3]==4, "No authenticated HTTP/2 settings frame" + result={{"authenticated_http2":True,"server_certificate_sha256":hashlib.sha256(s.getpeercert(binary_form=True)).hexdigest(),"protocol":s.version()}} + if result["server_certificate_sha256"]=="{expected}": + print(json.dumps(result));break + time.sleep(3) +else: raise RuntimeError("Renewed server certificate did not reach the running controller") +""" + pod = {'apiVersion':'v1','kind':'Pod','metadata':{'name':probe,'namespace':NS},'spec':{ + 'restartPolicy':'Never','automountServiceAccountToken':False, + 'containers':[{'name':'probe','image':IMAGE,'command':['python3','-c',script], + 'resources':{'requests':{'cpu':'10m','memory':'24Mi'},'limits':{'cpu':'100m','memory':'64Mi'}}, + 'volumeMounts':[{'name':'tls','mountPath':'/tls','readOnly':True}], + 'securityContext':{'allowPrivilegeEscalation':False,'readOnlyRootFilesystem':True,'capabilities':{'drop':['ALL']}}}], + 'volumes':[{'name':'tls','secret':{'secretName':'gateway-agent-tls'}}]}} + subprocess.run(K+['create','-f','-'],input=json.dumps(pod),capture_output=True,text=True,check=True,timeout=30) + try: + deadline = time.monotonic() + 240 + while True: + phase = get('-n', NS, 'get', 'pod', probe)['status']['phase'] + if phase == 'Succeeded': + break + if phase == 'Failed': + raise RuntimeError(run('-n', NS, 'logs', probe).stdout[-1500:]) + assert time.monotonic() < deadline, 'mTLS probe timeout' + time.sleep(3) + handshake=json.loads(run('-n',NS,'logs',probe).stdout) + assert handshake['server_certificate_sha256']==fingerprint(NS,'gateway-server-tls') + report['fresh_mtls_handshake']=handshake + finally: + run('-n',NS,'delete','pod',probe,'--wait=false') + current = get('-n', NS, 'get', 'deployment', 'registry-gateway-canary') + assert current['metadata']['generation'] == generation, 'Controller deployment changed during renewal' + assert controller_processes(current) == processes_before, 'Controller Pod or container restarted during renewal' + report['controller_processes_unchanged'] = True + report['controller_deployment_unchanged'] = True + report['agent_secret_propagated'] = True + report['finished'] = time.time() + args.output.write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/deploy/gateway-canary/collector_restart.py b/deploy/gateway-canary/collector_restart.py index d8f448883..2a2f95df2 100644 --- a/deploy/gateway-canary/collector_restart.py +++ b/deploy/gateway-canary/collector_restart.py @@ -7,7 +7,7 @@ a.add_argument('--output', type=Path, required=True) args = a.parse_args() ps = measure.pods() -proxy_nodes = {p['spec'].get('nodeName') for p in ps if p['metadata']['name'].startswith('registry-canary-registry-gateway-canary-')} +proxy_nodes = {p['spec'].get('nodeName') for p in ps if p['metadata']['name'].startswith(measure.gateway_target()[1] + '-')} old = next((p for p in ps if p['metadata']['name'].startswith('vector-') and p['spec'].get('nodeName') in proxy_nodes)) node = old['spec']['nodeName'] started = time.time() diff --git a/deploy/gateway-canary/detach_certificates.py b/deploy/gateway-canary/detach_certificates.py new file mode 100644 index 000000000..6bd240008 --- /dev/null +++ b/deploy/gateway-canary/detach_certificates.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Detach staging public Certificates after adoption and shim removal, before retirement. + +Uses the staging Pulumi passphrase from the environment; never prints stack state +or Secret values. UID/owner tests refuse concurrent changes. Safe to rerun after +retirement when the certificates already have no owners. +""" +import argparse,json,os,subprocess +from pathlib import Path +K=['kubectl','--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging','--request-timeout=30s'] +def run(*a):return subprocess.run(K+list(a),capture_output=True,text=True,check=True,timeout=60) +def get(*a):return json.loads(run(*a,'-o','json').stdout) +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output',type=Path,required=True) +args=parser.parse_args() +env=dict(os.environ,PULUMI_BACKEND_URL='gs://mcp-registry-staging-pulumi-state') +r=subprocess.run(['pulumi','stack','export','--stack','gcpStaging','--non-interactive'],capture_output=True,text=True,check=True,timeout=120,env=env) +resources=json.loads(r.stdout)['deployment']['resources'];del r +report=[] +for ns,name,ingress in [('default','mcp-registry-staging-tls','mcp-registry'),('monitoring','grafana-tls','grafana-ingress')]: + resource=next(x for x in resources if x['type']=='kubernetes:cert-manager.io/v1:Certificate' and x.get('id')==ns+'/'+name) + assert resource.get('protect') is True and resource['urn'].startswith('urn:pulumi:gcpStaging::') + assert resource['inputs']['metadata'].get('ownerReferences')==[] + cert=get('-n',ns,'get','certificate',name); uid=cert['metadata']['uid'];owners=cert['metadata'].get('ownerReferences',[]) + before=run('-n',ns,'get','secret',name,'-o','jsonpath={.data.tls\\.crt}').stdout + if owners: + old=get('-n',ns,'get','ingress',ingress) + assert 'cert-manager.io/cluster-issuer' not in old['metadata'].get('annotations',{}) + assert 'cert-manager.io/issuer' not in old['metadata'].get('annotations',{}) + assert len(owners)==1 and owners[0]['kind']=='Ingress' and owners[0]['uid']==old['metadata']['uid'] and owners[0]['name']==ingress + patch=[{'op':'test','path':'/metadata/uid','value':uid},{'op':'test','path':'/metadata/ownerReferences','value':owners},{'op':'remove','path':'/metadata/ownerReferences'}] + run('-n',ns,'patch','certificate',name,'--type=json','-p',json.dumps(patch)) + cert=get('-n',ns,'get','certificate',name) + assert cert['metadata']['uid']==uid and not cert['metadata'].get('ownerReferences') + assert run('-n',ns,'get','secret',name,'-o','jsonpath={.data.tls\\.crt}').stdout==before + report.append({'namespace':ns,'certificate':name,'pulumi_protected':True,'owner_detached':True,'certificate_unchanged':True}) +args.output.write_text(json.dumps(report,indent=2)+'\n') +print(json.dumps(report,indent=2)) diff --git a/deploy/gateway-canary/handoff.md b/deploy/gateway-canary/handoff.md index 8e45831f5..b877585a6 100644 --- a/deploy/gateway-canary/handoff.md +++ b/deploy/gateway-canary/handoff.md @@ -1,5 +1,9 @@ # Staging public-IP handoff +> Historical transition procedure. Staging now uses [standalone mode](standalone.md); +> the retained legacy controller and duplicate canary described below are gone. +> Recovery requires a reviewed redeployment from Git history. + The staging handoff is opt-in and restricted to `mcp-registry-staging`. Production and the image/release workflows are unchanged. `gatewayStagingHandoff: legacy` prepares the new data plane but leaves public traffic on ingress-nginx. The diff --git a/deploy/gateway-canary/measure.py b/deploy/gateway-canary/measure.py index 098fc7bae..cbfd9153d 100644 --- a/deploy/gateway-canary/measure.py +++ b/deploy/gateway-canary/measure.py @@ -1,10 +1,20 @@ #!/usr/bin/env python3 """Bounded matched fixture load. K8s reads plus exec in the dedicated generator.""" -import argparse, concurrent.futures, json, re, subprocess, threading, time, urllib.parse +import argparse, concurrent.futures, json, os, re, subprocess, threading, time, urllib.parse from pathlib import Path K = ['kubectl', '--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging', '--request-timeout=30s'] NS = 'registry-gateway-canary' + +def gateway_target(): + target = os.environ.get('REGISTRY_GATEWAY_TARGET', 'canary') + if target == 'public': + return 'ingress-nginx', 'registry-edge-registry-gateway-canary' + if target == 'canary': + return NS, 'registry-canary-registry-gateway-canary' + raise ValueError('REGISTRY_GATEWAY_TARGET must be public or canary') + + def get(*a): return json.loads(subprocess.check_output(K + list(a) + ['-o', 'json'])) @@ -12,7 +22,7 @@ def raw(path): return subprocess.check_output(K + ['get', '--raw', path], text=True) def pods(): - return get('-n', NS, 'get', 'pods')['items'] + return [p for ns in sorted({NS, gateway_target()[0]}) for p in get('-n', ns, 'get', 'pods')['items']] def vector(): out = {} @@ -34,7 +44,7 @@ def vector(): def sample(node): d = json.loads(raw('/api/v1/nodes/' + node + '/proxy/stats/summary')) - return {'node': node, 'cpu': d['node']['cpu'], 'memory': d['node']['memory'], 'pods': [{'name': p['podRef']['name'], 'containers': [{k: c[k] for k in ['name', 'cpu', 'memory']} for c in p['containers']]} for p in d['pods'] if p['podRef']['namespace'] == NS]} + return {'node': node, 'cpu': d['node']['cpu'], 'memory': d['node']['memory'], 'pods': [{'name': p['podRef']['name'], 'containers': [{k: c[k] for k in ['name', 'cpu', 'memory']} for c in p['containers']]} for p in d['pods'] if p['podRef']['namespace'] in {NS, gateway_target()[0]}]} def stamp(t): from datetime import datetime @@ -66,8 +76,8 @@ def main(): p.add_argument('--output', type=Path, required=True) a = p.parse_args() a.output.mkdir(parents=True, exist_ok=True) - service = 'registry-canary-baseline-controller' if a.arm == 'baseline' else 'registry-canary-registry-gateway-canary' - ip = get('-n', NS, 'get', 'svc', service)['spec']['clusterIP'] + gateway_namespace, service = (NS, 'registry-canary-baseline-controller') if a.arm == 'baseline' else gateway_target() + ip = get('-n', gateway_namespace, 'get', 'svc', service)['spec']['clusterIP'] inventory = pods() nodes = sorted({x['spec']['nodeName'] for x in inventory if x.get('spec', {}).get('nodeName')}) before = vector() @@ -126,10 +136,12 @@ def collect(): if len(pairs) > 1: (first, last) = (pairs[0], pairs[-1]) resources[name] = {'cpu_cores': (last[1]['cpu']['usageCoreNanoSeconds'] - first[1]['cpu']['usageCoreNanoSeconds']) / 1000000000.0 / (last[0] - first[0]), 'peak_working_set_bytes': max((x['memory'].get('workingSetBytes', 0) for x in points.values())), 'sample_seconds': last[0] - first[0]} - report = {'arm': a.arm, 'rate': a.rate, 'seconds': a.seconds, 'miss_percent': a.miss_percent, 'started_at': start, 'finished_at': end, 'client': client, 'metric_delta': delta, 'collection_observed_drain_seconds': drain, 'vm_client_count': vm, 'origin_reads': sum(o1.values()) - sum(o0.values()), 'resources': resources, 'placement': [{'pod': x['metadata']['name'], 'node': x['spec'].get('nodeName')} for x in inventory]} + report = {'gateway_namespace': gateway_namespace, 'gateway_service': service, 'arm': a.arm, 'rate': a.rate, 'seconds': a.seconds, 'miss_percent': a.miss_percent, 'started_at': start, 'finished_at': end, 'client': client, 'metric_delta': delta, 'collection_observed_drain_seconds': drain, 'vm_client_count': vm, 'origin_reads': sum(o1.values()) - sum(o0.values()), 'resources': resources, 'placement': [{'pod': x['metadata']['name'], 'node': x['spec'].get('nodeName')} for x in inventory]} (a.output / 'samples.json').write_text(json.dumps(samples)) (a.output / 'result.json').write_text(json.dumps(report, indent=2)) print(json.dumps(report, indent=2)) + assert client['p95_ms'] < 100, 'p95 latency exceeds the qualification gate' + assert client['p99_ms'] < 250, 'p99 latency exceeds the qualification gate' assert client['generator_dropped'] == 0, 'generator could not sustain offered rate' assert not client['errors'], client['errors'] assert client['statuses'] == {'200': client['offered']}, client['statuses'] diff --git a/deploy/gateway-canary/native_acme.py b/deploy/gateway-canary/native_acme.py new file mode 100644 index 000000000..1ad650ca0 --- /dev/null +++ b/deploy/gateway-canary/native_acme.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Prove native HTTP01 issuance/renewal without replacing public certificates.""" +import argparse +import hashlib +import json +import subprocess +import time +from pathlib import Path + +K = ['kubectl', '--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging', '--request-timeout=30s'] + + +def run(*args, **kw): + return subprocess.run(K + list(args), capture_output=True, text=True, check=True, timeout=60, **kw) + + +def get(*args): + return json.loads(run(*args, '-o', 'json').stdout) + + +def fingerprints(): + return {ns: hashlib.sha256(run('-n', ns, 'get', 'secret', name, '-o', 'jsonpath={.data.tls\\.crt}').stdout.encode()).hexdigest() + for ns, name in [('default', 'mcp-registry-staging-tls'), ('monitoring', 'grafana-tls')]} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cmctl', required=True) + parser.add_argument('--output', required=True) + args = parser.parse_args() + issuer = get('get', 'clusterissuer', 'letsencrypt-prod') + solver = issuer['spec']['acme']['solvers'] + assert len(solver) == 1 and 'gatewayHTTPRoute' in solver[0]['http01'] + parent = solver[0]['http01']['gatewayHTTPRoute']['parentRefs'][0] + assert all(parent.get(k) == v for k, v in {'kind': 'Gateway', 'name': 'registry-edge', 'namespace': 'ingress-nginx', 'sectionName': 'http'}.items()) + before = fingerprints() + report = {'native_gateway_solver': True, 'proofs': []} + name = 'gateway-native-proof-' + str(int(time.time())) + for ns, host in [('default', 'staging.registry.modelcontextprotocol.io'), ('monitoring', 'grafana.staging.registry.modelcontextprotocol.io')]: + created = [] + proof = {'namespace': ns, 'host': host, 'observed_http_routes': []} + try: + for kind, spec in [ + ('Issuer', {'acme': {'server': 'https://acme-staging-v02.api.letsencrypt.org/directory', 'privateKeySecretRef': {'name': name + '-account'}, 'solvers': solver}}), + ('Certificate', {'secretName': name + '-tls', 'dnsNames': [host], 'issuerRef': {'name': name, 'kind': 'Issuer'}, 'privateKey': {'rotationPolicy': 'Always'}}), + ]: + run('create', '-f', '-', input=json.dumps({'apiVersion': 'cert-manager.io/v1', 'kind': kind, 'metadata': {'name': name, 'namespace': ns}, 'spec': spec})) + created.append(kind) + for revision in [1, 2]: + if revision == 2: + subprocess.run([args.cmctl, 'renew', name, '-n', ns, '--context', K[1].split('=', 1)[1]], capture_output=True, check=True, timeout=30) + deadline = time.monotonic() + 360 + while time.monotonic() < deadline: + for route in get('-n', ns, 'get', 'httproutes')['items']: + if not route['metadata']['name'].startswith('cm-acme-http-solver-'): + continue + if host not in route['spec'].get('hostnames', []): + continue + assert route['spec']['parentRefs'][0]['name'] == parent['name'] + route_name = route['metadata']['name'] + if route_name not in proof['observed_http_routes']: + proof['observed_http_routes'].append(route_name) + status = get('-n', ns, 'get', 'certificate', name).get('status', {}) + conditions = {c['type']: c['status'] for c in status.get('conditions', [])} + if status.get('revision', 0) >= revision and conditions.get('Ready') == 'True' and conditions.get('Issuing') != 'True': + proof['revision_' + str(revision)] = {k: status[k] for k in ['revision', 'notBefore', 'notAfter', 'renewalTime']} + print(f'{ns}: native ACME revision {revision} ready', flush=True) + break + time.sleep(2) + else: + raise RuntimeError('Native certificate issuance timed out in ' + ns) + assert proof['observed_http_routes'], 'No native solver route observed' + finally: + for kind in reversed(created): + run('-n', ns, 'delete', kind, name, '--ignore-not-found', '--wait=true') + run('-n', ns, 'delete', 'secret', name + '-tls', name + '-account', '--ignore-not-found') + report['proofs'].append(proof) + report['public_certificates_unchanged'] = fingerprints() == before + Path(args.output).write_text(json.dumps(report, indent=2) + '\n') + assert report['public_certificates_unchanged'] + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/deploy/gateway-canary/public_probe.py b/deploy/gateway-canary/public_probe.py new file mode 100644 index 000000000..72ae7d457 --- /dev/null +++ b/deploy/gateway-canary/public_probe.py @@ -0,0 +1,70 @@ +"""Public HTTPS recovery observations for the staging disruption checks.""" +import concurrent.futures +import json +import subprocess +import threading +import time + +URLS = [ + 'https://staging.registry.modelcontextprotocol.io/v0/health', + 'https://staging.registry.modelcontextprotocol.io/v0.1/servers?limit=1', + 'https://grafana.staging.registry.modelcontextprotocol.io/api/health', +] + + +class PublicProbe: + def __init__(self, output): + self.output = output + self.rows = [] + self.stop = threading.Event() + self.thread = threading.Thread(target=self.collect, daemon=True) + + @staticmethod + def check(url): + result = subprocess.run(['curl', '--silent', '--show-error', '--max-time', '8', + '-o', '/dev/null', '-w', '%{http_code}', url], + capture_output=True, text=True, timeout=10) + return {'url': url, 'ok': result.returncode == 0 and result.stdout == '200', + 'status': result.stdout, 'exit': result.returncode} + + def collect(self): + with concurrent.futures.ThreadPoolExecutor(max_workers=len(URLS)) as pool: + while not self.stop.is_set(): + started = time.time() + try: + checks = list(pool.map(self.check, URLS)) + except Exception as error: + checks = [{'url': url, 'ok': False, 'error': type(error).__name__} for url in URLS] + self.rows.append({'at': started, 'checks': checks}) + self.stop.wait(max(0, 2 - (time.time() - started))) + + def start(self): + self.thread.start() + + def finish(self, allowed_recovery_seconds=0): + self.stop.set() + self.thread.join(timeout=15) + assert not self.thread.is_alive(), 'Public probe did not stop' + self.output.write_text(json.dumps(self.rows, indent=2) + '\n') + failures = 0 + longest = 0 + recovered = bool(self.rows) + for url in URLS: + last_good = None + outage = None + for row in self.rows: + check = next(c for c in row['checks'] if c['url'] == url) + if check['ok']: + if outage is not None: + longest = max(longest, row['at'] - outage) + outage = None + last_good = row['at'] + else: + failures += 1 + if outage is None: + outage = last_good if last_good is not None else row['at'] + recovered = recovered and outage is None and last_good is not None + return {'probe_rounds': len(self.rows), 'failed_checks': failures, + 'maximum_recovery_seconds': longest, 'recovered': recovered, + 'allowed_recovery_seconds': allowed_recovery_seconds, + 'passed': recovered and (failures == 0 or (allowed_recovery_seconds > 0 and longest <= allowed_recovery_seconds))} diff --git a/deploy/gateway-canary/resilience.py b/deploy/gateway-canary/resilience.py new file mode 100644 index 000000000..84faea587 --- /dev/null +++ b/deploy/gateway-canary/resilience.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Bounded public-Gateway recovery tests; always restore cordons and policy patches.""" +import argparse +import copy +import json +import os +import subprocess +import time +from pathlib import Path +from datetime import datetime +from measure import K, NS, get, gateway_target +from public_probe import PublicProbe + + +def run(*args, timeout=240, input=None): + return subprocess.run(K + list(args), input=input, capture_output=True, text=True, check=True, timeout=timeout) + + +def wait_rollout(namespace, name): + run('-n', namespace, 'rollout', 'status', 'deployment/' + name, '--timeout=360s', timeout=370) + + +def apply_policy(current, spec, manager): + # Preserve the existing SSA owner and use resourceVersion as an optimistic + # concurrency guard. No force override is allowed in qualification runs. + metadata = {k: current['metadata'][k] for k in ['name', 'namespace', 'resourceVersion', 'labels', 'annotations', 'ownerReferences'] if k in current['metadata']} + manifest = {'apiVersion': current['apiVersion'], 'kind': current['kind'], 'metadata': metadata, 'spec': spec} + run('apply', '--server-side', '--field-manager=' + manager, '-f', '-', input=json.dumps(manifest)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--scenario', required=True, choices=['proxy', 'controller', 'rollout', 'node']) + parser.add_argument('--rate', type=int, default=500) + parser.add_argument('--seconds', type=int, help='Defaults: rollout 600, node 300, other scenarios 180') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if args.seconds is None: + args.seconds = {'rollout': 600, 'node': 300}.get(args.scenario, 180) + assert os.environ.get('REGISTRY_GATEWAY_TARGET') == 'public' + assert 0 < args.rate <= 1500 and 150 <= args.seconds <= 600 + args.output.mkdir(parents=True, exist_ok=True) + edge_ns, edge_name = gateway_target() + assert get('get', 'ingress', '-A')['items'] == [], 'Retire legacy Ingress objects before qualification' + assert len(get('get', 'gateways', '-A')['items']) == 1, 'Remove the duplicate canary first' + nodes = get('get', 'nodes')['items'] + node_names = sorted(n['metadata']['name'] for n in nodes) + assert len(node_names) == 4 + gateway_nodes = [n for n in nodes if n['metadata']['labels'].get('registry-role') == 'gateway-canary'] + assert len(gateway_nodes) == 2 and not any(n['spec'].get('unschedulable') for n in gateway_nodes) + selector = 'gateway.networking.k8s.io/gateway-name=registry-edge' + proxy_pods = get('-n', edge_ns, 'get', 'pods', '-l', selector)['items'] + assert len({p['spec']['nodeName'] for p in proxy_pods if not p['metadata'].get('deletionTimestamp')}) == 2 + report = {'scenario': args.scenario, 'nodes_before': node_names, 'started': time.time()} + cordoned = None + original_patches = None + modified_patches = None + policy_uid = None + policy_before = None + policy_manager = None + modified_spec = None + failure = None + public = PublicProbe(args.output / 'public-probes.json') + public.start() + load_log = (args.output / 'load.log').open('w') + load = subprocess.Popen(['python3', 'gateway-canary/measure.py', '--arm', 'ngf', '--rate', str(args.rate), + '--seconds', str(args.seconds), '--workers', '256', '--epoch', 'public-' + args.scenario + '-' + str(int(time.time())), + '--output', str(args.output / 'load')], stdout=load_log, stderr=subprocess.STDOUT) + try: + time.sleep(30) + assert load.poll() is None, 'Load finished before disruption' + report['action_started'] = time.time() + if args.scenario in ['proxy', 'controller']: + namespace, name = (edge_ns, edge_name) if args.scenario == 'proxy' else (NS, 'registry-gateway-canary') + deployment = get('-n', namespace, 'get', 'deployment', name) + match = ','.join(k + '=' + v for k, v in deployment['spec']['selector']['matchLabels'].items()) + pod = next(p for p in get('-n', namespace, 'get', 'pods', '-l', match)['items'] if not p['metadata'].get('deletionTimestamp')) + report['deleted_pod'] = pod['metadata']['name'] + run('-n', namespace, 'delete', 'pod', pod['metadata']['name'], '--wait=true', '--timeout=90s') + wait_rollout(namespace, name) + elif args.scenario == 'rollout': + policy = get('-n', edge_ns, 'get', 'nginxproxy', 'registry-edge', '--show-managed-fields=true') + policy_before = copy.deepcopy(policy) + managers = {m['manager'] for m in policy['metadata']['managedFields'] if m['manager'].startswith('pulumi-kubernetes-') and m['operation'] == 'Apply' and 'f:spec' in m.get('fieldsV1', {})} + assert len(managers) == 1, 'Expected one Pulumi owner of the Gateway policy' + policy_manager = managers.pop() + (args.output / 'policy-before.json').write_text(json.dumps(policy_before, indent=2) + '\n') + policy_uid = policy['metadata']['uid'] + original_patches = policy['spec']['kubernetes']['deployment']['patches'] + modified_patches = copy.deepcopy(original_patches) + value = str(int(time.time())) + modified_patches[0]['value']['spec']['template'].setdefault('metadata', {}).setdefault('annotations', {})['registry.modelcontextprotocol.io/qualification'] = value + modified_spec = copy.deepcopy(policy['spec']) + modified_spec['kubernetes']['deployment']['patches'] = modified_patches + apply_policy(policy, modified_spec, policy_manager) + deadline = time.monotonic() + 60 + while get('-n', edge_ns, 'get', 'deployment', edge_name)['spec']['template']['metadata'].get('annotations', {}).get('registry.modelcontextprotocol.io/qualification') != value: + assert time.monotonic() < deadline, 'Controller did not apply rollout policy' + time.sleep(2) + wait_rollout(edge_ns, edge_name) + else: + cordoned = gateway_nodes[0]['metadata']['name'] + report['drained_node'] = cordoned + run('cordon', cordoned) + run('drain', cordoned, '--ignore-daemonsets', '--delete-emptydir-data', '--timeout=360s', timeout=370) + report['drained_at'] = time.time() + deadline = time.monotonic() + 90 + while True: + ready = [p for p in get('-n', edge_ns, 'get', 'pods', '-l', selector)['items'] + if not p['metadata'].get('deletionTimestamp') and any(c['type'] == 'Ready' and c['status'] == 'True' for c in p.get('status', {}).get('conditions', []))] + if ready and all(p['spec']['nodeName'] != cordoned for p in ready): + break + assert time.monotonic() < deadline, 'No ready proxy on surviving node' + time.sleep(2) + assert load.poll() is None + time.sleep(60) + assert load.poll() is None, 'Load did not cover a minute of one-node operation' + report['one_node_load_seconds'] = time.time() - report['drained_at'] + assert load.poll() is None, 'Load ended before the disruption recovered' + report['action_ready'] = time.time() + except Exception as error: + failure = repr(error) + finally: + cleanup_errors = [] + if cordoned: + report['one_node_ended'] = time.time() + try: + run('uncordon', cordoned) + wait_rollout(edge_ns, edge_name) + ready = [p for p in get('-n', edge_ns, 'get', 'pods', '-l', selector)['items'] if not p['metadata'].get('deletionTimestamp')] + if len({p['spec']['nodeName'] for p in ready}) != 2: + # Kubernetes does not rebalance existing replicas after uncordon. + # Replace one gracefully so the normal spread rule can place it. + run('-n', edge_ns, 'delete', 'pod', ready[0]['metadata']['name'], '--wait=true', '--timeout=90s') + wait_rollout(edge_ns, edge_name) + restored = get('-n', edge_ns, 'get', 'pods', '-l', selector)['items'] + assert len({p['spec']['nodeName'] for p in restored if not p['metadata'].get('deletionTimestamp')}) == 2, 'Gateway placement not restored' + except Exception as error: + cleanup_errors.append('uncordon: ' + repr(error)) + if modified_patches is not None: + try: + current = get('-n', edge_ns, 'get', 'nginxproxy', 'registry-edge') + assert current['metadata']['uid'] == policy_uid and current['spec'] == modified_spec, 'Concurrent Gateway policy change; refuse restoration' + apply_policy(current, policy_before['spec'], policy_manager) + report['pulumi_policy_ownership_preserved'] = True + deadline = time.monotonic() + 60 + while 'registry.modelcontextprotocol.io/qualification' in get('-n', edge_ns, 'get', 'deployment', edge_name)['spec']['template']['metadata'].get('annotations', {}): + assert time.monotonic() < deadline, 'Controller did not restore rollout policy' + time.sleep(2) + wait_rollout(edge_ns, edge_name) + except Exception as error: + cleanup_errors.append('restore policy: ' + repr(error)) + report['cleanup_finished'] = time.time() + report['cleanup_errors'] = cleanup_errors + try: + report['measurement_exit'] = load.wait(timeout=args.seconds + 100) + finally: + load_log.close() + report['public'] = public.finish(120 if args.scenario == 'node' else 0) + report['nodes_after'] = sorted(n['metadata']['name'] for n in get('get', 'nodes')['items']) + report['finished'] = time.time() + # The measurement driver remains alive while scraping/draining metrics. + # Qualify disruption overlap using the generator's own scheduling interval. + measurement = json.loads((args.output / 'load' / 'result.json').read_text()) + client = measurement['client'] + active_start = datetime.fromisoformat(client['started_at'].replace('Z', '+00:00')).timestamp() + active_end = min(active_start + client['seconds'], datetime.fromisoformat(client['finished_at'].replace('Z', '+00:00')).timestamp()) + report['generator_active_interval'] = [active_start, active_end] + report['action_covered_by_load'] = active_start <= report.get('action_started', 0) < report.get('action_ready', float('inf')) <= active_end + if not report['action_covered_by_load']: + failure = failure or 'Generator traffic did not cover the disruption and recovery' + if args.scenario == 'node': + overlap = max(0, min(active_end, report.get('one_node_ended', 0)) - max(active_start, report.get('drained_at', float('inf')))) + report['one_node_active_load_seconds'] = overlap + if overlap < 60: + failure = failure or 'Less than 60 seconds of actual one-node traffic' + if args.scenario == 'rollout' and report['cleanup_finished'] > active_end: + failure = failure or 'Generator traffic ended before the restoration rollout completed' + report['failure'] = failure + report['passed'] = failure is None and report['public']['passed'] and not report['cleanup_errors'] and report['measurement_exit'] == 0 and report['nodes_before'] == report['nodes_after'] + (args.output / 'operation.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + assert report['passed'], 'Recovery qualification failed; inspect operation.json and load/result.json' + + +if __name__ == '__main__': + main() diff --git a/deploy/gateway-canary/rollout.py b/deploy/gateway-canary/rollout.py index 4c48ac2ee..897876239 100644 --- a/deploy/gateway-canary/rollout.py +++ b/deploy/gateway-canary/rollout.py @@ -2,15 +2,17 @@ """Gracefully replace one canary proxy during a bounded fixture load.""" import argparse, json, subprocess, time from pathlib import Path +from measure import gateway_target a = argparse.ArgumentParser() a.add_argument('--output', type=Path, required=True) args = a.parse_args() args.output.mkdir(parents=True, exist_ok=True) -K = ['kubectl', '--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging', '-n', 'registry-gateway-canary'] +NS, NAME = gateway_target() +K = ['kubectl', '--context=gke_mcp-registry-staging_us-central1-b_mcp-registry-staging', '-n', NS] def pods(): return json.loads(subprocess.check_output(K + ['get', 'pods', '-o', 'json']))['items'] -ready = [p for p in pods() if p['metadata']['name'].startswith('registry-canary-registry-gateway-canary-') and all((x.get('ready') for x in p.get('status', {}).get('containerStatuses', [])))] +ready = [p for p in pods() if p['metadata']['name'].startswith(NAME + '-') and all((x.get('ready') for x in p.get('status', {}).get('containerStatuses', [])))] assert len(ready) == 2, len(ready) with open(args.output / 'load.log', 'w') as log: p = subprocess.Popen(['python3', 'gateway-canary/measure.py', '--arm', 'ngf', '--rate', '500', '--seconds', '90', '--epoch', 'canary-rollout', '--output', str(args.output / 'load')], stdout=log, stderr=subprocess.STDOUT) @@ -19,8 +21,8 @@ def pods(): old = ready[0]['metadata']['name'] started = time.time() subprocess.run(K + ['delete', 'pod', old, '--wait=false'], check=True) - subprocess.run(K + ['rollout', 'status', 'deployment/registry-canary-registry-gateway-canary', '--timeout=180s'], check=True) - report = {'deleted_pod': old, 'started_at': started, 'ready_at': time.time(), 'pods': [{'name': x['metadata']['name'], 'ready': all((c.get('ready') for c in x.get('status', {}).get('containerStatuses', [])))} for x in pods() if x['metadata']['name'].startswith('registry-canary-registry-gateway-canary-')]} + subprocess.run(K + ['rollout', 'status', 'deployment/' + NAME, '--timeout=180s'], check=True) + report = {'deleted_pod': old, 'started_at': started, 'ready_at': time.time(), 'pods': [{'name': x['metadata']['name'], 'ready': all((c.get('ready') for c in x.get('status', {}).get('containerStatuses', [])))} for x in pods() if x['metadata']['name'].startswith(NAME + '-')]} report['measurement_exit'] = p.wait(timeout=200) (args.output / 'operation.json').write_text(json.dumps(report, indent=2) + '\n') print(json.dumps(report, indent=2)) diff --git a/deploy/gateway-canary/standalone.md b/deploy/gateway-canary/standalone.md new file mode 100644 index 000000000..778409a57 --- /dev/null +++ b/deploy/gateway-canary/standalone.md @@ -0,0 +1,80 @@ +# Standalone staging Gateway qualification + +This is the staging completion gate before any production migration. Production +keeps its existing ingress and release flow. Staging uses OSS Gateway Fabric, +Vector, cert-manager, VictoriaMetrics and Grafana on the existing nodes. + +## Final topology and recovery + +`gatewayStagingStandalone: "true"` selects the final topology. Public TLS +Certificates are owned by Pulumi and renewed through native Gateway HTTP01 +routes. cert-manager also manages the internal CA and 30-day server/client +certificates, renewing leaves ten days before expiry. The staging CA is a local, +self-signed trust root; its long-lived root lifecycle needs an explicit rotation +procedure, not an assumption that renewing the CA automatically reissues leaves. + +The isolated canary Gateway/LB/address, ingress-nginx chart/controller, Ingress +objects, admission webhook, old solver and obsolete bootstrap resources are +removed before the main qualification run. The public Service and its namespace +retain their historical Kubernetes names to preserve Service UID, public IP and +GCP forwarding infrastructure. They are shared network resources adopted out of +the old chart, not an ingress-nginx runtime dependency. Stable metric names and +node-pool names are retained to preserve monitoring history and existing nodes. + +Before first enabling standalone mode, use full-stack applies to import the +public Certificates with `gatewayPublicCertificates: adopt`, then switch to +`managed` while retaining the legacy Ingress objects. That second apply removes +the cert-manager shim annotations. Run `detach_certificates.py --output ` +from `deploy/` with the staging Pulumi passphrase environment configured; it +checks Pulumi protection, verifies the old owner and shim annotations, removes +the owner with a UID-guarded JSON patch, and verifies unchanged Certificates/TLS. +Only then enable standalone and review its full saved plan. The standalone +preview reads both live Certificates and fails if either still has an owner. +An empty server-side-apply owner list alone is insufficient, and Pulumi protection +does not prevent Kubernetes garbage collection. These explicit detachment checks +were completed before the staging retirement recorded in the validation report. + +There is no running legacy fallback. Recover from a reviewed Git revision using +the normal full-stack preview/saved-plan workflow. Git restores configuration, +not resource identity or deleted data: retain the protected public IP and active +TLS Secrets. Inspect Service adoption, certificate ownership and obsolete +resource recreation before applying a reverse migration. Do not promise an +instant selector rollback after removing the old controller. + +Proxy rollouts use zero surge and replace one replica at a time. A 30-second +pre-stop delay covers endpoint propagation and the current GCP health check's +three 8-second failure intervals. The hook then gracefully quits NGINX and waits +for its workers before the image receives TERM; the total termination budget is +90 seconds. This follows Kubernetes' [NGINX lifecycle example](https://kubernetes.io/docs/tasks/configure-pod-container/attach-handler-lifecycle-event/) +and [NGINX signal semantics](https://nginx.org/en/docs/control.html). Recheck the +drain budget if the load-balancer health-check configuration changes. Terminating +pods can temporarily coexist with their replacement; the fixed node pool may +keep that replacement pending until the departing pod releases its reservation. + +## Qualification matrix + +Record actual results and any failed attempts; a healthy Pod is not a test result. +Run load scenarios sequentially with `REGISTRY_GATEWAY_TARGET=public`, so they +exercise the same data plane and policies as the public hostnames. The isolated +fixture has no database or publishing access. Its temporary hostname/listener, +Certificate, backend and generator must be removed after qualification. + +| Scenario | Pass criteria | +| --- | --- | +| Certificates | Native ACME issuance and forced renewal for both public hostnames; existing trusted TLS remains valid; public Certificates have no Ingress owner. | +| Internal TLS | Renew both leaves, observe agent Secret propagation, and complete a fresh mTLS handshake with the renewed server/client credentials without restarting the controller. | +| Feature contract | Cache MISS/HIT/offload, stale serving, refresh/locking, private/error bypass, gzip/identity/Vary, request/query/forwarded-header behavior, redirects, body limits and source-IP limiting pass. | +| Normal release | Main push builds/deploys its exact image digest; authenticated publishing/read/delete and Grafana access work. | +| Steady load | 500 and 1,000 requests/s, zero generator drops or unexpected statuses/errors, p95 under 100ms and p99 under 250ms for the controlled fixture. | +| Mixed traffic | 250 requests/s with 20% unique misses; same error/latency gates, origin work measured. | +| Burst and scaling | Bounded 1,500 requests/s burst measured separately; HPA reaches four ready replicas at peak and returns to two after cooldown; no added nodes. Do not report generator-limited load as gateway capacity. | +| Proxy/controller replacement | Replace a proxy and restart the controller under load; measure client errors, recovery and public health. | +| One-node operation | Cordon/drain one Gateway node, test 500 requests/s on the remaining node, keep public probes running, and uncordon in a finally block. Public recovery within two minutes; record convergence errors. This is node evacuation/unavailability, not a powered-off VM experiment. | +| Collector recovery | Same-node restart restores checkpoints without historical replay, followed by successful load/metric reconciliation. No exactly-once crash guarantee is claimed. | +| Monitoring | All live scrape targets healthy; client/exporter/database request and histogram counts reconcile; CRI/retry timing contract passes; Grafana alert fires/resolves and reaches the temporary internal webhook. | +| Cleanup and soak | No legacy workload, duplicate Gateway/LB, fixtures, test issuers, temporary alerts or cordoned nodes; reviewed previews and a clean normal GitOps apply; observe the final configuration for unexplained errors, restarts, memory growth or monitoring gaps. | + +No production apply is part of this work. Staging notification delivery is tested +through the previously approved temporary internal webhook; delivery to a real +person is a separate production gate. The earlier accepted staging interruption +budget does not establish a production availability target. diff --git a/deploy/pkg/k8s/autoscaling.go b/deploy/pkg/k8s/autoscaling.go index d22922d2f..9b66ae714 100644 --- a/deploy/pkg/k8s/autoscaling.go +++ b/deploy/pkg/k8s/autoscaling.go @@ -85,7 +85,7 @@ func burstResourceTransform(args *pulumi.ResourceTransformationArgs) *pulumi.Res } func withIngressScrape(ctx *pulumi.Context, configs pulumi.Array) pulumi.Array { - if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") { + if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") && !gatewaycanary.Standalone(ctx) { configs = append(configs, ingressScrapeConfig()) } if gatewaycanary.Enabled(ctx) { diff --git a/deploy/pkg/k8s/cert_manager.go b/deploy/pkg/k8s/cert_manager.go index cad75bb94..a5f83a32d 100644 --- a/deploy/pkg/k8s/cert_manager.go +++ b/deploy/pkg/k8s/cert_manager.go @@ -8,11 +8,12 @@ import ( metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/modelcontextprotocol/registry/deploy/infra/pkg/k8s/gatewaycanary" "github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers" ) // SetupCertManager sets up cert-manager for TLS certificates -func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo) error { +func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo, gatewayCRDs []pulumi.Resource) (*apiextensions.CustomResource, error) { // Create namespace for cert-manager certManagerNamespace, err := v1.NewNamespace(ctx, "cert-manager", &v1.NamespaceArgs{ Metadata: &metav1.ObjectMetaArgs{ @@ -20,9 +21,17 @@ func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo) erro }, }, pulumi.Provider(cluster.Provider)) if err != nil { - return err + return nil, err } + values := pulumi.Map{ + "installCRDs": pulumi.Bool(true), + "ingressShim": pulumi.Map{"defaultIssuerName": pulumi.String("letsencrypt-prod"), "defaultIssuerKind": pulumi.String("ClusterIssuer")}, + } + if gatewaycanary.NativeCertificates(ctx) { + // Use the supported flag without introducing a Helm ConfigMap output reference. + values["extraArgs"] = pulumi.StringArray{pulumi.String("--enable-gateway-api")} + } // Install cert-manager for TLS certificates certManager, err := helm.NewChart(ctx, "cert-manager", helm.ChartArgs{ Chart: pulumi.String("cert-manager"), @@ -31,19 +40,17 @@ func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo) erro Repo: pulumi.String("https://charts.jetstack.io"), }, Namespace: certManagerNamespace.Metadata.Name().Elem(), - Values: pulumi.Map{ - "installCRDs": pulumi.Bool(true), - "ingressShim": pulumi.Map{ - "defaultIssuerName": pulumi.String("letsencrypt-prod"), - "defaultIssuerKind": pulumi.String("ClusterIssuer"), - }, - }, - }, pulumi.Provider(cluster.Provider)) + Values: values, + }, pulumi.Provider(cluster.Provider), pulumi.DependsOn(gatewayCRDs)) if err != nil { - return err + return nil, err } - _, err = apiextensions.NewCustomResource(ctx, "letsencrypt-prod", &apiextensions.CustomResourceArgs{ + http01 := pulumi.Map{"ingress": pulumi.Map{"ingressClassName": pulumi.String("nginx")}} + if gatewaycanary.NativeCertificates(ctx) { + http01 = pulumi.Map{"gatewayHTTPRoute": pulumi.Map{"parentRefs": pulumi.Array{pulumi.Any(gatewaycanary.ACMEParent())}}} + } + issuer, err := apiextensions.NewCustomResource(ctx, "letsencrypt-prod", &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("cert-manager.io/v1"), Kind: pulumi.String("ClusterIssuer"), Metadata: &metav1.ObjectMetaArgs{ @@ -59,11 +66,7 @@ func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo) erro }, "solvers": pulumi.Array{ pulumi.Map{ - "http01": pulumi.Map{ - "ingress": pulumi.Map{ - "ingressClassName": pulumi.String("nginx"), - }, - }, + "http01": http01, }, }, }, @@ -71,8 +74,21 @@ func SetupCertManager(ctx *pulumi.Context, cluster *providers.ProviderInfo) erro }, }, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(certManager.Ready)) if err != nil { - return err + return nil, err } - return nil + if err := gatewaycanary.DeployPublicCertificates(ctx, cluster.Provider, issuer); err != nil { + return nil, err + } + // Depend on a concrete resource whose own dependencies await chart.Ready. + // Depending on the asynchronous Chart component can expand to different + // child sets between preview and apply, violating saved-plan constraints. + return issuer, nil +} + +func certificateIngressAnnotations(ctx *pulumi.Context, annotations pulumi.StringMap) pulumi.StringMap { + if gatewaycanary.PublicCertificates(ctx) != "managed" { + annotations["cert-manager.io/cluster-issuer"] = pulumi.String("letsencrypt-prod") + } + return annotations } diff --git a/deploy/pkg/k8s/deploy.go b/deploy/pkg/k8s/deploy.go index 932469181..770e31594 100644 --- a/deploy/pkg/k8s/deploy.go +++ b/deploy/pkg/k8s/deploy.go @@ -10,8 +10,12 @@ import ( // DeployAll orchestrates the complete deployment of the MCP Registry to Kubernetes func DeployAll(ctx *pulumi.Context, cluster *providers.ProviderInfo, backupStorage *providers.BackupStorageInfo, environment string) (service *corev1.Service, err error) { - // Setup cert-manager - err = SetupCertManager(ctx, cluster) + gatewayCRDs, err := gatewaycanary.DeployCRDs(ctx, cluster.Provider) + if err != nil { + return nil, err + } + // Setup cert-manager after the Gateway API definitions exist. + certificateIssuer, err := SetupCertManager(ctx, cluster, gatewayCRDs) if err != nil { return nil, err } @@ -46,7 +50,12 @@ func DeployAll(ctx *pulumi.Context, cluster *providers.ProviderInfo, backupStora return nil, err } - if err := gatewaycanary.Deploy(ctx, cluster.Provider, cluster.CanaryAddress, ingressNginx); err != nil { + gatewayDependencies := append(gatewayCRDs, certificateIssuer) + // Do not box a nil *helm.Chart into a non-nil pulumi.Resource interface. + if ingressNginx != nil { + gatewayDependencies = append(gatewayDependencies, ingressNginx) + } + if err := gatewaycanary.Deploy(ctx, cluster.Provider, cluster.CanaryAddress, gatewayDependencies...); err != nil { return nil, err } return service, nil diff --git a/deploy/pkg/k8s/gateway_service.go b/deploy/pkg/k8s/gateway_service.go new file mode 100644 index 000000000..e4e021c7c --- /dev/null +++ b/deploy/pkg/k8s/gateway_service.go @@ -0,0 +1,28 @@ +package k8s + +import ( + "github.com/modelcontextprotocol/registry/deploy/infra/pkg/k8s/gatewaycanary" + "github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers" + corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +// Adopt the existing network endpoint out of the retired Helm chart. Its +// historical Kubernetes name is retained to preserve the Service UID/GCP LB; +// there is no ingress-nginx controller, chart, solver or runtime dependency. +func setupPublicGatewayService(ctx *pulumi.Context, cluster *providers.ProviderInfo, namespace *corev1.Namespace) error { + oldURN := pulumi.URN("urn:pulumi:" + ctx.Stack() + "::" + ctx.Project() + "::kubernetes:helm.sh/v3:Chart$kubernetes:core/v1:Service::ingress-nginx/ingress-nginx-controller") + _, err := corev1.NewService(ctx, "registry-public-gateway", &corev1.ServiceArgs{ + Metadata: &metav1.ObjectMetaArgs{Name: pulumi.String("ingress-nginx-controller"), Namespace: namespace.Metadata.Name(), Labels: pulumi.StringMap{"app.kubernetes.io/name": pulumi.String("nginx-gateway-fabric")}}, + Spec: &corev1.ServiceSpecArgs{Type: pulumi.String("LoadBalancer"), LoadBalancerIP: cluster.StagingIngressAddress, ExternalTrafficPolicy: pulumi.String("Local"), HealthCheckNodePort: pulumi.Int(30780), Selector: pulumi.StringMap{"gateway.networking.k8s.io/gateway-name": pulumi.String(gatewaycanary.EdgeGateway)}, Ports: corev1.ServicePortArray{ + &corev1.ServicePortArgs{Name: pulumi.String("http"), Port: pulumi.Int(80), TargetPort: pulumi.Int(80), NodePort: pulumi.Int(32008), Protocol: pulumi.String("TCP")}, + &corev1.ServicePortArgs{Name: pulumi.String("https"), Port: pulumi.Int(443), TargetPort: pulumi.Int(443), NodePort: pulumi.Int(31296), Protocol: pulumi.String("TCP")}, + }}, + }, pulumi.Provider(cluster.Provider), pulumi.Protect(true), pulumi.Aliases([]pulumi.Alias{{URN: oldURN}})) + if err != nil { + return err + } + ctx.Export("ingressIps", pulumi.StringArray{cluster.StagingIngressAddress}) + return nil +} diff --git a/deploy/pkg/k8s/gatewaycanary/canary.go b/deploy/pkg/k8s/gatewaycanary/canary.go index 7082273f6..e77b089e2 100644 --- a/deploy/pkg/k8s/gatewaycanary/canary.go +++ b/deploy/pkg/k8s/gatewaycanary/canary.go @@ -1,4 +1,4 @@ -// Package gatewaycanary manages the staging-only, parallel OSS gateway. +// Package gatewaycanary manages the staging-only OSS Gateway and qualification modes. package gatewaycanary import ( @@ -19,11 +19,18 @@ const Namespace = "registry-gateway-canary" var assets embed.FS func Enabled(ctx *pulumi.Context) bool { - return config.New(ctx, "mcp-registry").GetBool("gatewayCanary") + return Standalone(ctx) || config.New(ctx, "mcp-registry").GetBool("gatewayCanary") } func Validate(ctx *pulumi.Context) error { c := config.New(ctx, "mcp-registry") + mode := PublicCertificates(ctx) + if mode != "" && (mode != "adopt" && mode != "managed" || !NativeCertificates(ctx)) { + return fmt.Errorf("public Gateway certificates require native renewal and adopt/managed mode") + } + if ManagedMTLS(ctx) && !NativeCertificates(ctx) { + return fmt.Errorf("managed Gateway mTLS requires native staging certificates") + } if err := validateHandoff(ctx); err != nil { return err } @@ -48,7 +55,7 @@ func objects(file string) ([]map[string]any, error) { // The upstream Helm render is registered as individual resources: all fixed // identities and inputs are visible in saved plans, without Helm output-ID holes. -func register(ctx *pulumi.Context, provider *kubernetes.Provider, o map[string]any, dependencies []pulumi.Resource) (*apiextensions.CustomResource, error) { +func register(ctx *pulumi.Context, provider *kubernetes.Provider, o map[string]any, dependencies []pulumi.Resource, extra ...pulumi.ResourceOption) (*apiextensions.CustomResource, error) { meta := o["metadata"].(map[string]any) name := meta["name"].(string) ns, _ := meta["namespace"].(string) @@ -62,35 +69,31 @@ func register(ctx *pulumi.Context, provider *kubernetes.Provider, o map[string]a } resourceName := "gateway-canary-" + strings.ToLower(o["kind"].(string)) + "-" + ns + "-" + name var result apiextensions.CustomResource - err := ctx.RegisterResource("kubernetes:"+api+":"+o["kind"].(string), resourceName, args, &result, pulumi.Provider(provider), pulumi.DependsOn(dependencies)) + err := ctx.RegisterResource("kubernetes:"+api+":"+o["kind"].(string), resourceName, args, &result, append([]pulumi.ResourceOption{pulumi.Provider(provider), pulumi.DependsOn(dependencies)}, extra...)...) return &result, err } -func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.StringOutput, ingress pulumi.Resource) error { +func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.StringOutput, prerequisites ...pulumi.Resource) error { if !Enabled(ctx) { return nil } if err := Validate(ctx); err != nil { return err } - crds, err := objects("crds.json") - if err != nil { - return err - } - deps := []pulumi.Resource{} - for _, o := range crds { - r, e := register(ctx, provider, o, nil) - if e != nil { - return e - } - deps = append(deps, r) - } + deps := append([]pulumi.Resource{}, prerequisites...) + namespace := map[string]any{"apiVersion": "v1", "kind": "Namespace", "metadata": map[string]any{"name": Namespace}} ns, err := register(ctx, provider, namespace, nil) if err != nil { return err } deps = append(deps, ns) + if ManagedMTLS(ctx) { + deps, err = deployInternalCertificates(ctx, provider, deps) + if err != nil { + return err + } + } control, err := objects("controller.json") if err != nil { return err @@ -101,21 +104,44 @@ func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.S return err } control = append(control, rbac...) - deps = append(deps, ingress) + public, err := handoffObjectsWithOptions(NativeCertificates(ctx), Standalone(ctx), false) + if err != nil { + return err + } + for _, o := range public { + if o["kind"] == "NginxProxy" { + control = append(control, o) + } + } } // RBAC precedes the certificate generator, which precedes the controller. for _, kinds := range [][]string{{"ServiceAccount", "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding", "Service"}, {"Job"}, {"GatewayClass", "NginxGateway", "NginxProxy"}, {"Deployment"}} { prior := append([]pulumi.Resource{}, deps...) for _, o := range control { + if ManagedMTLS(ctx) { + meta := o["metadata"].(map[string]any) + if strings.Contains(meta["name"].(string), "cert-generator") { + continue + } + if o["kind"] == "Deployment" { + managedControllerTLS(o) + } + } if !contains(kinds, o["kind"].(string)) { continue } - if o["kind"] == "NginxProxy" { + if o["kind"] == "NginxProxy" && o["metadata"].(map[string]any)["namespace"] == Namespace { o["spec"].(map[string]any)["kubernetes"].(map[string]any)["service"].(map[string]any)["loadBalancerIP"] = address - if config.New(ctx, "mcp-registry").GetBool("gatewayCanaryComparison") || HandoffEnabled(ctx) { + if Standalone(ctx) { + o["spec"].(map[string]any)["kubernetes"].(map[string]any)["service"] = map[string]any{"type": "ClusterIP"} + } + if config.New(ctx, "mcp-registry").GetBool("gatewayCanaryComparison") || (HandoffEnabled(ctx) && !Standalone(ctx)) { o["spec"].(map[string]any)["kubernetes"].(map[string]any)["deployment"].(map[string]any)["autoscaling"].(map[string]any)["maxReplicas"] = 2 } } + if o["kind"] == "NginxProxy" && NativeCertificates(ctx) { + boundedGatewayRollout(o) + } if HandoffEnabled(ctx) && o["kind"] == "Deployment" { containers := o["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any)["containers"].([]any) args := containers[0].(map[string]any)["args"].([]any) @@ -142,6 +168,11 @@ func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.S if !fixturesEnabled { continue } + if Standalone(ctx) && o["kind"] == "Deployment" { + spec := o["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any) + spec["nodeSelector"] = map[string]any{"registry-role": "worker"} + spec["tolerations"] = []any{} + } r, e := register(ctx, provider, o, deps) if e != nil { return e @@ -152,6 +183,9 @@ func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.S if err != nil { return err } + if Standalone(ctx) { + routes = nil + } // Install policy/filter/reference grants before exposing the routes. for _, group := range [][]string{{"Issuer"}, {"Certificate"}, {"ReferenceGrant", "SnippetsPolicy", "SnippetsFilter", "RateLimitPolicy", "ProxySettingsPolicy", "ClientSettingsPolicy"}, {"Gateway"}, {"HTTPRoute"}} { prior := append([]pulumi.Resource{}, deps...) @@ -184,14 +218,16 @@ func Deploy(ctx *pulumi.Context, provider *kubernetes.Provider, address pulumi.S } } if HandoffEnabled(ctx) { - if err := deployHandoff(ctx, provider, append(deps, ingress)); err != nil { + if err := deployHandoff(ctx, provider, deps); err != nil { return err } } if err := vector(ctx, provider, ns); err != nil { return err } - ctx.Export("gatewayCanaryIP", address) + if !Standalone(ctx) { + ctx.Export("gatewayCanaryIP", address) + } ctx.Export("gatewayCanaryHost", pulumi.String("staging.registry.modelcontextprotocol.io")) return nil } @@ -233,3 +269,23 @@ func contains(values []string, s string) bool { } return false } + +// Install API definitions before enabling cert-manager's Gateway controllers. +func DeployCRDs(ctx *pulumi.Context, provider *kubernetes.Provider) ([]pulumi.Resource, error) { + if !Enabled(ctx) { + return nil, nil + } + crds, err := objects("crds.json") + if err != nil { + return nil, err + } + deps := []pulumi.Resource{} + for _, o := range crds { + r, err := register(ctx, provider, o, nil) + if err != nil { + return nil, err + } + deps = append(deps, r) + } + return deps, nil +} diff --git a/deploy/pkg/k8s/gatewaycanary/certificates.go b/deploy/pkg/k8s/gatewaycanary/certificates.go new file mode 100644 index 000000000..c88d86c84 --- /dev/null +++ b/deploy/pkg/k8s/gatewaycanary/certificates.go @@ -0,0 +1,116 @@ +package gatewaycanary + +import ( + "fmt" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" +) + +func ManagedMTLS(ctx *pulumi.Context) bool { + return Standalone(ctx) || config.New(ctx, "mcp-registry").GetBool("gatewayManagedMTLS") +} + +func internalCertificates() []map[string]any { + object := func(kind, name string, spec map[string]any) map[string]any { + return map[string]any{"apiVersion": "cert-manager.io/v1", "kind": kind, "metadata": map[string]any{"name": name, "namespace": Namespace, "annotations": map[string]any{"pulumi.com/waitFor": "condition=Ready"}}, "spec": spec} + } + issuer := func(name string) map[string]any { + return map[string]any{"name": name, "kind": "Issuer", "group": "cert-manager.io"} + } + leaf := func(name, dns, usage string) map[string]any { + return object("Certificate", name, map[string]any{"secretName": name, "dnsNames": []any{dns}, "duration": "720h", "renewBefore": "240h", "privateKey": map[string]any{"algorithm": "RSA", "size": 2048, "rotationPolicy": "Always"}, "usages": []any{"digital signature", "key encipherment", usage}, "issuerRef": issuer("gateway-internal-ca")}) + } + return []map[string]any{ + object("Issuer", "gateway-bootstrap", map[string]any{"selfSigned": map[string]any{}}), + object("Certificate", "gateway-internal-ca", map[string]any{"secretName": "gateway-internal-ca", "isCA": true, "commonName": "registry-staging-gateway", "duration": "87600h", "renewBefore": "8760h", "privateKey": map[string]any{"algorithm": "RSA", "size": 2048, "rotationPolicy": "Never"}, "issuerRef": issuer("gateway-bootstrap")}), + object("Issuer", "gateway-internal-ca", map[string]any{"ca": map[string]any{"secretName": "gateway-internal-ca"}}), + leaf("gateway-server-tls", "registry-gateway-canary."+Namespace+".svc", "server auth"), + leaf("gateway-agent-tls", "*.cluster.local", "client auth"), + } +} + +func deployInternalCertificates(ctx *pulumi.Context, provider *kubernetes.Provider, deps []pulumi.Resource) ([]pulumi.Resource, error) { + for _, o := range internalCertificates() { + r, err := register(ctx, provider, o, deps) + if err != nil { + return nil, err + } + deps = append(deps, r) + } + return deps, nil +} + +func managedControllerTLS(o map[string]any) { + spec := o["spec"].(map[string]any)["template"].(map[string]any)["spec"].(map[string]any) + for _, c := range spec["containers"].([]any) { + args := c.(map[string]any)["args"].([]any) + for i, arg := range args { + if arg == "--agent-tls-secret=agent-tls" { + args[i] = "--agent-tls-secret=gateway-agent-tls" + } + } + } + for _, v := range spec["volumes"].([]any) { + volume := v.(map[string]any) + if volume["name"] == "nginx-agent-tls" { + volume["secret"].(map[string]any)["secretName"] = "gateway-server-tls" + } + } +} + +func PublicCertificates(ctx *pulumi.Context) string { + if Standalone(ctx) { + return "managed" + } + return config.New(ctx, "mcp-registry").Get("gatewayPublicCertificates") +} + +func detachedCertificate(meta *metav1.ObjectMeta) (bool, error) { + if meta == nil || len(meta.OwnerReferences) != 0 { + return false, fmt.Errorf("public Certificate still has an owner: run gateway-canary/detach_certificates.py before standalone retirement") + } + return true, nil +} + +// Import existing shim-created Certificates first, remove the shim annotations, +// then explicitly detach ownership using detach_certificates.py. An empty SSA +// ownerReferences list cannot remove references owned by the ingress shim. +// Standalone previews fail closed if a live Certificate still has an owner. +func DeployPublicCertificates(ctx *pulumi.Context, provider *kubernetes.Provider, dependencies ...pulumi.Resource) error { + mode := PublicCertificates(ctx) + if mode == "" { + return nil + } + for _, item := range []struct{ namespace, name, host string }{ + {"default", "mcp-registry-staging-tls", "staging.registry.modelcontextprotocol.io"}, + {"monitoring", "grafana-tls", "grafana.staging.registry.modelcontextprotocol.io"}, + } { + if Standalone(ctx) { + existing, err := apiextensions.GetCustomResource(ctx, "gateway-public-certificate-ownership-"+item.namespace, + pulumi.ID(item.namespace+"/"+item.name), &apiextensions.CustomResourceState{ + ApiVersion: pulumi.StringPtr("cert-manager.io/v1"), Kind: pulumi.StringPtr("Certificate"), + }, pulumi.Provider(provider)) + if err != nil { + return err + } + // Exporting the checked output makes a failed check fail the full + // saved-plan preview before any legacy resources can be deleted. + ctx.Export("gatewayPublicCertificateDetached-"+item.namespace, existing.Metadata.ApplyT(detachedCertificate)) + } + meta := map[string]any{"name": item.name, "namespace": item.namespace} + opts := []pulumi.ResourceOption{pulumi.Protect(true)} + if mode == "adopt" { + opts = append(opts, pulumi.Import(pulumi.ID(item.namespace+"/"+item.name)), pulumi.IgnoreChanges([]string{"metadata.ownerReferences"})) + } else { + meta["ownerReferences"] = []any{} + } + o := map[string]any{"apiVersion": "cert-manager.io/v1", "kind": "Certificate", "metadata": meta, "spec": map[string]any{"secretName": item.name, "dnsNames": []any{item.host}, "issuerRef": map[string]any{"name": "letsencrypt-prod", "kind": "ClusterIssuer", "group": "cert-manager.io"}, "usages": []any{"digital signature", "key encipherment"}}} + if _, err := register(ctx, provider, o, dependencies, opts...); err != nil { + return err + } + } + return nil +} diff --git a/deploy/pkg/k8s/gatewaycanary/certificates_test.go b/deploy/pkg/k8s/gatewaycanary/certificates_test.go new file mode 100644 index 000000000..3367b390a --- /dev/null +++ b/deploy/pkg/k8s/gatewaycanary/certificates_test.go @@ -0,0 +1,25 @@ +package gatewaycanary + +import ( + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "testing" +) + +func TestDetachedCertificate(t *testing.T) { + for _, test := range []struct { + name string + meta *metav1.ObjectMeta + want bool + }{ + {"missing metadata", nil, false}, + {"no owner", &metav1.ObjectMeta{}, true}, + {"shim owner", &metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{{Kind: "Ingress", Name: "mcp-registry"}}}, false}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := detachedCertificate(test.meta) + if got != test.want || (err == nil) != test.want { + t.Fatalf("got %v, %v; want %v", got, err, test.want) + } + }) + } +} diff --git a/deploy/pkg/k8s/gatewaycanary/handoff.go b/deploy/pkg/k8s/gatewaycanary/handoff.go index c4a871866..cefa200ea 100644 --- a/deploy/pkg/k8s/gatewaycanary/handoff.go +++ b/deploy/pkg/k8s/gatewaycanary/handoff.go @@ -3,6 +3,7 @@ package gatewaycanary import ( "fmt" "net" + "strings" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" @@ -17,12 +18,35 @@ const ( ) func HandoffEnabled(ctx *pulumi.Context) bool { - return config.New(ctx, "mcp-registry").Get("gatewayStagingHandoff") != "" + return Standalone(ctx) || config.New(ctx, "mcp-registry").Get("gatewayStagingHandoff") != "" +} + +// Standalone selects the final staging topology without legacy ingress or duplicate canary. +func Standalone(ctx *pulumi.Context) bool { + return config.New(ctx, "mcp-registry").GetBool("gatewayStagingStandalone") +} + +// NativeCertificates enables the Gateway HTTP01 solver before legacy retirement. +func NativeCertificates(ctx *pulumi.Context) bool { + return Standalone(ctx) || config.New(ctx, "mcp-registry").GetBool("gatewayNativeCertificates") +} + +func ACMEParent() map[string]any { + return map[string]any{"name": EdgeGateway, "namespace": EdgeNamespace, "kind": "Gateway", "sectionName": "http"} } func validateHandoff(ctx *pulumi.Context) error { c := config.New(ctx, "mcp-registry") mode, ip := c.Get("gatewayStagingHandoff"), c.Get("gatewayStagingPublicIP") + if Standalone(ctx) && mode == "" { + mode = "gateway" + } + if Standalone(ctx) && (!NativeCertificates(ctx) || !ManagedMTLS(ctx) || PublicCertificates(ctx) != "managed") { + return fmt.Errorf("standalone Gateway requires managed public/internal certificates") + } + if NativeCertificates(ctx) && mode != "gateway" { + return fmt.Errorf("native Gateway certificates require gateway-only staging traffic") + } if mode == "" && ip == "" { return nil } @@ -35,7 +59,7 @@ func validateHandoff(ctx *pulumi.Context) error { if address := net.ParseIP(ip); address == nil || address.To4() == nil { return fmt.Errorf("gatewayStagingPublicIP must be the current public IPv4 address") } - if c.GetBool("gatewayCanaryComparison") || c.GetBool("gatewayCanaryFixtures") { + if c.GetBool("gatewayCanaryComparison") || (c.GetBool("gatewayCanaryFixtures") && !Standalone(ctx)) { return fmt.Errorf("disable gateway test fixtures before public handoff") } return nil @@ -86,6 +110,14 @@ func ProtectPublicService(args *pulumi.ResourceTransformationArgs) *pulumi.Resou // Derive the public Gateway from the policies qualified on the canary. A Service // selects pods in its own namespace, so its new data plane must live beside it. func handoffObjects() ([]map[string]any, error) { + return handoffObjectsWithOptions(false, false, false) +} + +func handoffObjectsWithCertificates(native bool) ([]map[string]any, error) { + return handoffObjectsWithOptions(native, false, false) +} + +func handoffObjectsWithOptions(native, standalone, fixtures bool) ([]map[string]any, error) { control, err := objects("controller.json") if err != nil { return nil, err @@ -101,6 +133,9 @@ func handoffObjects() ([]map[string]any, error) { kube["service"] = map[string]any{"type": "ClusterIP", "loadBalancerIP": ""} deployment := kube["deployment"].(map[string]any) deployment["autoscaling"].(map[string]any)["maxReplicas"] = 2 + if standalone { + deployment["autoscaling"].(map[string]any)["maxReplicas"] = 4 + } for _, raw := range deployment["patches"].([]any) { template := raw.(map[string]any)["value"].(map[string]any)["spec"].(map[string]any)["template"].(map[string]any) template["metadata"] = map[string]any{"labels": map[string]any{EdgeLabel: "true"}} @@ -108,9 +143,12 @@ func handoffObjects() ([]map[string]any, error) { constraint.(map[string]any)["labelSelector"].(map[string]any)["matchLabels"].(map[string]any)["gateway.networking.k8s.io/gateway-name"] = EdgeGateway } } + if native { + boundedGatewayRollout(o) + } result = append(result, o) } - routes, err := routeObjects(false) + routes, err := routeObjects(fixtures) if err != nil { return nil, err } @@ -127,6 +165,14 @@ func handoffObjects() ([]map[string]any, error) { } if o["kind"] == "Gateway" { meta["name"] = EdgeGateway + if native { + for _, raw := range spec["listeners"].([]any) { + listener := raw.(map[string]any) + if listener["name"] == "http" { + listener["allowedRoutes"] = map[string]any{"namespaces": map[string]any{"from": "Selector", "selector": map[string]any{"matchExpressions": []any{map[string]any{"key": "kubernetes.io/metadata.name", "operator": "In", "values": []any{EdgeNamespace, "default", "monitoring"}}}}}} + } + } + } spec["infrastructure"] = map[string]any{"parametersRef": map[string]any{"group": "gateway.nginx.org", "kind": "NginxProxy", "name": EdgeGateway}} } if ref, ok := spec["targetRef"].(map[string]any); ok && ref["name"] == "registry-canary" { @@ -142,8 +188,21 @@ func handoffObjects() ([]map[string]any, error) { } } } + if fixtures && o["kind"] == "HTTPRoute" && strings.HasPrefix(meta["name"].(string), "fixture-") { + for _, raw := range spec["rules"].([]any) { + for _, backend := range raw.(map[string]any)["backendRefs"].([]any) { + backend.(map[string]any)["namespace"] = Namespace + } + } + } result = append(result, o) } + if fixtures { + result = append(result, map[string]any{"apiVersion": "gateway.networking.k8s.io/v1beta1", "kind": "ReferenceGrant", "metadata": map[string]any{"name": "public-gateway-fixture", "namespace": Namespace}, "spec": map[string]any{"from": []any{map[string]any{"group": "gateway.networking.k8s.io", "kind": "HTTPRoute", "namespace": EdgeNamespace}}, "to": []any{map[string]any{"group": "", "kind": "Service", "name": "fixture"}}}}) + } + if native { + return result, nil + } // Never route challenges to the public Service: that would loop after handoff. result = append(result, map[string]any{"apiVersion": "v1", "kind": "Service", "metadata": map[string]any{"name": "legacy-acme", "namespace": EdgeNamespace}, "spec": map[string]any{"type": "ClusterIP", "selector": legacySelector(), "ports": []any{map[string]any{"name": "http", "port": 80, "targetPort": 80}}}}) result = append(result, map[string]any{"apiVersion": "gateway.networking.k8s.io/v1", "kind": "HTTPRoute", "metadata": map[string]any{"name": "acme", "namespace": EdgeNamespace}, "spec": map[string]any{"parentRefs": []any{map[string]any{"name": EdgeGateway, "sectionName": "http"}}, "hostnames": []any{"staging.registry.modelcontextprotocol.io", "grafana.staging.registry.modelcontextprotocol.io"}, "rules": []any{map[string]any{"matches": []any{map[string]any{"path": map[string]any{"type": "PathPrefix", "value": "/.well-known/acme-challenge/"}}}, "backendRefs": []any{map[string]any{"name": "legacy-acme", "port": 80}}}}}}) @@ -151,11 +210,11 @@ func handoffObjects() ([]map[string]any, error) { } func deployHandoff(ctx *pulumi.Context, provider *kubernetes.Provider, dependencies []pulumi.Resource) error { - objects, err := handoffObjects() + objects, err := handoffObjectsWithOptions(NativeCertificates(ctx), Standalone(ctx), Standalone(ctx) && config.New(ctx, "mcp-registry").GetBool("gatewayCanaryFixtures")) if err != nil { return err } - for _, kinds := range [][]string{{"NginxProxy", "Service", "ReferenceGrant", "SnippetsPolicy", "SnippetsFilter", "RateLimitPolicy", "ProxySettingsPolicy", "ClientSettingsPolicy"}, {"Gateway"}, {"HTTPRoute"}} { + for _, kinds := range [][]string{{"Issuer"}, {"Certificate"}, {"Service", "ReferenceGrant", "SnippetsPolicy", "SnippetsFilter", "RateLimitPolicy", "ProxySettingsPolicy", "ClientSettingsPolicy"}, {"Gateway"}, {"HTTPRoute"}} { prior := append([]pulumi.Resource{}, dependencies...) for _, o := range objects { if !contains(kinds, o["kind"].(string)) { @@ -168,7 +227,12 @@ func deployHandoff(ctx *pulumi.Context, provider *kubernetes.Provider, dependenc dependencies = append(dependencies, r) } } - ctx.Export("gatewayStagingHandoff", pulumi.String(config.New(ctx, "mcp-registry").Get("gatewayStagingHandoff"))) + if !Standalone(ctx) { + ctx.Export("gatewayStagingHandoff", pulumi.String(config.New(ctx, "mcp-registry").Get("gatewayStagingHandoff"))) + } + if Standalone(ctx) { + ctx.Export("stagingGatewayMode", pulumi.String("standalone")) + } return nil } @@ -189,3 +253,24 @@ func handoffRBAC() ([]map[string]any, error) { } return result, nil } + +// A full four-replica deployment cannot surge a fifth 500m proxy on the fixed +// two-node pool. Replace at most one replica while preserving at least one. +func boundedGatewayRollout(proxy map[string]any) { + deployment := proxy["spec"].(map[string]any)["kubernetes"].(map[string]any)["deployment"].(map[string]any) + for _, raw := range deployment["patches"].([]any) { + spec := raw.(map[string]any)["value"].(map[string]any)["spec"].(map[string]any) + spec["strategy"] = map[string]any{"type": "RollingUpdate", "rollingUpdate": map[string]any{"maxSurge": 0, "maxUnavailable": 1}} + pod := spec["template"].(map[string]any)["spec"].(map[string]any) + // The public GCP health check takes up to 3 * 8s to remove a node. + // Keep serving while endpoints converge, then drain in-flight requests + // before the image entrypoint receives its fast-stop TERM signal. + pod["terminationGracePeriodSeconds"] = 90 + // Strategic merge joins this entry with the generated nginx container by + // name, retaining its image, ports, probes and resource reservations. + pod["containers"] = []any{map[string]any{ + "name": "nginx", + "lifecycle": map[string]any{"preStop": map[string]any{"exec": map[string]any{"command": []any{"/bin/sh", "-c", "sleep 30; nginx -s quit; while killall -0 nginx 2>/dev/null; do sleep 1; done"}}}}, + }} + } +} diff --git a/deploy/pkg/k8s/gatewaycanary/handoff_test.go b/deploy/pkg/k8s/gatewaycanary/handoff_test.go index b3743924e..bad6ed689 100644 --- a/deploy/pkg/k8s/gatewaycanary/handoff_test.go +++ b/deploy/pkg/k8s/gatewaycanary/handoff_test.go @@ -3,8 +3,11 @@ package gatewaycanary import ( "encoding/json" "strings" + "sync/atomic" "testing" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -133,3 +136,151 @@ func TestHandoffControllerCanWatchItsNewNamespace(t *testing.T) { } } } + +func TestNativeCertificatesRemoveLegacyChallengeDependency(t *testing.T) { + objects, err := handoffObjectsWithCertificates(true) + if err != nil { + t.Fatal(err) + } + found := false + for _, o := range objects { + data, _ := json.Marshal(o) + if strings.Contains(string(data), "legacy-acme") { + t.Fatal("native renewal still depends on ingress") + } + if o["kind"] == "Gateway" { + for _, raw := range o["spec"].(map[string]any)["listeners"].([]any) { + listener := raw.(map[string]any) + namespaces := listener["allowedRoutes"].(map[string]any)["namespaces"].(map[string]any) + if listener["name"] == "http" { + found = true + if namespaces["from"] != "Selector" { + t.Fatal("solver namespaces must be restricted") + } + encoded, _ := json.Marshal(namespaces) + for _, ns := range []string{EdgeNamespace, "default", "monitoring"} { + if !strings.Contains(string(encoded), ns) { + t.Fatal("missing solver namespace", ns) + } + } + } else if namespaces["from"] != "Same" { + t.Fatal("HTTPS route attachment was broadened") + } + } + } + } + if !found { + t.Fatal("missing HTTP solver listener") + } +} + +func TestManagedInternalCertificates(t *testing.T) { + objects := internalCertificates() + if len(objects) != 5 { + t.Fatal("missing CA or leaf lifecycle") + } + for _, o := range objects { + meta := o["metadata"].(map[string]any) + if meta["namespace"] != Namespace || meta["annotations"].(map[string]any)["pulumi.com/waitFor"] != "condition=Ready" { + t.Fatal("certificate dependency is not awaited") + } + if o["kind"] == "Certificate" && meta["name"] != "gateway-internal-ca" { + spec := o["spec"].(map[string]any) + if spec["duration"] != "720h" || spec["renewBefore"] != "240h" || spec["privateKey"].(map[string]any)["rotationPolicy"] != "Always" { + t.Fatal("leaf renewal policy") + } + } + } + control, _ := objectsForControllerTest() + managedControllerTLS(control) + data, _ := json.Marshal(control) + if !strings.Contains(string(data), "--agent-tls-secret=gateway-agent-tls") || !strings.Contains(string(data), `"secretName":"gateway-server-tls"`) { + t.Fatal("controller did not switch both credentials") + } +} + +func objectsForControllerTest() (map[string]any, error) { + control, err := objects("controller.json") + for _, o := range control { + if o["kind"] == "Deployment" { + return o, err + } + } + return nil, err +} + +func TestStandaloneGatewayUsesFinalTopologyForLoadTests(t *testing.T) { + manifests, err := handoffObjectsWithOptions(true, true, true) + if err != nil { + t.Fatal(err) + } + fixtureRoutes := 0 + for _, o := range manifests { + spec := o["spec"].(map[string]any) + name := o["metadata"].(map[string]any)["name"].(string) + if name == "legacy-acme" || (o["kind"] == "HTTPRoute" && name == "acme") { + t.Fatal("legacy solver retained") + } + if o["kind"] == "NginxProxy" && spec["kubernetes"].(map[string]any)["deployment"].(map[string]any)["autoscaling"].(map[string]any)["maxReplicas"] != 4 { + t.Fatal("final autoscaling not restored") + } + if o["kind"] == "HTTPRoute" && strings.HasPrefix(name, "fixture-") { + fixtureRoutes++ + if spec["parentRefs"].([]any)[0].(map[string]any)["name"] != EdgeGateway { + t.Fatal("load test bypasses public Gateway") + } + for _, rule := range spec["rules"].([]any) { + for _, b := range rule.(map[string]any)["backendRefs"].([]any) { + if b.(map[string]any)["namespace"] != Namespace { + t.Fatal("wrong isolated fixture backend") + } + } + } + } + } + if fixtureRoutes != 2 { + t.Fatal("missing fixture routes") + } +} + +type standaloneMocks struct { + mocks + t *testing.T + gateways atomic.Int32 +} + +func (m *standaloneMocks) NewResource(a pulumi.MockResourceArgs) (string, resource.PropertyMap, error) { + if a.TypeToken == "kubernetes:gateway.networking.k8s.io/v1:Gateway" { + m.gateways.Add(1) + meta := a.Inputs["metadata"].ObjectValue() + if meta["name"].StringValue() != EdgeGateway || meta["namespace"].StringValue() != EdgeNamespace { + m.t.Error("duplicate canary Gateway retained") + } + } + if strings.Contains(a.Name, "cert-generator") || strings.Contains(a.Name, "legacy-acme") { + m.t.Error("legacy resource retained", a.Name) + } + return m.mocks.NewResource(a) +} +func TestStandaloneRegistersWithoutLegacyIngress(t *testing.T) { + mock := &standaloneMocks{t: t} + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + provider, err := kubernetes.NewProvider(ctx, "staging-provider", &kubernetes.ProviderArgs{}) + if err != nil { + return err + } + crds, err := DeployCRDs(ctx, provider) + if err != nil { + return err + } + return Deploy(ctx, provider, pulumi.String("").ToStringOutput(), crds...) + }, pulumi.WithMocks("test", "staging", mock), func(info *pulumi.RunInfo) { + info.Config = map[string]string{"mcp-registry:gatewayStagingStandalone": "true", "mcp-registry:environment": "staging", "mcp-registry:provider": "gcp", "gcp:project": "mcp-registry-staging", "mcp-registry:gatewayStagingPublicIP": "35.222.36.75"} + }) + if err != nil { + t.Fatal(err) + } + if mock.gateways.Load() != 1 { + t.Fatal("expected exactly one public Gateway") + } +} diff --git a/deploy/pkg/k8s/ingress.go b/deploy/pkg/k8s/ingress.go index 6eb0c6d1f..3bbf957b5 100644 --- a/deploy/pkg/k8s/ingress.go +++ b/deploy/pkg/k8s/ingress.go @@ -38,6 +38,9 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo return nil, err } + if gatewaycanary.Standalone(ctx) { + return nil, setupPublicGatewayService(ctx, cluster, ingressNginxNamespace) + } // Usually we should expose the ingress to a LoadBalancer // This works in GCP and most local setups e.g. minikube (with minikube tunnel) // Kind unfortunately does not support LoadBalancer type, and hangs indefinitely. This is a workaround for that. diff --git a/deploy/pkg/k8s/monitoring.go b/deploy/pkg/k8s/monitoring.go index 3c54172f1..b4a36d151 100644 --- a/deploy/pkg/k8s/monitoring.go +++ b/deploy/pkg/k8s/monitoring.go @@ -671,15 +671,19 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor return err } + if gatewaycanary.Standalone(ctx) { + ctx.Export("grafanaUrl", pulumi.Sprintf("https://%s", grafanaHost)) + return nil + } // Create ingress for external access _, err = networkingv1.NewIngress(ctx, "grafana-ingress", &networkingv1.IngressArgs{ Metadata: &metav1.ObjectMetaArgs{ Name: pulumi.String("grafana-ingress"), Namespace: ns.Metadata.Name(), - Annotations: pulumi.StringMap{ - "cert-manager.io/cluster-issuer": pulumi.String("letsencrypt-prod"), - "kubernetes.io/ingress.class": pulumi.String("nginx"), - }, + Annotations: certificateIngressAnnotations(ctx, pulumi.StringMap{ + + "kubernetes.io/ingress.class": pulumi.String("nginx"), + }), }, Spec: &networkingv1.IngressSpecArgs{ Tls: networkingv1.IngressTLSArray{ diff --git a/deploy/pkg/k8s/registry.go b/deploy/pkg/k8s/registry.go index b43918cc7..0b74c96f0 100644 --- a/deploy/pkg/k8s/registry.go +++ b/deploy/pkg/k8s/registry.go @@ -16,6 +16,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" + "github.com/modelcontextprotocol/registry/deploy/infra/pkg/k8s/gatewaycanary" "github.com/modelcontextprotocol/registry/deploy/infra/pkg/providers" ) @@ -392,6 +393,10 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env return nil, err } + if gatewaycanary.Standalone(ctx) { + ctx.Export("ingressHosts", pulumi.StringArray{pulumi.String("staging.registry.modelcontextprotocol.io")}) + return service, nil + } // Create Ingress hosts := []string{ environment + ".registry.modelcontextprotocol.io", @@ -410,16 +415,16 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env "app": pulumi.String("mcp-registry"), "environment": pulumi.String(environment), }, - Annotations: pulumi.StringMap{ - "cert-manager.io/cluster-issuer": pulumi.String("letsencrypt-prod"), - "kubernetes.io/ingress.class": pulumi.String("nginx"), + Annotations: certificateIngressAnnotations(ctx, pulumi.StringMap{ + + "kubernetes.io/ingress.class": pulumi.String("nginx"), // Rate limiting to protect against abuse. The exact bulk-list paths get a // separate, higher budget below; all other routes retain this limit. // Allows 180 requests/minute (3 req/sec avg), with bursts up to 540 requests. // Status code 429 is set globally via NGINX ConfigMap (per-Ingress annotation doesn't work). "nginx.ingress.kubernetes.io/limit-rpm": pulumi.String("180"), "nginx.ingress.kubernetes.io/limit-burst-multiplier": pulumi.String("3"), - }, + }), }, Spec: &networkingv1.IngressSpecArgs{ Tls: networkingv1.IngressTLSArray{ diff --git a/deploy/pkg/providers/gcp/gateway_canary.go b/deploy/pkg/providers/gcp/gateway_canary.go index 8a52bee0c..432363f57 100644 --- a/deploy/pkg/providers/gcp/gateway_canary.go +++ b/deploy/pkg/providers/gcp/gateway_canary.go @@ -27,9 +27,12 @@ func createGatewayCanary(ctx *pulumi.Context, cluster *container.Cluster, projec if err != nil { return nil, nil, err } - address, err := compute.NewAddress(ctx, "registry-gateway-canary-ip", &compute.AddressArgs{Project: pulumi.String(project), Region: pulumi.String(region), AddressType: pulumi.String("EXTERNAL")}, opts...) - if err != nil { - return nil, nil, err + var address *compute.Address + if !config.New(ctx, "mcp-registry").GetBool("gatewayStagingStandalone") { + address, err = compute.NewAddress(ctx, "registry-gateway-canary-ip", &compute.AddressArgs{Project: pulumi.String(project), Region: pulumi.String(region), AddressType: pulumi.String("EXTERNAL")}, opts...) + if err != nil { + return nil, nil, err + } } _, err = logging.NewProjectExclusion(ctx, "registry-gateway-canary-fast-read-sampling", &logging.ProjectExclusionArgs{Project: pulumi.String(project), Name: pulumi.String("registry-gateway-canary-fast-read-sampling"), Description: pulumi.String("Keep 10% of fast successful canary listing logs; metrics use unsampled local logs."), Filter: pulumi.String(`resource.type="k8s_container" resource.labels.namespace_name=("registry-gateway-canary" OR "ingress-nginx") diff --git a/deploy/pkg/providers/gcp/provider.go b/deploy/pkg/providers/gcp/provider.go index 8eb921e5b..e1eefdb61 100644 --- a/deploy/pkg/providers/gcp/provider.go +++ b/deploy/pkg/providers/gcp/provider.go @@ -205,7 +205,7 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov return nil, err } - if config.New(ctx, "mcp-registry").Get("publicReadLogSampling") != "" { + if config.New(ctx, "mcp-registry").Get("publicReadLogSampling") != "" && !config.New(ctx, "mcp-registry").GetBool("gatewayStagingStandalone") { policy := traffic.ReadPolicy() opts := []pulumi.ResourceOption{} if gcpProvider != nil { @@ -331,14 +331,16 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov clusterDependencies = append(clusterDependencies, dbPool) } - var canaryAddress pulumi.StringOutput - if config.New(ctx, "mcp-registry").GetBool("gatewayCanary") { + canaryAddress := pulumi.String("").ToStringOutput() + if config.New(ctx, "mcp-registry").GetBool("gatewayCanary") || config.New(ctx, "mcp-registry").GetBool("gatewayStagingStandalone") { pool, address, err := createGatewayCanary(ctx, cluster, projectID, region, zone, gcpProvider) if err != nil { return nil, err } clusterDependencies = append(clusterDependencies, pool) - canaryAddress = address.Address + if address != nil { + canaryAddress = address.Address + } } stagingIngressAddress, err := reserveStagingIngressAddress(ctx, projectID, region, gcpProvider)