diff --git a/apps/api-go/internal/deploycli/deploy_schema_maintenance_host.go b/apps/api-go/internal/deploycli/deploy_schema_maintenance_host.go index 0f83a4a2d..7c9bc8005 100644 --- a/apps/api-go/internal/deploycli/deploy_schema_maintenance_host.go +++ b/apps/api-go/internal/deploycli/deploy_schema_maintenance_host.go @@ -711,10 +711,12 @@ func (h *productionSchemaMaintenanceHost) probeRestoredAdmission(ctx context.Con if err != nil { return err } - out, err := h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/status"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768}) + // Bind the provider to this host directly. The original edge IP policy may + // legitimately conceal /api/status from a loopback TLS client with a 404. + out, err := h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "http://127.0.0.1:3000/api/status"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768}) parts := bytes.SplitN(out, []byte("\r\n\r\n"), 2) if err != nil || len(parts) != 2 { - return errors.New("schema maintenance reopened direct-origin readiness is unavailable") + return errors.New("schema maintenance current loopback provider readiness is unavailable") } status := strings.Fields(strings.SplitN(string(parts[0]), "\r\n", 2)[0]) var body struct { @@ -724,7 +726,24 @@ func (h *productionSchemaMaintenanceHost) probeRestoredAdmission(ctx context.Con } `json:"data"` } if len(status) < 2 || status[1] != "200" || json.Unmarshal(parts[1], &body) != nil || !body.Success || body.Data.Version != version { - return errors.New("schema maintenance reopened direct-origin provider differs") + return errors.New("schema maintenance current loopback provider differs") + } + // /api/scripts is already public in both its exact and named nginx + // locations. A successful response proves TLS admission is restored without + // changing the IP policy or inventing edge headers. The maintenance barrier + // still precedes this location and returns 503 while admission is closed. + out, err = h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/scripts"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768}) + parts = bytes.SplitN(out, []byte("\r\n\r\n"), 2) + if err != nil || len(parts) != 2 { + return errors.New("schema maintenance reopened direct-origin public readiness is unavailable") + } + status = strings.Fields(strings.SplitN(string(parts[0]), "\r\n", 2)[0]) + var public struct { + Success bool `json:"success"` + Data []json.RawMessage `json:"data"` + } + if len(status) < 2 || status[1] != "200" || json.Unmarshal(parts[1], &public) != nil || !public.Success || public.Data == nil { + return errors.New("schema maintenance reopened direct-origin public response differs") } return nil } diff --git a/apps/api-go/internal/deploycli/deploy_schema_maintenance_host_test.go b/apps/api-go/internal/deploycli/deploy_schema_maintenance_host_test.go index 875f7f3cd..39eefb74c 100644 --- a/apps/api-go/internal/deploycli/deploy_schema_maintenance_host_test.go +++ b/apps/api-go/internal/deploycli/deploy_schema_maintenance_host_test.go @@ -7,11 +7,84 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "testing" "time" ) +func TestSchemaMaintenanceRestoredAdmissionRequiresLocalVersionAndPublicTLS(t *testing.T) { + response := func(status, body string) []byte { + return []byte("HTTP/1.1 " + status + "\r\nContent-Type: application/json\r\n\r\n" + body) + } + own := response("200 OK", `{"success":true,"data":{"version":"0.2.111"}}`) + public := response("200 OK", `{"success":true,"data":[]}`) + for _, tc := range []struct { + name string + own, edge []byte + failedCall int + wantCalls int + wantError bool + }{ + {name: "empty public catalog", own: own, edge: public, wantCalls: 2}, + {name: "populated public catalog", own: own, edge: response("200 OK", `{"success":true,"data":[{"name":"install.sh"}]}`), wantCalls: 2}, + {name: "wrong local version", own: response("200 OK", `{"success":true,"data":{"version":"0.2.110"}}`), wantCalls: 1, wantError: true}, + {name: "local unsuccessful", own: response("200 OK", `{"success":false,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true}, + {name: "local malformed", own: response("200 OK", `{`), wantCalls: 1, wantError: true}, + {name: "local non-200", own: response("503 Service Unavailable", `{"success":true,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true}, + {name: "local missing HTTP header", own: []byte(`{"success":true,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true}, + {name: "local transport failed despite body", own: own, failedCall: 1, wantCalls: 1, wantError: true}, + {name: "concealed policy rejection", own: own, edge: response("404 Not Found", `{"message":"Not Found"}`), wantCalls: 2, wantError: true}, + {name: "maintenance barrier", own: own, edge: response("503 Service Unavailable", "lmm-schema-maintenance:schema-test"), wantCalls: 2, wantError: true}, + {name: "edge unsuccessful", own: own, edge: response("200 OK", `{"success":false,"data":[]}`), wantCalls: 2, wantError: true}, + {name: "edge missing data", own: own, edge: response("200 OK", `{"success":true}`), wantCalls: 2, wantError: true}, + {name: "edge null data", own: own, edge: response("200 OK", `{"success":true,"data":null}`), wantCalls: 2, wantError: true}, + {name: "edge object data", own: own, edge: response("200 OK", `{"success":true,"data":{}}`), wantCalls: 2, wantError: true}, + {name: "edge string data", own: own, edge: response("200 OK", `{"success":true,"data":"[]"}`), wantCalls: 2, wantError: true}, + {name: "edge non-JSON", own: own, edge: response("200 OK", "public script fixture"), wantCalls: 2, wantError: true}, + {name: "edge trailing JSON", own: own, edge: response("200 OK", `{"success":true,"data":[]} {}`), wantCalls: 2, wantError: true}, + {name: "edge missing HTTP header", own: own, edge: []byte(`{"success":true,"data":[]}`), wantCalls: 2, wantError: true}, + {name: "edge transport failed despite body", own: own, edge: public, failedCall: 2, wantCalls: 2, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + h := &productionSchemaMaintenanceHost{runtime: &productionRuntime{runner: existingSchemaTestRunner{run: func(command productionCommand) ([]byte, error) { + calls++ + args := []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*"} + body := tc.own + if calls == 1 { + args = append(args, "http://127.0.0.1:3000/api/status") + } else if calls == 2 { + args = append(args, "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/scripts") + body = tc.edge + } else { + t.Fatal("unexpected extra readiness command") + } + if command.Name != "/usr/bin/curl" || !slices.Equal(command.Args, args) || !slices.Equal(command.Env, []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}) || command.Timeout != 7*time.Second || command.OutputLimit != 32768 { + t.Fatalf("probe changed endpoint, TLS, proxy, header or resource bounds: %+v", command) + } + if calls == tc.failedCall { + return body, errors.New("transport failed") + } + return body, nil + }}}} + capsule := productionMerchantStoreCapsule{Candidate: productionReleasePackagePlan{Version: "0.2.111-1"}} + if err := h.probeRestoredAdmission(context.Background(), capsule); (err != nil) != tc.wantError || calls != tc.wantCalls { + t.Fatalf("error=%v calls=%d, want rejected=%v calls=%d", err, calls, tc.wantError, tc.wantCalls) + } + }) + } + t.Run("invalid sealed version", func(t *testing.T) { + h := &productionSchemaMaintenanceHost{runtime: &productionRuntime{runner: existingSchemaTestRunner{run: func(productionCommand) ([]byte, error) { + t.Fatal("invalid sealed version reached a readiness probe") + return nil, nil + }}}} + if err := h.probeRestoredAdmission(context.Background(), productionMerchantStoreCapsule{Candidate: productionReleasePackagePlan{Version: "invalid"}}); err == nil { + t.Fatal("accepted invalid sealed provider version") + } + }) +} + func TestSchemaMaintenanceAdmissionCannotAdoptOtherOwner(t *testing.T) { original := []byte("location @lmm_api_backend {\n proxy_pass http://127.0.0.1:3000;\n}\n") barrier, err := schemaMaintenanceAdmissionBarrier(original, "schema-test") diff --git a/scripts/nginx-service-errors.test.mjs b/scripts/nginx-service-errors.test.mjs index 92b4184e2..923b848d3 100644 --- a/scripts/nginx-service-errors.test.mjs +++ b/scripts/nginx-service-errors.test.mjs @@ -94,3 +94,126 @@ http { await new Promise(resolve=>backend.close(resolve));fs.rmSync(dir,{recursive:true,force:true}) } }) + +test('TLS public script admission survives denied status policy and remains behind the maintenance barrier', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lmm-nginx-restored-tls-')) + const templates = path.join(root, 'packaging/common/lmm-api/edge-policy/nginx') + const requests = [] + let scripts = [] + const backend = http.createServer((req, res) => { + requests.push({ url: req.url, headers: req.headers }) + if (req.url === '/internal/access-ip-policy') { + res.writeHead(403, { 'X-LMM-Access-Policy': 'denied' }).end() + } else if (req.url === '/internal/errors/access-policy') { + assert.equal(req.headers['x-lmm-internal-error'], 'access-policy') + assert.equal(req.headers['x-lmm-access-policy'], 'denied') + res.writeHead(404, { 'Content-Type': 'application/json' }).end('{"message":"Not Found"}') + } else if (req.url === '/api/scripts') { + res.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify({ success: true, data: scripts })) + } else if (req.url === '/api/status') { + res.writeHead(200, { 'Content-Type': 'application/json' }).end('{"success":true,"data":{"version":"0.2.111"}}') + } else { + res.writeHead(404).end() + } + }) + let nginx + try { + execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=LMM fixture CA', '-addext', 'basicConstraints=critical,CA:TRUE', '-keyout', `${dir}/ca.key`, '-out', `${dir}/ca.crt`], { stdio: 'pipe' }) + execFileSync('openssl', ['req', '-new', '-newkey', 'rsa:2048', '-nodes', '-subj', '/CN=api.lmm.best', '-keyout', `${dir}/tls.key`, '-out', `${dir}/tls.csr`], { stdio: 'pipe' }) + fs.writeFileSync(`${dir}/tls.ext`, 'subjectAltName=DNS:api.lmm.best\nextendedKeyUsage=serverAuth\nkeyUsage=digitalSignature,keyEncipherment\n') + execFileSync('openssl', ['x509', '-req', '-in', `${dir}/tls.csr`, '-CA', `${dir}/ca.crt`, '-CAkey', `${dir}/ca.key`, '-CAcreateserial', '-days', '1', '-extfile', `${dir}/tls.ext`, '-out', `${dir}/tls.crt`], { stdio: 'pipe' }) + backend.listen(0, '127.0.0.1');await once(backend, 'listening') + const reservation = net.createServer();reservation.listen(0, '127.0.0.1');await once(reservation, 'listening') + const port = reservation.address().port;await new Promise(resolve => reservation.close(resolve)) + const source = fs.readFileSync(`${templates}/lmm-api-locations.conf`, 'utf8') + fs.writeFileSync(`${dir}/locations.conf`, source.replaceAll('/var/log/nginx/access.log', `${dir}/access.log`).replaceAll('/etc/nginx/lmm-api-mime.types', `${templates}/mime.types`).replaceAll('/srv/lmm-api-frontend', `${dir}/frontend`)) + const maps = fs.readFileSync(`${templates}/http-map.conf`, 'utf8') + .replace(/geoip2[^]*?\n\}/, '') + .replaceAll('127.0.0.1:3000', `127.0.0.1:${backend.address().port}`) + .replace('include /etc/lmm-api/nginx/peers/*.conf;', '') + const tlsRequest = route => new Promise((resolve, reject) => { + const child = spawn('curl', ['--disable', '--silent', '--show-error', '--noproxy', '*', '--max-time', '2', '--cacert', `${dir}/ca.crt`, '--resolve', `api.lmm.best:${port}:127.0.0.1`, '--write-out', '\n%{http_code}', `https://api.lmm.best:${port}${route}`]) + let stdout = '', stderr = '' + child.stdout.on('data', chunk => { stdout += chunk }) + child.stderr.on('data', chunk => { stderr += chunk }) + child.on('error', reject) + child.on('close', code => { + if (code !== 0) { reject(new Error(`TLS curl failed (${code}): ${stderr}`));return } + const boundary = stdout.lastIndexOf('\n') + resolve({ status: Number(stdout.slice(boundary + 1)), body: stdout.slice(0, boundary) }) + }) + }) + const stop = async () => { + if (nginx && nginx.exitCode === null) { nginx.kill('SIGTERM');await once(nginx, 'exit') } + } + const start = async barrier => { + await stop() + fs.writeFileSync(`${dir}/nginx.conf`, `worker_processes 1; +pid ${dir}/nginx.pid; +error_log ${dir}/error.log; +events {worker_connections 64;} +http { + access_log off; + client_body_temp_path ${dir}/body; + proxy_temp_path ${dir}/proxy; + fastcgi_temp_path ${dir}/fastcgi; + uwsgi_temp_path ${dir}/uwsgi; + scgi_temp_path ${dir}/scgi; + map $host $lmm_geoip_country_code {default "";} + ${maps} + server { + listen 127.0.0.1:${port} ssl; + server_name api.lmm.best; + ssl_certificate ${dir}/tls.crt; + ssl_certificate_key ${dir}/tls.key; + ${barrier ? "return 503 'lmm-schema-maintenance:fixture';" : ''} + include ${templates}/lmm-api-region-policy.conf; + include ${dir}/locations.conf; + } +}`) + execFileSync('nginx', ['-t', '-p', dir, '-c', `${dir}/nginx.conf`], { stdio: 'pipe' }) + nginx = spawn('nginx', ['-p', dir, '-c', `${dir}/nginx.conf`, '-g', 'daemon off; master_process off;'], { stdio: 'ignore' }) + for (let attempt = 0; attempt < 50; attempt++) { + try { return await tlsRequest('/api/scripts') } catch (error) { + if (attempt === 49) throw error + await new Promise(resolve => setTimeout(resolve, 30)) + } + } + } + // Local process version and TLS admission are independent observations. + const localStatus = await fetch(`http://127.0.0.1:${backend.address().port}/api/status`) + assert.equal(localStatus.status, 200) + assert.deepEqual(await localStatus.json(), { success: true, data: { version: '0.2.111' } }) + let response = await start(false) + assert.equal(response.status, 200) + assert.deepEqual(JSON.parse(response.body), { success: true, data: [] }) + response = await tlsRequest('/api/status') + assert.equal(response.status, 404) + assert.deepEqual(JSON.parse(response.body), { message: 'Not Found' }) + assert.equal(requests.filter(request => request.url === '/api/status').length, 1, 'denied TLS status never reaches the status handler') + const policyRequest = requests.find(request => request.url === '/internal/access-ip-policy') + assert.equal(policyRequest.headers['x-original-client-ip'], '127.0.0.1') + assert.equal(policyRequest.headers['x-lmm-edge-country'], undefined) + scripts = [{ name: 'install.sh', size: 17, updated: '2026-10-12T00:00:00Z', fetches: 0 }] + const policyAttempts = requests.filter(request => request.url === '/internal/access-ip-policy').length + response = await tlsRequest('/api/scripts') + assert.equal(response.status, 200) + assert.deepEqual(JSON.parse(response.body), { success: true, data: scripts }) + assert.equal(requests.filter(request => request.url === '/internal/access-ip-policy').length, policyAttempts, 'script named redirect retains its existing auth exemption') + const beforeBarrier = requests.length + response = await start(true) + assert.equal(response.status, 503) + assert.equal(response.body, 'lmm-schema-maintenance:fixture') + assert.equal((await tlsRequest('/api/status')).status, 503) + assert.equal(requests.length, beforeBarrier, 'server barrier executes before public routing and auth_request') + response = await start(false) + assert.equal(response.status, 200) + assert.deepEqual(JSON.parse(response.body), { success: true, data: scripts }) + assert.equal((await tlsRequest('/api/status')).status, 404, 'restoration retains the original access policy') + } finally { + if (nginx && nginx.exitCode === null) { nginx.kill('SIGTERM');await once(nginx, 'exit') } + backend.closeAllConnections() + if (backend.listening) await new Promise(resolve => backend.close(resolve)) + fs.rmSync(dir, { recursive: true, force: true }) + } +})