Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions apps/api-go/internal/deploycli/deploy_schema_maintenance_host.go
Original file line number Diff line number Diff line change
Expand Up @@ -711,10 +711,12 @@ func (h *productionSchemaMaintenanceHost) probeRestoredAdmission(ctx context.Con
if err != nil {
return err
}
out, err := h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/status"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768})
// Bind the provider to this host directly. The original edge IP policy may
// legitimately conceal /api/status from a loopback TLS client with a 404.
out, err := h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "http://127.0.0.1:3000/api/status"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768})
parts := bytes.SplitN(out, []byte("\r\n\r\n"), 2)
if err != nil || len(parts) != 2 {
return errors.New("schema maintenance reopened direct-origin readiness is unavailable")
return errors.New("schema maintenance current loopback provider readiness is unavailable")
}
status := strings.Fields(strings.SplitN(string(parts[0]), "\r\n", 2)[0])
var body struct {
Expand All @@ -724,7 +726,24 @@ func (h *productionSchemaMaintenanceHost) probeRestoredAdmission(ctx context.Con
} `json:"data"`
}
if len(status) < 2 || status[1] != "200" || json.Unmarshal(parts[1], &body) != nil || !body.Success || body.Data.Version != version {
return errors.New("schema maintenance reopened direct-origin provider differs")
return errors.New("schema maintenance current loopback provider differs")
}
// /api/scripts is already public in both its exact and named nginx
// locations. A successful response proves TLS admission is restored without
// changing the IP policy or inventing edge headers. The maintenance barrier
// still precedes this location and returns 503 while admission is closed.
out, err = h.runtime.runner.Run(ctx, productionCommand{Name: "/usr/bin/curl", Args: []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*", "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/scripts"}, Env: []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}, Timeout: 7 * time.Second, OutputLimit: 32768})
parts = bytes.SplitN(out, []byte("\r\n\r\n"), 2)
if err != nil || len(parts) != 2 {
return errors.New("schema maintenance reopened direct-origin public readiness is unavailable")
}
status = strings.Fields(strings.SplitN(string(parts[0]), "\r\n", 2)[0])
var public struct {
Success bool `json:"success"`
Data []json.RawMessage `json:"data"`
}
if len(status) < 2 || status[1] != "200" || json.Unmarshal(parts[1], &public) != nil || !public.Success || public.Data == nil {
return errors.New("schema maintenance reopened direct-origin public response differs")
}
return nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,84 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
)

func TestSchemaMaintenanceRestoredAdmissionRequiresLocalVersionAndPublicTLS(t *testing.T) {
response := func(status, body string) []byte {
return []byte("HTTP/1.1 " + status + "\r\nContent-Type: application/json\r\n\r\n" + body)
}
own := response("200 OK", `{"success":true,"data":{"version":"0.2.111"}}`)
public := response("200 OK", `{"success":true,"data":[]}`)
for _, tc := range []struct {
name string
own, edge []byte
failedCall int
wantCalls int
wantError bool
}{
{name: "empty public catalog", own: own, edge: public, wantCalls: 2},
{name: "populated public catalog", own: own, edge: response("200 OK", `{"success":true,"data":[{"name":"install.sh"}]}`), wantCalls: 2},
{name: "wrong local version", own: response("200 OK", `{"success":true,"data":{"version":"0.2.110"}}`), wantCalls: 1, wantError: true},
{name: "local unsuccessful", own: response("200 OK", `{"success":false,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true},
{name: "local malformed", own: response("200 OK", `{`), wantCalls: 1, wantError: true},
{name: "local non-200", own: response("503 Service Unavailable", `{"success":true,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true},
{name: "local missing HTTP header", own: []byte(`{"success":true,"data":{"version":"0.2.111"}}`), wantCalls: 1, wantError: true},
{name: "local transport failed despite body", own: own, failedCall: 1, wantCalls: 1, wantError: true},
{name: "concealed policy rejection", own: own, edge: response("404 Not Found", `{"message":"Not Found"}`), wantCalls: 2, wantError: true},
{name: "maintenance barrier", own: own, edge: response("503 Service Unavailable", "lmm-schema-maintenance:schema-test"), wantCalls: 2, wantError: true},
{name: "edge unsuccessful", own: own, edge: response("200 OK", `{"success":false,"data":[]}`), wantCalls: 2, wantError: true},
{name: "edge missing data", own: own, edge: response("200 OK", `{"success":true}`), wantCalls: 2, wantError: true},
{name: "edge null data", own: own, edge: response("200 OK", `{"success":true,"data":null}`), wantCalls: 2, wantError: true},
{name: "edge object data", own: own, edge: response("200 OK", `{"success":true,"data":{}}`), wantCalls: 2, wantError: true},
{name: "edge string data", own: own, edge: response("200 OK", `{"success":true,"data":"[]"}`), wantCalls: 2, wantError: true},
{name: "edge non-JSON", own: own, edge: response("200 OK", "public script fixture"), wantCalls: 2, wantError: true},
{name: "edge trailing JSON", own: own, edge: response("200 OK", `{"success":true,"data":[]} {}`), wantCalls: 2, wantError: true},
{name: "edge missing HTTP header", own: own, edge: []byte(`{"success":true,"data":[]}`), wantCalls: 2, wantError: true},
{name: "edge transport failed despite body", own: own, edge: public, failedCall: 2, wantCalls: 2, wantError: true},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
h := &productionSchemaMaintenanceHost{runtime: &productionRuntime{runner: existingSchemaTestRunner{run: func(command productionCommand) ([]byte, error) {
calls++
args := []string{"--disable", "--silent", "--show-error", "--include", "--max-time", "5", "--noproxy", "*"}
body := tc.own
if calls == 1 {
args = append(args, "http://127.0.0.1:3000/api/status")
} else if calls == 2 {
args = append(args, "--connect-to", "api.lmm.best:443:127.0.0.1:443", "https://api.lmm.best/api/scripts")
body = tc.edge
} else {
t.Fatal("unexpected extra readiness command")
}
if command.Name != "/usr/bin/curl" || !slices.Equal(command.Args, args) || !slices.Equal(command.Env, []string{"PATH=/usr/bin:/bin", "LANG=C", "LC_ALL=C"}) || command.Timeout != 7*time.Second || command.OutputLimit != 32768 {
t.Fatalf("probe changed endpoint, TLS, proxy, header or resource bounds: %+v", command)
}
if calls == tc.failedCall {
return body, errors.New("transport failed")
}
return body, nil
}}}}
capsule := productionMerchantStoreCapsule{Candidate: productionReleasePackagePlan{Version: "0.2.111-1"}}
if err := h.probeRestoredAdmission(context.Background(), capsule); (err != nil) != tc.wantError || calls != tc.wantCalls {
t.Fatalf("error=%v calls=%d, want rejected=%v calls=%d", err, calls, tc.wantError, tc.wantCalls)
}
})
}
t.Run("invalid sealed version", func(t *testing.T) {
h := &productionSchemaMaintenanceHost{runtime: &productionRuntime{runner: existingSchemaTestRunner{run: func(productionCommand) ([]byte, error) {
t.Fatal("invalid sealed version reached a readiness probe")
return nil, nil
}}}}
if err := h.probeRestoredAdmission(context.Background(), productionMerchantStoreCapsule{Candidate: productionReleasePackagePlan{Version: "invalid"}}); err == nil {
t.Fatal("accepted invalid sealed provider version")
}
})
}

func TestSchemaMaintenanceAdmissionCannotAdoptOtherOwner(t *testing.T) {
original := []byte("location @lmm_api_backend {\n proxy_pass http://127.0.0.1:3000;\n}\n")
barrier, err := schemaMaintenanceAdmissionBarrier(original, "schema-test")
Expand Down
123 changes: 123 additions & 0 deletions scripts/nginx-service-errors.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -94,3 +94,126 @@ http {
await new Promise(resolve=>backend.close(resolve));fs.rmSync(dir,{recursive:true,force:true})
}
})

test('TLS public script admission survives denied status policy and remains behind the maintenance barrier', async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lmm-nginx-restored-tls-'))
const templates = path.join(root, 'packaging/common/lmm-api/edge-policy/nginx')
const requests = []
let scripts = []
const backend = http.createServer((req, res) => {
requests.push({ url: req.url, headers: req.headers })
if (req.url === '/internal/access-ip-policy') {
res.writeHead(403, { 'X-LMM-Access-Policy': 'denied' }).end()
} else if (req.url === '/internal/errors/access-policy') {
assert.equal(req.headers['x-lmm-internal-error'], 'access-policy')
assert.equal(req.headers['x-lmm-access-policy'], 'denied')
res.writeHead(404, { 'Content-Type': 'application/json' }).end('{"message":"Not Found"}')
} else if (req.url === '/api/scripts') {
res.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify({ success: true, data: scripts }))
} else if (req.url === '/api/status') {
res.writeHead(200, { 'Content-Type': 'application/json' }).end('{"success":true,"data":{"version":"0.2.111"}}')
} else {
res.writeHead(404).end()
}
})
let nginx
try {
execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-subj', '/CN=LMM fixture CA', '-addext', 'basicConstraints=critical,CA:TRUE', '-keyout', `${dir}/ca.key`, '-out', `${dir}/ca.crt`], { stdio: 'pipe' })
execFileSync('openssl', ['req', '-new', '-newkey', 'rsa:2048', '-nodes', '-subj', '/CN=api.lmm.best', '-keyout', `${dir}/tls.key`, '-out', `${dir}/tls.csr`], { stdio: 'pipe' })
fs.writeFileSync(`${dir}/tls.ext`, 'subjectAltName=DNS:api.lmm.best\nextendedKeyUsage=serverAuth\nkeyUsage=digitalSignature,keyEncipherment\n')
execFileSync('openssl', ['x509', '-req', '-in', `${dir}/tls.csr`, '-CA', `${dir}/ca.crt`, '-CAkey', `${dir}/ca.key`, '-CAcreateserial', '-days', '1', '-extfile', `${dir}/tls.ext`, '-out', `${dir}/tls.crt`], { stdio: 'pipe' })
backend.listen(0, '127.0.0.1');await once(backend, 'listening')
const reservation = net.createServer();reservation.listen(0, '127.0.0.1');await once(reservation, 'listening')
const port = reservation.address().port;await new Promise(resolve => reservation.close(resolve))
const source = fs.readFileSync(`${templates}/lmm-api-locations.conf`, 'utf8')
fs.writeFileSync(`${dir}/locations.conf`, source.replaceAll('/var/log/nginx/access.log', `${dir}/access.log`).replaceAll('/etc/nginx/lmm-api-mime.types', `${templates}/mime.types`).replaceAll('/srv/lmm-api-frontend', `${dir}/frontend`))
const maps = fs.readFileSync(`${templates}/http-map.conf`, 'utf8')
.replace(/geoip2[^]*?\n\}/, '')
.replaceAll('127.0.0.1:3000', `127.0.0.1:${backend.address().port}`)
.replace('include /etc/lmm-api/nginx/peers/*.conf;', '')
const tlsRequest = route => new Promise((resolve, reject) => {
const child = spawn('curl', ['--disable', '--silent', '--show-error', '--noproxy', '*', '--max-time', '2', '--cacert', `${dir}/ca.crt`, '--resolve', `api.lmm.best:${port}:127.0.0.1`, '--write-out', '\n%{http_code}', `https://api.lmm.best:${port}${route}`])
let stdout = '', stderr = ''
child.stdout.on('data', chunk => { stdout += chunk })
child.stderr.on('data', chunk => { stderr += chunk })
child.on('error', reject)
child.on('close', code => {
if (code !== 0) { reject(new Error(`TLS curl failed (${code}): ${stderr}`));return }
const boundary = stdout.lastIndexOf('\n')
resolve({ status: Number(stdout.slice(boundary + 1)), body: stdout.slice(0, boundary) })
})
})
const stop = async () => {
if (nginx && nginx.exitCode === null) { nginx.kill('SIGTERM');await once(nginx, 'exit') }
}
const start = async barrier => {
await stop()
fs.writeFileSync(`${dir}/nginx.conf`, `worker_processes 1;
pid ${dir}/nginx.pid;
error_log ${dir}/error.log;
events {worker_connections 64;}
http {
access_log off;
client_body_temp_path ${dir}/body;
proxy_temp_path ${dir}/proxy;
fastcgi_temp_path ${dir}/fastcgi;
uwsgi_temp_path ${dir}/uwsgi;
scgi_temp_path ${dir}/scgi;
map $host $lmm_geoip_country_code {default "";}
${maps}
server {
listen 127.0.0.1:${port} ssl;
server_name api.lmm.best;
ssl_certificate ${dir}/tls.crt;
ssl_certificate_key ${dir}/tls.key;
${barrier ? "return 503 'lmm-schema-maintenance:fixture';" : ''}
include ${templates}/lmm-api-region-policy.conf;
include ${dir}/locations.conf;
}
}`)
execFileSync('nginx', ['-t', '-p', dir, '-c', `${dir}/nginx.conf`], { stdio: 'pipe' })
nginx = spawn('nginx', ['-p', dir, '-c', `${dir}/nginx.conf`, '-g', 'daemon off; master_process off;'], { stdio: 'ignore' })
for (let attempt = 0; attempt < 50; attempt++) {
try { return await tlsRequest('/api/scripts') } catch (error) {
if (attempt === 49) throw error
await new Promise(resolve => setTimeout(resolve, 30))
}
}
}
// Local process version and TLS admission are independent observations.
const localStatus = await fetch(`http://127.0.0.1:${backend.address().port}/api/status`)
assert.equal(localStatus.status, 200)
assert.deepEqual(await localStatus.json(), { success: true, data: { version: '0.2.111' } })
let response = await start(false)
assert.equal(response.status, 200)
assert.deepEqual(JSON.parse(response.body), { success: true, data: [] })
response = await tlsRequest('/api/status')
assert.equal(response.status, 404)
assert.deepEqual(JSON.parse(response.body), { message: 'Not Found' })
assert.equal(requests.filter(request => request.url === '/api/status').length, 1, 'denied TLS status never reaches the status handler')
const policyRequest = requests.find(request => request.url === '/internal/access-ip-policy')
assert.equal(policyRequest.headers['x-original-client-ip'], '127.0.0.1')
assert.equal(policyRequest.headers['x-lmm-edge-country'], undefined)
scripts = [{ name: 'install.sh', size: 17, updated: '2026-10-12T00:00:00Z', fetches: 0 }]
const policyAttempts = requests.filter(request => request.url === '/internal/access-ip-policy').length
response = await tlsRequest('/api/scripts')
assert.equal(response.status, 200)
assert.deepEqual(JSON.parse(response.body), { success: true, data: scripts })
assert.equal(requests.filter(request => request.url === '/internal/access-ip-policy').length, policyAttempts, 'script named redirect retains its existing auth exemption')
const beforeBarrier = requests.length
response = await start(true)
assert.equal(response.status, 503)
assert.equal(response.body, 'lmm-schema-maintenance:fixture')
assert.equal((await tlsRequest('/api/status')).status, 503)
assert.equal(requests.length, beforeBarrier, 'server barrier executes before public routing and auth_request')
response = await start(false)
assert.equal(response.status, 200)
assert.deepEqual(JSON.parse(response.body), { success: true, data: scripts })
assert.equal((await tlsRequest('/api/status')).status, 404, 'restoration retains the original access policy')
} finally {
if (nginx && nginx.exitCode === null) { nginx.kill('SIGTERM');await once(nginx, 'exit') }
backend.closeAllConnections()
if (backend.listening) await new Promise(resolve => backend.close(resolve))
fs.rmSync(dir, { recursive: true, force: true })
}
})
Loading