Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 62 additions & 38 deletions .github/workflows/catalog-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ env:
PROD_API_URL: https://api.osc.earth/nemar
DEV_API_URL: https://nemar-api-dev.shirazi-10f.workers.dev
NEMAR_API_URL: https://nemar.org/api/dataexplorer/datapipeline
CLI_VERSION: '0.7.22'

jobs:
sync:
Expand All @@ -38,31 +37,62 @@ jobs:
- name: Fetch nemar.org catalog
id: fetch
run: |
# nemar.org requires GET with JSON body (non-standard)
RESPONSE=$(curl -s -X GET \
"${{ env.NEMAR_API_URL }}/records" \
-H 'Content-Type: application/json' \
-d '{"table_name":"dataexplorer_dataset","start":0,"limit":1000}')

# Validate response
SUCCESS=$(echo "$RESPONSE" | jq -r '.success // false')
if [ "$SUCCESS" != "true" ]; then
echo "::error::nemar.org API returned unsuccessful response"
echo "$RESPONSE" | jq -r '.error // "unknown error"'
exit 1
fi

TOTAL=$(echo "$RESPONSE" | jq -r '.total')
echo "Fetched $TOTAL records from nemar.org"

# Convert entries object to array, filter out empty IDs
RECORDS=$(echo "$RESPONSE" | jq '[.entries | to_entries | .[].value | select(.id != null and .id != "")]')
COUNT=$(echo "$RECORDS" | jq 'length')
echo "Parsed $COUNT valid records"

# Save to file (too large for env var)
echo "$RECORDS" > /tmp/catalog-records.json
echo "count=$COUNT" >> $GITHUB_OUTPUT
# nemar.org requires GET with JSON body (non-standard).
# Some curl versions strip the body on GET; use Python urllib as fallback.
python3 << 'PYEOF'
import json, sys, subprocess, os

# Try curl first (some versions support GET with body)
result = subprocess.run(
["curl", "-s", "-X", "GET",
f"{os.environ.get('NEMAR_API_URL', 'https://nemar.org/api/dataexplorer/datapipeline')}/records",
"-H", "Content-Type: application/json",
"-d", '{"table_name":"dataexplorer_dataset","start":0,"limit":1000}'],
capture_output=True, text=True, timeout=60
)

data = None
if result.stdout.strip():
try:
data = json.loads(result.stdout)
except json.JSONDecodeError:
print(f"curl response not valid JSON, trying http.client...")

# Fallback: use http.client which supports GET with body
if not data or not data.get("success"):
import http.client
conn = http.client.HTTPSConnection("nemar.org", timeout=60)
body = json.dumps({"table_name": "dataexplorer_dataset", "start": 0, "limit": 1000})
conn.request("GET", "/api/dataexplorer/datapipeline/records", body=body,
headers={"Content-Type": "application/json",
"User-Agent": "nemar-cli/catalog-sync"})
resp = conn.getresponse()
raw = resp.read().decode()
print(f"http.client response: status={resp.status}, length={len(raw)}")
if not raw.strip():
print(f"::error::nemar.org returned empty response (HTTP {resp.status})")
print(f"Response headers: {dict(resp.getheaders())}")
sys.exit(1)
data = json.loads(raw)
conn.close()

if not data.get("success"):
print(f"::error::nemar.org API returned unsuccessful response: {data.get('error', 'unknown')}")
sys.exit(1)

entries = data.get("entries", {})
records = [v for v in entries.values() if v.get("id")]
print(f"Fetched {data.get('total', 0)} records, {len(records)} valid")

# Wrap in {"records": [...]} for the admin endpoint
with open("/tmp/catalog-records.json", "w") as f:
json.dump({"records": records}, f)

# Write count to GITHUB_OUTPUT
import os
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
f.write(f"count={len(records)}\n")
PYEOF

- name: Push to production backend
if: fromJSON(steps.fetch.outputs.count) > 0
Expand All @@ -74,33 +104,27 @@ jobs:
-H "Authorization: Bearer $ADMIN_API_KEY" \
-H "X-CLI-Version: ${{ steps.version.outputs.cli_version }}" \
-H "Content-Type: application/json" \
-d "{\"records\": $(cat /tmp/catalog-records.json)}" \
-d @/tmp/catalog-records.json \
--max-time 120)

RESPONSE=$(cat /tmp/prod-response.json)
echo "Production sync response (HTTP $HTTP_CODE):"
echo "$RESPONSE" | jq .
echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE"

if [ "$HTTP_CODE" -ne 200 ]; then
echo "::error::Production backend returned HTTP $HTTP_CODE"
exit 1
fi

SYNCED=$(echo "$RESPONSE" | jq -r '.records_synced // 0')
INDEXED=$(echo "$RESPONSE" | jq -r '.records_indexed // 0')
SYNCED=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('records_synced',0))")
INDEXED=$(echo "$RESPONSE" | python3 -c "import sys,json; print(json.load(sys.stdin).get('records_indexed',0))")
echo "Synced: $SYNCED, Indexed: $INDEXED"

if [ "$SYNCED" -eq 0 ]; then
echo "::error::Production sync reported 0 records synced"
exit 1
fi

ERRORS=$(echo "$RESPONSE" | jq -r '.errors | length // 0')
if [ "$ERRORS" -gt 0 ]; then
echo "::warning::Sync completed with $ERRORS error(s)"
echo "$RESPONSE" | jq -r '.errors[]'
fi

- name: Push to dev backend
if: fromJSON(steps.fetch.outputs.count) > 0
env:
Expand All @@ -111,12 +135,12 @@ jobs:
-H "Authorization: Bearer $ADMIN_API_KEY" \
-H "X-CLI-Version: ${{ steps.version.outputs.cli_version }}" \
-H "Content-Type: application/json" \
-d "{\"records\": $(cat /tmp/catalog-records.json)}" \
-d @/tmp/catalog-records.json \
--max-time 120)

RESPONSE=$(cat /tmp/dev-response.json)
echo "Dev sync response (HTTP $HTTP_CODE):"
echo "$RESPONSE" | jq .
echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE"

if [ "$HTTP_CODE" -ne 200 ]; then
echo "::warning::Dev backend returned HTTP $HTTP_CODE (non-fatal)"
Expand Down
Loading