Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ README.md

docker/
!docker/entrypoint-*.sh
!docker/scripts/
.venv/
.vscode/
.ollama/
Expand Down
12 changes: 6 additions & 6 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,12 @@ REDIS_URL='redis://redis:6379/0' # Redis URL for Socket.IO message queue (e.g.,
# GenVM Configuration
GENVM_BIN="/genvm/bin"
GENVMROOT="/genvm"
# GenVM binding for Docker builds — exactly one of GENVM_TAG / GENVM_REF.
# Resolution order in CI: env override (e2e matrix / release train) wins;
# otherwise the checked-in default third_party/genvm/version applies
# (^v[0-9] = release tag -> tarball build, anything else = git ref -> source build).
GENVM_TAG="" # Release tag, e.g. v0.3.0-rc3. Mutually exclusive with GENVM_REF.
GENVM_REF="" # GenVM git ref/SHA to build from source. CI source builds support linux/amd64.
# Acquisition mode: prebuilt, source, release, or empty for auto selection.
# Auto precedence is prebuilt E2E tree > <branch>:<commit> source pin > release.
# Release is the default and uses third_party/genvm/version unless GENVM_TAG overrides it.
GENVM_SOURCE_MODE=""
GENVM_TAG="" # Exact release tag, e.g. v0.6.0-rc0. Mutually exclusive with GENVM_REF.
GENVM_REF="" # Source git ref/SHA; set mode=source, or use <branch>:<commit> for auto source mode.
GENVM_EXECUTOR_VERSION_NAME="" # Optional source-build executor label, e.g. v0.3.0-e2e-dev.
GENVM_LLM_DEBUG="0"
GENVM_WEB_DEBUG="0"
Expand Down
124 changes: 102 additions & 22 deletions .github/workflows/backend_integration_tests_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,15 @@ jobs:
# the GenVM this branch's host protocol speaks; it is flipped to the
# release tag at release cut. The e2e pipeline / release train can
# override by injecting GENVM_TAG or GENVM_REF into the environment.
# Auto-detection for the file value: ^v[0-9] => release tag (tarball
# build), anything else => git ref (source build, pinned to a SHA so
# caches invalidate exactly when genvm moves).
# Exact vX.Y.Z[-suffix] values use release assets. Source pins must be
# explicit <branch>:<commit> values so builds cannot drift with a branch.
if [[ -n "${GENVM_TAG:-}" && -n "${GENVM_REF:-}" ]]; then
echo "::error::GENVM_TAG and GENVM_REF are both set; they are mutually exclusive"
exit 1
fi
GENVM_TAG_VALUE="${GENVM_TAG:-}"
GENVM_REF_VALUE="${GENVM_REF:-}"
GENVM_SOURCE_MODE_VALUE="${GENVM_SOURCE_MODE:-}"
if [[ -z "$GENVM_TAG_VALUE" && -z "$GENVM_REF_VALUE" ]]; then
if [[ ! -f third_party/genvm/version ]]; then
echo "::error::third_party/genvm/version is missing and no GENVM_TAG/GENVM_REF env override is set"
Expand All @@ -76,21 +76,50 @@ jobs:
echo "::error::third_party/genvm/version is empty"
exit 1
fi
if [[ "$BINDING" =~ ^v[0-9] ]]; then
if [[ "$BINDING" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
GENVM_TAG_VALUE="$BINDING"
else
elif [[ "$BINDING" == *-dev ]]; then
echo "::error::Bare GenVM branch pin '$BINDING' is not supported; pin an exact vX.Y.Z release or '<branch>:<commit>'"
exit 1
elif [[ "$BINDING" =~ ^.+:[0-9a-fA-F]{7,40}$ ]]; then
GENVM_REF_VALUE="$BINDING"
GENVM_SOURCE_MODE_VALUE="source"
else
echo "::error::Invalid GenVM pin '$BINDING'; expected vX.Y.Z[-suffix] or '<branch>:<commit>'"
exit 1
fi
fi
if [[ -n "$GENVM_TAG_VALUE" && ! "$GENVM_TAG_VALUE" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
echo "::error::Invalid GENVM_TAG '$GENVM_TAG_VALUE'; expected vX.Y.Z[-suffix]"
exit 1
fi
if [[ -n "$GENVM_REF_VALUE" ]]; then
GENVM_REF_VALUE="$(gh api "repos/genlayerlabs/genvm-manager/commits/$GENVM_REF_VALUE" --jq .sha)"
if [[ "$GENVM_REF_VALUE" == *-dev ]]; then
echo "::error::Bare GenVM branch pin '$GENVM_REF_VALUE' is not supported; pin an exact vX.Y.Z release or '<branch>:<commit>'"
exit 1
elif [[ "$GENVM_REF_VALUE" =~ ^(.+):([0-9a-fA-F]{7,40})$ ]]; then
GENVM_REF_BRANCH="${BASH_REMATCH[1]}"
GENVM_REF_COMMIT="$(gh api "repos/genlayerlabs/genvm-manager/commits/${BASH_REMATCH[2]}" --jq .sha)"
GENVM_REF_VALUE="$GENVM_REF_BRANCH:$GENVM_REF_COMMIT"
elif [[ "$GENVM_SOURCE_MODE_VALUE" == "source" ]]; then
GENVM_REF_VALUE="$(gh api "repos/genlayerlabs/genvm-manager/commits/$GENVM_REF_VALUE" --jq .sha)"
else
echo "::error::GENVM_REF must be '<branch>:<commit>' unless GENVM_SOURCE_MODE=source is explicit"
exit 1
fi
GENVM_SOURCE_MODE_VALUE="source"
fi
sed -i "s|^GENVM_TAG=.*|GENVM_TAG=\"$GENVM_TAG_VALUE\"|" .env
sed -i "s|^GENVM_REF=.*|GENVM_REF=\"$GENVM_REF_VALUE\"|" .env
echo "GENVM_TAG=$GENVM_TAG_VALUE" >> "$GITHUB_ENV"
echo "GENVM_REF=$GENVM_REF_VALUE" >> "$GITHUB_ENV"
sed -i "s|^GENVM_SOURCE_MODE=.*|GENVM_SOURCE_MODE=\"$GENVM_SOURCE_MODE_VALUE\"|" .env
{
echo "GENVM_TAG=$GENVM_TAG_VALUE"
echo "GENVM_REF=$GENVM_REF_VALUE"
echo "GENVM_SOURCE_MODE=$GENVM_SOURCE_MODE_VALUE"
} >> "$GITHUB_ENV"
grep "^GENVM_TAG=" .env
grep "^GENVM_REF=" .env
grep "^GENVM_SOURCE_MODE=" .env

- name: Configure LLM provider
env:
Expand Down Expand Up @@ -150,8 +179,9 @@ jobs:
GENVM_REF_VALUE="$(sed -n 's/^GENVM_REF=["'\'']\{0,1\}\([^"'\'']*\)["'\'']\{0,1\}.*/\1/p' .env | head -n1)"
fi
if [[ -n "$GENVM_REF_VALUE" ]]; then
echo "tag=ref-$GENVM_REF_VALUE" >> "$GITHUB_OUTPUT"
echo "bake_cache=,scope=genvm-$GENVM_REF_VALUE" >> "$GITHUB_OUTPUT"
GENVM_CACHE_REF="${GENVM_REF_VALUE##*:}"
echo "binding=ref-$GENVM_CACHE_REF" >> "$GITHUB_OUTPUT"
echo "bake_cache=,scope=genvm-$GENVM_CACHE_REF" >> "$GITHUB_OUTPUT"
else
GENVM_TAG_VALUE="${GENVM_TAG:-}"
if [[ -z "$GENVM_TAG_VALUE" && -f .env ]]; then
Expand All @@ -161,10 +191,16 @@ jobs:
echo "::warning::GENVM_TAG not set (this job runs prebuilt images) — using shared 'untagged' precompile cache key that will not invalidate across GenVM upgrades"
GENVM_TAG_VALUE="untagged"
fi
echo "tag=$GENVM_TAG_VALUE" >> "$GITHUB_OUTPUT"
echo "binding=$GENVM_TAG_VALUE" >> "$GITHUB_OUTPUT"
echo "bake_cache=" >> "$GITHUB_OUTPUT"
fi

- name: Restore GenVM precompile cache
uses: actions/cache@v5
with:
path: ${{ env.GENVM_CACHE_DIR }}
key: genvm-precompile-${{ runner.os }}-${{ runner.arch }}-${{ steps.genvm.outputs.binding }}

- name: Prepare GenVM cache directory
run: |
sudo mkdir -p "$GENVM_CACHE_DIR/pc"
Expand All @@ -191,6 +227,10 @@ jobs:
consensus-worker.tags=genlayer-studio-consensus-worker:latest
load: true

- name: Precompile GenVM cache
timeout-minutes: 30
run: docker/scripts/precompile_genvm.sh

# Start services without rebuilding images; CI override provides fast healthcheck timings.
# --wait blocks until all healthchecks pass (jsonrpc /ready + consensus-worker /health).
- name: Run Docker Compose
Expand Down Expand Up @@ -253,15 +293,15 @@ jobs:
# the GenVM this branch's host protocol speaks; it is flipped to the
# release tag at release cut. The e2e pipeline / release train can
# override by injecting GENVM_TAG or GENVM_REF into the environment.
# Auto-detection for the file value: ^v[0-9] => release tag (tarball
# build), anything else => git ref (source build, pinned to a SHA so
# caches invalidate exactly when genvm moves).
# Exact vX.Y.Z[-suffix] values use release assets. Source pins must be
# explicit <branch>:<commit> values so builds cannot drift with a branch.
if [[ -n "${GENVM_TAG:-}" && -n "${GENVM_REF:-}" ]]; then
echo "::error::GENVM_TAG and GENVM_REF are both set; they are mutually exclusive"
exit 1
fi
GENVM_TAG_VALUE="${GENVM_TAG:-}"
GENVM_REF_VALUE="${GENVM_REF:-}"
GENVM_SOURCE_MODE_VALUE="${GENVM_SOURCE_MODE:-}"
if [[ -z "$GENVM_TAG_VALUE" && -z "$GENVM_REF_VALUE" ]]; then
if [[ ! -f third_party/genvm/version ]]; then
echo "::error::third_party/genvm/version is missing and no GENVM_TAG/GENVM_REF env override is set"
Expand All @@ -276,21 +316,50 @@ jobs:
echo "::error::third_party/genvm/version is empty"
exit 1
fi
if [[ "$BINDING" =~ ^v[0-9] ]]; then
if [[ "$BINDING" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
GENVM_TAG_VALUE="$BINDING"
else
elif [[ "$BINDING" == *-dev ]]; then
echo "::error::Bare GenVM branch pin '$BINDING' is not supported; pin an exact vX.Y.Z release or '<branch>:<commit>'"
exit 1
elif [[ "$BINDING" =~ ^.+:[0-9a-fA-F]{7,40}$ ]]; then
GENVM_REF_VALUE="$BINDING"
GENVM_SOURCE_MODE_VALUE="source"
else
echo "::error::Invalid GenVM pin '$BINDING'; expected vX.Y.Z[-suffix] or '<branch>:<commit>'"
exit 1
fi
fi
if [[ -n "$GENVM_TAG_VALUE" && ! "$GENVM_TAG_VALUE" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$ ]]; then
echo "::error::Invalid GENVM_TAG '$GENVM_TAG_VALUE'; expected vX.Y.Z[-suffix]"
exit 1
fi
if [[ -n "$GENVM_REF_VALUE" ]]; then
GENVM_REF_VALUE="$(gh api "repos/genlayerlabs/genvm-manager/commits/$GENVM_REF_VALUE" --jq .sha)"
if [[ "$GENVM_REF_VALUE" == *-dev ]]; then
echo "::error::Bare GenVM branch pin '$GENVM_REF_VALUE' is not supported; pin an exact vX.Y.Z release or '<branch>:<commit>'"
exit 1
elif [[ "$GENVM_REF_VALUE" =~ ^(.+):([0-9a-fA-F]{7,40})$ ]]; then
GENVM_REF_BRANCH="${BASH_REMATCH[1]}"
GENVM_REF_COMMIT="$(gh api "repos/genlayerlabs/genvm-manager/commits/${BASH_REMATCH[2]}" --jq .sha)"
GENVM_REF_VALUE="$GENVM_REF_BRANCH:$GENVM_REF_COMMIT"
elif [[ "$GENVM_SOURCE_MODE_VALUE" == "source" ]]; then
GENVM_REF_VALUE="$(gh api "repos/genlayerlabs/genvm-manager/commits/$GENVM_REF_VALUE" --jq .sha)"
else
echo "::error::GENVM_REF must be '<branch>:<commit>' unless GENVM_SOURCE_MODE=source is explicit"
exit 1
fi
GENVM_SOURCE_MODE_VALUE="source"
fi
sed -i "s|^GENVM_TAG=.*|GENVM_TAG=\"$GENVM_TAG_VALUE\"|" .env
sed -i "s|^GENVM_REF=.*|GENVM_REF=\"$GENVM_REF_VALUE\"|" .env
echo "GENVM_TAG=$GENVM_TAG_VALUE" >> "$GITHUB_ENV"
echo "GENVM_REF=$GENVM_REF_VALUE" >> "$GITHUB_ENV"
sed -i "s|^GENVM_SOURCE_MODE=.*|GENVM_SOURCE_MODE=\"$GENVM_SOURCE_MODE_VALUE\"|" .env
{
echo "GENVM_TAG=$GENVM_TAG_VALUE"
echo "GENVM_REF=$GENVM_REF_VALUE"
echo "GENVM_SOURCE_MODE=$GENVM_SOURCE_MODE_VALUE"
} >> "$GITHUB_ENV"
grep "^GENVM_TAG=" .env
grep "^GENVM_REF=" .env
grep "^GENVM_SOURCE_MODE=" .env

# TODO: we should also add also heuristai and anthropic keys to the e2e tests and test all providers

Expand Down Expand Up @@ -353,8 +422,9 @@ jobs:
GENVM_REF_VALUE="$(sed -n 's/^GENVM_REF=["'\'']\{0,1\}\([^"'\'']*\)["'\'']\{0,1\}.*/\1/p' .env | head -n1)"
fi
if [[ -n "$GENVM_REF_VALUE" ]]; then
echo "tag=ref-$GENVM_REF_VALUE" >> "$GITHUB_OUTPUT"
echo "bake_cache=,scope=genvm-$GENVM_REF_VALUE" >> "$GITHUB_OUTPUT"
GENVM_CACHE_REF="${GENVM_REF_VALUE##*:}"
echo "binding=ref-$GENVM_CACHE_REF" >> "$GITHUB_OUTPUT"
echo "bake_cache=,scope=genvm-$GENVM_CACHE_REF" >> "$GITHUB_OUTPUT"
else
GENVM_TAG_VALUE="${GENVM_TAG:-}"
if [[ -z "$GENVM_TAG_VALUE" && -f .env ]]; then
Expand All @@ -364,10 +434,16 @@ jobs:
echo "::warning::GENVM_TAG not set (this job runs prebuilt images) — using shared 'untagged' precompile cache key that will not invalidate across GenVM upgrades"
GENVM_TAG_VALUE="untagged"
fi
echo "tag=$GENVM_TAG_VALUE" >> "$GITHUB_OUTPUT"
echo "binding=$GENVM_TAG_VALUE" >> "$GITHUB_OUTPUT"
echo "bake_cache=" >> "$GITHUB_OUTPUT"
fi

- name: Restore GenVM precompile cache
uses: actions/cache@v5
with:
path: ${{ env.GENVM_CACHE_DIR }}
key: genvm-precompile-${{ runner.os }}-${{ runner.arch }}-${{ steps.genvm.outputs.binding }}

- name: Prepare GenVM cache directory
run: |
sudo mkdir -p "$GENVM_CACHE_DIR/pc"
Expand All @@ -393,6 +469,10 @@ jobs:
consensus-worker.tags=genlayer-studio-consensus-worker:latest
load: true

- name: Precompile GenVM cache
timeout-minutes: 30
run: docker/scripts/precompile_genvm.sh

# Start services without rebuilding images; CI override provides fast healthcheck timings.
# --wait blocks until all healthchecks pass (jsonrpc /ready + consensus-worker /health).
- name: Run Docker Compose
Expand Down
Loading
Loading