diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..ce212be4 --- /dev/null +++ b/.env.example @@ -0,0 +1,35 @@ +# Copy this file to .env in the project root before running sandbox examples +# or sandbox integration tests. + +# Sandbox API key auth. +QINIU_SANDBOX_API_KEY= + +# Optional custom endpoint. +# QINIU_SANDBOX_ENDPOINT is the preferred name; QINIU_SANDBOX_API_URL and +# E2B_API_URL are also accepted by the SDK for compatibility. +QINIU_SANDBOX_ENDPOINT= + +# Optional template alias or ID. Defaults to base. +QINIU_SANDBOX_TEMPLATE=base + +# Required only for injection-rule and Kodo resource examples/tests. +QINIU_SANDBOX_ACCESS_KEY= +QINIU_SANDBOX_SECRET_KEY= + +# Optional Git remote examples. +GIT_REPO_URL= +GIT_USERNAME= +GIT_PASSWORD= + +# Optional Git repository resource example. +GITHUB_TOKEN= +QINIU_SANDBOX_GIT_MOUNT_PATH=/workspace/repo + +# Optional Kodo resource example. +QINIU_SANDBOX_KODO_BUCKET= +QINIU_SANDBOX_KODO_MOUNT_PATH=/workspace/kodo +QINIU_SANDBOX_KODO_PREFIX= + +# Optional request injection examples. +QINIU_SANDBOX_HTTP_INJECTION_TOKEN=real_token +QINIU_SANDBOX_OPENAI_API_KEY= diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index fca8936e..8237820c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,5 +1,11 @@ -on: [push] name: Run Test Cases +on: + - push + - pull_request + +permissions: + contents: read + jobs: test: strategy: @@ -29,6 +35,9 @@ jobs: run: | MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) + if ! python -m pip --version >/dev/null 2>&1; then + conda install -y pip + fi # reinstall pip by some python(<3.7) not compatible if ! [[ $MAJOR -ge 3 && $MINOR -ge 7 ]]; then cd /tmp @@ -49,7 +58,31 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" + - name: Run public cases + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + shell: bash -el {0} + env: + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + run: | + flake8 --show-source --max-line-length=160 ./qiniu + python -m pytest \ + tests/cases/test_auth.py \ + tests/cases/test_http/test_endpoint.py \ + tests/cases/test_http/test_endpoints_retry_policy.py \ + tests/cases/test_http/test_middleware.py \ + tests/cases/test_http/test_qiniu_conf.py \ + tests/cases/test_http/test_region.py \ + tests/cases/test_http/test_regions_retry_policy.py \ + tests/cases/test_http/test_resp.py \ + tests/cases/test_http/test_single_flight.py \ + tests/cases/test_retry \ + tests/cases/test_utils.py \ + tests/cases/test_services/test_sandbox/test_client.py \ + tests/cases/test_services/test_sandbox/test_config.py \ + tests/cases/test_services/test_sandbox/test_envd.py \ + tests/cases/test_services/test_sandbox/test_example_config.py - name: Run cases + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} shell: bash -el {0} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} @@ -75,6 +108,7 @@ jobs: run: | cat py-mock-server.log - name: Upload results to Codecov + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -106,6 +140,10 @@ jobs: PYTHON_VERSION: ${{ matrix.python_version }} PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip run: | + $null = python -m pip --version + if ($LASTEXITCODE -ne 0) { + conda install -y pip + } # reinstall pip by some python(<3.7) not compatible $pyversion = [Version]"$ENV:PYTHON_VERSION" if ($pyversion -lt [Version]"3.7") { @@ -117,7 +155,47 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" + - name: Run public cases + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + env: + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + PYTHONPATH: "$PYTHONPATH:." + run: | + Write-Host "======== Setup Mock Server =========" + conda create -y -n mock-server python=3.10 + conda activate mock-server + python --version + $processOptions = @{ + FilePath="python" + ArgumentList="tests\mock_server\main.py", "--port", "9000" + PassThru=$true + RedirectStandardOutput="py-mock-server.log" + } + $mocksrvp = Start-Process @processOptions + $mocksrvp.Id | Out-File -FilePath "mock-server.pid" + conda deactivate + Sleep 3 + Write-Host "======== Running Public Test =========" + python --version + flake8 --show-source --max-line-length=160 ./qiniu + python -m pytest ` + tests/cases/test_auth.py ` + tests/cases/test_http/test_endpoint.py ` + tests/cases/test_http/test_endpoints_retry_policy.py ` + tests/cases/test_http/test_middleware.py ` + tests/cases/test_http/test_qiniu_conf.py ` + tests/cases/test_http/test_region.py ` + tests/cases/test_http/test_regions_retry_policy.py ` + tests/cases/test_http/test_resp.py ` + tests/cases/test_http/test_single_flight.py ` + tests/cases/test_retry ` + tests/cases/test_utils.py ` + tests/cases/test_services/test_sandbox/test_client.py ` + tests/cases/test_services/test_sandbox/test_config.py ` + tests/cases/test_services/test_sandbox/test_envd.py ` + tests/cases/test_services/test_sandbox/test_example_config.py - name: Run cases + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} @@ -148,6 +226,7 @@ jobs: python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml - name: Post Setup mock server if: ${{ always() }} + continue-on-error: true run: | Try { $mocksrvpid = Get-Content -Path "mock-server.pid" @@ -158,9 +237,11 @@ jobs: } - name: Print mock server log if: ${{ failure() }} + continue-on-error: true run: | Get-Content -Path "py-mock-server.log" | Write-Host - name: Upload results to Codecov + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 41be9369..c17651eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store *.swp *.pyc +.env *.py[cod] diff --git a/examples/sandbox_commands.py b/examples/sandbox_commands.py new file mode 100644 index 00000000..af3fcb1f --- /dev/null +++ b/examples/sandbox_commands.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_commands'}) + try: + stdout = [] + result = sandbox.commands.run( + 'printf command-callback', + on_stdout=stdout.append, + ) + print('run:', result.stdout) + print('callback:', ''.join(stdout)) + + handle = sandbox.commands.start( + 'read line; echo "stdin:$line"', + stdin=True, + tag='qiniu-python-sdk-commands', + timeout=30, + ) + print('started pid:', handle.pid) + print('processes:', [ + item.pid for item in sandbox.commands.list() + if item.pid == handle.pid or item.tag == 'qiniu-python-sdk-commands' + ]) + try: + sandbox.commands.send_stdin(handle.pid, 'hello\n') + sandbox.commands.close_stdin(handle.pid) + print('stdin result:', handle.wait().stdout.strip()) + except SandboxError as err: + print('stdin skipped:', err) + try: + sandbox.commands.kill(handle.pid) + except SandboxError: + pass + + sleeper = sandbox.commands.run( + 'sleep 30', + background=True, + tag='qiniu-python-sdk-kill', + ) + print('background pid:', sleeper.pid) + try: + connected = sandbox.commands.connect(sleeper.pid) + print('connect running pid:', connected.pid) + except SandboxError as err: + print('connect running process skipped:', err) + try: + print('kill background:', sandbox.commands.kill(sleeper.pid)) + except SandboxError as err: + print('kill background skipped:', err) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_common.py b/examples/sandbox_common.py new file mode 100644 index 00000000..b5a3f94b --- /dev/null +++ b/examples/sandbox_common.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import os + +from qiniu.services.sandbox import Sandbox +from qiniu.services.sandbox.config import ( + env, + load_dotenv_if_present, + required_env, + sandbox_client, + sandbox_template, +) + + +__all__ = [ + 'cleanup_sandbox', + 'create_sandbox', + 'env', + 'load_example_env', + 'required_env', + 'run_example', + 'sandbox_client', + 'sandbox_template', +] + + +def load_example_env(): + root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + load_dotenv_if_present( + os.path.join(root, '.env'), + os.path.join(os.getcwd(), '.env'), + ) + + +def create_sandbox(**options): + client = options.pop('client', None) or sandbox_client() + template = options.pop('template', sandbox_template()) + options.setdefault('timeout', 300) + sandbox = Sandbox.create(template, client=client, **options) + print('Sandbox created:', sandbox.sandbox_id) + return sandbox + + +def cleanup_sandbox(sandbox): + if sandbox is None: + return + try: + sandbox.kill() + print('Sandbox killed:', sandbox.sandbox_id) + except Exception as err: + print('Failed to kill sandbox:', sandbox.sandbox_id, err) + + +def run_example(fn): + load_example_env() + try: + fn() + except Exception as err: + print(err) + raise diff --git a/examples/sandbox_connect.py b/examples/sandbox_connect.py new file mode 100644 index 00000000..1c85a0f0 --- /dev/null +++ b/examples/sandbox_connect.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import Sandbox +from sandbox_common import ( + cleanup_sandbox, + create_sandbox, + run_example, + sandbox_client, +) + + +def main(): + client = sandbox_client() + sandbox = create_sandbox( + client=client, + timeout=300, + metadata={'example': 'sandbox_connect'}, + ) + try: + items = Sandbox.list(client=client, limit=10).next_items() + print('list:', [item.sandbox_id for item in items]) + + connected = Sandbox.connect( + sandbox.sandbox_id, + client=client, + timeout=300, + ) + print('connected:', connected.sandbox_id) + print('envd:', connected.envd_url()) + print('uptime:', connected.commands.run('uptime').stdout) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_create.py b/examples/sandbox_create.py new file mode 100644 index 00000000..4d4c37d8 --- /dev/null +++ b/examples/sandbox_create.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox( + metadata={'example': 'sandbox_create'}, + envs={'HELLO': 'qiniu'}, + ) + try: + print('sandbox:', sandbox.sandbox_id) + result = sandbox.commands.run('printf "$HELLO"') + print('stdout:', result.stdout) + + sandbox.files.write('/tmp/qiniu.txt', 'hello from qiniu sandbox') + print('file:', sandbox.files.read_text('/tmp/qiniu.txt')) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_filesystem.py b/examples/sandbox_filesystem.py new file mode 100644 index 00000000..b7f10e01 --- /dev/null +++ b/examples/sandbox_filesystem.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from io import BytesIO + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_filesystem'}) + base = '/tmp/qiniu-python-sdk-files' + renamed_path = base + '/renamed.txt' + try: + sandbox.files.make_dir(base) + print('exists base:', sandbox.files.exists(base)) + + text_info = sandbox.files.write(base + '/hello.txt', 'hello') + print('write text:', text_info.path if text_info else 'ok') + print('read text:', sandbox.files.read_text(base + '/hello.txt')) + + try: + sandbox.files.write( + base + '/bytes.bin', + BytesIO(b'\x00qiniu\xff'), + use_octet_stream=True, + ) + except SandboxError as err: + print('octet-stream write skipped:', err) + sandbox.files.write(base + '/bytes.bin', BytesIO(b'\x00qiniu\xff')) + print('read bytes:', list(sandbox.files.read( + base + '/bytes.bin', + format='bytes', + ))) + + sandbox.files.write_files([ + {'path': base + '/a.txt', 'data': 'a'}, + {'path': base + '/b.txt', 'data': 'b'}, + ]) + print('list:', [entry.name for entry in sandbox.files.list(base)]) + + info = sandbox.files.get_info(base + '/hello.txt') + print('stat:', info.name, info.type, info.size) + + chunks = sandbox.files.read( + base + '/hello.txt', + format='stream', + chunk_size=2, + ) + print('stream:', b''.join(chunks).decode('utf-8')) + + moved = sandbox.files.rename(base + '/hello.txt', renamed_path) + print('renamed:', moved.path) + sandbox.files.remove(renamed_path) + print('exists renamed:', sandbox.files.exists(renamed_path)) + finally: + try: + sandbox.files.remove(base) + except Exception: + pass + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py new file mode 100644 index 00000000..8cc4c5be --- /dev/null +++ b/examples/sandbox_git.py @@ -0,0 +1,234 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +import os +import time + +from qiniu.services.sandbox import CommandExitError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + +try: + from urllib.parse import quote, urlparse, urlunparse +except ImportError: + from urllib import quote + from urlparse import urlparse, urlunparse + + +def is_git_ok(result): + message = result.stderr or result.stdout or result.error or '' + return result.exit_code == 0 or ( + result.exit_code == -1 and + not result.error and + 'fatal:' not in message.lower() and + 'error:' not in message.lower() + ) + + +def assert_git_ok(step, result): + message = result.stderr or result.stdout or result.error or '' + if not is_git_ok(result): + raise RuntimeError( + '{0} failed with exit {1}: {2}'.format( + step, + result.exit_code, + message, + ) + ) + print(step + ':', result.exit_code) + return result + + +def is_retryable_git_network_error(result): + message = (result.stderr or result.stdout or result.error or '').lower() + return result.exit_code != 0 and ( + 'gnutls' in message or + 'tls connection' in message or + 'unable to access' in message or + 'the remote end hung up unexpectedly' in message + ) + + +def assert_git_network_ok(step, run, attempts=5): + result = None + for attempt in range(attempts): + result = run() + if result.exit_code == 0: + print(step + ':', result.exit_code) + return result + if ( + not is_retryable_git_network_error(result) or + attempt == attempts - 1): + return assert_git_ok(step, result) + print('{0}: retry {1}/{2}'.format(step, attempt + 2, attempts)) + time.sleep(attempt + 1) + + +def remote_git_config(): + repo_url = os.getenv('GIT_REPO_URL') + username = os.getenv('GIT_USERNAME') + password = os.getenv('GIT_PASSWORD') or os.getenv('GITHUB_TOKEN') + if password and not username: + username = 'x-access-token' + if not repo_url or not username or not password: + return None + return repo_url, username, password + + +def credentialed_repo_url(repo_url, username, password): + parsed = urlparse(repo_url) + if parsed.scheme not in ('http', 'https'): + raise RuntimeError( + 'remote git push: only http(s) URLs support credentials') + netloc = parsed.netloc.split('@', 1)[-1] + auth = '{0}:{1}@'.format( + quote(username, safe=''), + quote(password, safe=''), + ) + return urlunparse(( + parsed.scheme, + auth + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + )) + + +def clone_remote_with_credentials(sandbox, repo_url, repo_path, + username, password, attempts=5): + auth_url = credentialed_repo_url(repo_url, username, password) + result = None + for attempt in range(attempts): + sandbox.commands.run('rm -rf {0}'.format(repo_path)) + result = sandbox.git.clone( + auth_url, + repo_path, + depth=1, + timeout=180, + request_timeout=180, + ) + if is_git_ok(result): + print('git clone remote:', result.exit_code) + return result + if ( + not is_retryable_git_network_error(result) or + attempt == attempts - 1): + raise RuntimeError( + 'git clone remote failed with exit {0}'.format( + result.exit_code, + ) + ) + print('git clone remote: retry {0}/{1}'.format( + attempt + 2, + attempts, + )) + time.sleep(attempt + 1) + + +def run_remote_push_demo(sandbox): + config = remote_git_config() + if not config: + print('remote git push: skipped, GIT_REPO_URL/Git credentials missing') + return + + repo_url, username, password = config + branch = 'python-sdk-example-{0}'.format(int(time.time() * 1000)) + repo_path = '/tmp/qiniu-python-sdk-git/remote' + + assert_git_ok( + 'git http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + + try: + clone_remote_with_credentials( + sandbox, + repo_url, + repo_path, + username, + password, + ) + except Exception as err: + print('remote git push skipped:', err) + return + assert_git_ok( + 'reset remote url', + sandbox.git.remote_add(repo_path, 'origin', repo_url, overwrite=True), + ) + assert_git_ok( + 'configure remote user', + sandbox.git.configure_user( + repo_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_git_ok( + 'checkout remote branch', + sandbox.git.checkout_branch(repo_path, branch, create=True), + ) + sandbox.files.write( + repo_path + '/python-sdk-example.txt', + 'qiniu-python-sdk example push {0}\n'.format(branch), + ) + assert_git_ok('git add remote', sandbox.git.add(repo_path, all=True)) + assert_git_ok( + 'git commit remote', + sandbox.git.commit(repo_path, 'test: qiniu python sdk example push'), + ) + assert_git_network_ok( + 'git push remote', + lambda: sandbox.git.push( + repo_path, + 'origin', + 'HEAD:refs/heads/{0}'.format(branch), + username=username, + password=password, + timeout=180, + request_timeout=180, + ), + ) + print('remote git branch:', branch) + + +def main(): + repo_path = '/tmp/qiniu-python-sdk-git/repo' + clone_path = '/tmp/qiniu-python-sdk-git/clone' + sandbox = create_sandbox(timeout=300) + try: + sandbox.commands.run( + 'mkdir -p /tmp/qiniu-python-sdk-git/repo' + ) + assert_git_ok( + 'git init', + sandbox.git.init(repo_path, initial_branch='main'), + ) + assert_git_ok( + 'configure user', + sandbox.git.configure_user( + repo_path, + 'Sandbox Demo', + 'sandbox-demo@example.com', + ), + ) + sandbox.files.write(repo_path + '/README.md', '# sandbox git demo\n') + assert_git_ok('git add', sandbox.git.add(repo_path, all=True)) + assert_git_ok( + 'git commit', + sandbox.git.commit(repo_path, 'feat: initial commit'), + ) + assert_git_ok('git clone', sandbox.git.clone(repo_path, clone_path)) + try: + status = sandbox.git.status(clone_path) + print('clone status clean:', status.is_clean) + except CommandExitError as err: + print('git status skipped:', err) + run_remote_push_demo(sandbox) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_injection_rules.py b/examples/sandbox_injection_rules.py new file mode 100644 index 00000000..ac438c73 --- /dev/null +++ b/examples/sandbox_injection_rules.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import run_example, sandbox_client + + +def main(): + client = sandbox_client() + rule_id = None + try: + rule = client.create_injection_rule( + name='python-sdk-example', + injection={ + 'type': 'http', + 'base_url': 'https://api.example.com', + 'headers': {'X-From-Sandbox': 'qiniu-python-sdk'}, + }, + ) + print('created:', rule) + rules = client.list_injection_rules() + print('rules count:', len(rules)) + rule_id = rule.get('id') or rule.get('ruleID') + print('get:', client.get_injection_rule(rule_id)) + print('updated:', client.update_injection_rule( + rule_id, + name='python-sdk-example-updated', + injection={ + 'type': 'http', + 'base_url': 'https://api.example.com', + 'headers': {'X-Updated-From-Sandbox': 'qiniu-python-sdk'}, + }, + )) + finally: + if rule_id: + client.delete_injection_rule(rule_id) + print('deleted:', rule_id) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_lifecycle.py b/examples/sandbox_lifecycle.py new file mode 100644 index 00000000..9df5335a --- /dev/null +++ b/examples/sandbox_lifecycle.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def print_optional(label, fn): + try: + value = fn() + if label == 'logs' and isinstance(value, dict): + print(label + ':', len(value.get('logs') or [])) + else: + print(label + ':', value) + except SandboxError as err: + print(label + ' skipped:', err) + + +def main(): + sandbox = create_sandbox(timeout=300) + try: + print('created:', sandbox.sandbox_id) + sandbox.set_timeout(600) + sandbox.refresh(duration=600) + info = sandbox.get_info() + print('state:', info.get('state')) + print_optional('metrics', sandbox.get_metrics) + print_optional('logs', sandbox.get_logs) + sandbox.pause() + print('paused:', sandbox.sandbox_id) + sandbox.resume(timeout=300) + print('resumed:', sandbox.sandbox_id) + try: + print('network:', sandbox.update_network({'allowInternet': True})) + except Exception as err: + print('update network skipped:', err) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_observability.py b/examples/sandbox_observability.py new file mode 100644 index 00000000..23355550 --- /dev/null +++ b/examples/sandbox_observability.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def print_optional(label, fn): + try: + value = fn() + if label == 'logs' and isinstance(value, dict): + print(label + ':', len(value.get('logs') or [])) + else: + print(label + ':', value) + except SandboxError as err: + print(label + ' skipped:', err) + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_observability'}) + try: + sandbox.wait_for_ready(timeout=60) + print('running:', sandbox.is_running(request_timeout=3)) + print('envd url:', sandbox.envd_url()) + print('envd host:', sandbox.get_host(49983)) + print('mcp url:', sandbox.get_mcp_url()) + print('mcp token present:', bool(sandbox.get_mcp_token())) + print('download url present:', bool(sandbox.download_url('/tmp/demo.txt'))) + print('upload url present:', bool(sandbox.upload_url('/tmp/demo.txt'))) + print_optional('metrics', lambda: sandbox.get_metrics()) + print_optional('logs', lambda: sandbox.get_logs()) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py new file mode 100644 index 00000000..d77f75da --- /dev/null +++ b/examples/sandbox_resources.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +import time + +from qiniu.services.sandbox import ( + GitRepositoryResource, + KodoResource, + SandboxError, +) +from qiniu.services.sandbox.util import shell_quote +from sandbox_common import ( + cleanup_sandbox, + create_sandbox, + env, + run_example, +) + +try: + from urllib.parse import quote, urlparse, urlunparse +except ImportError: + from urllib import quote + from urlparse import urlparse, urlunparse + + +def git_ok(result): + message = result.stderr or result.stdout or result.error or '' + return result.exit_code == 0 or ( + result.exit_code == -1 and + not result.error and + 'fatal:' not in message.lower() and + 'error:' not in message.lower() + ) + + +def assert_git_ok(step, result): + if not git_ok(result): + raise RuntimeError( + '{0} failed with exit {1}: {2}'.format( + step, + result.exit_code, + result.stderr or result.stdout or result.error, + ) + ) + print(step + ':', result.exit_code) + return result + + +def credentialed_repo_url(repo_url, username, password): + parsed = urlparse(repo_url) + if parsed.scheme not in ('http', 'https'): + raise RuntimeError( + 'GitHub repository resource push requires an http(s) URL') + netloc = parsed.netloc.split('@', 1)[-1] + auth = '{0}:{1}@'.format( + quote(username, safe=''), + quote(password, safe=''), + ) + return urlunparse(( + parsed.scheme, + auth + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + )) + + +def push_branch_with_credentials(sandbox, repo_path, repo_url, branch, + username, password): + auth_url = credentialed_repo_url(repo_url, username, password) + command = ( + 'cd {repo_path} && ' + 'git push {auth_url} HEAD:refs/heads/{branch} && ' + 'git ls-remote --heads {auth_url} {branch}' + ).format( + repo_path=shell_quote(repo_path), + auth_url=shell_quote(auth_url), + branch=shell_quote(branch), + ) + result = sandbox.commands.run( + command, + timeout=180, + request_timeout=180, + ) + if branch not in (result.stdout or ''): + raise RuntimeError( + 'git resource push verification failed with exit {0}'.format( + result.exit_code, + ) + ) + print('git resource push:', result.exit_code) + return result + + +def is_optional_resource_error(err): + status_code = getattr(err, 'status_code', None) + if status_code in (404, 408, 409, 429, 500, 502, 503, 504): + return True + message = str(err).lower() + return ( + 'timeout' in message or + 'timed out' in message or + 'not found' in message or + 'temporarily unavailable' in message + ) + + +def run_git_resource_example(): + repo_url = env('GIT_REPO_URL') + repo_token = env('GITHUB_TOKEN') + repo_username = env('GIT_USERNAME') or 'x-access-token' + repo_password = env('GIT_PASSWORD') or repo_token + if not repo_url or not repo_token: + print( + 'Skip GitHub repository resource: ' + 'set GIT_REPO_URL and GITHUB_TOKEN.' + ) + return + + mount_path = env('QINIU_SANDBOX_GIT_MOUNT_PATH', '/workspace/repo') + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_git'}, + resources=[ + GitRepositoryResource( + url=repo_url, + mount_path=mount_path, + authorization_token=repo_token, + ) + ], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip GitHub repository resource:', err) + return + raise + try: + print('Git resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + shell_quote(mount_path) + )).stdout) + branch = 'python-sdk-example-{0}'.format(int(time.time() * 1000)) + assert_git_ok( + 'git resource http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + assert_git_ok( + 'git resource configure user', + sandbox.git.configure_user( + mount_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_git_ok( + 'git resource checkout branch', + sandbox.git.checkout_branch(mount_path, branch, create=True), + ) + sandbox.files.write( + mount_path + '/python-sdk-resource-example.txt', + 'qiniu-python-sdk resource push {0}\n'.format(branch), + ) + assert_git_ok( + 'git resource add', + sandbox.git.add(mount_path, all=True), + ) + assert_git_ok( + 'git resource commit', + sandbox.git.commit( + mount_path, + 'test: qiniu python sdk resource example push', + ), + ) + push_branch_with_credentials( + sandbox, + mount_path, + repo_url, + branch, + repo_username, + repo_password, + ) + print('git resource branch:', branch) + finally: + cleanup_sandbox(sandbox) + + +def run_kodo_resource_example(): + bucket = env('QINIU_SANDBOX_KODO_BUCKET') + if not bucket: + print('Skip Kodo resource: set QINIU_SANDBOX_KODO_BUCKET.') + return + + mount_path = env('QINIU_SANDBOX_KODO_MOUNT_PATH', '/mnt/kodo') + resource = KodoResource( + bucket=bucket, + mount_path=mount_path, + prefix=env('QINIU_SANDBOX_KODO_PREFIX') or None, + ) + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_kodo'}, + resources=[resource], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip Kodo resource:', err) + return + raise + try: + print('Kodo resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + shell_quote(mount_path) + )).stdout) + test_path = mount_path + '/qiniu-python-sdk-resource-test.txt' + result = sandbox.commands.run( + 'sh -c {0}'.format(shell_quote( + 'echo qiniu-python-sdk > {0} && cat {0}'.format( + shell_quote(test_path) + ) + )) + ) + if ( + result.exit_code != 0 and + 'qiniu-python-sdk' not in (result.stdout or '')): + raise RuntimeError(result.stderr or result.stdout) + print('Kodo write/read:', result.stdout.strip()) + sandbox.commands.run('rm -f {0}'.format(shell_quote(test_path))) + finally: + cleanup_sandbox(sandbox) + + read_only = KodoResource( + bucket=bucket, + mount_path=mount_path, + prefix=env('QINIU_SANDBOX_KODO_PREFIX') or None, + read_only=True, + ) + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_kodo_read_only'}, + resources=[read_only], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip read-only Kodo resource:', err) + return + raise + try: + print('Read-only Kodo resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + shell_quote(mount_path) + )).stdout) + finally: + cleanup_sandbox(sandbox) + + +def main(): + run_git_resource_example() + run_kodo_resource_example() + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_runtime.py b/examples/sandbox_runtime.py new file mode 100644 index 00000000..1a211cfc --- /dev/null +++ b/examples/sandbox_runtime.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import time + +from qiniu.services.sandbox import PtySize, SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_runtime'}) + watcher = None + try: + stdout = [] + result = sandbox.commands.run( + 'printf runtime', + on_stdout=stdout.append, + ) + print('command stdout:', result.stdout) + print('callback stdout:', ''.join(stdout)) + + sandbox.files.write_files([ + {'path': '/tmp/qiniu-runtime-a.txt', 'data': 'a'}, + {'path': '/tmp/qiniu-runtime-b.txt', 'data': 'b'}, + ]) + print('write_files: ok') + + try: + watcher = sandbox.files.watch_dir('/tmp') + sandbox.files.write('/tmp/qiniu-runtime-watch.txt', 'watch') + time.sleep(1) + for event in watcher.get_new_events(): + print('watch event:', event.name, event.type) + except Exception as err: + print('watch_dir skipped:', err) + finally: + if watcher is not None: + try: + watcher.stop() + except SandboxError as err: + print('failed to stop watcher:', err) + + try: + pty = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + print('pty pid:', pty.pid) + sandbox.pty.send_stdin(pty.pid, 'echo pty-ready\n') + sandbox.pty.send_stdin(pty.pid, 'exit\n') + pty_result = pty.wait() + print('pty stdout:', pty_result.stdout) + except SandboxError as err: + print('pty skipped:', err) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_template_lifecycle.py b/examples/sandbox_template_lifecycle.py new file mode 100644 index 00000000..5866cc22 --- /dev/null +++ b/examples/sandbox_template_lifecycle.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +import time + +from qiniu.services.sandbox import ( + SandboxError, + Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) +from sandbox_common import run_example, sandbox_client + + +def pick(data, *keys): + for key in keys: + if isinstance(data, dict) and data.get(key) is not None: + return data.get(key) + return None + + +def main(): + client = sandbox_client() + name = 'qiniu-python-sdk-example-{0}'.format(int(time.time() * 1000)) + target = '{0}:v1'.format(name) + template_id = None + + builder_demo = ( + Template() + .from_template('base') + .copy('README.md', '/tmp/README.md', chmod='0644') + .set_ready_cmd(wait_for_file('/tmp/README.md')) + ) + print('builder demo:', builder_demo.to_dict()) + + config_demo = ( + Template() + .from_image('python:3.11') + .run_cmd(['python', '--version']) + .set_env('PYTHONUNBUFFERED', '1') + .set_start_cmd( + 'python -m http.server 8000', + ready_cmd=wait_for_port(8000), + ) + .set_ready_cmd(wait_for_timeout(1000)) + .to_dict() + ) + print('config demo:', config_demo) + print('ready helpers:', [ + wait_for_file('/tmp/README.md').get_cmd(), + wait_for_process('python').get_cmd(), + wait_for_url('http://127.0.0.1:8000').get_cmd(), + ]) + + try: + created = client.create_template(name=target) + template_id = pick( + created, + 'templateID', + 'templateId', + 'template_id', + 'id', + ) or name + build_id = pick(created, 'buildID', 'buildId', 'build_id') + print('created:', template_id) + + print('default templates:', len(client.list_default_templates())) + print('templates:', len(client.list_templates())) + print('get:', client.get_template(template_id)) + + updated = client.update_template( + template_id, + description='Created by qiniu-python-sdk example', + ) + print('updated:', updated) + + if build_id: + print('build status:', client.get_template_build_status( + template_id, + build_id, + )) + print('build logs:', client.get_template_build_logs( + template_id, + build_id, + )) + client.start_template_build( + template_id, + build_id, + fromTemplate='base', + ) + print('build started:', build_id) + final_build = client.wait_for_build( + template_id, + build_id, + interval=3, + timeout=120, + ) + print( + 'wait build:', + final_build.get('status'), + 'logs:', + len(final_build.get('logs') or []), + ) + + client.assign_template_tags( + target=target, + tags=['example'], + ) + print('tagged:', template_id) + client.delete_template_tags(name=name, tags=['example']) + print('untagged:', template_id) + except SandboxError as err: + print('template lifecycle skipped:', err) + finally: + if template_id: + try: + client.delete_template(template_id) + print('deleted:', template_id) + except SandboxError as err: + print('delete template skipped:', err) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_templates.py b/examples/sandbox_templates.py new file mode 100644 index 00000000..aabaef46 --- /dev/null +++ b/examples/sandbox_templates.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +import time + +from qiniu.services.sandbox import SandboxError +from qiniu.services.sandbox import Template +from sandbox_common import run_example, sandbox_client + + +def main(): + client = sandbox_client() + template_id = None + template = ( + Template() + .from_image('python:3.11') + .run_cmd('python --version') + .set_env('PYTHONUNBUFFERED', '1') + ) + + try: + created = client.create_template( + name='qiniu-python-sdk-example-{0}'.format( + int(time.time() * 1000), + ), + buildConfig=template.to_dict(), + ) + template_id = ( + created.get('templateID') or + created.get('templateId') or + created.get('id') + ) + print('template:', created) + finally: + if template_id: + try: + client.delete_template(template_id) + print('deleted:', template_id) + except SandboxError as err: + print('delete template skipped:', err) + + +if __name__ == '__main__': + run_example(main) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 85d8ce35..a2090ab7 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -26,6 +26,31 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient +from .services.sandbox import ( + EntryInfo, + FileType, + GitAuthException, + GitBranches, + GitFileStatus, + FilesystemEventType, + GitRepositoryResource, + GitStatus, + GitUpstreamException, + KodoResource, + PtySize, + ReadyCmd, + Sandbox, + SandboxClient, + Template, + WatchHandle, + WriteEntry, + WriteInfo, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, decode_entry, canonical_mime_header_key diff --git a/qiniu/services/sandbox/__init__.py b/qiniu/services/sandbox/__init__.py new file mode 100644 index 00000000..7ce0f9d8 --- /dev/null +++ b/qiniu/services/sandbox/__init__.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- + +from .client import SandboxClient +from .config import ( + env, + load_dotenv_if_present, + required_env, + sandbox_client, + sandbox_template, +) +from .commands import ( + CommandExitError, + CommandHandle, + CommandResult, + Commands, + ProcessInfo, +) +from .constants import ( + DEFAULT_ENDPOINT, + DEFAULT_TEMPLATE, + DEFAULT_USER, + ENVD_PORT, +) +from .errors import ( + FileNotFoundException, + GitAuthException, + GitUpstreamException, + InvalidArgumentException, + SandboxError, + TemplateBuildError, +) +from .filesystem import ( + EntryInfo, + FileType, + Filesystem, + FilesystemEvent, + FilesystemEventType, + WatchHandle, + WriteEntry, + WriteInfo, +) +from .git import Git, GitBranches, GitFileStatus, GitStatus +from .pty import Pty, PtySize +from .resources import GitRepositoryResource, KodoResource +from .sandbox import Sandbox, SandboxPaginator +from .template import ( + ReadyCmd, + Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) + +__all__ = [ + 'CommandExitError', + 'CommandHandle', + 'CommandResult', + 'Commands', + 'DEFAULT_ENDPOINT', + 'DEFAULT_TEMPLATE', + 'DEFAULT_USER', + 'ENVD_PORT', + 'EntryInfo', + 'FileType', + 'FileNotFoundException', + 'Filesystem', + 'FilesystemEvent', + 'FilesystemEventType', + 'Git', + 'GitAuthException', + 'GitBranches', + 'GitFileStatus', + 'GitRepositoryResource', + 'GitStatus', + 'GitUpstreamException', + 'InvalidArgumentException', + 'KodoResource', + 'ProcessInfo', + 'Pty', + 'PtySize', + 'ReadyCmd', + 'Sandbox', + 'SandboxClient', + 'SandboxError', + 'SandboxPaginator', + 'Template', + 'TemplateBuildError', + 'WatchHandle', + 'WriteEntry', + 'WriteInfo', + 'env', + 'load_dotenv_if_present', + 'required_env', + 'sandbox_client', + 'sandbox_template', + 'wait_for_file', + 'wait_for_port', + 'wait_for_process', + 'wait_for_timeout', + 'wait_for_url', +] diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py new file mode 100644 index 00000000..252832ff --- /dev/null +++ b/qiniu/services/sandbox/client.py @@ -0,0 +1,622 @@ +# -*- coding: utf-8 -*- +import os +import time + +import requests + +from qiniu.auth import QiniuMacAuth, QiniuMacRequestsAuth +from qiniu.compat import basestring, urlencode + +from .constants import DEFAULT_TEMPLATE +from .errors import SandboxError, TemplateBuildError +from .resources import KodoResource +from .util import ( + encode_path, + json_dumps, + normalize_endpoint, + parse_json_response, +) + + +_UNSET = object() + + +try: + from time import monotonic as _monotonic_time +except ImportError: + _monotonic_time = time.time + + +def _to_dict(value): + if hasattr(value, 'to_dict'): + return value.to_dict() + return value + + +def _normalize_injection(injection): + if hasattr(injection, 'to_dict'): + injection = injection.to_dict() + if not isinstance(injection, dict): + return injection + data = dict(injection) + if 'apiKey' in data and 'api_key' not in data: + data['api_key'] = data.pop('apiKey') + if 'baseUrl' in data and 'base_url' not in data: + data['base_url'] = data.pop('baseUrl') + if 'ruleId' in data and 'ruleID' not in data: + data['ruleID'] = data.pop('ruleId') + if 'rule_id' in data and 'ruleID' not in data: + data['ruleID'] = data.pop('rule_id') + return data + + +def _normalize_resources(resources): + if resources is None: + return None + return [_to_dict(resource) for resource in resources] + + +def _has_kodo_resource(resources): + for resource in resources or []: + if isinstance(resource, KodoResource): + return True + data = _to_dict(resource) + if isinstance(data, dict) and data.get('type') == 'kodo': + return True + return False + + +def _has_saved_injection_rule(injections): + for injection in injections or []: + data = _normalize_injection(injection) + if not isinstance(data, dict): + continue + if data.get('type') == 'id': + return True + if data.get('ruleID') or data.get('rule_id') or data.get('id'): + return True + return False + + +def _normalize_sandbox_create_options(template=None, **opts): + body = {'templateID': ( + opts.pop('templateID', None) or + opts.pop('template_id', None) or + opts.pop('template', None) or + template or + DEFAULT_TEMPLATE + )} + for key in ( + 'timeout', + 'autoPause', + 'secure', + 'network', + 'metadata', + 'mcp', + ): + if opts.get(key) is not None: + body[key] = opts.get(key) + if opts.get('auto_pause') is not None: + body['autoPause'] = opts.get('auto_pause') + allow_internet_access = _single_alias_value( + opts, 'allow_internet_access', 'allowInternetAccess') + if allow_internet_access is not None: + body['allow_internet_access'] = allow_internet_access + envs = _single_alias_value(opts, 'envs', 'envVars') + if envs is not None: + body['envVars'] = envs + if opts.get('lifecycle') is not None: + lifecycle = opts.get('lifecycle') or {} + on_timeout = lifecycle.get('on_timeout') or lifecycle.get('onTimeout') + if on_timeout == 'pause': + body['autoPause'] = True + if opts.get('injections') is not None: + body['injections'] = [_normalize_injection( + item) for item in opts.get('injections')] + if opts.get('resources') is not None: + body['resources'] = _normalize_resources(opts.get('resources')) + return body + + +def _single_alias_value(opts, *names): + present = [name for name in names if opts.get(name) is not None] + if len(present) > 1: + raise SandboxError( + 'Conflicting sandbox create options: {0}'.format( + ', '.join(present))) + if present: + return opts.get(present[0]) + return None + + +def _normalize_list_options(opts): + opts = dict(opts or {}) + query = opts.pop('query', None) or {} + metadata = query.get('metadata') + if metadata is not None: + opts['metadata'] = metadata + if isinstance(opts.get('metadata'), dict): + opts['metadata'] = urlencode(opts.get('metadata')) + if query.get('state') is not None: + opts['state'] = query.get('state') + return opts + + +def _sandbox_api_key_from_env(): + return ( + os.getenv('QINIU_SANDBOX_API_KEY') or + os.getenv('QINIU_API_KEY') or + os.getenv('E2B_API_KEY') + ) + + +class SandboxClient(object): + def __init__(self, endpoint=None, api_url=None, api_key=None, + access_token=None, mac=None, access_key=None, + secret_key=None, session=None, timeout=None, **opts): + access_key = access_key or os.getenv('QINIU_SANDBOX_ACCESS_KEY') + secret_key = secret_key or os.getenv('QINIU_SANDBOX_SECRET_KEY') + if (access_key and not secret_key) or (secret_key and not access_key): + raise SandboxError( + 'Both access_key and secret_key must be provided') + self.endpoint = normalize_endpoint(endpoint or api_url) + self.api_key = api_key or _sandbox_api_key_from_env() + self.access_token = access_token or os.getenv( + 'QINIU_SANDBOX_ACCESS_TOKEN') + self.mac = mac + if self.mac is None and access_key and secret_key: + self.mac = QiniuMacAuth(access_key, secret_key) + self.session = session or requests.Session() + self.timeout = timeout if timeout is not None else 30 + + def _headers(self, auth_type=None): + headers = {'Content-Type': 'application/json'} + if self.api_key: + headers['X-API-Key'] = self.api_key + if auth_type == 'qiniu': + return headers + if auth_type == 'accessToken': + if not self.access_token: + raise SandboxError( + 'access_token is required for this operation') + headers['Authorization'] = 'Bearer {0}'.format( + self.access_token) + return headers + if self.api_key: + headers['Authorization'] = 'Bearer {0}'.format(self.api_key) + elif self.access_token: + headers['Authorization'] = 'Bearer {0}'.format(self.access_token) + return headers + + def _auth(self, auth_type=None): + if auth_type == 'qiniu' or ( + not self.api_key and not self.access_token and self.mac): + if not self.mac: + raise SandboxError( + 'Qiniu AK/SK credentials are required for this operation' + ) + return QiniuMacRequestsAuth(self.mac) + return None + + def _request(self, method, path, params=None, body=_UNSET, + auth_type=None, empty=False): + url = self.endpoint + path + data = None if body is _UNSET else json_dumps(body) + headers = self._headers(auth_type) + auth = self._auth(auth_type) + request = requests.Request( + method=method, + url=url, + params=params, + data=data, + headers=headers, + auth=auth, + ) + prepared = self.session.prepare_request(request) + try: + response = self.session.send(prepared, timeout=self.timeout) + except requests.RequestException as err: + raise SandboxError('Sandbox API request failed: {0}'.format(err)) + if response.status_code < 200 or response.status_code >= 300: + response_data = None + try: + response_data = response.json() + except ValueError: + try: + response_data = response.text + except Exception: + response_data = getattr(response, 'content', None) + message = 'Sandbox API request failed with status {0}'.format( + response.status_code + ) + if isinstance(response_data, dict): + err_msg = response_data.get( + 'message') or response_data.get('error') + if isinstance(err_msg, dict): + err_msg = err_msg.get('message') or err_msg.get('error') + if err_msg: + message += ': {0}'.format(err_msg) + elif isinstance(response_data, basestring) and response_data: + if len(response_data) > 200: + response_data = response_data[:200] + '...' + message += ': {0}'.format(response_data) + raise SandboxError(message, response, response_data) + if empty: + return None + return parse_json_response(response) + + def list_sandboxes(self, **opts): + return self._request('GET', '/sandboxes', params=opts) + + listSandboxes = list_sandboxes + + def list_sandboxes_v2(self, **opts): + return self._request( + 'GET', + '/v2/sandboxes', + params=_normalize_list_options(opts)) + + listSandboxesV2 = list_sandboxes_v2 + list = list_sandboxes_v2 + + def create_sandbox(self, template=None, **opts): + body = _normalize_sandbox_create_options(template, **opts) + auth_type = 'qiniu' if ( + _has_kodo_resource(body.get('resources')) or + _has_saved_injection_rule(body.get('injections')) + ) else None + return self._request( + 'POST', + '/sandboxes', + body=body, + auth_type=auth_type) + + createSandbox = create_sandbox + create = create_sandbox + + def get_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'GET', '/sandboxes/{0}'.format(encode_path(sandbox_id))) + + getSandbox = get_sandbox + get_info = get_sandbox + getInfo = get_sandbox + + def delete_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'DELETE', + '/sandboxes/{0}'.format(encode_path(sandbox_id)), + empty=True, + ) + + deleteSandbox = delete_sandbox + kill_sandbox = delete_sandbox + killSandbox = delete_sandbox + kill = delete_sandbox + + def pause_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'POST', + '/sandboxes/{0}/pause'.format(encode_path(sandbox_id)), + body={}, + empty=True, + ) + + pauseSandbox = pause_sandbox + + def resume_sandbox(self, sandbox_id, **opts): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'POST', + '/sandboxes/{0}/resume'.format(encode_path(sandbox_id)), + body=opts, + ) + + resumeSandbox = resume_sandbox + + def connect_sandbox(self, sandbox_id, timeout=15): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'POST', + '/sandboxes/{0}/connect'.format(encode_path(sandbox_id)), + body={'timeout': timeout}, + ) + + connectSandbox = connect_sandbox + connect = connect_sandbox + + def update_sandbox_timeout(self, sandbox_id, timeout=None, **opts): + if timeout is None: + timeout = opts.get('timeout') + return self._request( + 'POST', + '/sandboxes/{0}/timeout'.format(encode_path(sandbox_id)), + body={'timeout': timeout}, + empty=True, + ) + + updateSandboxTimeout = update_sandbox_timeout + set_timeout = update_sandbox_timeout + setTimeout = update_sandbox_timeout + + def refresh_sandbox(self, sandbox_id, **opts): + return self._request( + 'POST', + '/sandboxes/{0}/refreshes'.format(encode_path(sandbox_id)), + body=opts, + empty=True, + ) + + refreshSandbox = refresh_sandbox + + def update_sandbox(self, sandbox_id, **opts): + return self._request( + 'PATCH', + '/sandboxes/{0}'.format(encode_path(sandbox_id)), + body=opts, + ) + + updateSandbox = update_sandbox + + def get_sandbox_metrics(self, sandbox_id, **opts): + return self._request( + 'GET', + '/sandboxes/{0}/metrics'.format(encode_path(sandbox_id)), + params=opts, + ) + + getSandboxMetrics = get_sandbox_metrics + get_metrics = get_sandbox_metrics + getMetrics = get_sandbox_metrics + + def get_sandboxes_metrics(self, sandbox_ids): + if not sandbox_ids: + raise SandboxError('At least one sandbox ID must be provided') + values = sandbox_ids + if isinstance(sandbox_ids, dict): + values = sandbox_ids.get( + 'sandbox_ids') or sandbox_ids.get('sandboxIDs') + if values is None: + values = [sandbox_ids] + if values is None: + raise SandboxError('At least one sandbox ID must be provided') + if isinstance(values, (list, tuple, set)): + values = list(values) + elif hasattr(values, '__iter__') and not isinstance( + values, (basestring, dict)): + values = list(values) + else: + values = [values] + ids = [] + for value in values: + if isinstance(value, basestring): + ids.append(value) + elif isinstance(value, dict): + ids.append(value.get('sandboxId') or value.get( + 'sandboxID') or value.get('id')) + elif hasattr(value, 'sandbox_id'): + ids.append(value.sandbox_id) + ids = [item for item in ids if item] + if not ids: + raise SandboxError('At least one sandbox ID must be provided') + return self._request( + 'GET', + '/sandboxes/metrics', + params={ + 'sandbox_ids': ','.join(ids)}) + + getSandboxesMetrics = get_sandboxes_metrics + + def get_sandbox_logs(self, sandbox_id, **opts): + return self._request( + 'GET', + '/sandboxes/{0}/logs'.format(encode_path(sandbox_id)), + params=opts, + ) + + getSandboxLogs = get_sandbox_logs + get_logs = get_sandbox_logs + getLogs = get_sandbox_logs + + def create_template(self, **opts): + return self._request('POST', '/v3/templates', body=opts) + + createTemplate = create_template + createTemplateV3 = create_template + + def create_template_v2(self, **opts): + return self._request('POST', '/v2/templates', body=opts) + + createTemplateV2 = create_template_v2 + + def list_templates(self, **opts): + return self._request('GET', '/templates', params=opts) + + listTemplates = list_templates + + def list_default_templates(self): + return self._request('GET', '/default-templates') + + listDefaultTemplates = list_default_templates + + def get_template(self, template_id, **opts): + return self._request( + 'GET', + '/templates/{0}'.format(encode_path(template_id)), + params=opts, + ) + + getTemplate = get_template + + def delete_template(self, template_id): + if not template_id: + raise SandboxError('template_id is required') + return self._request( + 'DELETE', + '/templates/{0}'.format(encode_path(template_id)), + empty=True, + ) + + deleteTemplate = delete_template + + def update_template(self, template_id, **opts): + return self._request( + 'PATCH', + '/templates/{0}'.format(encode_path(template_id)), + body=opts, + ) + + updateTemplate = update_template + + def rebuild_template(self, template_id, **opts): + return self._request( + 'POST', + '/templates/{0}'.format(encode_path(template_id)), + body=opts, + auth_type='accessToken', + ) + + rebuildTemplate = rebuild_template + + def get_template_build_status(self, template_id, build_id, **opts): + return self._request( + 'GET', + '/templates/{0}/builds/{1}/status'.format( + encode_path(template_id), + encode_path(build_id), + ), + params=opts, + ) + + getTemplateBuildStatus = get_template_build_status + + def get_template_build_logs(self, template_id, build_id, **opts): + return self._request( + 'GET', + '/templates/{0}/builds/{1}/logs'.format( + encode_path(template_id), + encode_path(build_id), + ), + params=opts, + ) + + getTemplateBuildLogs = get_template_build_logs + + def start_template_build(self, template_id, build_id, **opts): + return self._request( + 'POST', + '/v2/templates/{0}/builds/{1}'.format( + encode_path(template_id), + encode_path(build_id), + ), + body=opts, + empty=True, + ) + + startTemplateBuild = start_template_build + startTemplateBuildV2 = start_template_build + + def assign_template_tags(self, **opts): + return self._request('POST', '/templates/tags', body=opts) + + assignTemplateTags = assign_template_tags + + def delete_template_tags(self, **opts): + return self._request( + 'DELETE', + '/templates/tags', + body=opts, + empty=True) + + deleteTemplateTags = delete_template_tags + + def get_template_by_alias(self, alias): + return self._request( + 'GET', '/templates/aliases/{0}'.format(encode_path(alias))) + + getTemplateByAlias = get_template_by_alias + + def list_injection_rules(self): + return self._request('GET', '/injection-rules', auth_type='qiniu') + + listInjectionRules = list_injection_rules + + def create_injection_rule(self, **opts): + if opts.get('injection') is not None: + opts['injection'] = _normalize_injection(opts.get('injection')) + return self._request( + 'POST', + '/injection-rules', + body=opts, + auth_type='qiniu', + ) + + createInjectionRule = create_injection_rule + + def get_injection_rule(self, rule_id): + return self._request( + 'GET', + '/injection-rules/{0}'.format(encode_path(rule_id)), + auth_type='qiniu', + ) + + getInjectionRule = get_injection_rule + + def update_injection_rule(self, rule_id, **opts): + if opts.get('injection') is not None: + opts['injection'] = _normalize_injection(opts.get('injection')) + return self._request( + 'PUT', + '/injection-rules/{0}'.format(encode_path(rule_id)), + body=opts, + auth_type='qiniu', + ) + + updateInjectionRule = update_injection_rule + + def delete_injection_rule(self, rule_id): + return self._request( + 'DELETE', + '/injection-rules/{0}'.format(encode_path(rule_id)), + auth_type='qiniu', + empty=True, + ) + + deleteInjectionRule = delete_injection_rule + + def wait_for_build(self, template_id, build_id, interval=1, timeout=60): + start = _monotonic_time() + while True: + try: + info = self.get_template_build_status(template_id, build_id) + if info and info.get('status') in ('ready', 'error'): + if info.get('status') == 'error': + message = ( + ( + isinstance(info.get('error'), dict) and + info.get('error').get('message') + ) or + info.get('error') or + info.get('message') or + 'Sandbox template build failed' + ) + raise TemplateBuildError(message, data=info) + return info + except SandboxError as err: + if isinstance(err, TemplateBuildError) or ( + err.status_code is not None and + err.status_code >= 400 and err.status_code < 500): + raise + if _monotonic_time() - start >= timeout: + raise SandboxError('Sandbox template build polling timed out') + time.sleep(interval) + + waitForBuild = wait_for_build diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py new file mode 100644 index 00000000..2d3dcaad --- /dev/null +++ b/qiniu/services/sandbox/commands.py @@ -0,0 +1,292 @@ +# -*- coding: utf-8 -*- +import base64 + +from qiniu.compat import basestring, bytes as bytes_type + +from .envd import connect_rpc, connect_stream_rpc +from .errors import CommandExitError, SandboxError + + +class ProcessInfo(object): + def __init__(self, pid=None, tag=None, cmd=None, args=None, envs=None, + cwd=None): + self.pid = pid + self.tag = tag + self.cmd = cmd + self.args = args or [] + self.envs = envs or {} + self.cwd = cwd + + def to_dict(self): + return { + 'pid': self.pid, + 'tag': self.tag, + 'cmd': self.cmd, + 'args': self.args, + 'envs': self.envs, + 'cwd': self.cwd, + } + + +class CommandResult(object): + def __init__(self, pid=0, exit_code=0, stdout='', stderr='', error=''): + self.pid = pid + self.exit_code = exit_code + self.exitCode = exit_code + self.stdout = stdout or '' + self.stderr = stderr or '' + self.error = error or '' + + def to_dict(self): + return { + 'pid': self.pid, + 'exitCode': self.exit_code, + 'stdout': self.stdout, + 'stderr': self.stderr, + 'error': self.error, + } + + +def _decode_bytes(value): + if value is None: + return '' + if isinstance(value, list): + return bytearray(value).decode('utf-8', 'replace') + if isinstance(value, bytes_type): + return value.decode('utf-8', 'replace') + if isinstance(value, basestring): + return base64.b64decode(value).decode('utf-8', 'replace') + return str(value) + + +def command_result_from_events(events, on_stdout=None, on_stderr=None, + on_pty=None): + pid = 0 + stdout = '' + stderr = '' + exit_code = -1 + error = '' + for raw in events or []: + event = raw.get('event', raw) if isinstance(raw, dict) else {} + start = event.get('start') + data = event.get('data') + end = event.get('end') + if start: + pid = start.get('pid') or pid + if data: + stdout_chunk = _decode_bytes(data.get('stdout')) + stderr_chunk = _decode_bytes(data.get('stderr')) + pty_chunk = _decode_bytes(data.get('pty')) + if stdout_chunk and on_stdout: + on_stdout(stdout_chunk) + if stderr_chunk and on_stderr: + on_stderr(stderr_chunk) + if pty_chunk and on_pty: + on_pty(pty_chunk) + stdout += stdout_chunk + stderr += stderr_chunk + stdout += pty_chunk + if end: + if end.get('exitCode') is not None: + exit_code = end.get('exitCode') + error = end.get('error') or '' + return CommandResult(pid, exit_code, stdout, stderr, error) + + +class CommandHandle(object): + def __init__(self, commands, result=None, throw_on_error=False, + events=None, on_stdout=None, on_stderr=None): + self.commands = commands + result = result or CommandResult() + self.result = result + self.pid = result.pid + self.exit_code = result.exit_code + self.exitCode = result.exit_code + self.stdout = result.stdout + self.stderr = result.stderr + self.throw_on_error = throw_on_error + self._events = events + self._on_stdout = on_stdout + self._on_stderr = on_stderr + + def wait(self, on_stdout=None, on_stderr=None): + if self._events is not None: + result = command_result_from_events( + self._events, + on_stdout=on_stdout or self._on_stdout, + on_stderr=on_stderr or self._on_stderr, + ) + if not result.pid: + result.pid = self.pid + result.stdout = self.result.stdout + result.stdout + result.stderr = self.result.stderr + result.stderr + if not result.error: + result.error = self.result.error + self.result = result + self.pid = result.pid + self.exit_code = result.exit_code + self.exitCode = result.exit_code + self.stdout = result.stdout + self.stderr = result.stderr + self._events = None + if self.throw_on_error and self.result.exit_code: + raise CommandExitError(self.result) + return self.result + + def kill(self): + return self.commands.kill(self.pid) + + +class Commands(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, + tag=None, background=False, throw_on_error=False, + timeout=None, request_timeout=None, on_stdout=None, + on_stderr=None, **opts): + handle = self.start( + cmd, + cwd=cwd, + envs=envs, + user=user, + stdin=stdin, + tag=tag, + throw_on_error=throw_on_error, + timeout=timeout, + request_timeout=request_timeout, + on_stdout=on_stdout, + on_stderr=on_stderr, + **opts + ) + return handle if background else handle.wait() + + def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, + tag=None, throw_on_error=False, timeout=None, + request_timeout=None, on_stdout=None, on_stderr=None, **opts): + process = { + 'cmd': '/bin/bash', + 'args': ['-l', '-c', cmd], + } + if cwd: + process['cwd'] = cwd + if envs: + process['envs'] = envs + body = { + 'process': process, + 'stdin': stdin, + } + if tag: + body['tag'] = tag + if timeout is not None: + body['timeout'] = timeout + stream_timeout = request_timeout + if stream_timeout is None: + stream_timeout = timeout + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Start', + body, + user=user, + timeout=stream_timeout, + stream=True, + ) + events = iter(events) + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events( + [first_event] if first_event else [], + on_stdout=on_stdout, + on_stderr=on_stderr, + ) + return CommandHandle( + self, + result, + throw_on_error=throw_on_error, + events=events, + on_stdout=on_stdout, + on_stderr=on_stderr, + ) + + def list(self, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/process.Process/List', + {}, + user=user, + timeout=timeout) + processes = data.get('processes', []) if isinstance(data, dict) else [] + result = [] + for process in processes: + config = process.get('config') or {} + result.append(ProcessInfo( + pid=process.get('pid'), + tag=process.get('tag'), + cmd=config.get('cmd'), + args=config.get('args'), + envs=config.get('envs'), + cwd=config.get('cwd'), + )) + return result + + def connect(self, pid, tag=None, user=None, timeout=None, + throw_on_error=False): + selector = {'pid': pid} + if tag is not None: + selector = {'tag': tag} + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Connect', + {'process': {'selector': selector}}, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events([first_event]) + return CommandHandle( + self, + result, + events=events, + throw_on_error=throw_on_error, + ) + + def send_stdin(self, pid, data, user=None, timeout=None): + if isinstance(data, bytearray): + data = bytes(data) + elif not isinstance(data, bytes): + if hasattr(data, 'encode'): + data = data.encode('utf-8') + else: + data = str(data).encode('utf-8') + return connect_rpc(self.sandbox, '/process.Process/SendInput', { + 'process': {'selector': {'pid': pid}}, + 'input': {'stdin': base64.b64encode(data).decode('ascii')}, + }, user=user, timeout=timeout) + + sendStdin = send_stdin + + def close_stdin(self, pid, user=None, timeout=None): + return connect_rpc(self.sandbox, '/process.Process/CloseStdin', { + 'process': {'selector': {'pid': pid}}, + }, user=user, timeout=timeout) + + closeStdin = close_stdin + + def kill(self, pid, user=None, timeout=None): + try: + connect_rpc(self.sandbox, '/process.Process/SendSignal', { + 'process': {'selector': {'pid': pid}}, + 'signal': 'SIGNAL_SIGKILL', + }, user=user, timeout=timeout) + return True + except SandboxError as err: + if getattr(err, 'status_code', None) == 404: + return False + raise diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py new file mode 100644 index 00000000..f96a1033 --- /dev/null +++ b/qiniu/services/sandbox/config.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +import io +import os + +from qiniu.compat import is_py2, str as text_type + +from .client import SandboxClient + + +def load_dotenv_if_present(*paths): + if not paths: + paths = ( + os.path.join(os.getcwd(), '.env'), + ) + + for path in paths: + if not path or not os.path.exists(path): + continue + with io.open(path, 'r', encoding='utf-8') as f: + for raw_line in f: + line = raw_line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + key, value = line.split('=', 1) + key = key.strip() + value = _strip_inline_comment(value.strip()).strip() + if ( + len(value) >= 2 and + value[0] == value[-1] and + value[0] in ('"', "'") + ): + value = value[1:-1] + key, value = _native_env_pair(key, value) + if key and key not in os.environ: + os.environ[key] = value + + +def _native_env_pair(key, value): + if is_py2: + if isinstance(key, text_type): + key = key.encode('utf-8') + if isinstance(value, text_type): + value = value.encode('utf-8') + return key, value + + +def _strip_inline_comment(value): + quote = None + escaped = False + for index, char in enumerate(value): + if escaped: + escaped = False + continue + if char == '\\': + escaped = True + continue + if quote: + if char == quote: + quote = None + continue + if char in ('"', "'"): + quote = char + continue + if char == '#' and (index == 0 or value[index - 1].isspace()): + return value[:index] + return value + + +def env(key, fallback=None): + return os.getenv(key) or fallback + + +def first_env(*keys): + for key in keys: + value = os.getenv(key) + if value: + return value + return None + + +def required_env(key): + value = os.getenv(key) + if value: + return value + raise RuntimeError('Please set {0}'.format(key)) + + +def sandbox_endpoint(): + return env('QINIU_SANDBOX_ENDPOINT') + + +def sandbox_api_key(): + value = first_env( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_API_KEY', + 'E2B_API_KEY', + ) + if value: + return value + raise RuntimeError( + 'Please set QINIU_SANDBOX_API_KEY, QINIU_API_KEY, or E2B_API_KEY') + + +def sandbox_template(): + return env('QINIU_SANDBOX_TEMPLATE', 'base') + + +def sandbox_client(**options): + defaults = { + 'endpoint': sandbox_endpoint(), + 'api_key': sandbox_api_key(), + } + access_key = env('QINIU_SANDBOX_ACCESS_KEY') + secret_key = env('QINIU_SANDBOX_SECRET_KEY') + if access_key and secret_key: + defaults['access_key'] = access_key + defaults['secret_key'] = secret_key + defaults.update(options) + return SandboxClient(**defaults) diff --git a/qiniu/services/sandbox/constants.py b/qiniu/services/sandbox/constants.py new file mode 100644 index 00000000..3d7c4c7b --- /dev/null +++ b/qiniu/services/sandbox/constants.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +DEFAULT_ENDPOINT = 'https://cn-yangzhou-1-sandbox.qiniuapi.com' +DEFAULT_TEMPLATE = 'base' +DEFAULT_USER = 'user' +ENVD_PORT = 49983 +MCP_PORT = 50005 diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py new file mode 100644 index 00000000..e75acaa0 --- /dev/null +++ b/qiniu/services/sandbox/envd.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +import json +import struct + +import requests + +from .constants import DEFAULT_USER +from .errors import SandboxError +from .util import basic_auth + + +MAX_CONNECT_ENVELOPE_BYTES = 10 * 1024 * 1024 + + +def envd_headers(sandbox, user=None, extra=None): + headers = { + 'Authorization': basic_auth(user or DEFAULT_USER), + } + if sandbox.envd_access_token: + headers['X-Access-Token'] = sandbox.envd_access_token + if extra: + headers.update(extra) + return headers + + +def connect_rpc(sandbox, procedure, body=None, user=None, timeout=None): + if timeout is None: + timeout = sandbox.client.timeout + url = sandbox.envd_url() + procedure + headers = envd_headers(sandbox, user, {'Content-Type': 'application/json'}) + try: + response = sandbox.client.session.post( + url, + data=json.dumps(body or {}, separators=(',', ':')), + headers=headers, + timeout=timeout, + ) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + if not response.content: + return None + data = response.json() + if isinstance(data, dict) and 'result' in data: + return data.get('result') + return data + + +def encode_connect_envelope(message): + payload = json.dumps(message or {}, separators=(',', ':')).encode('utf-8') + return struct.pack('>BI', 0, len(payload)) + payload + + +def decode_connect_envelopes(data): + if not data: + return [] + if not isinstance(data, bytes): + data = data.encode('utf-8') + messages = [] + offset = 0 + while offset + 5 <= len(data): + flags, length = struct.unpack('>BI', data[offset:offset + 5]) + offset += 5 + if length > MAX_CONNECT_ENVELOPE_BYTES: + raise SandboxError( + 'Sandbox envd stream envelope too large: {0}'.format(length) + ) + if offset + length > len(data): + raise SandboxError('Sandbox envd stream truncated unexpectedly') + payload = data[offset:offset + length] + offset += length + if flags & 2: + if _is_connect_end(payload): + continue + _raise_connect_error(payload) + if payload: + messages.append(json.loads(payload.decode('utf-8'))) + if offset < len(data): + raise SandboxError('Sandbox envd stream truncated unexpectedly') + return messages + + +def _raise_connect_error(payload): + error = None + message = 'Sandbox envd stream failed' + if payload: + try: + data = json.loads(payload.decode('utf-8')) + if isinstance(data, dict): + error = data.get('error') + if isinstance(error, dict) and error.get('message'): + message = error.get('message') + except (TypeError, ValueError): + pass + raise SandboxError(message, data=error) + + +def _is_connect_end(payload): + if not payload: + return False + try: + data = json.loads(payload.decode('utf-8')) + except (TypeError, ValueError): + return False + return data == {} + + +def iter_connect_envelopes(chunks, response=None): + try: + buffer = b'' + for chunk in chunks: + if not chunk: + continue + if not isinstance(chunk, bytes): + chunk = chunk.encode('utf-8') + buffer += chunk + while len(buffer) >= 5: + flags, length = struct.unpack('>BI', buffer[:5]) + if length > MAX_CONNECT_ENVELOPE_BYTES: + raise SandboxError( + 'Sandbox envd stream envelope too large: {0}'.format( + length) + ) + if len(buffer) < 5 + length: + break + payload = buffer[5:5 + length] + buffer = buffer[5 + length:] + if flags & 2: + if _is_connect_end(payload): + continue + _raise_connect_error(payload) + if payload: + yield json.loads(payload.decode('utf-8')) + if buffer: + raise SandboxError('Sandbox envd stream truncated unexpectedly') + finally: + if response is not None: + try: + response.close() + except Exception: + pass + + +def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, + stream=False): + if timeout is None: + timeout = sandbox.client.timeout + url = sandbox.envd_url() + procedure + headers = envd_headers(sandbox, user, { + 'Content-Type': 'application/connect+json', + 'Keepalive-Ping-Interval': '50', + }) + request_opts = { + 'data': encode_connect_envelope(body), + 'headers': headers, + 'timeout': timeout, + } + if stream: + request_opts['stream'] = True + try: + try: + response = sandbox.client.session.post(url, **request_opts) + except TypeError: + request_opts.pop('stream', None) + response = sandbox.client.session.post(url, **request_opts) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + + if stream and hasattr(response, 'iter_content'): + return iter_connect_envelopes( + response.iter_content(chunk_size=8192), + response=response, + ) + + content_type = response.headers.get('Content-Type', '') + if 'application/connect+json' in content_type: + return decode_connect_envelopes(response.content) + + if not response.content: + return [] + data = response.json() + if isinstance(data, dict) and 'result' in data: + data = data.get('result') + if isinstance(data, dict) and 'events' in data: + return data.get('events') + if isinstance(data, dict) and 'event' in data: + return [data] + return data + + +def raw_envd_request(sandbox, method, url, **kwargs): + if kwargs.get('timeout') is None: + kwargs['timeout'] = sandbox.client.timeout + try: + response = sandbox.client.session.request(method, url, **kwargs) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + return response diff --git a/qiniu/services/sandbox/errors.py b/qiniu/services/sandbox/errors.py new file mode 100644 index 00000000..1635fabe --- /dev/null +++ b/qiniu/services/sandbox/errors.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + + +class SandboxError(Exception): + def __init__(self, message, response=None, data=None): + super(SandboxError, self).__init__(message) + self.response = response + self.data = data + self.status_code = getattr(response, 'status_code', None) + + +class TemplateBuildError(SandboxError): + pass + + +class CommandExitError(SandboxError): + def __init__(self, result): + self.result = result + super(CommandExitError, self).__init__( + 'Command exited with code {0}'.format(result.exit_code) + ) + + +class InvalidArgumentException(SandboxError): + pass + + +class FileNotFoundException(SandboxError): + pass + + +class GitAuthException(SandboxError): + pass + + +class GitUpstreamException(SandboxError): + pass diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py new file mode 100644 index 00000000..41dc85f7 --- /dev/null +++ b/qiniu/services/sandbox/filesystem.py @@ -0,0 +1,398 @@ +# -*- coding: utf-8 -*- +from io import IOBase, TextIOBase + +from qiniu.compat import basestring + +from .errors import InvalidArgumentException, SandboxError +from .envd import connect_rpc, envd_headers, raw_envd_request +from .util import file_basename + + +class _EncodedTextReader(object): + def __init__(self, stream, encoding): + self.stream = stream + self.encoding = encoding + + def read(self, size=-1): + chunk = self.stream.read(size) + if isinstance(chunk, basestring): + return chunk.encode(self.encoding) + return chunk + + def __getattr__(self, name): + if name in ('seek', 'tell', 'getvalue', 'len'): + raise AttributeError(name) + return getattr(self.stream, name) + + +class FileType(object): + FILE = 'file' + DIR = 'dir' + DIRECTORY = DIR + + +class FilesystemEventType(object): + CREATE = 'create' + WRITE = 'write' + REMOVE = 'remove' + RENAME = 'rename' + CHMOD = 'chmod' + + +class FilesystemEvent(object): + def __init__(self, name=None, type=None): + self.name = name + self.type = type + + def to_dict(self): + return {'name': self.name, 'type': self.type} + + +class WriteEntry(dict): + def __init__(self, path=None, data=None, **kwargs): + dict.__init__(self, path=path, data=data, **kwargs) + self.path = path + self.data = data + + +class WriteInfo(object): + def __init__(self, name=None, type=None, path=None): + self.name = name + self.type = type + self.path = path + + def to_dict(self): + return { + 'name': self.name, + 'type': self.type, + 'path': self.path, + } + + def __getitem__(self, key): + return self.to_dict()[key] + + def __eq__(self, other): + if isinstance(other, dict): + data = self.to_dict() + return all(data.get(key) == value for key, value in other.items()) + return object.__eq__(self, other) + + +class EntryInfo(WriteInfo): + def __init__(self, name=None, type=None, path=None, size=None, mode=None, + permissions=None, owner=None, group=None, modified_time=None, + symlink_target=None): + WriteInfo.__init__(self, name=name, type=type, path=path) + self.size = size + self.mode = mode + self.permissions = permissions + self.owner = owner + self.group = group + self.modified_time = modified_time + self.symlink_target = symlink_target + + def to_dict(self): + data = WriteInfo.to_dict(self) + data.update({ + 'size': self.size, + 'mode': self.mode, + 'permissions': self.permissions, + 'owner': self.owner, + 'group': self.group, + 'modified_time': self.modified_time, + 'symlink_target': self.symlink_target, + }) + return data + + +def normalize_event_type(event_type): + mapping = { + 'EVENT_TYPE_CREATE': FilesystemEventType.CREATE, + 'EVENT_TYPE_WRITE': FilesystemEventType.WRITE, + 'EVENT_TYPE_REMOVE': FilesystemEventType.REMOVE, + 'EVENT_TYPE_RENAME': FilesystemEventType.RENAME, + 'EVENT_TYPE_CHMOD': FilesystemEventType.CHMOD, + 1: FilesystemEventType.CREATE, + 2: FilesystemEventType.WRITE, + 3: FilesystemEventType.REMOVE, + 4: FilesystemEventType.RENAME, + 5: FilesystemEventType.CHMOD, + } + return mapping.get(event_type, event_type) + + +def normalize_entry_type(entry_type): + if entry_type in ('FILE_TYPE_DIRECTORY', 'DIRECTORY', 'dir'): + return FileType.DIR + if entry_type in ('FILE_TYPE_FILE', 'FILE', 'file'): + return FileType.FILE + return entry_type + + +def normalize_entry(entry, extended=False): + entry = entry or {} + entry_type = entry.get('type') + entry_type = normalize_entry_type(entry_type) + info_cls = EntryInfo if extended else WriteInfo + return info_cls( + name=entry.get('name'), + type=entry_type, + path=entry.get('path'), + size=entry.get('size'), + mode=entry.get('mode'), + permissions=entry.get('permissions'), + owner=entry.get('owner'), + group=entry.get('group'), + modified_time=entry.get('modifiedTime') or entry.get('modified_time'), + symlink_target=entry.get('symlinkTarget') or entry.get( + 'symlink_target'), + ) if extended else info_cls( + name=entry.get('name'), + type=entry_type, + path=entry.get('path'), + ) + + +def to_upload_body(data, encoding='utf-8'): + if isinstance(data, bytearray): + return bytes(data) + if isinstance(data, bytes): + return data + if isinstance(data, basestring): + return data.encode(encoding) + if isinstance(data, TextIOBase): + return _EncodedTextReader(data, encoding) + if isinstance(data, IOBase) or hasattr(data, 'read'): + return data + raise InvalidArgumentException( + 'Unsupported data type for filesystem write: {0}'.format(type(data))) + + +def _response_stream(response, chunk_size=8192): + try: + for chunk in response.iter_content(chunk_size=chunk_size): + yield chunk + finally: + response.close() + + +class WatchHandle(object): + def __init__(self, filesystem, watcher_id): + self.filesystem = filesystem + self.watcher_id = watcher_id + self.watcherID = watcher_id + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + return False + + def get_new_events(self, user=None, timeout=None): + if self._closed: + raise SandboxError('The watcher is already stopped') + data = connect_rpc( + self.filesystem.sandbox, + '/filesystem.Filesystem/GetWatcherEvents', + {'watcherId': self.watcher_id}, + user=user, + timeout=timeout, + ) + events = [] + for event in (data or {}).get('events', []): + events.append(FilesystemEvent( + name=event.get('name'), + type=normalize_event_type(event.get('type')), + )) + return events + + getNewEvents = get_new_events + + def stop(self, user=None, timeout=None): + if self._closed: + return None + connect_rpc( + self.filesystem.sandbox, + '/filesystem.Filesystem/RemoveWatcher', + {'watcherId': self.watcher_id}, + user=user, + timeout=timeout, + ) + self._closed = True + return None + + +class Filesystem(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def _request_timeout(self, opts): + if opts.get('request_timeout') is not None: + return opts.get('request_timeout') + return opts.get('timeout') + + def read(self, path, user=None, fmt=None, **opts): + if fmt is None: + fmt = opts.pop('format', 'text') + url = self.sandbox.download_url(path, user=user) + stream_mode = fmt == 'stream' + response = raw_envd_request( + self.sandbox, + 'GET', + url, + headers=envd_headers(self.sandbox, user), + timeout=self._request_timeout(opts), + stream=stream_mode, + ) + if fmt == 'stream': + if hasattr(response, 'iter_content'): + return _response_stream( + response, + chunk_size=opts.get('chunk_size', 8192), + ) + return iter([response.content]) + if fmt == 'bytes': + return bytearray(response.content) + return response.content.decode(opts.get('encoding', 'utf-8')) + + def read_text(self, path, user=None, **opts): + return self.read(path, user=user, format='text', **opts) + + readText = read_text + + def write(self, path, data, user=None, **opts): + data = to_upload_body(data, opts.get('encoding', 'utf-8')) + url = self.sandbox.upload_url(path, user=user) + if opts.get('use_octet_stream'): + response = raw_envd_request( + self.sandbox, + 'POST', + url, + data=data, + headers=envd_headers( + self.sandbox, + user, + {'Content-Type': 'application/octet-stream'}, + ), + timeout=self._request_timeout(opts), + ) + return self._format_write_response(response) + + response = raw_envd_request( + self.sandbox, + 'POST', + url, + files={'file': (file_basename(path), data)}, + headers=envd_headers(self.sandbox, user), + timeout=self._request_timeout(opts), + ) + return self._format_write_response(response) + + def _format_write_response(self, response): + if not response.content: + return None + data = response.json() + if isinstance(data, list): + return normalize_entry(data[0] if data else {}) + return normalize_entry(data) + + def write_files(self, files, user=None, **opts): + result = [] + for item in files or []: + if isinstance(item, dict): + path = item.get('path') + data = item.get('data', item.get('content', '')) + else: + path, data = item + result.append(self.write(path, data, user=user, **opts)) + return result + + writeFiles = write_files + + def get_info(self, path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Stat', + {'path': path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry'), extended=True) + + getInfo = get_info + stat = get_info + + def list(self, path, depth=1, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/ListDir', + {'path': path, 'depth': depth}, + user=user, + timeout=timeout, + ) + return [ + normalize_entry(entry, extended=True) for entry in ( + data or {}).get( + 'entries', [])] + + def exists(self, path, user=None, timeout=None): + try: + self.get_info(path, user=user, timeout=timeout) + return True + except SandboxError as err: + response = getattr(err, 'response', None) + if response is not None and getattr( + response, 'status_code', None) == 404: + return False + raise + + def make_dir(self, path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/MakeDir', + {'path': path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry'), extended=True) + + makeDir = make_dir + mkdir = make_dir + + def remove(self, path, user=None, timeout=None): + connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Remove', + {'path': path}, + user=user, + timeout=timeout, + ) + return None + + def rename(self, old_path, new_path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Move', + {'source': old_path, 'destination': new_path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry'), extended=True) + + def watch_dir(self, path, recursive=False, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/CreateWatcher', + {'path': path, 'recursive': recursive}, + user=user, + timeout=timeout, + ) + watcher_id = ( + (data or {}).get('watcherId') or + (data or {}).get('watcher_id') + ) + return WatchHandle(self, watcher_id) + + watchDir = watch_dir diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py new file mode 100644 index 00000000..065d2b3f --- /dev/null +++ b/qiniu/services/sandbox/git.py @@ -0,0 +1,807 @@ +# -*- coding: utf-8 -*- +import uuid + +from qiniu.compat import basestring + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + +from .errors import ( + CommandExitError, + GitAuthException, + GitUpstreamException, + InvalidArgumentException, + SandboxError, +) +from .util import shell_quote + + +def _normalize_paths(paths): + if paths is None: + return None + if isinstance(paths, basestring): + return [paths] + return paths + + +RESET_MODES = set(['soft', 'mixed', 'hard', 'merge', 'keep']) + + +def _remove_credential_file(filesystem, path): + if not path: + return + try: + filesystem.remove(path) + except Exception: + pass + + +def _cleanup_after_wait(wait, filesystem, path): + def wrapped_wait(*args, **kwargs): + try: + return wait(*args, **kwargs) + finally: + _remove_credential_file(filesystem, path) + return wrapped_wait + + +class GitFileStatus(object): + def __init__(self, name, status, index_status, working_tree_status, + staged, renamed_from=None): + self.name = name + self.status = status + self.index_status = index_status + self.working_tree_status = working_tree_status + self.staged = staged + self.renamed_from = renamed_from + + +class GitStatus(object): + def __init__(self, current_branch=None, upstream=None, ahead=0, behind=0, + detached=False, file_status=None): + self.current_branch = current_branch + self.upstream = upstream + self.ahead = ahead + self.behind = behind + self.detached = detached + self.file_status = file_status or [] + + @property + def is_clean(self): + return len(self.file_status) == 0 + + @property + def has_changes(self): + return len(self.file_status) > 0 + + @property + def has_staged(self): + return any(item.staged for item in self.file_status) + + @property + def has_untracked(self): + return any(item.status == 'untracked' for item in self.file_status) + + @property + def has_conflicts(self): + return any(item.status == 'conflict' for item in self.file_status) + + @property + def total_count(self): + return len(self.file_status) + + @property + def staged_count(self): + return sum(1 for item in self.file_status if item.staged) + + @property + def unstaged_count(self): + return sum(1 for item in self.file_status if not item.staged) + + @property + def untracked_count(self): + return sum(1 for item in self.file_status + if item.status == 'untracked') + + @property + def conflict_count(self): + return sum(1 for item in self.file_status + if item.status == 'conflict') + + +class GitBranches(object): + def __init__(self, branches=None, current_branch=None): + self.branches = branches or [] + self.current_branch = current_branch + + +def _parse_ahead_behind(segment): + ahead = 0 + behind = 0 + if not segment: + return ahead, behind + if 'ahead' in segment: + try: + ahead = int(segment.split('ahead')[1].split(',')[0].strip()) + except Exception: + ahead = 0 + if 'behind' in segment: + try: + behind = int(segment.split('behind')[1].split(',')[0].strip()) + except Exception: + behind = 0 + return ahead, behind + + +def _normalize_branch_name(name): + if name.startswith('HEAD (detached at '): + return name.replace('HEAD (detached at ', '').rstrip(')') + return ( + name.replace('HEAD (no branch)', 'HEAD') + .replace('No commits yet on ', '') + .replace('Initial commit on ', '') + ) + + +def _derive_status(index_status, working_status): + statuses = set([index_status, working_status]) + if 'U' in statuses: + return 'conflict' + if 'R' in statuses: + return 'renamed' + if 'C' in statuses: + return 'copied' + if 'D' in statuses: + return 'deleted' + if 'A' in statuses: + return 'added' + if 'M' in statuses: + return 'modified' + if 'T' in statuses: + return 'typechange' + if '?' in statuses: + return 'untracked' + return 'unknown' + + +def parse_git_status(output): + lines = [line.rstrip() for line in (output or '').split('\n') + if line.strip()] + current_branch = None + upstream = None + ahead = 0 + behind = 0 + detached = False + file_status = [] + + if not lines: + return GitStatus(file_status=file_status) + + branch_line = lines[0] + if branch_line.startswith('## '): + branch_info = branch_line[3:] + ahead_start = branch_info.find(' [') + branch_part = branch_info if ahead_start == -1 else branch_info[ + :ahead_start] + ahead_part = None if ahead_start == -1 else branch_info[ + ahead_start + 2:-1] + normalized = _normalize_branch_name(branch_part) + is_detached = branch_part.startswith('HEAD (detached at ') or ( + 'detached' in branch_part) + if is_detached or normalized.startswith('HEAD'): + detached = True + elif '...' in normalized: + current_branch, upstream = normalized.split('...', 1) + current_branch = current_branch or None + upstream = upstream or None + else: + current_branch = normalized or None + ahead, behind = _parse_ahead_behind(ahead_part) + + for line in lines[1:]: + if line.startswith('?? '): + file_status.append(GitFileStatus( + name=line[3:], + status='untracked', + index_status='?', + working_tree_status='?', + staged=False, + )) + continue + if len(line) < 3: + continue + index_status = line[0] + working_status = line[1] + path = line[3:] + renamed_from = None + name = path + if (index_status == 'R' or working_status == 'R') and ' -> ' in path: + renamed_from, name = path.split(' -> ', 1) + file_status.append(GitFileStatus( + name=name, + status=_derive_status(index_status, working_status), + index_status=index_status, + working_tree_status=working_status, + staged=index_status not in (' ', '?'), + renamed_from=renamed_from, + )) + + return GitStatus( + current_branch=current_branch, + upstream=upstream, + ahead=ahead, + behind=behind, + detached=detached, + file_status=file_status, + ) + + +def parse_git_branches(output): + branches = [] + current_branch = None + for line in [line.strip() for line in (output or '').split('\n') + if line.strip()]: + if '\t' in line: + name, marker = line.split('\t', 1) + marker = marker.strip() + else: + marker = '*' if line.startswith('* ') else '' + name = line[2:] if line.startswith('* ') else line + branches.append(name) + if marker == '*': + current_branch = name + return GitBranches(branches=branches, current_branch=current_branch) + + +def _is_auth_failure(result): + message = '{0}\n{1}\n{2}'.format( + getattr(result, 'stderr', ''), + getattr(result, 'stdout', ''), + getattr(result, 'error', ''), + ).lower() + snippets = ( + 'authentication failed', + 'terminal prompts disabled', + 'could not read username', + 'invalid username or password', + 'access denied', + 'permission denied', + 'not authorized', + ) + return any(snippet in message for snippet in snippets) + + +def _is_missing_upstream(result): + message = '{0}\n{1}\n{2}'.format( + getattr(result, 'stderr', ''), + getattr(result, 'stdout', ''), + getattr(result, 'error', ''), + ).lower() + snippets = ( + 'has no upstream branch', + 'no upstream branch', + 'no upstream configured', + 'no tracking information for the current branch', + 'no tracking information', + 'set the remote as upstream', + 'set the upstream branch', + 'please specify which branch you want to merge with', + ) + return any(snippet in message for snippet in snippets) + + +def _validate_git_url_credentials(url, username, password): + if not username and not password: + return None + if not username or not password: + raise InvalidArgumentException( + 'Both username and password are required when using Git ' + 'credentials.') + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https'): + raise InvalidArgumentException( + 'Only http(s) Git URLs support username/password credentials.') + return parsed + + +def _askpass_script(): + return ( + '#!/bin/sh\n' + 'case "$1" in\n' + '*Username*) printf "%s\\n" "$GIT_USERNAME" ;;\n' + '*Password*) printf "%s\\n" "$GIT_PASSWORD" ;;\n' + '*) printf "\\n" ;;\n' + 'esac\n' + ) + + +class Git(object): + def __init__(self, commands): + self.commands = commands + + def _run_git(self, repo_path, args, **opts): + if repo_path: + opts['cwd'] = repo_path + return self.commands.run('git {0}'.format(' '.join(args)), **opts) + + def _get_remote_url(self, repo_path, remote, **opts): + opts = dict(opts) + opts.pop('background', None) + result = self._run_git( + repo_path, + ['remote', 'get-url', shell_quote(remote)], + **opts + ) + url = (getattr(result, 'stdout', '') or '').strip() + if not url: + raise InvalidArgumentException( + 'Remote "{0}" URL not found in repository.'.format(remote)) + return url + + def _resolve_remote_name(self, repo_path, remote=None, **opts): + if remote: + return remote + opts = dict(opts) + opts.pop('background', None) + result = self._run_git(repo_path, ['remote'], **opts) + remotes = [ + line.strip() + for line in (getattr(result, 'stdout', '') or '').splitlines() + if line.strip() + ] + if len(remotes) == 1: + return remotes[0] + if len(remotes) == 0: + raise InvalidArgumentException( + 'No remotes found in the repository.') + raise InvalidArgumentException( + 'Remote is required when using username/password and the ' + 'repository has multiple remotes.') + + def _with_remote_credentials(self, repo_path, remote, username, password, + operation, **opts): + original_url = self._get_remote_url(repo_path, remote, **opts) + _validate_git_url_credentials(original_url, username, password) + sandbox = getattr(self.commands, 'sandbox', None) + filesystem = getattr(sandbox, 'files', None) + if filesystem is None: + raise SandboxError( + 'Sandbox filesystem is required for credentialed Git ' + 'operations.') + temp_dir = '/tmp/qiniu-git-auth' + setup_opts = dict(opts) + setup_opts.pop('background', None) + prepare_result = self.commands.run( + 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), + **setup_opts + ) + if getattr(prepare_result, 'exit_code', 0): + return prepare_result + askpass_path = '{0}/qiniu-git-askpass-{1}'.format( + temp_dir, + uuid.uuid4().hex, + ) + try: + filesystem.write(askpass_path, _askpass_script()) + chmod_result = self.commands.run( + 'chmod 700 {0}'.format(shell_quote(askpass_path)), + **setup_opts + ) + if getattr(chmod_result, 'exit_code', 0): + _remove_credential_file(filesystem, askpass_path) + return chmod_result + + auth_opts = dict(opts) + envs = dict(auth_opts.get('envs') or {}) + envs.update({ + 'GIT_ASKPASS': askpass_path, + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_USERNAME': username, + 'GIT_PASSWORD': password, + }) + auth_opts['envs'] = envs + + result = operation(auth_opts) + if opts.get('background') and hasattr(result, 'wait'): + result.wait = _cleanup_after_wait( + result.wait, filesystem, askpass_path) + else: + _remove_credential_file(filesystem, askpass_path) + return result + except BaseException as err: + _remove_credential_file(filesystem, askpass_path) + raise err + + def _raise_known_result_error( + self, result, operation, throw_on_error=False): + if (getattr(result, 'exit_code', None) == -1 and + hasattr(result, 'wait')): + if throw_on_error: + result.wait = self._raise_after_wait(result.wait, operation) + return + if not hasattr(result, 'exit_code'): + return + if result.exit_code: + if _is_auth_failure(result): + raise GitAuthException( + 'Git {0} requires credentials for private repositories.' + .format(operation)) + if _is_missing_upstream(result): + raise GitUpstreamException( + 'Git {0} failed because no upstream branch is configured.' + .format(operation)) + if throw_on_error: + raise CommandExitError(result) + + def _raise_after_wait(self, wait, operation): + def wrapped_wait(*args, **kwargs): + result = wait(*args, **kwargs) + self._raise_known_result_error( + result, operation, throw_on_error=True) + return result + return wrapped_wait + + def clone(self, repo_url, path=None, branch=None, depth=None, **opts): + args = ['clone'] + if depth: + args.extend(['--depth', shell_quote(depth)]) + if branch: + args.extend(['--branch', shell_quote(branch)]) + args.append(shell_quote(repo_url)) + if path: + args.append(shell_quote(path)) + return self._run_git(None, args, **opts) + + def init(self, repo_path, bare=False, initial_branch=None, **opts): + args = ['init'] + if bare: + args.append('--bare') + if initial_branch: + args.extend(['--initial-branch', shell_quote(initial_branch)]) + return self._run_git(repo_path, args, **opts) + + def status(self, repo_path, **opts): + result = self._run_git( + repo_path, [ + 'status', '--porcelain=1', '-b'], **opts) + if result.exit_code: + raise CommandExitError(result) + return parse_git_status(result.stdout) + + def add(self, repo_path, files=None, all=False, **opts): + args = ['add'] + if all: + args.append('--all') + else: + args.append('--') + for path in _normalize_paths(files) or ['.']: + args.append(shell_quote(path)) + return self._run_git(repo_path, args, **opts) + + def commit(self, repo_path, message, author_name=None, author_email=None, + allow_empty=False, **opts): + args = [] + if author_name: + args.extend(['-c', shell_quote('user.name={0}'.format( + author_name))]) + if author_email: + args.extend(['-c', shell_quote('user.email={0}'.format( + author_email))]) + args.extend(['commit', '-m', shell_quote(message)]) + if allow_empty: + args.append('--allow-empty') + return self._run_git(repo_path, args, **opts) + + def configure_user(self, repo_path, name, email, **opts): + name_result = self._run_git( + repo_path, + ['config', 'user.name', shell_quote(name)], + **opts + ) + if name_result.exit_code != 0: + return name_result + return self._run_git( + repo_path, + ['config', 'user.email', shell_quote(email)], + **opts + ) + + configureUser = configure_user + + def pull(self, repo_path, remote=None, branch=None, username=None, + password=None, **opts): + throw_on_error = opts.pop('throw_on_error', False) + if password and not username: + raise InvalidArgumentException( + 'Git pull requires username when password is provided') + remote_name = None + if username and password: + remote_name = self._resolve_remote_name(repo_path, remote, **opts) + + args = ['pull'] + target_remote = remote_name or remote + if target_remote: + args.append(shell_quote(target_remote)) + if branch: + args.append(shell_quote(branch)) + if username and password: + result = self._with_remote_credentials( + repo_path, + remote_name, + username, + password, + lambda auth_opts: self._run_git( + repo_path, args, throw_on_error=False, **auth_opts), + **opts + ) + self._raise_known_result_error( + result, 'pull', throw_on_error=throw_on_error) + return result + + result = self._run_git(repo_path, args, throw_on_error=False, **opts) + self._raise_known_result_error( + result, 'pull', throw_on_error=throw_on_error) + return result + + def push(self, repo_path, remote=None, branch=None, set_upstream=True, + username=None, password=None, **opts): + throw_on_error = opts.pop('throw_on_error', False) + if password and not username: + raise InvalidArgumentException( + 'Git push requires username when password is provided') + remote_name = None + if username and password: + remote_name = self._resolve_remote_name(repo_path, remote, **opts) + + args = ['push'] + target_remote = remote_name or remote + if set_upstream and target_remote: + args.append('--set-upstream') + if target_remote: + args.append(shell_quote(target_remote)) + if branch: + args.append(shell_quote(branch)) + if username and password: + result = self._with_remote_credentials( + repo_path, + remote_name, + username, + password, + lambda auth_opts: self._run_git( + repo_path, args, throw_on_error=False, **auth_opts), + **opts + ) + self._raise_known_result_error( + result, 'push', throw_on_error=throw_on_error) + return result + + result = self._run_git(repo_path, args, throw_on_error=False, **opts) + self._raise_known_result_error( + result, 'push', throw_on_error=throw_on_error) + return result + + def dangerously_authenticate( + self, + username, + password, + host='github.com', + protocol='https', + **opts): + opts = dict(opts) + opts.pop('background', None) + if not username: + raise InvalidArgumentException('username is required') + if not password: + raise InvalidArgumentException('password is required') + + result = self._run_git( + None, + ['config', '--global', 'credential.helper', 'store'], + **opts + ) + if result.exit_code != 0: + return result + + credential = ( + 'protocol={0}\n' + 'host={1}\n' + 'username={2}\n' + 'password={3}\n\n' + ).format(protocol, host, username, password) + handle = self.commands.run( + 'git credential approve', + stdin=True, + background=True, + **opts + ) + self.commands.send_stdin(handle.pid, credential) + self.commands.close_stdin(handle.pid) + return handle.wait() + + dangerouslyAuthenticate = dangerously_authenticate + + def remote_add(self, repo_path, name, url, **opts): + fetch = opts.pop('fetch', False) + overwrite = opts.pop('overwrite', False) + if overwrite: + self._run_git(repo_path, [ + 'remote', + 'remove', + shell_quote(name), + ], **opts) + result = self._run_git(repo_path, [ + 'remote', + 'add', + shell_quote(name), + shell_quote(url), + ], **opts) + if result.exit_code != 0 or not fetch: + return result + return self._run_git(repo_path, [ + 'fetch', + shell_quote(name), + ], **opts) + + remoteAdd = remote_add + + def remote_get(self, repo_path, name='origin', **opts): + return self._run_git(repo_path, [ + 'remote', + 'get-url', + shell_quote(name), + ], **opts) + + remoteGet = remote_get + + def branches(self, repo_path, **opts): + result = self._run_git( + repo_path, + ['branch', shell_quote('--format=%(refname:short)\t%(HEAD)')], + **opts + ) + return parse_git_branches(result.stdout) + + def create_branch(self, repo_path, name, start_point=None, **opts): + args = ['checkout', '-b', shell_quote(name)] + if start_point: + args.append(shell_quote(start_point)) + return self._run_git(repo_path, args, **opts) + + createBranch = create_branch + + def checkout_branch(self, repo_path, name, create=False, **opts): + args = ['checkout'] + if create: + args.append('-b') + args.append(shell_quote(name)) + return self._run_git(repo_path, args, **opts) + + checkoutBranch = checkout_branch + + def delete_branch(self, repo_path, name, force=True, **opts): + return self._run_git(repo_path, [ + 'branch', + '-D' if force else '-d', + shell_quote(name), + ], **opts) + + deleteBranch = delete_branch + + def reset(self, repo_path, target=None, mode=None, **opts): + args = ['reset'] + mode = mode or opts.get('reset_type') or opts.get('resetType') + if mode: + if mode not in RESET_MODES: + raise InvalidArgumentException( + 'Unsupported git reset mode: {0}'.format(mode) + ) + args.append('--{0}'.format(mode)) + if target: + args.append(shell_quote(target)) + return self._run_git(repo_path, args, **opts) + + def restore(self, repo_path, paths=None, staged=False, source=None, + **opts): + args = ['restore'] + if staged: + args.append('--staged') + if source: + args.extend(['--source', shell_quote(source)]) + paths = paths if paths is not None else opts.get('files') + args.append('--') + for path in _normalize_paths(paths) or ['.']: + args.append(shell_quote(path)) + return self._run_git(repo_path, args, **opts) + + def set_config(self, key, value, scope='global', path=None, **opts): + """Set a Git config value. + + Preferred signature: + set_config(key, value, scope='global', path=None, **opts) + + For local config, pass scope='local' and path=repo_path. For backward + compatibility this method also accepts the deprecated signature: + set_config(repo_path, key, value, global_config=False, **opts) + """ + key, value, scope, path = self._normalize_set_config_args( + key, value, scope, path, opts) + scope_flag, repo_path = self._resolve_config_scope(scope, path) + args = ['config'] + if scope_flag: + args.append(scope_flag) + args.extend([shell_quote(key), shell_quote(value)]) + return self._run_git(repo_path, args, **opts) + + setConfig = set_config + + def get_config(self, key, scope='global', path=None, **opts): + """Get a Git config value. + + Preferred signature: + get_config(key, scope='global', path=None, **opts) + + For local config, pass scope='local' and path=repo_path. For backward + compatibility this method also accepts the deprecated signature: + get_config(repo_path, key, global_config=False, **opts) + """ + key, scope, path = self._normalize_get_config_args( + key, scope, path, opts) + scope_flag, repo_path = self._resolve_config_scope(scope, path) + args = ['config'] + if scope_flag: + args.append(scope_flag) + args.extend(['--get', shell_quote(key)]) + return self._run_git(repo_path, args, **opts) + + getConfig = get_config + + def _normalize_set_config_args(self, key, value, scope, path, opts): + global_config = opts.pop( + 'global_config', opts.pop('globalConfig', False)) + if global_config or self._is_legacy_config_call(key, scope): + repo_path = key + key = value + value = scope + scope = 'global' if global_config or repo_path is None else 'local' + path = None if scope == 'global' else repo_path + return key, value, scope, path + + def _normalize_get_config_args(self, key, scope, path, opts): + global_config = opts.pop( + 'global_config', opts.pop('globalConfig', False)) + if global_config or self._is_legacy_config_call(key, scope): + repo_path = key + key = scope + scope = 'global' if global_config or repo_path is None else 'local' + path = None if scope == 'global' else repo_path + return key, scope, path + + def _is_legacy_config_call(self, key, scope): + if scope is None: + return False + scope_name = str(scope).strip().lower() + if scope_name not in ('global', 'local', 'system'): + return True + key_text = str(key or '') + return ( + '/' in key_text or + '\\' in key_text or + key_text in ('.', '..') or + '.' not in key_text + ) + + def _resolve_config_scope(self, scope=None, path=None): + scope_name = (scope or 'global').strip().lower() + if scope_name not in ('global', 'local', 'system'): + raise InvalidArgumentException( + 'Git config scope must be global, local, or system') + if scope_name == 'local': + if not path: + raise InvalidArgumentException( + 'Repository path is required for local scope') + return '--local', path + if scope_name == 'system': + return '--system', None + return '--global', None diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py new file mode 100644 index 00000000..500c7a52 --- /dev/null +++ b/qiniu/services/sandbox/pty.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +import base64 + +from .commands import CommandHandle, command_result_from_events +from .envd import connect_rpc, connect_stream_rpc +from .errors import SandboxError + + +class PtySize(object): + def __init__(self, rows=24, cols=80): + self.rows = rows + self.cols = cols + + def to_dict(self): + return {'rows': self.rows, 'cols': self.cols} + + +def _normalize_size(size=None, rows=None, cols=None): + if size is None: + return {'rows': rows or 24, 'cols': cols or 80} + if hasattr(size, 'to_dict'): + data = size.to_dict() + elif isinstance(size, dict): + data = size + else: + data = { + 'rows': getattr(size, 'rows', None), + 'cols': getattr(size, 'cols', None), + } + return { + 'rows': data.get('rows') or rows or 24, + 'cols': data.get('cols') or cols or 80, + } + + +class Pty(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def create(self, size=None, user=None, cwd=None, envs=None, timeout=None, + rows=None, cols=None, **opts): + envs = dict(envs or {}) + envs.setdefault('TERM', 'xterm-256color') + envs.setdefault('LANG', 'C.UTF-8') + envs.setdefault('LC_ALL', 'C.UTF-8') + process = { + 'cmd': '/bin/bash', + 'args': ['-i', '-l'], + 'envs': envs, + } + if cwd: + process['cwd'] = cwd + body = { + 'process': process, + 'pty': {'size': _normalize_size(size, rows=rows, cols=cols)}, + } + if opts.get('tag'): + body['tag'] = opts.get('tag') + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Start', + body, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events([first_event]) + return CommandHandle(self, result, events=events) + + def connect(self, pid, user=None, timeout=None, throw_on_error=False): + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Connect', + {'process': {'selector': {'pid': pid}}}, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + try: + first_event = next(events) + except StopIteration: + first_event = None + result = command_result_from_events([first_event]) + return CommandHandle( + self, + result, + events=events, + throw_on_error=throw_on_error, + ) + + def send_stdin(self, pid, data, user=None, timeout=None): + if isinstance(data, bytearray): + data = bytes(data) + elif not isinstance(data, bytes): + if hasattr(data, 'encode'): + data = data.encode('utf-8') + else: + data = str(data).encode('utf-8') + return connect_rpc(self.sandbox, '/process.Process/SendInput', { + 'process': {'selector': {'pid': pid}}, + 'input': {'pty': base64.b64encode(data).decode('ascii')}, + }, user=user, timeout=timeout) + + sendStdin = send_stdin + send_input = send_stdin + sendInput = send_stdin + + def resize(self, pid, size=None, user=None, timeout=None, rows=None, + cols=None): + return connect_rpc(self.sandbox, '/process.Process/Update', { + 'process': {'selector': {'pid': pid}}, + 'pty': {'size': _normalize_size(size, rows=rows, cols=cols)}, + }, user=user, timeout=timeout) + + def kill(self, pid, user=None, timeout=None): + try: + connect_rpc(self.sandbox, '/process.Process/SendSignal', { + 'process': {'selector': {'pid': pid}}, + 'signal': 'SIGNAL_SIGKILL', + }, user=user, timeout=timeout) + return True + except SandboxError as err: + if err.status_code == 404: + return False + raise diff --git a/qiniu/services/sandbox/resources.py b/qiniu/services/sandbox/resources.py new file mode 100644 index 00000000..05456ff4 --- /dev/null +++ b/qiniu/services/sandbox/resources.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + + +class GitRepositoryResource(object): + def __init__(self, url, mount_path, authorization_token=None, + repository_type='github_repository'): + self.url = url + self.mount_path = mount_path + self.authorization_token = authorization_token + self.repository_type = repository_type + + def to_dict(self): + data = { + 'type': self.repository_type, + 'url': self.url, + 'mount_path': self.mount_path, + } + if self.authorization_token is not None: + data['authorization_token'] = self.authorization_token + return data + + +class KodoResource(object): + def __init__(self, bucket, mount_path, prefix=None, read_only=None): + self.bucket = bucket + self.mount_path = mount_path + self.prefix = prefix + self.read_only = read_only + + def to_dict(self): + data = { + 'type': 'kodo', + 'bucket': self.bucket, + 'mount_path': self.mount_path, + } + if self.prefix is not None: + data['prefix'] = self.prefix + if self.read_only is not None: + data['read_only'] = self.read_only + return data diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py new file mode 100644 index 00000000..f95305cf --- /dev/null +++ b/qiniu/services/sandbox/sandbox.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- +import time + +import requests + +from .client import SandboxClient +from .commands import Commands +from .constants import DEFAULT_USER, ENVD_PORT, MCP_PORT +from .errors import SandboxError +from .filesystem import Filesystem +from .git import Git +from .pty import Pty +from .util import ( + append_query, + file_signature, + get_info_value, + utc_timestamp_after, +) + + +try: + from time import monotonic as _monotonic_time +except ImportError: + _monotonic_time = time.time + + +class _ConnectDescriptor(object): + def __get__(self, obj, cls): + if obj is None: + def class_connect(sandbox_id, client=None, timeout=15, **opts): + client = client or SandboxClient(**opts) + info = client.connect_sandbox(sandbox_id, timeout=timeout) + sandbox = cls( + client=client, + info=info, + sandbox_id=sandbox_id, + **opts + ) + sandbox.refresh_envd_token_if_needed() + return sandbox + return class_connect + + def instance_connect(timeout=15): + info = obj.client.connect_sandbox( + obj.sandbox_id, timeout=timeout) + obj.update_info(info) + obj.refresh_envd_token_if_needed() + return obj + return instance_connect + + +class SandboxPaginator(object): + def __init__(self, client=None, **opts): + opts = dict(opts) + client_opts = {} + for key in ('endpoint', 'api_url', 'api_key', 'access_token', + 'mac', 'access_key', 'secret_key', 'session', 'timeout'): + if key in opts: + client_opts[key] = opts.pop(key) + self.client = client or SandboxClient(**client_opts) + self.next_token = opts.pop('nextToken', None) or opts.pop( + 'next_token', None) + self.opts = opts + self._has_next = True + + @property + def has_next(self): + return bool(self.next_token) or self._has_next + + hasNext = has_next + + def next_items(self, **opts): + request_opts = dict(self.opts) + request_opts.update(opts) + if self.next_token and request_opts.get('nextToken') is None: + request_opts['nextToken'] = self.next_token + data = self.client.list_sandboxes_v2(**request_opts) or {} + items = data if isinstance(data, list) else ( + data.get('items') or data.get('sandboxes') or [] + ) + self.next_token = None if isinstance(data, list) else ( + data.get('nextToken') or data.get('next_token') + ) + self._has_next = bool(self.next_token) + return [Sandbox(client=self.client, info=item) for item in items] + + nextItems = next_items + + +class Sandbox(object): + connect = _ConnectDescriptor() + + def __init__(self, client=None, info=None, sandbox_id=None, sandboxID=None, + envd_url=None, envdAccessToken=None, **client_opts): + envd_url = envd_url or client_opts.pop('envdUrl', None) + self.client = client or SandboxClient(**client_opts) + self.info = info or {} + self.sandbox_id = ( + sandbox_id or sandboxID or + self.info.get('sandboxID') or + self.info.get('sandboxId') or + self.info.get('sandbox_id') or + self.info.get('id') + ) + self.sandboxID = self.sandbox_id + self.template_id = ( + self.info.get('templateID') or + self.info.get('templateId') or + self.info.get('template_id') + ) + self.templateID = self.template_id + self.domain = ( + self.info.get('domain') or + self.info.get('sandboxDomain') or + self.info.get('sandbox_domain') + ) + self.sandbox_domain = self.domain + self.sandboxDomain = self.domain + self.envd_version = get_info_value( + self.info, 'envdVersion', 'envd_version') + self.envdVersion = self.envd_version + self.envd_access_token = ( + envdAccessToken or + get_info_value(self.info, 'envdAccessToken', 'envd_access_token') + ) + self.envdAccessToken = self.envd_access_token + self.traffic_access_token = get_info_value( + self.info, 'trafficAccessToken', 'traffic_access_token' + ) + self.trafficAccessToken = self.traffic_access_token + self._envd_url = envd_url + self.files = Filesystem(self) + self.filesystem = self.files + self.commands = Commands(self) + self.pty = Pty(self) + self.git = Git(self.commands) + + @classmethod + def create(cls, template=None, client=None, timeout=None, metadata=None, + envs=None, secure=True, allow_internet_access=True, mcp=None, + network=None, lifecycle=None, resources=None, injections=None, + **opts): + client_opts = {} + for key in ('endpoint', 'api_url', 'api_key', 'access_token', + 'mac', 'access_key', 'secret_key', 'session'): + if key in opts: + client_opts[key] = opts.pop(key) + client = client or SandboxClient(**client_opts) + info = client.create_sandbox( + template=template, + timeout=timeout, + metadata=metadata, + envs=envs, + secure=secure, + allow_internet_access=allow_internet_access, + mcp=mcp, + network=network, + lifecycle=lifecycle, + resources=resources, + injections=injections, + **opts + ) + sandbox = cls(client=client, info=info) + sandbox.refresh_envd_token_if_needed() + return sandbox + + @classmethod + def list(cls, client=None, **opts): + return SandboxPaginator(client=client, **opts) + + def update_info(self, info): + if not info: + return self + self.info = info + self.sandbox_id = ( + info.get('sandboxID') or info.get('sandboxId') or + info.get('sandbox_id') or self.sandbox_id + ) + self.sandboxID = self.sandbox_id + self.template_id = ( + info.get('templateID') or info.get('templateId') or + info.get('template_id') or self.template_id + ) + self.templateID = self.template_id + self.domain = ( + info.get('domain') or info.get('sandboxDomain') or + info.get('sandbox_domain') or self.domain + ) + self.sandbox_domain = self.domain + self.sandboxDomain = self.domain + self.envd_access_token = ( + get_info_value(info, 'envdAccessToken', 'envd_access_token') or + self.envd_access_token + ) + self.envdAccessToken = self.envd_access_token + self.traffic_access_token = ( + get_info_value( + info, + 'trafficAccessToken', + 'traffic_access_token', + ) or self.traffic_access_token + ) + self.trafficAccessToken = self.traffic_access_token + self.envd_version = ( + get_info_value(info, 'envdVersion', 'envd_version') or + self.envd_version + ) + self.envdVersion = self.envd_version + return self + + updateInfo = update_info + + def refresh_envd_token_if_needed(self): + if self.envd_access_token or not self.sandbox_id: + return self + try: + self.update_info(self.get_info()) + except SandboxError: + pass + return self + + refreshEnvdTokenIfNeeded = refresh_envd_token_if_needed + + def kill(self): + return self.client.delete_sandbox(self.sandbox_id) + + def set_timeout(self, timeout): + return self.client.update_sandbox_timeout( + self.sandbox_id, timeout=timeout) + + setTimeout = set_timeout + + def refresh(self, duration=None, **opts): + if duration is not None: + opts['duration'] = duration + return self.client.refresh_sandbox(self.sandbox_id, **opts) + + def pause(self): + return self.client.pause_sandbox(self.sandbox_id) + + beta_pause = pause + betaPause = pause + + def resume(self, **opts): + info = self.client.resume_sandbox(self.sandbox_id, **opts) + self.update_info(info) + return self + + def update_network(self, network): + return self.client.update_sandbox(self.sandbox_id, network=network) + + updateNetwork = update_network + + def get_info(self): + return self.client.get_sandbox(self.sandbox_id) + + getInfo = get_info + + def get_metrics(self, **opts): + return self.client.get_sandbox_metrics(self.sandbox_id, **opts) + + getMetrics = get_metrics + + def get_logs(self, **opts): + return self.client.get_sandbox_logs(self.sandbox_id, **opts) + + getLogs = get_logs + + def get_host(self, port): + if not self.domain: + return '' + return '{0}-{1}.{2}'.format(port, self.sandbox_id, self.domain) + + getHost = get_host + + def envd_url(self): + if self._envd_url: + return self._envd_url + if not self.domain: + raise SandboxError('Sandbox domain is not available') + return 'https://{0}'.format(self.get_host(ENVD_PORT)) + + envdUrl = envd_url + + def get_mcp_url(self): + return 'https://{0}/mcp'.format(self.get_host(MCP_PORT)) + + getMcpUrl = get_mcp_url + + def get_mcp_token(self): + return self.traffic_access_token + + getMcpToken = get_mcp_token + + def file_url( + self, + path, + operation, + user=None, + signature_expiration=300, + **opts): + user = user or DEFAULT_USER + query = { + 'path': path, + 'username': user, + } + if self.envd_access_token: + expiration = opts.get('signatureExpiration', signature_expiration) + if expiration is not None and expiration < 1000000000: + expiration = utc_timestamp_after(expiration) + query['signature'] = file_signature( + path, + operation, + user, + self.envd_access_token, + expiration, + ) + query['signature_expiration'] = expiration + return self.envd_url() + append_query('/files', query) + + fileUrl = file_url + + def download_url(self, path, **opts): + return self.file_url(path, 'read', **opts) + + downloadUrl = download_url + DownloadURL = download_url + + def upload_url(self, path, **opts): + return self.file_url(path, 'write', **opts) + + uploadUrl = upload_url + UploadURL = upload_url + + def wait_for_ready(self, timeout=60, interval=1): + started = _monotonic_time() + while True: + elapsed = _monotonic_time() - started + remaining = None if timeout is None else max(timeout - elapsed, 0) + request_timeout = 5 + if remaining is not None: + request_timeout = max(0.1, min(5, remaining)) + try: + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) + if response.status_code >= 200 and response.status_code < 300: + return self + except requests.RequestException: + pass + if timeout is not None and elapsed >= timeout: + raise SandboxError('Sandbox envd did not become ready') + sleep_time = interval + if remaining is not None: + sleep_time = min(interval, remaining) + if sleep_time > 0: + time.sleep(sleep_time) + + waitForReady = wait_for_ready + + def is_running(self, request_timeout=None): + if not self.domain: + return False + try: + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) + except requests.RequestException: + return False + if response.status_code == 502: + return False + if response.status_code >= 200 and response.status_code < 300: + return True + return False + + isRunning = is_running diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py new file mode 100644 index 00000000..e719100e --- /dev/null +++ b/qiniu/services/sandbox/template.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +import json + +from .util import shell_quote + + +class ReadyCmd(object): + def __init__(self, cmd): + self._cmd = cmd + + def get_cmd(self): + return self._cmd + + +def wait_for_port(port): + port = int(port) + return ReadyCmd( + "ss -tuln | awk '{{print $5}}' | grep -E '(^|:){0}$'".format( + port)) + + +def wait_for_url(url, status_code=200): + status_code = int(status_code) + return ReadyCmd( + '[ "$(curl -s -o /dev/null -w "%{{http_code}}" {0})" = "{1}" ]'.format( + shell_quote(url), + status_code, + ) + ) + + +def wait_for_process(process_name): + return ReadyCmd('pgrep {0} > /dev/null'.format(shell_quote(process_name))) + + +def wait_for_file(filename): + return ReadyCmd('[ -f {0} ]'.format(shell_quote(filename))) + + +def wait_for_timeout(timeout): + seconds = max(1, int(timeout) // 1000) + return ReadyCmd('sleep {0}'.format(seconds)) + + +def _ready_cmd_value(command): + if hasattr(command, 'get_cmd'): + return command.get_cmd() + return command + + +class Template(object): + def __init__(self): + self.build_config = {'steps': []} + + def from_image(self, image, credentials=None): + self.build_config['fromImage'] = image + self.build_config.pop('fromTemplate', None) + if credentials: + self.build_config['fromImageRegistry'] = credentials + return self + + fromImage = from_image + + def from_template(self, template_id): + self.build_config['fromTemplate'] = template_id + self.build_config.pop('fromImage', None) + self.build_config.pop('fromImageRegistry', None) + return self + + fromTemplate = from_template + + def add_step(self, step_type, args, **extra): + step = {'type': step_type, 'args': [str(arg) for arg in args]} + step.update(extra) + self.build_config['steps'].append(step) + return self + + addStep = add_step + + def run_cmd(self, command, user=None): + if isinstance(command, (list, tuple)): + command = ' '.join(shell_quote(arg) for arg in command) + args = [command] + if user: + args.append(user) + return self.add_step('RUN', args) + + run = run_cmd + runCmd = run_cmd + + def copy(self, src, dest, chmod=None, chown=None): + extra = {} + if chmod is not None: + extra['chmod'] = str(chmod) + if chown is not None: + extra['chown'] = chown + return self.add_step('COPY', [src, dest], **extra) + + def set_env(self, key, value): + return self.add_step('ENV', [key, value]) + + setEnv = set_env + + def set_start_cmd(self, command, ready_cmd=None): + self.build_config['startCmd'] = command + if ready_cmd is not None: + self.build_config['readyCmd'] = _ready_cmd_value(ready_cmd) + return self + + setStartCmd = set_start_cmd + + def set_ready_cmd(self, command): + self.build_config['readyCmd'] = _ready_cmd_value(command) + return self + + setReadyCmd = set_ready_cmd + + def to_dict(self): + data = dict(self.build_config) + data['steps'] = list(self.build_config.get('steps', [])) + return data + + def to_json(self): + return json.dumps(self.to_dict(), separators=(',', ':')) + + toJSON = to_json diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py new file mode 100644 index 00000000..3139c871 --- /dev/null +++ b/qiniu/services/sandbox/util.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +import base64 +import hashlib +import json as std_json +import os +import posixpath +import time + +from qiniu.compat import bytes as bytes_type +from qiniu.compat import is_py2, str as text_type, urlencode + +from .constants import DEFAULT_ENDPOINT, DEFAULT_USER + + +def normalize_endpoint(endpoint=None): + endpoint = ( + endpoint or + os.getenv('QINIU_SANDBOX_API_URL') or + os.getenv('QINIU_SANDBOX_ENDPOINT') or + os.getenv('E2B_API_URL') or + DEFAULT_ENDPOINT + ) + return endpoint.rstrip('/') + + +def encode_path(value): + if is_py2: + from urllib import quote + if isinstance(value, text_type): + value = value.encode('utf-8') + else: + value = str(value) + else: + from urllib.parse import quote + value = str(value) + return quote(value, safe='') + + +def append_query(path, query=None): + query = query or {} + normalized = {} + for key, value in query.items(): + if value is None: + continue + if isinstance(value, (list, tuple)): + value = ','.join([str(item) for item in value]) + normalized[key] = value + if not normalized: + return path + return path + '?' + urlencode(normalized) + + +def json_dumps(data): + return std_json.dumps(data, separators=(',', ':')) + + +def basic_auth(user=None): + user = user or DEFAULT_USER + raw = ('{0}:'.format(user)).encode('utf-8') + return 'Basic {0}'.format(base64.b64encode(raw).decode('ascii')) + + +def file_signature(path, operation, user, access_token, expiration): + if expiration is None: + expiration = '' + components = [path, operation, user, access_token, expiration] + raw = b':'.join([_to_utf8_bytes(component) + for component in components]) + return 'v1_{0}'.format(hashlib.sha256(raw).hexdigest()) + + +def _to_utf8_bytes(value): + if isinstance(value, bytes_type): + return value + if isinstance(value, text_type): + return value.encode('utf-8') + return str(value).encode('utf-8') + + +def file_basename(path): + basename = posixpath.basename(path.rstrip('/')) or 'file' + if is_py2 and isinstance(basename, text_type): + return basename.encode('utf-8') + return basename + + +def shell_quote(value): + try: + from shlex import quote + except ImportError: + from pipes import quote + if is_py2: + if isinstance(value, text_type): + value = value.encode('utf-8') + else: + value = str(value) + else: + value = str(value) + return quote(value) + + +def utc_timestamp_after(seconds): + return int(time.time()) + int(seconds) + + +def parse_json_response(response): + if response.content in (None, b'', ''): + return None + return response.json() + + +def get_info_value(info, camel_key, snake_key=None): + info = info or {} + if camel_key in info: + return info.get(camel_key) + if snake_key and snake_key in info: + return info.get(snake_key) + return None diff --git a/tests/cases/test_services/test_sandbox/__init__.py b/tests/cases/test_services/test_sandbox/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py new file mode 100644 index 00000000..6e91a1ad --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -0,0 +1,1637 @@ +# -*- coding: utf-8 -*- +import json + +import pytest +import requests +import qiniu.services.sandbox.sandbox as sandbox_module + +try: + from urllib.parse import parse_qs, urlparse +except ImportError: + from urlparse import parse_qs, urlparse + +from qiniu.services.sandbox import ( + CommandExitError, + DEFAULT_ENDPOINT, + ENVD_PORT, + GitAuthException, + GitBranches, + GitStatus, + Git, + InvalidArgumentException, + KodoResource, + ReadyCmd, + Sandbox, + SandboxClient, + SandboxError, + SandboxPaginator, + Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) +from qiniu.services.sandbox.util import encode_path, file_signature + + +class DummyResponse(object): + def __init__(self, status_code=200, body=None): + self.status_code = status_code + self.content = b'' if body is None else json.dumps( + body).encode('utf-8') + self.text = self.content.decode('utf-8') + self.headers = {'Content-Type': 'application/json'} + + def json(self): + if not self.content: + return None + return json.loads(self.content.decode('utf-8')) + + +class ErrorResponse(object): + def __init__(self, status_code=502): + self.status_code = status_code + self.content = b'' + self.text = '' + self.headers = {} + + def json(self): + return None + + +class RecordingSession(requests.Session): + def __init__(self, responses=None): + super(RecordingSession, self).__init__() + self.responses = list(responses or [DummyResponse(body={})]) + self.requests = [] + + def send(self, request, **kwargs): + self.requests.append(request) + if self.responses: + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response + return DummyResponse(body={}) + + def get(self, url, **kwargs): + self.requests.append(type('Request', (object,), { + 'method': 'GET', + 'url': url, + 'kwargs': kwargs, + })()) + if self.responses: + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response + return DummyResponse(body={}) + + +def body_of(request): + if request.body is None: + return None + if isinstance(request.body, bytes): + return json.loads(request.body.decode('utf-8')) + return json.loads(request.body) + + +def test_client_uses_default_endpoint_and_api_key_headers(): + session = RecordingSession([DummyResponse(201, { + 'sandboxID': 'sbx123', + 'templateID': 'base', + 'domain': 'example.test', + 'envdAccessToken': 'envd-token', + })]) + client = SandboxClient(api_key='api-key', session=session) + + info = client.create_sandbox(template='base', timeout=60, envs={'A': 'B'}) + + assert info['sandboxID'] == 'sbx123' + req = session.requests[0] + assert req.method == 'POST' + assert req.url == DEFAULT_ENDPOINT + '/sandboxes' + assert req.headers['X-API-Key'] == 'api-key' + assert req.headers['Authorization'] == 'Bearer api-key' + assert body_of(req) == { + 'templateID': 'base', + 'timeout': 60, + 'envVars': {'A': 'B'}, + } + + +def test_create_sandbox_rejects_conflicting_option_aliases(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as env_err: + client.create_sandbox(envs={'A': 'B'}, envVars={'A': 'C'}) + with pytest.raises(SandboxError) as network_err: + client.create_sandbox( + allow_internet_access=True, + allowInternetAccess=False, + ) + + assert 'envs, envVars' in str(env_err.value) + assert 'allow_internet_access, allowInternetAccess' in str( + network_err.value) + + +def test_client_reads_documented_api_key_env_fallbacks(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_API_KEY', raising=False) + monkeypatch.delenv('QINIU_API_KEY', raising=False) + monkeypatch.delenv('E2B_API_KEY', raising=False) + + monkeypatch.setenv('QINIU_API_KEY', 'qiniu-api-key') + assert SandboxClient(session=RecordingSession()).api_key == 'qiniu-api-key' + + monkeypatch.delenv('QINIU_API_KEY') + monkeypatch.setenv('E2B_API_KEY', 'e2b-api-key') + assert SandboxClient(session=RecordingSession()).api_key == 'e2b-api-key' + + monkeypatch.setenv('QINIU_SANDBOX_API_KEY', 'sandbox-api-key') + assert SandboxClient(session=RecordingSession()).api_key == ( + 'sandbox-api-key') + + +def test_client_reads_qiniu_mac_credentials_from_env(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_ACCESS_KEY', raising=False) + monkeypatch.delenv('QINIU_SANDBOX_SECRET_KEY', raising=False) + monkeypatch.setenv('QINIU_SANDBOX_ACCESS_KEY', 'ak') + monkeypatch.setenv('QINIU_SANDBOX_SECRET_KEY', 'sk') + + client = SandboxClient(session=RecordingSession()) + + assert client.mac is not None + + +def test_util_helpers_encode_unicode_values_safely(): + assert encode_path(u'目录/文件.txt') + assert file_signature( + u'/tmp/文件.txt', + 'read', + u'用户', + u'token', + 1893456000, + ).startswith('v1_') + + +def test_create_with_kodo_resource_requires_qiniu_credentials(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.create_sandbox( + resources=[ + KodoResource( + bucket='bucket', + mount_path='/mnt/bucket')]) + + assert 'Qiniu AK/SK' in str(err.value) + + +def test_create_with_kodo_resource_uses_qiniu_signature(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient( + api_key='api-key', + access_key='ak', + secret_key='sk', + session=session, + ) + + client.create_sandbox( + resources=[ + KodoResource( + bucket='bucket', + mount_path='/mnt/bucket')]) + + req = session.requests[0] + assert req.headers['X-API-Key'] == 'api-key' + assert req.headers['Authorization'].startswith('Qiniu ak:') + assert 'X-Qiniu-Date' in req.headers + assert body_of(req)['resources'] == [{ + 'type': 'kodo', + 'bucket': 'bucket', + 'mount_path': '/mnt/bucket', + }] + + +def test_create_with_saved_injection_rule_requires_qiniu_credentials(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.create_sandbox(injections=[{ + 'type': 'id', + 'ruleID': 'rule-1', + }]) + + assert 'Qiniu AK/SK' in str(err.value) + + +def test_create_with_saved_injection_rule_uses_qiniu_signature(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient(access_key='ak', secret_key='sk', session=session) + + client.create_sandbox(injections=[{ + 'type': 'id', + 'ruleId': 'rule-1', + }]) + + req = session.requests[0] + assert req.headers['Authorization'].startswith('Qiniu ak:') + assert body_of(req)['injections'] == [{ + 'type': 'id', + 'ruleID': 'rule-1', + }] + + +def test_create_with_saved_injection_rule_accepts_snake_case_rule_id(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient(access_key='ak', secret_key='sk', session=session) + + client.create_sandbox(injections=[{ + 'type': 'id', + 'rule_id': 'rule-1', + }]) + + assert body_of(session.requests[0])['injections'] == [{ + 'type': 'id', + 'ruleID': 'rule-1', + }] + + +def test_sandbox_create_signature_matches_e2b_style(): + session = RecordingSession([ + DummyResponse(201, { + 'sandboxID': 'sbx123', + 'templateID': 'python', + 'domain': 'example.test', + 'envdAccessToken': 'envd-token', + }) + ]) + client = SandboxClient(api_key='api-key', session=session) + + sandbox = Sandbox.create( + 'python', + timeout=120, + metadata={'app': 'tests'}, + envs={'HELLO': 'world'}, + client=client, + ) + + assert sandbox.sandbox_id == 'sbx123' + assert sandbox.sandboxID == 'sbx123' + assert sandbox.template_id == 'python' + assert sandbox.files is sandbox.filesystem + assert sandbox.commands.sandbox is sandbox + assert body_of(session.requests[0])['templateID'] == 'python' + + +def test_client_uses_default_http_timeout(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + custom = SandboxClient( + api_key='api-key', + session=RecordingSession(), + timeout=12, + ) + + assert client.timeout == 30 + assert custom.timeout == 12 + + +def test_client_wraps_request_exceptions_in_sandbox_error(): + client = SandboxClient( + api_key='api-key', + session=RecordingSession([requests.Timeout('timed out')])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert 'Sandbox API request failed' in str(err.value) + assert 'timed out' in str(err.value) + + +def test_client_truncates_long_text_error_responses(): + class TextErrorResponse(ErrorResponse): + def json(self): + raise ValueError('not json') + + client = SandboxClient( + api_key='api-key', + session=RecordingSession([TextErrorResponse(502)])) + client.session.responses[0].text = '' + ('x' * 300) + '' + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert len(str(err.value)) < 260 + assert str(err.value).endswith('...') + + +def test_client_falls_back_to_content_when_error_text_fails(): + class BrokenTextResponse(object): + status_code = 502 + content = b'raw error bytes' + headers = {} + + def json(self): + raise ValueError('not json') + + @property + def text(self): + raise UnicodeDecodeError('utf-8', b'\xff', 0, 1, 'bad') + + client = SandboxClient( + api_key='api-key', + session=RecordingSession([BrokenTextResponse()])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert err.value.data == b'raw error bytes' + + +def test_client_includes_nested_error_message_in_api_errors(): + client = SandboxClient( + api_key='api-key', + session=RecordingSession([DummyResponse(400, { + 'error': {'message': 'bad sandbox request'}, + })])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert 'bad sandbox request' in str(err.value) + + +def test_access_token_auth_requires_access_token(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.rebuild_template('tmpl123') + + assert 'access_token is required' in str(err.value) + + +def test_sandbox_instance_lifecycle_methods_call_control_plane(): + session = RecordingSession([ + DummyResponse(204, None), + DummyResponse(204, None), + DummyResponse(200, { + 'sandboxID': 'sbx123', + 'templateID': 'base', + 'envdAccessToken': 'envd-token', + }), + ]) + client = SandboxClient(api_key='api-key', session=session) + sandbox = Sandbox( + client=client, + info={ + 'sandboxID': 'sbx123', + 'templateID': 'base'}) + + assert sandbox.kill() is None + assert sandbox.set_timeout(30) is None + sandbox.connect(timeout=45) + + assert [ + req.method for req in session.requests] == [ + 'DELETE', + 'POST', + 'POST'] + assert session.requests[0].url.endswith('/sandboxes/sbx123') + assert session.requests[1].url.endswith('/sandboxes/sbx123/timeout') + assert body_of(session.requests[1]) == {'timeout': 30} + assert session.requests[2].url.endswith('/sandboxes/sbx123/connect') + assert body_of(session.requests[2]) == {'timeout': 45} + + +def test_connect_sandbox_uses_timeout_body_only(): + session = RecordingSession([DummyResponse(200, {'sandboxID': 'sbx123'})]) + client = SandboxClient(api_key='api-key', session=session) + + client.connect_sandbox('sbx123', timeout=9) + + assert body_of(session.requests[0]) == {'timeout': 9} + + +def test_sandbox_control_methods_require_sandbox_id(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError): + client.pause_sandbox('') + with pytest.raises(SandboxError): + client.resume_sandbox(None) + with pytest.raises(SandboxError): + client.connect_sandbox('') + + +def test_sandbox_connect_forwards_envd_options_to_instance(): + session = RecordingSession([DummyResponse(200, {'sandboxID': 'sbx123'})]) + client = SandboxClient(api_key='api-key', session=session) + + sandbox = Sandbox.connect( + 'sbx123', + client=client, + envd_url='http://envd.local', + envdAccessToken='envd-token', + ) + + assert sandbox.envd_url() == 'http://envd.local' + assert sandbox.envdAccessToken == 'envd-token' + assert body_of(session.requests[0]) == {'timeout': 15} + + +def test_delete_template_requires_template_id(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.delete_template(None) + + assert 'template_id is required' in str(err.value) + + +def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + + assert sandbox.get_host(ENVD_PORT) == '49983-sbx123.example.test' + assert sandbox.envd_url() == 'https://49983-sbx123.example.test' + + url = sandbox.download_url( + '/tmp/hello.txt', + signature_expiration=1893456000) + parsed = urlparse(url) + query = parse_qs(parsed.query) + + assert parsed.scheme == 'https' + assert parsed.netloc == '49983-sbx123.example.test' + assert parsed.path == '/files' + assert query['path'] == ['/tmp/hello.txt'] + assert query['username'] == ['user'] + assert query['signature_expiration'] == ['1893456000'] + assert query['signature'][0] + + +def test_envd_url_requires_domain_without_override(): + sandbox = Sandbox(info={'sandboxID': 'sbx123'}) + + with pytest.raises(SandboxError): + sandbox.envd_url() + + +def test_file_url_accepts_none_signature_expiration(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + + url = sandbox.download_url('/tmp/hello.txt', signatureExpiration=None) + query = parse_qs(urlparse(url).query) + + assert query['signature'][0] + assert query['signature'][0] == file_signature( + '/tmp/hello.txt', + 'read', + 'user', + 'token', + '', + ) + assert 'signature_expiration' not in query + + +def test_wait_for_ready_passes_request_timeout_to_health_check(): + session = RecordingSession([DummyResponse(200, {})]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=10, interval=2) + + assert session.requests[0].url == sandbox.envd_url() + '/health' + assert session.requests[0].kwargs['timeout'] == 5 + + +def test_wait_for_ready_caps_request_timeout_to_remaining_timeout(): + session = RecordingSession([DummyResponse(200, {})]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=3, interval=2) + + assert session.requests[0].kwargs['timeout'] <= 3 + + +def test_wait_for_ready_caps_sleep_to_remaining_timeout(monkeypatch): + session = RecordingSession([ + DummyResponse(503, {}), + DummyResponse(503, {}), + ]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + times = iter([0, 2.5, 3]) + sleeps = [] + monkeypatch.setattr( + sandbox_module, '_monotonic_time', lambda: next(times)) + monkeypatch.setattr(sandbox_module.time, 'sleep', sleeps.append) + + with pytest.raises(SandboxError): + sandbox.wait_for_ready(timeout=3, interval=2) + + assert sleeps == [0.5] + + +def test_wait_for_ready_ignores_startup_request_errors_until_ready(): + session = RecordingSession([ + requests.exceptions.ConnectionError('envd is starting'), + DummyResponse(200, {}), + ]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=10, interval=0) + + assert len(session.requests) == 2 + + +def test_wait_for_ready_raises_sandbox_error_on_timeout(): + session = RecordingSession([ + requests.exceptions.ConnectionError('envd is starting'), + ]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + with pytest.raises(SandboxError): + sandbox.wait_for_ready(timeout=0, interval=0) + assert len(session.requests) == 1 + + +def test_update_info_refreshes_traffic_access_token(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'trafficAccessToken': 'old-token', + }) + + sandbox.update_info({'trafficAccessToken': 'new-token'}) + + assert sandbox.traffic_access_token == 'new-token' + assert sandbox.trafficAccessToken == 'new-token' + + +class RecordingSandboxListClient(object): + def __init__(self): + self.calls = [] + + def list_sandboxes_v2(self, **opts): + self.calls.append(opts) + if len(self.calls) == 1: + return {'items': [], 'nextToken': 'next-page'} + return {'items': []} + + +def test_sandbox_paginator_does_not_reuse_initial_next_token(): + client = RecordingSandboxListClient() + paginator = SandboxPaginator(client=client, next_token='saved-page') + + paginator.next_items() + paginator.next_items() + + assert client.calls[0]['nextToken'] == 'saved-page' + assert 'next_token' not in client.calls[0] + assert client.calls[1]['nextToken'] == 'next-page' + + +def test_sandbox_paginator_does_not_send_client_credentials_as_filters(): + client = RecordingSandboxListClient() + paginator = SandboxPaginator( + client=client, + api_key='api-key', + access_token='access-token', + endpoint='https://sandbox.example.test', + metadata={'app': 'tests'}, + ) + + paginator.next_items() + + assert client.calls[0] == {'metadata': {'app': 'tests'}} + + +def test_list_sandboxes_v2_serializes_metadata_dict_as_query_string(): + session = RecordingSession([DummyResponse(200, {'items': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes_v2(query={ + 'metadata': {'app': 'tests'}, + 'state': 'running', + }) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == ['app=tests'] + assert query['state'] == ['running'] + + +def test_list_sandboxes_v2_accepts_metadata_string(): + session = RecordingSession([DummyResponse(200, {'items': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes_v2(metadata='user=abc&app=prod') + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == ['user=abc&app=prod'] + + +def test_wait_for_build_retries_transient_sandbox_errors(monkeypatch): + class BuildClient(SandboxClient): + def __init__(self): + super(BuildClient, self).__init__( + api_key='api-key', + session=RecordingSession(), + ) + self.calls = 0 + + def get_template_build_status(self, template_id, build_id, **opts): + del template_id, build_id, opts + self.calls += 1 + if self.calls == 1: + raise SandboxError('temporary gateway error') + return {'status': 'ready', 'templateID': 'tmpl123'} + + monkeypatch.setattr( + 'qiniu.services.sandbox.client.time.sleep', + lambda x: x, + ) + client = BuildClient() + + assert client.wait_for_build('tmpl123', 'build123', interval=0) == { + 'status': 'ready', + 'templateID': 'tmpl123', + } + assert client.calls == 2 + + +def test_wait_for_build_reraises_permanent_sandbox_errors(monkeypatch): + class BuildClient(SandboxClient): + def __init__(self): + super(BuildClient, self).__init__( + api_key='api-key', + session=RecordingSession(), + ) + self.calls = 0 + + def get_template_build_status(self, template_id, build_id, **opts): + del template_id, build_id, opts + self.calls += 1 + raise SandboxError('missing build', DummyResponse(404, {})) + + monkeypatch.setattr( + 'qiniu.services.sandbox.client.time.sleep', + lambda x: x, + ) + client = BuildClient() + + with pytest.raises(SandboxError) as err: + client.wait_for_build('tmpl123', 'build123', interval=0) + + assert err.value.status_code == 404 + assert client.calls == 1 + + +def test_is_running_matches_e2b_health_check_semantics(): + running_session = RecordingSession([DummyResponse(200, {})]) + running = Sandbox(client=SandboxClient( + api_key='api-key', + session=running_session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + stopped_session = RecordingSession([ErrorResponse(502)]) + stopped = Sandbox(client=SandboxClient( + api_key='api-key', + session=stopped_session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + assert running.is_running(request_timeout=3) is True + assert running_session.requests[0].kwargs['timeout'] == 3 + assert stopped.is_running() is False + + +def test_is_running_returns_false_for_envd_request_errors(): + session = RecordingSession([requests.Timeout('timed out')]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + assert sandbox.is_running(request_timeout=1) is False + assert session.requests[0].kwargs['timeout'] == 1 + + +def test_is_running_returns_false_without_domain(): + session = RecordingSession() + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={'sandboxID': 'sbx123'}) + + assert sandbox.is_running() is False + assert session.requests == [] + + +def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(['sbx1', 'sbx2']) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx1,sbx2'] + + +def test_get_sandboxes_metrics_accepts_set_values(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(set(['sbx1', 'sbx2'])) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert set(query['sandbox_ids'][0].split(',')) == set(['sbx1', 'sbx2']) + + +def test_get_sandboxes_metrics_accepts_generator_values(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(item for item in ['sbx1', 'sbx2']) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx1,sbx2'] + + +def test_get_sandboxes_metrics_accepts_single_sandbox_dict(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics({'id': 'sbx123'}) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx123'] + + +def test_get_sandboxes_metrics_rejects_empty_dict_values(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError): + client.get_sandboxes_metrics({}) + with pytest.raises(SandboxError): + client.get_sandboxes_metrics({'sandboxIDs': None}) + + +def test_template_builder_outputs_build_config(): + template = ( + Template() + .from_image('python:3.11') + .run_cmd('pip install qiniu') + .run_cmd(['python', '-m', 'pip', 'install', 'pytest']) + .copy('/local/app.py', '/app/app.py') + .set_env('PYTHONUNBUFFERED', '1') + .set_start_cmd('python /app/app.py') + ) + + assert template.to_dict() == { + 'fromImage': 'python:3.11', + 'steps': [ + {'type': 'RUN', 'args': ['pip install qiniu']}, + {'type': 'RUN', 'args': ['python -m pip install pytest']}, + {'type': 'COPY', 'args': ['/local/app.py', '/app/app.py']}, + {'type': 'ENV', 'args': ['PYTHONUNBUFFERED', '1']}, + ], + 'startCmd': 'python /app/app.py', + } + + +def test_template_ready_cmd_helpers_align_with_e2b(): + ready = wait_for_port(8000) + assert isinstance(ready, ReadyCmd) + assert ready.get_cmd() == ( + "ss -tuln | awk '{print $5}' | grep -E '(^|:)8000$'") + assert wait_for_url( + 'http://localhost:3000/health', + status_code=204, + ).get_cmd() == ( + '[ "$(curl -s -o /dev/null -w "%{http_code}" ' + 'http://localhost:3000/health)" = "204" ]' + ) + assert wait_for_process('nginx').get_cmd() == 'pgrep nginx > /dev/null' + assert wait_for_file('/tmp/ready').get_cmd() == '[ -f /tmp/ready ]' + assert wait_for_timeout(500).get_cmd() == 'sleep 1' + + template = ( + Template() + .from_image('python:3.11') + .set_start_cmd('python app.py', wait_for_port(8000)) + ) + + assert template.to_dict()['startCmd'] == 'python app.py' + assert template.to_dict()['readyCmd'] == ( + "ss -tuln | awk '{print $5}' | grep -E '(^|:)8000$'") + + +def test_template_ready_cmd_helpers_quote_shell_inputs(): + assert wait_for_url( + 'http://localhost:3000/health; touch /tmp/pwn', + status_code='204', + ).get_cmd() == ( + '[ "$(curl -s -o /dev/null -w "%{http_code}" ' + '\'http://localhost:3000/health; touch /tmp/pwn\')" = "204" ]' + ) + assert wait_for_process('nginx; touch /tmp/pwn').get_cmd() == ( + "pgrep 'nginx; touch /tmp/pwn' > /dev/null" + ) + assert wait_for_file('/tmp/ready; touch /tmp/pwn').get_cmd() == ( + "[ -f '/tmp/ready; touch /tmp/pwn' ]" + ) + with pytest.raises(ValueError): + wait_for_port('8000; touch /tmp/pwn') + with pytest.raises(ValueError): + wait_for_url('http://localhost:3000', status_code='200; true') + + +class RecordingCommands(object): + def __init__(self): + self.calls = [] + self.results = [] + + def run(self, cmd, **opts): + self.calls.append((cmd, opts)) + result = self.results.pop(0) if self.results else type( + 'Result', (object,), { + 'pid': 12, + 'exit_code': 0, + 'stdout': 'origin https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })() + if result.exit_code and opts.get('throw_on_error'): + from qiniu.services.sandbox import CommandExitError + raise CommandExitError(result) + if opts.get('background'): + return type('Handle', (object,), { + 'pid': getattr(result, 'pid', 12), + 'exit_code': -1, + 'wait': lambda self: result, + })() + return result + + def send_stdin(self, pid, data): + self.calls.append(('send_stdin', {'pid': pid, 'data': data})) + return None + + def close_stdin(self, pid): + self.calls.append(('close_stdin', {'pid': pid})) + return None + + +class RecordingFiles(object): + def __init__(self): + self.writes = [] + self.removes = [] + + def write(self, path, data, **opts): + self.writes.append((path, data, opts)) + return None + + def remove(self, path, **opts): + self.removes.append((path, opts)) + return None + + +def attach_recording_files(commands): + files = RecordingFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + return files + + +def test_git_helpers_align_with_e2b_method_names(): + commands = RecordingCommands() + git = Git(commands) + + git.remote_add('/repo', 'origin', 'https://github.com/qiniu/repo.git') + git.remote_get('/repo', 'origin') + git.branches('/repo') + git.create_branch('/repo', 'feature') + git.checkout_branch('/repo', 'main') + git.delete_branch('/repo', 'old') + git.reset('/repo', 'HEAD~1', mode='hard') + git.restore('/repo', paths=['a.txt', 'b.txt']) + git.set_config('user.name', 'tester', scope='local', path='/repo') + git.get_config('user.name', scope='local', path='/repo') + + assert commands.calls[0][0] == ( + 'git remote add origin https://github.com/qiniu/repo.git') + assert commands.calls[1][0] == 'git remote get-url origin' + assert commands.calls[2][0] == ( + "git branch '--format=%(refname:short)\t%(HEAD)'") + assert commands.calls[3][0] == 'git checkout -b feature' + assert commands.calls[4][0] == 'git checkout main' + assert commands.calls[5][0] == 'git branch -D old' + assert commands.calls[6][0] == "git reset --hard 'HEAD~1'" + assert commands.calls[7][0] == 'git restore -- a.txt b.txt' + assert commands.calls[8][0] == 'git config --local user.name tester' + assert commands.calls[8][1]['cwd'] == '/repo' + assert commands.calls[9][0] == 'git config --local --get user.name' + assert commands.calls[9][1]['cwd'] == '/repo' + + +def test_git_remote_add_supports_overwrite_and_fetch_options(): + commands = RecordingCommands() + git = Git(commands) + + git.remote_add( + '/repo', + 'origin', + 'https://github.com/qiniu/repo.git', + overwrite=True, + fetch=True, + ) + + assert commands.calls[0][0] == 'git remote remove origin' + assert commands.calls[1][0] == ( + 'git remote add origin https://github.com/qiniu/repo.git') + assert commands.calls[2][0] == 'git fetch origin' + + +def test_git_add_and_restore_accept_single_string_path(): + commands = RecordingCommands() + git = Git(commands) + + git.add('/repo', files='README.md') + git.restore('/repo', paths='README.md') + git.restore('/repo', files='setup.py') + + assert commands.calls[0][0] == 'git add -- README.md' + assert commands.calls[1][0] == 'git restore -- README.md' + assert commands.calls[2][0] == 'git restore -- setup.py' + + +def test_git_add_and_restore_use_path_separator_for_dash_paths(): + commands = RecordingCommands() + git = Git(commands) + + git.add('/repo', files='--intent-to-add') + git.restore('/repo', paths='--worktree') + + assert commands.calls[0][0] == "git add -- --intent-to-add" + assert commands.calls[1][0] == "git restore -- --worktree" + + +def test_git_reset_rejects_unsupported_mode(): + commands = RecordingCommands() + git = Git(commands) + + with pytest.raises(InvalidArgumentException): + git.reset('/repo', 'HEAD', mode='hard; touch /tmp/pwn') + + assert commands.calls == [] + + +def test_git_dangerously_authenticate_aligns_with_e2b(): + commands = RecordingCommands() + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-token', + host='github.com', + protocol='https', + ) + + assert commands.calls[0][0] == ( + 'git config --global credential.helper store') + assert commands.calls[1][0] == 'git credential approve' + assert commands.calls[1][1]['stdin'] is True + assert commands.calls[1][1]['background'] is True + assert commands.calls[2] == ('send_stdin', { + 'pid': 12, + 'data': ( + 'protocol=https\nhost=github.com\n' + 'username=git-user\npassword=secret-token\n\n' + ), + }) + assert commands.calls[3] == ('close_stdin', {'pid': 12}) + + +def test_git_dangerously_authenticate_uses_stdin_with_real_sandbox(): + commands = RecordingCommands() + files = attach_recording_files(commands) + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-%-token', + host='github.com', + protocol='https', + ) + + assert files.writes == [] + assert files.removes == [] + assert commands.calls[1][0] == 'git credential approve' + assert commands.calls[2][1]['data'] == ( + 'protocol=https\nhost=github.com\n' + 'username=git-user\npassword=secret-%-token\n\n' + ) + + +def test_git_dangerously_authenticate_ignores_background_for_stdin_flow(): + commands = RecordingCommands() + files = attach_recording_files(commands) + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-token', + background=True, + ) + + assert 'background' not in commands.calls[0][1] + assert commands.calls[1][1]['background'] is True + assert 'background' not in commands.calls[2][1] + assert 'background' not in commands.calls[3][1] + assert files.writes == [] + assert files.removes == [] + + +def test_git_status_and_branches_return_structured_e2b_types(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': ( + '## main...origin/main [ahead 2, behind 1]\n' + ' M changed.txt\n' + 'A staged.txt\n' + ' M my -> file.txt\n' + 'R old.txt -> renamed.txt\n' + '?? new.txt\n' + ), + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'main\t*\nfeature\t\n', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + status = git.status('/repo') + branches = git.branches('/repo') + + assert isinstance(status, GitStatus) + assert status.current_branch == 'main' + assert status.upstream == 'origin/main' + assert status.ahead == 2 + assert status.behind == 1 + assert status.has_changes is True + assert status.has_staged is True + assert status.has_untracked is True + assert status.file_status[0].name == 'changed.txt' + assert status.file_status[0].status == 'modified' + assert status.file_status[2].name == 'my -> file.txt' + assert status.file_status[2].renamed_from is None + assert status.file_status[3].name == 'renamed.txt' + assert status.file_status[3].renamed_from == 'old.txt' + assert isinstance(branches, GitBranches) + assert branches.branches == ['main', 'feature'] + assert branches.current_branch == 'main' + + +def test_git_status_raises_when_git_command_fails(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: not a git repository', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(CommandExitError): + git.status('/not-a-repo') + + +def test_git_push_maps_auth_failure_to_e2b_exception(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push('/repo') + + +def test_git_credential_remote_requires_existing_remote(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(InvalidArgumentException) as err: + git.push('/repo', username='git-user', password='secret') + + assert 'No remotes found' in str(err.value) + + +def test_git_credentials_require_username_with_password(): + git = Git(RecordingCommands()) + + with pytest.raises(InvalidArgumentException): + git.pull('/repo', password='secret') + with pytest.raises(InvalidArgumentException): + git.push('/repo', password='secret') + + +def test_git_push_maps_known_errors_before_throw_on_error(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push('/repo', throw_on_error=True) + + +def test_git_push_respects_throw_on_error_for_unknown_errors(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'unexpected failure', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(CommandExitError): + git.push('/repo', throw_on_error=True) + + +def test_git_push_with_credentials_uses_askpass_without_remote_url_leak(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + git.push( + '/repo', + remote='origin', + branch='main', + username='git:user', + password='secret:%@token', + request_timeout=7, + ) + + assert commands.calls[0][0] == 'git remote get-url origin' + assert commands.calls[1][0] == 'install -d -m 700 /tmp/qiniu-git-auth' + assert files.writes[0][0].startswith( + '/tmp/qiniu-git-auth/qiniu-git-askpass-') + assert commands.calls[2][0].startswith( + 'chmod 700 /tmp/qiniu-git-auth/qiniu-git-askpass-') + assert commands.calls[3][0] == 'git push --set-upstream origin main' + assert 'secret:%@token' not in commands.calls[3][0] + assert commands.calls[3][1]['request_timeout'] == 7 + assert commands.calls[3][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] + assert commands.calls[3][1]['envs']['GIT_USERNAME'] == 'git:user' + assert commands.calls[3][1]['envs']['GIT_PASSWORD'] == 'secret:%@token' + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_pull_with_credentials_resolves_single_remote(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + ) + + assert commands.calls[0][0] == 'git remote' + assert commands.calls[2][0] == 'install -d -m 700 /tmp/qiniu-git-auth' + assert commands.calls[4][0] == 'git pull origin main' + assert 'secret-token' not in commands.calls[4][0] + assert commands.calls[4][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_credential_helpers_ignore_background_when_reading_remote(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + handle = git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + background=True, + ) + + assert 'background' not in commands.calls[0][1] + assert 'background' not in commands.calls[1][1] + assert commands.calls[4][1]['background'] is True + assert files.removes == [] + handle.wait() + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_background_throw_on_error_waits_before_raising(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'fatal: failed', + 'error': '', + })(), + ] + git = Git(commands) + + handle = git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + background=True, + throw_on_error=True, + ) + + assert files.removes == [] + with pytest.raises(CommandExitError): + handle.wait() + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push( + '/repo', + remote='origin', + username='git-user', + password='bad-token', + ) + + assert commands.calls[-1][0] == 'git push --set-upstream origin' + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_push_cleans_askpass_after_operation_exception(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError) as err: + git._with_remote_credentials( + '/repo', + 'origin', + 'git-user', + 'bad-token', + lambda auth_opts: (_ for _ in ()).throw( + SandboxError('rpc timed out')), + ) + + assert 'rpc timed out' in str(err.value) + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_credentials_clean_askpass_when_chmod_raises(): + class ChmodFailCommands(RecordingCommands): + def run(self, cmd, **opts): + if cmd.startswith('chmod 700 '): + raise SandboxError('chmod rpc failed') + return super(ChmodFailCommands, self).run(cmd, **opts) + + commands = ChmodFailCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError): + git.push( + '/repo', + remote='origin', + username='git-user', + password='secret-token', + ) + + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_credentials_clean_askpass_when_write_raises(): + class FailingWriteFiles(RecordingFiles): + def write(self, path, data, **opts): + super(FailingWriteFiles, self).write(path, data, **opts) + raise SandboxError('write rpc failed') + + commands = RecordingCommands() + files = FailingWriteFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError): + git.push( + '/repo', + remote='origin', + username='git-user', + password='secret-token', + ) + + assert files.removes == [(files.writes[0][0], {})] + + +def test_git_helpers_accept_e2b_style_signatures(): + commands = RecordingCommands() + git = Git(commands) + + git.create_branch('/repo', 'feature') + git.commit( + '/repo', + 'feat: demo', + author_name='Demo User', + author_email='demo@example.com', + allow_empty=True, + ) + git.set_config('user.name', 'Demo User', scope='local', path='/repo') + git.get_config('user.name', scope='local', path='/repo') + + assert commands.calls[0][0] == 'git checkout -b feature' + assert commands.calls[1][0] == ( + "git -c 'user.name=Demo User' -c user.email=demo@example.com " + "commit -m 'feat: demo' --allow-empty" + ) + assert commands.calls[2][0] == "git config --local user.name 'Demo User'" + assert commands.calls[2][1]['cwd'] == '/repo' + assert commands.calls[3][0] == 'git config --local --get user.name' + assert commands.calls[3][1]['cwd'] == '/repo' + + +def test_git_config_helpers_accept_legacy_repo_path_signatures(): + commands = RecordingCommands() + git = Git(commands) + + git.set_config(None, 'http.version', 'HTTP/1.1', global_config=True) + git.set_config('/repo', 'user.name', 'Sandbox Demo') + git.set_config('/repo', 'gitreview.username', 'global') + git.set_config('repo', 'user.email', 'global') + git.set_config('.', 'core.editor', 'global') + git.get_config('/repo', 'user.name') + + assert commands.calls[0][0] == "git config --global http.version HTTP/1.1" + assert 'cwd' not in commands.calls[0][1] + assert commands.calls[1][0] == ( + "git config --local user.name 'Sandbox Demo'") + assert commands.calls[1][1]['cwd'] == '/repo' + assert commands.calls[2][0] == ( + 'git config --local gitreview.username global') + assert commands.calls[2][1]['cwd'] == '/repo' + assert commands.calls[3][0] == 'git config --local user.email global' + assert commands.calls[3][1]['cwd'] == 'repo' + assert commands.calls[4][0] == 'git config --local core.editor global' + assert commands.calls[4][1]['cwd'] == '.' + assert commands.calls[5][0] == 'git config --local --get user.name' + assert commands.calls[5][1]['cwd'] == '/repo' diff --git a/tests/cases/test_services/test_sandbox/test_config.py b/tests/cases/test_services/test_sandbox/test_config.py new file mode 100644 index 00000000..35f1191c --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_config.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +import os + +import qiniu.services.sandbox.config as sandbox_config +from qiniu.services.sandbox.config import ( + load_dotenv_if_present, + sandbox_client, +) + + +def test_load_dotenv_if_present_reads_key_values_without_overwriting(tmpdir): + dotenv = tmpdir.join('.env') + dotenv.write( + '\n'.join([ + 'QINIU_SANDBOX_API_KEY=from-file', + 'QINIU_SANDBOX_TEMPLATE="python:3.11"', + 'QINIU_SANDBOX_COMMENTED=value # inline comment', + 'QINIU_SANDBOX_HASH="value # not comment"', + 'EXISTING=value-from-file', + '# ignored', + '', + ]) + ) + + old = { + key: os.environ.get(key) + for key in ( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_SANDBOX_TEMPLATE', + 'QINIU_SANDBOX_COMMENTED', + 'QINIU_SANDBOX_HASH', + 'EXISTING', + ) + } + try: + os.environ.pop('QINIU_SANDBOX_API_KEY', None) + os.environ.pop('QINIU_SANDBOX_TEMPLATE', None) + os.environ.pop('QINIU_SANDBOX_COMMENTED', None) + os.environ.pop('QINIU_SANDBOX_HASH', None) + os.environ['EXISTING'] = 'already-set' + + load_dotenv_if_present(str(dotenv)) + + assert os.environ['QINIU_SANDBOX_API_KEY'] == 'from-file' + assert os.environ['QINIU_SANDBOX_TEMPLATE'] == 'python:3.11' + assert os.environ['QINIU_SANDBOX_COMMENTED'] == 'value' + assert os.environ['QINIU_SANDBOX_HASH'] == 'value # not comment' + assert os.environ['EXISTING'] == 'already-set' + finally: + for key, value in old.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def test_sandbox_client_uses_loaded_env(tmpdir): + dotenv = tmpdir.join('.env') + dotenv.write('\n'.join([ + 'QINIU_SANDBOX_API_KEY=api-key', + 'QINIU_SANDBOX_ENDPOINT=https://sandbox.example.test', + 'QINIU_SANDBOX_ACCESS_KEY=ak', + 'QINIU_SANDBOX_SECRET_KEY=sk', + ])) + + old = { + key: os.environ.get(key) + for key in ( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_SANDBOX_ENDPOINT', + 'QINIU_SANDBOX_ACCESS_KEY', + 'QINIU_SANDBOX_SECRET_KEY', + ) + } + try: + for key in old: + os.environ.pop(key, None) + load_dotenv_if_present(str(dotenv)) + + client = sandbox_client() + + assert client.endpoint == 'https://sandbox.example.test' + assert client.api_key == 'api-key' + assert client.mac is not None + finally: + for key, value in old.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def test_sandbox_client_uses_documented_api_key_env_fallbacks(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_API_KEY', raising=False) + monkeypatch.delenv('QINIU_API_KEY', raising=False) + monkeypatch.delenv('E2B_API_KEY', raising=False) + + monkeypatch.setenv('QINIU_API_KEY', 'qiniu-api-key') + assert sandbox_client().api_key == 'qiniu-api-key' + + monkeypatch.delenv('QINIU_API_KEY') + monkeypatch.setenv('E2B_API_KEY', 'e2b-api-key') + assert sandbox_client().api_key == 'e2b-api-key' + + +def test_native_env_pair_encodes_unicode_on_python2(monkeypatch): + monkeypatch.setattr(sandbox_config, 'is_py2', True) + + key, value = sandbox_config._native_env_pair(u'KEY', u'值') + + assert key == b'KEY' + assert value == u'值'.encode('utf-8') diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py new file mode 100644 index 00000000..cc173056 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -0,0 +1,613 @@ +# -*- coding: utf-8 -*- +import base64 +from io import BytesIO, StringIO +import json +import struct + +import pytest +import requests + +from qiniu.services.sandbox import ( + EntryInfo, + FileType, + FilesystemEventType, + PtySize, + Sandbox, + SandboxClient, + SandboxError, +) +from qiniu.services.sandbox.envd import ( + decode_connect_envelopes, + encode_connect_envelope, + iter_connect_envelopes, +) +from qiniu.services.sandbox.commands import command_result_from_events +import qiniu.services.sandbox.util as sandbox_util + + +class DummyResponse(object): + def __init__(self, status_code=200, body=None, raw=None, + content_type='application/json'): + self.status_code = status_code + self.content = raw if raw is not None else json.dumps( + body or {}).encode('utf-8') + self.text = self.content.decode('utf-8') + self.headers = {'Content-Type': content_type} + self.closed = False + + def json(self): + return json.loads(self.content.decode('utf-8')) + + def iter_content(self, chunk_size=8192): + del chunk_size + yield self.content + + def close(self): + self.closed = True + + +class EnvdSession(object): + def __init__(self): + self.posts = [] + self.requests = [] + self.empty_stream_paths = set() + self.post_exception = None + self.request_exception = None + + def post(self, url, data=None, headers=None, timeout=None, stream=False): + if self.post_exception is not None: + raise self.post_exception + if headers.get('Content-Type') == 'application/connect+json': + decoded = decode_connect_envelopes(data)[0] + else: + decoded = json.loads( + data.decode('utf-8') if isinstance(data, bytes) else data + ) + self.posts.append({ + 'url': url, + 'data': decoded, + 'headers': headers, + 'timeout': timeout, + 'stream': stream, + }) + if any(url.endswith(path) for path in self.empty_stream_paths): + return DummyResponse( + raw=b'', + content_type='application/connect+json', + ) + if url.endswith('/process.Process/Start'): + decoded = self.posts[-1]['data'] + output_key = 'pty' if decoded.get('pty') else 'stdout' + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + output_key: base64.b64encode(b'hello\n').decode('ascii'), + 'stderr': base64.b64encode(b'').decode('ascii'), + }}}), + encode_connect_envelope({'event': {'end': {'exitCode': 0}}}), + ]) + return DummyResponse( + raw=raw, + content_type='application/connect+json', + ) + if url.endswith('/process.Process/Connect'): + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'connected\n').decode( + 'ascii'), + }}}), + encode_connect_envelope({'event': {'end': {'exitCode': 0}}}), + ]) + return DummyResponse( + raw=raw, + content_type='application/connect+json', + ) + if url.endswith('/filesystem.Filesystem/Stat'): + return DummyResponse( + body={ + 'result': { + 'entry': { + 'name': 'hello.txt', + 'type': 'FILE'}}}) + if url.endswith('/filesystem.Filesystem/ListDir'): + return DummyResponse( + body={'result': {'entries': [{ + 'name': 'hello.txt', + 'type': 'FILE', + }]}}) + if url.endswith('/filesystem.Filesystem/CreateWatcher'): + return DummyResponse(body={'result': {'watcherId': 'watch-1'}}) + if url.endswith('/filesystem.Filesystem/GetWatcherEvents'): + return DummyResponse(body={'result': {'events': [{ + 'name': 'hello.txt', + 'type': 'EVENT_TYPE_WRITE', + }]}}) + return DummyResponse(body={'result': {}}) + + def request(self, method, url, **kwargs): + if self.request_exception is not None: + raise self.request_exception + self.requests.append({'method': method, 'url': url, 'kwargs': kwargs}) + if method == 'GET': + response = DummyResponse(raw=b'hello') + self.requests[-1]['response'] = response + return response + response = DummyResponse(body=[{ + 'name': 'hello.txt', + 'type': 'FILE', + 'path': '/tmp/hello.txt', + }]) + self.requests[-1]['response'] = response + return response + + +def sandbox_with_envd_session(): + session = EnvdSession() + client = SandboxClient(api_key='api-key', session=session) + sandbox = Sandbox(client=client, info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + return sandbox, session + + +def test_iter_connect_envelopes_decodes_chunked_stream_frames(): + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'hello').decode('ascii'), + }}}), + ]) + + assert list(iter_connect_envelopes([ + raw[:3], + raw[3:9], + raw[9:], + ])) == [ + {'event': {'start': {'pid': 12}}}, + {'event': {'data': { + 'stdout': base64.b64encode(b'hello').decode('ascii'), + }}}, + ] + + +def test_decode_connect_envelopes_rejects_trailing_bytes(): + raw = encode_connect_envelope({'event': {'start': {'pid': 12}}}) + b'x' + + try: + decode_connect_envelopes(raw) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream truncated unexpectedly' + + +def test_iter_connect_envelopes_closes_stream_response(): + raw = encode_connect_envelope({'event': {'start': {'pid': 12}}}) + response = DummyResponse(raw=raw, content_type='application/connect+json') + + assert list(iter_connect_envelopes( + response.iter_content(chunk_size=8192), + response=response, + )) == [{'event': {'start': {'pid': 12}}}] + assert response.closed is True + + +def test_command_event_decode_handles_base64_and_non_utf8_output(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': base64.b64encode(b'YWJj').decode('ascii'), + 'stderr': base64.b64encode(b'\xff\xff\xff').decode('ascii'), + }}, + }]) + + assert result.stdout == 'YWJj' + assert result.stderr == u'\ufffd\ufffd\ufffd' + + +def test_command_event_decode_expects_base64_encoded_strings(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': base64.b64encode(b'test').decode('ascii'), + }}, + }]) + + assert result.stdout == 'test' + + +def test_command_event_decode_decodes_plain_word_base64(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': base64.b64encode(b'text').decode('ascii'), + }}, + }]) + + assert result.stdout == 'text' + + +def test_command_event_decode_keeps_unknown_exit_code(): + result = command_result_from_events([{ + 'event': {'end': {}}, + }]) + + assert result.exit_code == -1 + + +def test_command_event_decode_handles_bytes_values(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': base64.b64encode(b'abc'), + 'stderr': b'plain bytes', + }}, + }]) + + assert result.stdout == 'YWJj' + assert result.stderr == 'plain bytes' + + +def test_connect_error_envelopes_raise_default_sandbox_error(): + empty_error = struct.pack('>BI', 2, 0) + missing_message = ( + struct.pack('>BI', 2, len(b'{"error":{}}')) + b'{"error":{}}' + ) + + for data in (empty_error, missing_message): + try: + decode_connect_envelopes(data) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream failed' + + try: + list(iter_connect_envelopes([empty_error])) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream failed' + + +def test_connect_rpc_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.post_exception = requests.Timeout('timed out') + + with pytest.raises(SandboxError) as err: + sandbox.commands.run('echo hello') + + assert 'Sandbox envd request failed' in str(err.value) + assert 'timed out' in str(err.value) + + +def test_connect_stream_rpc_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.post_exception = requests.ConnectionError('connection reset') + + with pytest.raises(SandboxError) as err: + sandbox.commands.connect(12) + + assert 'Sandbox envd request failed' in str(err.value) + assert 'connection reset' in str(err.value) + + +def test_raw_envd_request_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.request_exception = requests.Timeout('timed out') + + with pytest.raises(SandboxError) as err: + sandbox.files.read_text('/tmp/hello.txt') + + assert 'Sandbox envd request failed' in str(err.value) + assert 'timed out' in str(err.value) + + +def test_filesystem_write_accepts_unicode_text(): + sandbox, session = sandbox_with_envd_session() + + sandbox.files.write('/tmp/unicode.txt', u'你好') + sandbox.files.write('/tmp/bytes.txt', bytearray(b'abc')) + sandbox.files.write('/tmp/text-stream.txt', StringIO(u'你好')) + + assert session.requests[0]['kwargs']['files']['file'] == ( + 'unicode.txt', + u'你好'.encode('utf-8'), + ) + assert session.requests[1]['kwargs']['files']['file'] == ( + 'bytes.txt', + b'abc', + ) + stream_upload = session.requests[2]['kwargs']['files']['file'] + assert stream_upload[0] == 'text-stream.txt' + assert stream_upload[1].read() == u'你好'.encode('utf-8') + for attr in ('seek', 'tell', 'getvalue', 'len'): + with pytest.raises(AttributeError): + getattr(stream_upload[1], attr) + + +def test_file_basename_encodes_unicode_on_python2(monkeypatch): + monkeypatch.setattr(sandbox_util, 'is_py2', True) + + assert sandbox_util.file_basename(u'/tmp/文件.txt') == ( + u'文件.txt'.encode('utf-8')) + + +def test_filesystem_write_accepts_duck_typed_file_like_objects(): + class FileLike(object): + def read(self): + return b'hello' + + sandbox, session = sandbox_with_envd_session() + file_like = FileLike() + + sandbox.files.write('/tmp/file-like.txt', file_like) + + assert session.requests[0]['kwargs']['files']['file'] == ( + 'file-like.txt', + file_like, + ) + + +def test_commands_run_posts_process_start_and_decodes_events(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.commands.run('echo hello', cwd='/tmp', envs={'A': 'B'}) + + assert result.pid == 12 + assert result.exit_code == 0 + assert result.stdout == 'hello\n' + assert session.posts[0]['url'].endswith('/process.Process/Start') + assert ( + session.posts[0]['headers']['Content-Type'] == + 'application/connect+json' + ) + assert session.posts[0]['headers']['Authorization'] == 'Basic dXNlcjo=' + assert session.posts[0]['headers']['X-Access-Token'] == 'token' + assert session.posts[0]['timeout'] == 30 + assert session.posts[0]['data']['process'] == { + 'cmd': '/bin/bash', + 'args': ['-l', '-c', 'echo hello'], + 'cwd': '/tmp', + 'envs': {'A': 'B'}, + } + + +def test_commands_connect_returns_handle_for_running_process(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.commands.connect(12) + result = handle.wait() + + assert result.pid == 12 + assert result.stdout == 'connected\n' + assert session.posts[0]['timeout'] == 30 + assert session.posts[0]['url'].endswith('/process.Process/Connect') + assert session.posts[0]['data'] == { + 'process': {'selector': {'pid': 12}}, + } + + +def test_commands_connect_handles_empty_event_stream(): + sandbox, session = sandbox_with_envd_session() + session.empty_stream_paths.add('/process.Process/Connect') + + handle = sandbox.commands.connect(12) + result = handle.wait() + + assert result.pid == 0 + assert result.exit_code == -1 + assert result.stdout == '' + + +def test_commands_send_stdin_encodes_unicode_text(): + sandbox, session = sandbox_with_envd_session() + + sandbox.commands.send_stdin(12, u'你好') + sandbox.commands.send_stdin(12, bytearray(b'abc')) + + raw = session.posts[0]['data']['input']['stdin'] + assert base64.b64decode(raw).decode('utf-8') == u'你好' + raw = session.posts[1]['data']['input']['stdin'] + assert base64.b64decode(raw) == b'abc' + + +def test_commands_run_supports_e2b_callbacks_and_request_timeout(): + sandbox, session = sandbox_with_envd_session() + stdout = [] + stderr = [] + + result = sandbox.commands.run( + 'echo hello', + on_stdout=stdout.append, + on_stderr=stderr.append, + timeout=3, + request_timeout=7, + ) + + assert result.stdout == 'hello\n' + assert stdout == ['hello\n'] + assert stderr == [] + assert session.posts[0]['data']['timeout'] == 3 + assert session.posts[0]['timeout'] == 7 + + +def test_commands_run_uses_command_timeout_for_stream_request_by_default(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.commands.run('sleep 60', timeout=120) + + assert result.exit_code == 0 + assert session.posts[0]['data']['timeout'] == 120 + assert session.posts[0]['timeout'] == 120 + + +def test_commands_run_background_returns_before_command_finishes(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.commands.run('sleep 30', background=True) + + assert session.posts[0]['stream'] is True + assert handle.pid == 12 + assert handle.exit_code == -1 + assert handle.stdout == '' + assert handle.wait().stdout == 'hello\n' + + +def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.pty.create(PtySize(rows=24, cols=80), cwd='/workspace') + sandbox.pty.send_stdin(handle.pid, u'你好\n') + sandbox.pty.send_stdin(handle.pid, bytearray(b'abc')) + sandbox.pty.resize(handle.pid, {'rows': 30, 'cols': 100}) + connected = sandbox.pty.connect(handle.pid) + assert sandbox.pty.kill(handle.pid) is True + + assert handle.wait().stdout == 'hello\n' + assert connected.wait().stdout == 'connected\n' + assert session.posts[0]['url'].endswith('/process.Process/Start') + assert session.posts[0]['data']['process']['args'] == ['-i', '-l'] + assert session.posts[0]['data']['process']['cwd'] == '/workspace' + assert session.posts[0]['data']['process']['envs']['TERM'] == ( + 'xterm-256color') + assert session.posts[0]['data']['pty'] == { + 'size': {'rows': 24, 'cols': 80}, + } + assert session.posts[1]['url'].endswith('/process.Process/SendInput') + assert session.posts[1]['data']['input']['pty'] + assert base64.b64decode( + session.posts[1]['data']['input']['pty'] + ).decode('utf-8') == u'你好\n' + assert session.posts[2]['url'].endswith('/process.Process/SendInput') + assert base64.b64decode( + session.posts[2]['data']['input']['pty']) == b'abc' + assert session.posts[3]['url'].endswith('/process.Process/Update') + assert session.posts[3]['data']['pty'] == { + 'size': {'rows': 30, 'cols': 100}, + } + assert session.posts[4]['url'].endswith('/process.Process/Connect') + assert session.posts[5]['url'].endswith('/process.Process/SendSignal') + + +def test_pty_create_and_connect_handle_empty_event_streams(): + sandbox, session = sandbox_with_envd_session() + session.empty_stream_paths.add('/process.Process/Start') + + handle = sandbox.pty.create(PtySize(rows=24, cols=80)) + result = handle.wait() + + assert result.pid == 0 + assert result.exit_code == -1 + assert result.stdout == '' + + session.empty_stream_paths = set(['/process.Process/Connect']) + connected = sandbox.pty.connect(12) + + assert connected.wait().stdout == '' + + +def test_filesystem_uses_envd_rpc_and_signed_file_urls(): + sandbox, session = sandbox_with_envd_session() + + assert sandbox.files.read_text('/tmp/hello.txt') == 'hello' + assert sandbox.files.write('/tmp/hello.txt', 'hello')['type'] == 'file' + assert sandbox.files.stat('/tmp/hello.txt')['type'] == 'file' + assert sandbox.files.list( + '/tmp') == [{'name': 'hello.txt', 'type': 'file'}] + + assert session.requests[0]['method'] == 'GET' + assert '/files?' in session.requests[0]['url'] + assert session.requests[1]['method'] == 'POST' + assert 'Content-Type' not in session.requests[1]['kwargs']['headers'] + assert session.requests[1]['kwargs']['files']['file'] == ( + 'hello.txt', + b'hello', + ) + assert session.posts[0]['url'].endswith('/filesystem.Filesystem/Stat') + assert session.posts[1]['url'].endswith('/filesystem.Filesystem/ListDir') + + +def test_filesystem_returns_e2b_style_entry_objects_and_streams(): + sandbox, session = sandbox_with_envd_session() + + stream = sandbox.files.read('/tmp/hello.txt', format='stream') + written = sandbox.files.write('/tmp/hello.txt', BytesIO(b'hello')) + info = sandbox.files.stat('/tmp/hello.txt') + entries = sandbox.files.list('/tmp') + + assert b''.join(stream) == b'hello' + assert session.requests[0]['response'].closed is True + assert written.name == 'hello.txt' + assert written.path == '/tmp/hello.txt' + assert written.type == FileType.FILE + assert isinstance(info, EntryInfo) + assert info.name == 'hello.txt' + assert info.type == FileType.FILE + assert entries[0].name == 'hello.txt' + assert entries[0].type == FileType.FILE + assert session.requests[0]['kwargs']['stream'] is True + written_data = session.requests[1]['kwargs']['files']['file'][1] + assert session.requests[1]['kwargs']['files']['file'] == ( + 'hello.txt', + written_data, + ) + assert isinstance(written_data, BytesIO) + + +def test_filesystem_read_write_pass_request_timeout_to_file_requests(): + sandbox, session = sandbox_with_envd_session() + + sandbox.files.read_text('/tmp/hello.txt', timeout=3, request_timeout=7) + sandbox.files.write('/tmp/hello.txt', 'hello', timeout=5) + sandbox.files.write( + '/tmp/octet.txt', + b'hello', + use_octet_stream=True, + request_timeout=11, + ) + + assert session.requests[0]['kwargs']['timeout'] == 7 + assert session.requests[1]['kwargs']['timeout'] == 5 + assert session.requests[2]['kwargs']['timeout'] == 11 + + +def test_filesystem_write_files_accepts_e2b_style_file_list(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.files.write_files([ + {'path': '/tmp/a.txt', 'data': 'a'}, + {'path': '/tmp/b.txt', 'data': b'b'}, + ]) + + assert [entry['type'] for entry in result] == ['file', 'file'] + assert session.requests[0]['method'] == 'POST' + assert session.requests[1]['method'] == 'POST' + + +def test_filesystem_watch_dir_returns_e2b_style_watch_handle(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.files.watch_dir('/tmp', recursive=True) + events = handle.get_new_events() + handle.stop() + + assert handle.watcher_id == 'watch-1' + assert events[0].name == 'hello.txt' + assert events[0].type == FilesystemEventType.WRITE + assert FileType.FILE == 'file' + assert session.posts[0]['url'].endswith( + '/filesystem.Filesystem/CreateWatcher') + assert session.posts[0]['data'] == {'path': '/tmp', 'recursive': True} + assert session.posts[1]['url'].endswith( + '/filesystem.Filesystem/GetWatcherEvents') + assert session.posts[1]['data'] == {'watcherId': 'watch-1'} + assert session.posts[2]['url'].endswith( + '/filesystem.Filesystem/RemoveWatcher') + + +def test_filesystem_watch_dir_handle_supports_context_manager(): + sandbox, session = sandbox_with_envd_session() + + with sandbox.files.watch_dir('/tmp') as handle: + assert handle.watcher_id == 'watch-1' + + assert handle._closed is True + assert session.posts[1]['url'].endswith( + '/filesystem.Filesystem/RemoveWatcher') diff --git a/tests/cases/test_services/test_sandbox/test_example_config.py b/tests/cases/test_services/test_sandbox/test_example_config.py new file mode 100644 index 00000000..a222e8a1 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_example_config.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +import os + + +ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', '..') +) + + +def read_project_file(*parts): + with open(os.path.join(ROOT, *parts), 'r') as f: + return f.read() + + +def test_env_example_only_contains_connection_and_resource_inputs(): + content = read_project_file('.env.example') + + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in content + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in content + assert 'QINIU_SANDBOX_ID' not in content + assert 'QINIU_SANDBOX_KODO_READ_ONLY' not in content + + +def test_examples_handle_runtime_branches_in_code(): + connect = read_project_file('examples', 'sandbox_connect.py') + resources = read_project_file('examples', 'sandbox_resources.py') + runtime = read_project_file('examples', 'sandbox_runtime.py') + integration = read_project_file( + 'tests', + 'cases', + 'test_services', + 'test_sandbox', + 'test_integration.py', + ) + + assert 'QINIU_SANDBOX_ID' not in connect + assert 'QINIU_SANDBOX_KODO_READ_ONLY' not in resources + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in runtime + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in runtime + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in integration + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in integration + + +def test_examples_cover_primary_sandbox_surfaces(): + expected_examples = [ + 'sandbox_commands.py', + 'sandbox_connect.py', + 'sandbox_create.py', + 'sandbox_filesystem.py', + 'sandbox_git.py', + 'sandbox_injection_rules.py', + 'sandbox_lifecycle.py', + 'sandbox_observability.py', + 'sandbox_resources.py', + 'sandbox_runtime.py', + 'sandbox_template_lifecycle.py', + 'sandbox_templates.py', + ] + for name in expected_examples: + assert os.path.exists(os.path.join(ROOT, 'examples', name)) + + checks = { + 'sandbox_commands.py': [ + 'commands.start', + 'commands.list', + 'commands.send_stdin', + 'commands.close_stdin', + 'commands.kill', + 'commands.connect', + ], + 'sandbox_filesystem.py': [ + 'files.make_dir', + 'files.write(', + 'files.read(', + 'files.write_files', + 'files.get_info', + 'files.list', + 'files.rename', + 'files.remove', + ], + 'sandbox_observability.py': [ + 'wait_for_ready', + 'is_running', + 'get_host', + 'get_mcp_url', + 'get_mcp_token', + 'download_url', + 'upload_url', + 'get_metrics', + 'get_logs', + ], + 'sandbox_template_lifecycle.py': [ + 'list_default_templates', + 'list_templates', + 'get_template', + 'update_template', + 'assign_template_tags', + 'delete_template_tags', + 'get_template_build_status', + 'get_template_build_logs', + 'start_template_build', + 'wait_for_build', + 'delete_template', + ], + 'sandbox_injection_rules.py': [ + 'create_injection_rule', + 'list_injection_rules', + 'get_injection_rule', + 'update_injection_rule', + 'delete_injection_rule', + ], + 'sandbox_lifecycle.py': [ + 'set_timeout', + 'refresh', + 'pause', + 'resume', + 'update_network', + ], + 'sandbox_resources.py': [ + 'GitRepositoryResource', + 'push_branch_with_credentials', + 'KodoResource', + 'read_only=True', + ], + } + + for name, snippets in checks.items(): + content = read_project_file('examples', name) + for snippet in snippets: + assert snippet in content diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py new file mode 100644 index 00000000..f4a647f2 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +import os +import time + +import pytest + +from qiniu.services.sandbox import ( + PtySize, + Sandbox, + SandboxClient, + SandboxError, +) +from qiniu.services.sandbox.config import load_dotenv_if_present + + +load_dotenv_if_present() + + +def integration_client(): + if not os.getenv('QINIU_SANDBOX_API_KEY'): + pytest.skip('QINIU_SANDBOX_API_KEY is required') + return SandboxClient() + + +def is_unsupported_runtime_error(err): + response = getattr(err, 'response', None) + status_code = getattr(response, 'status_code', None) + if status_code in (404, 501): + return True + message = str(err).lower() + return ( + 'unimplemented' in message or + 'not implemented' in message or + 'not found' in message + ) + + +def create_integration_sandbox(test_name): + return Sandbox.create( + os.getenv('QINIU_SANDBOX_TEMPLATE', 'base'), + timeout=300, + metadata={'sdk': 'python', 'test': test_name}, + envs={'QINIU_SANDBOX_TEST': '1'}, + client=integration_client(), + ) + + +def assert_command_ok(step, result): + assert result.exit_code == 0, '{0} failed: {1}'.format( + step, result.stderr or result.stdout or result.error) + return result + + +def is_retryable_git_network_error(result): + message = (result.stderr or result.stdout or result.error or '').lower() + return result.exit_code != 0 and ( + 'gnutls' in message or + 'tls connection' in message or + 'connection was non-properly terminated' in message or + 'the remote end hung up unexpectedly' in message + ) + + +def assert_git_network_ok(step, run, attempts=5): + result = None + for attempt in range(attempts): + result = run() + if result.exit_code == 0: + return result + if not is_retryable_git_network_error(result): + return assert_command_ok(step, result) + if attempt == attempts - 1: + pytest.skip('{0} skipped due to external Git network error: {1}' + .format(step, result.stderr or result.stdout)) + time.sleep(attempt + 1) + return assert_command_ok(step, result) + + +def remote_git_credentials(): + repo_url = os.getenv('GIT_REPO_URL') + username = os.getenv('GIT_USERNAME') + password = os.getenv('GIT_PASSWORD') or os.getenv('GITHUB_TOKEN') + if password and not username: + username = 'x-access-token' + if not repo_url or not username or not password: + pytest.skip('GIT_REPO_URL and Git credentials are required') + return repo_url, username, password + + +def test_create_run_filesystem_and_kill_sandbox(): + sandbox = None + client = integration_client() + template = os.getenv('QINIU_SANDBOX_TEMPLATE', 'base') + try: + sandbox = Sandbox.create( + template, + timeout=300, + metadata={'sdk': 'python', 'test': 'integration'}, + envs={'QINIU_SANDBOX_TEST': '1'}, + client=client, + ) + result = sandbox.commands.run('printf hello') + assert result.exit_code == 0 + assert result.stdout == 'hello' + + sandbox.files.write('/tmp/qiniu-python-sdk.txt', 'hello') + assert sandbox.files.read_text('/tmp/qiniu-python-sdk.txt') == 'hello' + finally: + if sandbox is not None: + sandbox.kill() + + +def test_git_remote_push_when_credentials_are_configured(): + repo_url, username, password = remote_git_credentials() + sandbox = None + branch = 'python-sdk-it-{0}'.format(int(time.time() * 1000)) + repo_path = '/tmp/qiniu-python-sdk-remote-git' + try: + sandbox = create_integration_sandbox('remote_git') + assert_command_ok( + 'git authenticate', + sandbox.git.dangerously_authenticate(username, password), + ) + assert_command_ok( + 'git http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + assert_git_network_ok( + 'git clone', + lambda: ( + sandbox.commands.run('rm -rf {0}'.format(repo_path)), + sandbox.git.clone(repo_url, repo_path, depth=1), + )[1], + ) + assert_command_ok( + 'configure user', + sandbox.git.configure_user( + repo_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_command_ok( + 'checkout branch', + sandbox.git.checkout_branch(repo_path, branch, create=True), + ) + sandbox.files.write( + repo_path + '/python-sdk-integration.txt', + 'qiniu-python-sdk remote push {0}\n'.format(branch), + ) + assert_command_ok('git add', sandbox.git.add(repo_path, all=True)) + assert_command_ok( + 'git commit', + sandbox.git.commit( + repo_path, + 'test: qiniu python sdk remote push', + ), + ) + assert_git_network_ok( + 'git push', + lambda: sandbox.git.push( + repo_path, + 'origin', + 'HEAD:refs/heads/{0}'.format(branch), + ), + ) + finally: + if sandbox is not None: + sandbox.kill() + + +def test_runtime_commands_filesystem_and_git_helpers(): + sandbox = None + try: + sandbox = create_integration_sandbox('runtime') + + stdout = [] + result = sandbox.commands.run( + 'printf runtime', + on_stdout=stdout.append, + ) + assert result.exit_code == 0 + assert result.stdout == 'runtime' + assert stdout == ['runtime'] + + sandbox.files.write_files([ + {'path': '/tmp/qiniu-python-sdk-a.txt', 'data': 'a'}, + {'path': '/tmp/qiniu-python-sdk-b.txt', 'data': 'b'}, + ]) + assert sandbox.files.read_text('/tmp/qiniu-python-sdk-a.txt') == 'a' + assert sandbox.files.read_text('/tmp/qiniu-python-sdk-b.txt') == 'b' + + repo_path = '/tmp/qiniu-python-sdk-runtime-repo' + sandbox.commands.run('rm -rf {0} && mkdir -p {0}'.format(repo_path)) + assert sandbox.git.init(repo_path).exit_code == 0 + assert sandbox.git.set_config( + repo_path, 'user.name', 'Sandbox Demo').exit_code == 0 + assert sandbox.git.set_config( + repo_path, 'user.email', 'sandbox-demo@example.com').exit_code == 0 + sandbox.files.write(repo_path + '/README.md', '# runtime\n') + assert sandbox.git.add(repo_path, all=True).exit_code == 0 + assert sandbox.git.commit( + repo_path, 'feat: runtime', allow_empty=True).exit_code == 0 + assert sandbox.git.create_branch(repo_path, 'feature').exit_code == 0 + assert sandbox.git.checkout_branch(repo_path, 'feature').exit_code == 0 + branches = sandbox.git.branches(repo_path) + assert 'feature' in branches.branches + assert branches.current_branch == 'feature' + assert sandbox.git.get_config(repo_path, 'user.name').exit_code == 0 + assert sandbox.git.reset(repo_path, 'HEAD', mode='hard').exit_code == 0 + assert sandbox.git.restore( + repo_path, + paths=['README.md'], + ).exit_code == 0 + finally: + if sandbox is not None: + sandbox.kill() + + +def test_filesystem_watch_dir_when_envd_supports_it(): + sandbox = None + watcher = None + try: + sandbox = create_integration_sandbox('watch_dir') + try: + watcher = sandbox.files.watch_dir('/tmp') + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd does not support filesystem watcher') + raise + + sandbox.files.write('/tmp/qiniu-python-sdk-watch.txt', 'watch') + events = [] + for _ in range(5): + events = watcher.get_new_events() + if events: + break + time.sleep(1) + assert isinstance(events, list) + assert all(hasattr(event, 'name') for event in events) + finally: + if watcher is not None: + watcher.stop() + if sandbox is not None: + sandbox.kill() + + +def test_pty_when_envd_supports_it(): + sandbox = None + try: + sandbox = create_integration_sandbox('pty') + try: + handle = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd does not support pty') + raise + + try: + sandbox.pty.send_stdin(handle.pid, 'echo qiniu-pty-ready\n') + sandbox.pty.send_stdin(handle.pid, 'exit\n') + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd can create pty but does not support input') + raise + result = handle.wait() + assert 'qiniu-pty-ready' in result.stdout + finally: + if sandbox is not None: + sandbox.kill() + + +def test_list_and_connect_existing_sandbox(): + client = integration_client() + page = Sandbox.list(client=client, limit=5).next_items() + assert isinstance(page, list) + if not page: + pytest.skip('No sandbox available to connect') + + connected = Sandbox.connect(page[0].sandbox_id, client=client, timeout=60) + assert connected.sandbox_id == page[0].sandbox_id