From a5adf91d5e287d5eb1e13fe5520f2b9de008c190 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 16:05:33 +0800 Subject: [PATCH 01/45] feat(sandbox): add sandbox module Add an E2B-style sandbox client with lifecycle, envd commands, filesystem, git, template, resource, and injection-rule support. Include .env-based examples plus unit and integration coverage for the new sandbox surface. --- .env.example | 35 ++ .gitignore | 1 + examples/sandbox_common.py | 60 ++ examples/sandbox_connect.py | 37 ++ examples/sandbox_create.py | 24 + examples/sandbox_git.py | 50 ++ examples/sandbox_injection_rules.py | 26 + examples/sandbox_lifecycle.py | 20 + examples/sandbox_resources.py | 83 +++ examples/sandbox_templates.py | 25 + qiniu/__init__.py | 1 + qiniu/services/sandbox/__init__.py | 54 ++ qiniu/services/sandbox/client.py | 540 ++++++++++++++++++ qiniu/services/sandbox/commands.py | 170 ++++++ qiniu/services/sandbox/config.py | 68 +++ qiniu/services/sandbox/constants.py | 7 + qiniu/services/sandbox/envd.py | 121 ++++ qiniu/services/sandbox/errors.py | 21 + qiniu/services/sandbox/filesystem.py | 165 ++++++ qiniu/services/sandbox/git.py | 83 +++ qiniu/services/sandbox/resources.py | 40 ++ qiniu/services/sandbox/sandbox.py | 312 ++++++++++ qiniu/services/sandbox/template.py | 77 +++ qiniu/services/sandbox/util.py | 108 ++++ .../test_services/test_sandbox/__init__.py | 0 .../test_services/test_sandbox/test_client.py | 219 +++++++ .../test_services/test_sandbox/test_config.py | 81 +++ .../test_services/test_sandbox/test_envd.py | 129 +++++ .../test_sandbox/test_example_config.py | 38 ++ .../test_sandbox/test_integration.py | 50 ++ 30 files changed, 2645 insertions(+) create mode 100644 .env.example create mode 100644 examples/sandbox_common.py create mode 100644 examples/sandbox_connect.py create mode 100644 examples/sandbox_create.py create mode 100644 examples/sandbox_git.py create mode 100644 examples/sandbox_injection_rules.py create mode 100644 examples/sandbox_lifecycle.py create mode 100644 examples/sandbox_resources.py create mode 100644 examples/sandbox_templates.py create mode 100644 qiniu/services/sandbox/__init__.py create mode 100644 qiniu/services/sandbox/client.py create mode 100644 qiniu/services/sandbox/commands.py create mode 100644 qiniu/services/sandbox/config.py create mode 100644 qiniu/services/sandbox/constants.py create mode 100644 qiniu/services/sandbox/envd.py create mode 100644 qiniu/services/sandbox/errors.py create mode 100644 qiniu/services/sandbox/filesystem.py create mode 100644 qiniu/services/sandbox/git.py create mode 100644 qiniu/services/sandbox/resources.py create mode 100644 qiniu/services/sandbox/sandbox.py create mode 100644 qiniu/services/sandbox/template.py create mode 100644 qiniu/services/sandbox/util.py create mode 100644 tests/cases/test_services/test_sandbox/__init__.py create mode 100644 tests/cases/test_services/test_sandbox/test_client.py create mode 100644 tests/cases/test_services/test_sandbox/test_config.py create mode 100644 tests/cases/test_services/test_sandbox/test_envd.py create mode 100644 tests/cases/test_services/test_sandbox/test_example_config.py create mode 100644 tests/cases/test_services/test_sandbox/test_integration.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..ce212be4 --- /dev/null +++ b/.env.example @@ -0,0 +1,35 @@ +# Copy this file to .env in the project root before running sandbox examples +# or sandbox integration tests. + +# Sandbox API key auth. +QINIU_SANDBOX_API_KEY= + +# Optional custom endpoint. +# QINIU_SANDBOX_ENDPOINT is the preferred name; QINIU_SANDBOX_API_URL and +# E2B_API_URL are also accepted by the SDK for compatibility. +QINIU_SANDBOX_ENDPOINT= + +# Optional template alias or ID. Defaults to base. +QINIU_SANDBOX_TEMPLATE=base + +# Required only for injection-rule and Kodo resource examples/tests. +QINIU_SANDBOX_ACCESS_KEY= +QINIU_SANDBOX_SECRET_KEY= + +# Optional Git remote examples. +GIT_REPO_URL= +GIT_USERNAME= +GIT_PASSWORD= + +# Optional Git repository resource example. +GITHUB_TOKEN= +QINIU_SANDBOX_GIT_MOUNT_PATH=/workspace/repo + +# Optional Kodo resource example. +QINIU_SANDBOX_KODO_BUCKET= +QINIU_SANDBOX_KODO_MOUNT_PATH=/workspace/kodo +QINIU_SANDBOX_KODO_PREFIX= + +# Optional request injection examples. +QINIU_SANDBOX_HTTP_INJECTION_TOKEN=real_token +QINIU_SANDBOX_OPENAI_API_KEY= diff --git a/.gitignore b/.gitignore index 41be9369..c17651eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store *.swp *.pyc +.env *.py[cod] diff --git a/examples/sandbox_common.py b/examples/sandbox_common.py new file mode 100644 index 00000000..b5a3f94b --- /dev/null +++ b/examples/sandbox_common.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import os + +from qiniu.services.sandbox import Sandbox +from qiniu.services.sandbox.config import ( + env, + load_dotenv_if_present, + required_env, + sandbox_client, + sandbox_template, +) + + +__all__ = [ + 'cleanup_sandbox', + 'create_sandbox', + 'env', + 'load_example_env', + 'required_env', + 'run_example', + 'sandbox_client', + 'sandbox_template', +] + + +def load_example_env(): + root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + load_dotenv_if_present( + os.path.join(root, '.env'), + os.path.join(os.getcwd(), '.env'), + ) + + +def create_sandbox(**options): + client = options.pop('client', None) or sandbox_client() + template = options.pop('template', sandbox_template()) + options.setdefault('timeout', 300) + sandbox = Sandbox.create(template, client=client, **options) + print('Sandbox created:', sandbox.sandbox_id) + return sandbox + + +def cleanup_sandbox(sandbox): + if sandbox is None: + return + try: + sandbox.kill() + print('Sandbox killed:', sandbox.sandbox_id) + except Exception as err: + print('Failed to kill sandbox:', sandbox.sandbox_id, err) + + +def run_example(fn): + load_example_env() + try: + fn() + except Exception as err: + print(err) + raise diff --git a/examples/sandbox_connect.py b/examples/sandbox_connect.py new file mode 100644 index 00000000..1c85a0f0 --- /dev/null +++ b/examples/sandbox_connect.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import Sandbox +from sandbox_common import ( + cleanup_sandbox, + create_sandbox, + run_example, + sandbox_client, +) + + +def main(): + client = sandbox_client() + sandbox = create_sandbox( + client=client, + timeout=300, + metadata={'example': 'sandbox_connect'}, + ) + try: + items = Sandbox.list(client=client, limit=10).next_items() + print('list:', [item.sandbox_id for item in items]) + + connected = Sandbox.connect( + sandbox.sandbox_id, + client=client, + timeout=300, + ) + print('connected:', connected.sandbox_id) + print('envd:', connected.envd_url()) + print('uptime:', connected.commands.run('uptime').stdout) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_create.py b/examples/sandbox_create.py new file mode 100644 index 00000000..4d4c37d8 --- /dev/null +++ b/examples/sandbox_create.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox( + metadata={'example': 'sandbox_create'}, + envs={'HELLO': 'qiniu'}, + ) + try: + print('sandbox:', sandbox.sandbox_id) + result = sandbox.commands.run('printf "$HELLO"') + print('stdout:', result.stdout) + + sandbox.files.write('/tmp/qiniu.txt', 'hello from qiniu sandbox') + print('file:', sandbox.files.read_text('/tmp/qiniu.txt')) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py new file mode 100644 index 00000000..96b436b2 --- /dev/null +++ b/examples/sandbox_git.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def assert_git_ok(step, result): + if result.exit_code != 0: + raise RuntimeError( + '{0} failed with exit {1}: {2}'.format( + step, + result.exit_code, + result.stderr or result.stdout, + ) + ) + print(step + ':', result.exit_code) + return result + + +def main(): + repo_path = '/tmp/qiniu-python-sdk-git/repo' + clone_path = '/tmp/qiniu-python-sdk-git/clone' + sandbox = create_sandbox(timeout=300) + try: + sandbox.commands.run( + 'mkdir -p /tmp/qiniu-python-sdk-git/repo' + ) + assert_git_ok('git init', sandbox.git.init(repo_path)) + assert_git_ok( + 'configure user', + sandbox.git.configure_user( + repo_path, + 'Sandbox Demo', + 'sandbox-demo@example.com', + ), + ) + sandbox.files.write(repo_path + '/README.md', '# sandbox git demo\n') + assert_git_ok('git add', sandbox.git.add(repo_path, all=True)) + assert_git_ok( + 'git commit', + sandbox.git.commit(repo_path, 'feat: initial commit'), + ) + assert_git_ok('git clone', sandbox.git.clone(repo_path, clone_path)) + print(sandbox.git.status(clone_path).stdout) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_injection_rules.py b/examples/sandbox_injection_rules.py new file mode 100644 index 00000000..c664c9df --- /dev/null +++ b/examples/sandbox_injection_rules.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import run_example, sandbox_client + + +def main(): + client = sandbox_client() + rule = client.create_injection_rule( + name='python-sdk-example', + injection={ + 'type': 'http', + 'base_url': 'https://api.example.com', + 'headers': {'X-From-Sandbox': 'qiniu-python-sdk'}, + }, + ) + print('created:', rule) + rules = client.list_injection_rules() + print('rules count:', len(rules)) + rule_id = rule.get('id') or rule.get('ruleID') + client.delete_injection_rule(rule_id) + print('deleted:', rule_id) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_lifecycle.py b/examples/sandbox_lifecycle.py new file mode 100644 index 00000000..09a83e85 --- /dev/null +++ b/examples/sandbox_lifecycle.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(timeout=300) + try: + print('created:', sandbox.sandbox_id) + sandbox.set_timeout(600) + sandbox.refresh(duration=600) + info = sandbox.get_info() + print('state:', info.get('state')) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py new file mode 100644 index 00000000..f31c2b59 --- /dev/null +++ b/examples/sandbox_resources.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import GitRepositoryResource, KodoResource +from sandbox_common import ( + cleanup_sandbox, + create_sandbox, + env, + run_example, +) + + +def run_git_resource_example(): + repo_url = env('GIT_REPO_URL') + repo_token = env('GITHUB_TOKEN') + if not repo_url or not repo_token: + print( + 'Skip GitHub repository resource: ' + 'set GIT_REPO_URL and GITHUB_TOKEN.' + ) + return + + mount_path = env('QINIU_SANDBOX_GIT_MOUNT_PATH', '/workspace/repo') + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_git'}, + resources=[ + GitRepositoryResource( + url=repo_url, + mount_path=mount_path, + authorization_token=repo_token, + ) + ], + ) + try: + print('Git resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + mount_path + )).stdout) + finally: + cleanup_sandbox(sandbox) + + +def run_kodo_resource_example(): + bucket = env('QINIU_SANDBOX_KODO_BUCKET') + if not bucket: + print('Skip Kodo resource: set QINIU_SANDBOX_KODO_BUCKET.') + return + + mount_path = env('QINIU_SANDBOX_KODO_MOUNT_PATH', '/mnt/kodo') + resource = KodoResource( + bucket=bucket, + mount_path=mount_path, + prefix=env('QINIU_SANDBOX_KODO_PREFIX') or None, + ) + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_kodo'}, + resources=[resource], + ) + try: + print('Kodo resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + mount_path + )).stdout) + test_path = mount_path + '/qiniu-python-sdk-resource-test.txt' + result = sandbox.commands.run( + 'sh -c "echo qiniu-python-sdk > {0} && cat {0}"'.format(test_path) + ) + if result.exit_code != 0: + raise RuntimeError(result.stderr or result.stdout) + print('Kodo write/read:', result.stdout.strip()) + finally: + cleanup_sandbox(sandbox) + + +def main(): + run_git_resource_example() + run_kodo_resource_example() + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_templates.py b/examples/sandbox_templates.py new file mode 100644 index 00000000..3f88d0bc --- /dev/null +++ b/examples/sandbox_templates.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import Template +from sandbox_common import run_example, sandbox_client + + +def main(): + client = sandbox_client() + template = ( + Template() + .from_image('python:3.11') + .run_cmd('python --version') + .set_env('PYTHONUNBUFFERED', '1') + ) + + created = client.create_template( + name='qiniu-python-sdk-example', + buildConfig=template.to_dict(), + ) + print('template:', created) + + +if __name__ == '__main__': + run_example(main) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 85d8ce35..204b215e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -26,6 +26,7 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient +from .services.sandbox import Sandbox, SandboxClient, Template, KodoResource, GitRepositoryResource from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, decode_entry, canonical_mime_header_key diff --git a/qiniu/services/sandbox/__init__.py b/qiniu/services/sandbox/__init__.py new file mode 100644 index 00000000..912edce5 --- /dev/null +++ b/qiniu/services/sandbox/__init__.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- + +from .client import SandboxClient +from .config import ( + env, + load_dotenv_if_present, + required_env, + sandbox_client, + sandbox_template, +) +from .commands import ( + CommandExitError, + CommandHandle, + CommandResult, + Commands, +) +from .constants import ( + DEFAULT_ENDPOINT, + DEFAULT_TEMPLATE, + DEFAULT_USER, + ENVD_PORT, +) +from .errors import SandboxError, TemplateBuildError +from .filesystem import Filesystem +from .git import Git +from .resources import GitRepositoryResource, KodoResource +from .sandbox import Sandbox, SandboxPaginator +from .template import Template + +__all__ = [ + 'CommandExitError', + 'CommandHandle', + 'CommandResult', + 'Commands', + 'DEFAULT_ENDPOINT', + 'DEFAULT_TEMPLATE', + 'DEFAULT_USER', + 'ENVD_PORT', + 'Filesystem', + 'Git', + 'GitRepositoryResource', + 'KodoResource', + 'Sandbox', + 'SandboxClient', + 'SandboxError', + 'SandboxPaginator', + 'Template', + 'TemplateBuildError', + 'env', + 'load_dotenv_if_present', + 'required_env', + 'sandbox_client', + 'sandbox_template', +] diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py new file mode 100644 index 00000000..7e9e0db4 --- /dev/null +++ b/qiniu/services/sandbox/client.py @@ -0,0 +1,540 @@ +# -*- coding: utf-8 -*- +import os + +import requests + +from qiniu.auth import QiniuMacAuth, QiniuMacRequestsAuth + +from .constants import DEFAULT_TEMPLATE +from .errors import SandboxError, TemplateBuildError +from .resources import KodoResource +from .util import ( + encode_path, + json_dumps, + normalize_endpoint, + parse_json_response, +) + + +_UNSET = object() + + +def _to_dict(value): + if hasattr(value, 'to_dict'): + return value.to_dict() + return value + + +def _normalize_injection(injection): + if hasattr(injection, 'to_dict'): + injection = injection.to_dict() + if not isinstance(injection, dict): + return injection + data = dict(injection) + if 'apiKey' in data and 'api_key' not in data: + data['api_key'] = data.pop('apiKey') + if 'baseUrl' in data and 'base_url' not in data: + data['base_url'] = data.pop('baseUrl') + if 'ruleId' in data and 'ruleID' not in data: + data['ruleID'] = data.pop('ruleId') + return data + + +def _normalize_resources(resources): + if resources is None: + return None + return [_to_dict(resource) for resource in resources] + + +def _has_kodo_resource(resources): + for resource in resources or []: + if isinstance(resource, KodoResource): + return True + data = _to_dict(resource) + if isinstance(data, dict) and data.get('type') == 'kodo': + return True + return False + + +def _normalize_sandbox_create_options(template=None, **opts): + body = {'templateID': ( + opts.pop('templateID', None) or + opts.pop('template_id', None) or + opts.pop('template', None) or + template or + DEFAULT_TEMPLATE + )} + for key in ( + 'timeout', + 'autoPause', + 'secure', + 'network', + 'metadata', + 'mcp'): + if opts.get(key) is not None: + body[key] = opts.get(key) + if opts.get('auto_pause') is not None: + body['autoPause'] = opts.get('auto_pause') + if opts.get('allow_internet_access') is not None: + body['allow_internet_access'] = opts.get('allow_internet_access') + if opts.get('allowInternetAccess') is not None: + body['allow_internet_access'] = opts.get('allowInternetAccess') + if opts.get('envs') is not None: + body['envVars'] = opts.get('envs') + if opts.get('envVars') is not None: + body['envVars'] = opts.get('envVars') + if opts.get('lifecycle') is not None: + lifecycle = opts.get('lifecycle') or {} + on_timeout = lifecycle.get('on_timeout') or lifecycle.get('onTimeout') + if on_timeout == 'pause': + body['autoPause'] = True + if opts.get('injections') is not None: + body['injections'] = [_normalize_injection( + item) for item in opts.get('injections')] + if opts.get('resources') is not None: + body['resources'] = _normalize_resources(opts.get('resources')) + return body + + +def _normalize_list_options(opts): + opts = dict(opts or {}) + query = opts.pop('query', None) or {} + metadata = query.get('metadata') + if metadata: + for key, value in metadata.items(): + opts['metadata[{0}]'.format(key)] = value + if query.get('state') is not None: + opts['state'] = query.get('state') + return opts + + +class SandboxClient(object): + def __init__(self, endpoint=None, api_url=None, api_key=None, + access_token=None, mac=None, access_key=None, + secret_key=None, session=None, timeout=None, **opts): + if (access_key and not secret_key) or (secret_key and not access_key): + raise SandboxError( + 'Both access_key and secret_key must be provided') + self.endpoint = normalize_endpoint(endpoint or api_url) + self.api_key = api_key or os.getenv('QINIU_SANDBOX_API_KEY') + self.access_token = access_token or os.getenv( + 'QINIU_SANDBOX_ACCESS_TOKEN') + self.mac = mac + if self.mac is None and access_key and secret_key: + self.mac = QiniuMacAuth(access_key, secret_key) + self.session = session or requests.Session() + self.timeout = timeout + + def _headers(self, auth_type=None): + headers = {'Content-Type': 'application/json'} + if auth_type == 'qiniu': + return headers + if auth_type == 'accessToken': + if self.access_token: + headers['Authorization'] = 'Bearer {0}'.format( + self.access_token) + return headers + if self.api_key: + headers['X-API-Key'] = self.api_key + headers['Authorization'] = 'Bearer {0}'.format(self.api_key) + elif self.access_token: + headers['Authorization'] = 'Bearer {0}'.format(self.access_token) + return headers + + def _auth(self, auth_type=None): + if auth_type == 'qiniu' or ( + not self.api_key and not self.access_token and self.mac): + if not self.mac: + raise SandboxError( + 'Qiniu AK/SK credentials are required for this operation' + ) + return QiniuMacRequestsAuth(self.mac) + return None + + def _request(self, method, path, params=None, body=_UNSET, + auth_type=None, empty=False): + url = self.endpoint + path + data = None if body is _UNSET else json_dumps(body) + headers = self._headers(auth_type) + auth = self._auth(auth_type) + request = requests.Request( + method=method, + url=url, + params=params, + data=data, + headers=headers, + auth=auth, + ) + prepared = self.session.prepare_request(request) + response = self.session.send(prepared, timeout=self.timeout) + if response.status_code < 200 or response.status_code >= 300: + response_data = None + try: + response_data = response.json() + except Exception: + response_data = getattr(response, 'text', None) + message = 'Sandbox API request failed with status {0}'.format( + response.status_code + ) + if isinstance( + response_data, + dict) and response_data.get('message'): + message += ': {0}'.format(response_data.get('message')) + elif isinstance(response_data, str) and response_data: + message += ': {0}'.format(response_data) + raise SandboxError(message, response, response_data) + if empty: + return None + return parse_json_response(response) + + def list_sandboxes(self, **opts): + return self._request('GET', '/sandboxes', params=opts) + + listSandboxes = list_sandboxes + + def list_sandboxes_v2(self, **opts): + return self._request( + 'GET', + '/v2/sandboxes', + params=_normalize_list_options(opts)) + + listSandboxesV2 = list_sandboxes_v2 + list = list_sandboxes_v2 + + def create_sandbox(self, template=None, **opts): + body = _normalize_sandbox_create_options(template, **opts) + auth_type = 'qiniu' if _has_kodo_resource( + opts.get('resources')) else None + return self._request( + 'POST', + '/sandboxes', + body=body, + auth_type=auth_type) + + createSandbox = create_sandbox + create = create_sandbox + + def get_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'GET', '/sandboxes/{0}'.format(encode_path(sandbox_id))) + + getSandbox = get_sandbox + get_info = get_sandbox + getInfo = get_sandbox + + def delete_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') + return self._request( + 'DELETE', + '/sandboxes/{0}'.format(encode_path(sandbox_id)), + empty=True, + ) + + deleteSandbox = delete_sandbox + kill_sandbox = delete_sandbox + killSandbox = delete_sandbox + kill = delete_sandbox + + def pause_sandbox(self, sandbox_id): + return self._request( + 'POST', + '/sandboxes/{0}/pause'.format(encode_path(sandbox_id)), + body={}, + empty=True, + ) + + pauseSandbox = pause_sandbox + + def resume_sandbox(self, sandbox_id, **opts): + return self._request( + 'POST', + '/sandboxes/{0}/resume'.format(encode_path(sandbox_id)), + body=opts, + ) + + resumeSandbox = resume_sandbox + + def connect_sandbox(self, sandbox_id, timeout=15, **opts): + if timeout is None: + timeout = opts.pop('timeout', 15) + return self._request( + 'POST', + '/sandboxes/{0}/connect'.format(encode_path(sandbox_id)), + body={'timeout': timeout}, + ) + + connectSandbox = connect_sandbox + connect = connect_sandbox + + def update_sandbox_timeout(self, sandbox_id, timeout=None, **opts): + if timeout is None: + timeout = opts.get('timeout') + return self._request( + 'POST', + '/sandboxes/{0}/timeout'.format(encode_path(sandbox_id)), + body={'timeout': timeout}, + empty=True, + ) + + updateSandboxTimeout = update_sandbox_timeout + set_timeout = update_sandbox_timeout + setTimeout = update_sandbox_timeout + + def refresh_sandbox(self, sandbox_id, **opts): + return self._request( + 'POST', + '/sandboxes/{0}/refreshes'.format(encode_path(sandbox_id)), + body=opts, + empty=True, + ) + + refreshSandbox = refresh_sandbox + + def update_sandbox(self, sandbox_id, **opts): + return self._request( + 'PATCH', + '/sandboxes/{0}'.format(encode_path(sandbox_id)), + body=opts, + ) + + updateSandbox = update_sandbox + + def get_sandbox_metrics(self, sandbox_id, **opts): + return self._request( + 'GET', + '/sandboxes/{0}/metrics'.format(encode_path(sandbox_id)), + params=opts, + ) + + getSandboxMetrics = get_sandbox_metrics + get_metrics = get_sandbox_metrics + getMetrics = get_sandbox_metrics + + def get_sandboxes_metrics(self, sandbox_ids): + values = sandbox_ids + if isinstance(sandbox_ids, dict): + values = sandbox_ids.get( + 'sandbox_ids') or sandbox_ids.get('sandboxIDs') + if not isinstance(values, (list, tuple)): + values = [values] + ids = [] + for value in values: + if isinstance(value, str): + ids.append(value) + elif isinstance(value, dict): + ids.append(value.get('sandboxId') or value.get( + 'sandboxID') or value.get('id')) + elif hasattr(value, 'sandbox_id'): + ids.append(value.sandbox_id) + ids = [item for item in ids if item] + if not ids: + raise SandboxError('At least one sandbox ID must be provided') + return self._request( + 'GET', + '/sandboxes/metrics', + params={ + 'sandbox_ids': ids}) + + getSandboxesMetrics = get_sandboxes_metrics + + def get_sandbox_logs(self, sandbox_id, **opts): + return self._request( + 'GET', + '/sandboxes/{0}/logs'.format(encode_path(sandbox_id)), + params=opts, + ) + + getSandboxLogs = get_sandbox_logs + get_logs = get_sandbox_logs + getLogs = get_sandbox_logs + + def create_template(self, **opts): + return self._request('POST', '/v3/templates', body=opts) + + createTemplate = create_template + createTemplateV3 = create_template + + def create_template_v2(self, **opts): + return self._request('POST', '/v2/templates', body=opts) + + createTemplateV2 = create_template_v2 + + def list_templates(self, **opts): + return self._request('GET', '/templates', params=opts) + + listTemplates = list_templates + + def list_default_templates(self): + return self._request('GET', '/default-templates') + + listDefaultTemplates = list_default_templates + + def get_template(self, template_id, **opts): + return self._request( + 'GET', + '/templates/{0}'.format(encode_path(template_id)), + params=opts, + ) + + getTemplate = get_template + + def delete_template(self, template_id): + return self._request( + 'DELETE', + '/templates/{0}'.format(encode_path(template_id)), + empty=True, + ) + + deleteTemplate = delete_template + + def update_template(self, template_id, **opts): + return self._request( + 'PATCH', + '/templates/{0}'.format(encode_path(template_id)), + body=opts, + ) + + updateTemplate = update_template + + def rebuild_template(self, template_id, **opts): + return self._request( + 'POST', + '/templates/{0}'.format(encode_path(template_id)), + body=opts, + auth_type='accessToken', + ) + + rebuildTemplate = rebuild_template + + def get_template_build_status(self, template_id, build_id, **opts): + return self._request( + 'GET', + '/templates/{0}/builds/{1}/status'.format( + encode_path(template_id), + encode_path(build_id), + ), + params=opts, + ) + + getTemplateBuildStatus = get_template_build_status + + def get_template_build_logs(self, template_id, build_id, **opts): + return self._request( + 'GET', + '/templates/{0}/builds/{1}/logs'.format( + encode_path(template_id), + encode_path(build_id), + ), + params=opts, + ) + + getTemplateBuildLogs = get_template_build_logs + + def start_template_build(self, template_id, build_id, **opts): + return self._request( + 'POST', + '/v2/templates/{0}/builds/{1}'.format( + encode_path(template_id), + encode_path(build_id), + ), + body=opts, + empty=True, + ) + + startTemplateBuild = start_template_build + startTemplateBuildV2 = start_template_build + + def assign_template_tags(self, **opts): + return self._request('POST', '/templates/tags', body=opts) + + assignTemplateTags = assign_template_tags + + def delete_template_tags(self, **opts): + return self._request( + 'DELETE', + '/templates/tags', + body=opts, + empty=True) + + deleteTemplateTags = delete_template_tags + + def get_template_by_alias(self, alias): + return self._request( + 'GET', '/templates/aliases/{0}'.format(encode_path(alias))) + + getTemplateByAlias = get_template_by_alias + + def list_injection_rules(self): + return self._request('GET', '/injection-rules', auth_type='qiniu') + + listInjectionRules = list_injection_rules + + def create_injection_rule(self, **opts): + if opts.get('injection') is not None: + opts['injection'] = _normalize_injection(opts.get('injection')) + return self._request( + 'POST', + '/injection-rules', + body=opts, + auth_type='qiniu', + ) + + createInjectionRule = create_injection_rule + + def get_injection_rule(self, rule_id): + return self._request( + 'GET', + '/injection-rules/{0}'.format(encode_path(rule_id)), + auth_type='qiniu', + ) + + getInjectionRule = get_injection_rule + + def update_injection_rule(self, rule_id, **opts): + if opts.get('injection') is not None: + opts['injection'] = _normalize_injection(opts.get('injection')) + return self._request( + 'PUT', + '/injection-rules/{0}'.format(encode_path(rule_id)), + body=opts, + auth_type='qiniu', + ) + + updateInjectionRule = update_injection_rule + + def delete_injection_rule(self, rule_id): + return self._request( + 'DELETE', + '/injection-rules/{0}'.format(encode_path(rule_id)), + auth_type='qiniu', + empty=True, + ) + + deleteInjectionRule = delete_injection_rule + + def wait_for_build(self, template_id, build_id, interval=1, timeout=60): + import time + start = time.time() + while True: + info = self.get_template_build_status(template_id, build_id) + if info and info.get('status') in ('ready', 'error'): + if info.get('status') == 'error': + message = ( + ( + isinstance(info.get('error'), dict) and + info.get('error').get('message') + ) or + info.get('error') or + info.get('message') or + 'Sandbox template build failed' + ) + raise TemplateBuildError(message, data=info) + return info + if time.time() - start >= timeout: + raise SandboxError('Sandbox template build polling timed out') + time.sleep(interval) + + waitForBuild = wait_for_build diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py new file mode 100644 index 00000000..b8754281 --- /dev/null +++ b/qiniu/services/sandbox/commands.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- +import base64 + +from .envd import connect_rpc, connect_stream_rpc +from .errors import CommandExitError + + +class CommandResult(object): + def __init__(self, pid=0, exit_code=0, stdout='', stderr='', error=''): + self.pid = pid + self.exit_code = exit_code + self.exitCode = exit_code + self.stdout = stdout or '' + self.stderr = stderr or '' + self.error = error or '' + + def to_dict(self): + return { + 'pid': self.pid, + 'exitCode': self.exit_code, + 'stdout': self.stdout, + 'stderr': self.stderr, + 'error': self.error, + } + + +def _decode_bytes(value): + if value is None: + return '' + if isinstance(value, list): + return bytearray(value).decode('utf-8') + if isinstance(value, str): + try: + return base64.b64decode(value).decode('utf-8') + except Exception: + return value + return str(value) + + +def command_result_from_events(events): + pid = 0 + stdout = '' + stderr = '' + exit_code = -1 + error = '' + for raw in events or []: + event = raw.get('event', raw) if isinstance(raw, dict) else {} + start = event.get('start') + data = event.get('data') + end = event.get('end') + if start: + pid = start.get('pid') or pid + if data: + stdout += _decode_bytes(data.get('stdout')) + stderr += _decode_bytes(data.get('stderr')) + if end: + exit_code = 0 if end.get( + 'exitCode') is None else end.get('exitCode') + error = end.get('error') or '' + return CommandResult(pid, exit_code, stdout, stderr, error) + + +class CommandHandle(object): + def __init__(self, commands, result, throw_on_error=False): + self.commands = commands + self.result = result + self.pid = result.pid + self.stdout = result.stdout + self.stderr = result.stderr + self.throw_on_error = throw_on_error + + def wait(self): + if self.throw_on_error and self.result.exit_code: + raise CommandExitError(self.result) + return self.result + + def kill(self): + return self.commands.kill(self.pid) + + +class Commands(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, + tag=None, background=False, throw_on_error=False, + timeout=None, **opts): + handle = self.start( + cmd, + cwd=cwd, + envs=envs, + user=user, + stdin=stdin, + tag=tag, + throw_on_error=throw_on_error, + timeout=timeout, + **opts + ) + return handle if background else handle.wait() + + def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, + tag=None, throw_on_error=False, timeout=None, **opts): + process = { + 'cmd': '/bin/bash', + 'args': ['-l', '-c', cmd], + } + if cwd: + process['cwd'] = cwd + if envs: + process['envs'] = envs + body = { + 'process': process, + 'stdin': stdin, + } + if tag: + body['tag'] = tag + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Start', + body, + user=user, + timeout=timeout, + ) + result = command_result_from_events(events) + return CommandHandle(self, result, throw_on_error=throw_on_error) + + def list(self, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/process.Process/List', + {}, + user=user, + timeout=timeout) + processes = data.get('processes', []) if isinstance(data, dict) else [] + result = [] + for process in processes: + config = process.get('config') or {} + result.append({ + 'pid': process.get('pid'), + 'tag': process.get('tag'), + 'cmd': config.get('cmd'), + 'args': config.get('args'), + 'envs': config.get('envs'), + 'cwd': config.get('cwd'), + }) + return result + + def send_stdin(self, pid, data, user=None, timeout=None): + if not isinstance(data, bytes): + data = str(data).encode('utf-8') + return connect_rpc(self.sandbox, '/process.Process/SendInput', { + 'process': {'selector': {'pid': pid}}, + 'input': {'stdin': base64.b64encode(data).decode('ascii')}, + }, user=user, timeout=timeout) + + sendStdin = send_stdin + + def close_stdin(self, pid, user=None, timeout=None): + return connect_rpc(self.sandbox, '/process.Process/CloseStdin', { + 'process': {'selector': {'pid': pid}}, + }, user=user, timeout=timeout) + + closeStdin = close_stdin + + def kill(self, pid, user=None, timeout=None): + connect_rpc(self.sandbox, '/process.Process/SendSignal', { + 'process': {'selector': {'pid': pid}}, + 'signal': 'SIGNAL_SIGKILL', + }, user=user, timeout=timeout) + return None diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py new file mode 100644 index 00000000..3818a47f --- /dev/null +++ b/qiniu/services/sandbox/config.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +import os + +from .client import SandboxClient + + +def load_dotenv_if_present(*paths): + if not paths: + paths = ( + os.path.join(os.getcwd(), '.env'), + ) + + for path in paths: + if not path or not os.path.exists(path): + continue + with open(path, 'r') as f: + for raw_line in f: + line = raw_line.strip() + if not line or line.startswith('#') or '=' not in line: + continue + key, value = line.split('=', 1) + key = key.strip() + value = value.strip() + if ( + len(value) >= 2 and + value[0] == value[-1] and + value[0] in ('"', "'") + ): + value = value[1:-1] + if key and key not in os.environ: + os.environ[key] = value + + +def env(key, fallback=None): + return os.getenv(key) or fallback + + +def required_env(key): + value = os.getenv(key) + if value: + return value + raise RuntimeError('Please set {0}'.format(key)) + + +def sandbox_endpoint(): + return env('QINIU_SANDBOX_ENDPOINT') + + +def sandbox_api_key(): + return required_env('QINIU_SANDBOX_API_KEY') + + +def sandbox_template(): + return env('QINIU_SANDBOX_TEMPLATE', 'base') + + +def sandbox_client(**options): + defaults = { + 'endpoint': sandbox_endpoint(), + 'api_key': sandbox_api_key(), + } + access_key = env('QINIU_SANDBOX_ACCESS_KEY') + secret_key = env('QINIU_SANDBOX_SECRET_KEY') + if access_key and secret_key: + defaults['access_key'] = access_key + defaults['secret_key'] = secret_key + defaults.update(options) + return SandboxClient(**defaults) diff --git a/qiniu/services/sandbox/constants.py b/qiniu/services/sandbox/constants.py new file mode 100644 index 00000000..3d7c4c7b --- /dev/null +++ b/qiniu/services/sandbox/constants.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +DEFAULT_ENDPOINT = 'https://cn-yangzhou-1-sandbox.qiniuapi.com' +DEFAULT_TEMPLATE = 'base' +DEFAULT_USER = 'user' +ENVD_PORT = 49983 +MCP_PORT = 50005 diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py new file mode 100644 index 00000000..6b685de8 --- /dev/null +++ b/qiniu/services/sandbox/envd.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +import json +import struct + +from .constants import DEFAULT_USER +from .errors import SandboxError +from .util import basic_auth + + +MAX_CONNECT_ENVELOPE_BYTES = 10 * 1024 * 1024 + + +def envd_headers(sandbox, user=None, extra=None): + headers = { + 'Authorization': basic_auth(user or DEFAULT_USER), + } + if sandbox.envd_access_token: + headers['X-Access-Token'] = sandbox.envd_access_token + if extra: + headers.update(extra) + return headers + + +def connect_rpc(sandbox, procedure, body=None, user=None, timeout=None): + url = sandbox.envd_url() + procedure + headers = envd_headers(sandbox, user, {'Content-Type': 'application/json'}) + response = sandbox.client.session.post( + url, + data=json.dumps(body or {}, separators=(',', ':')), + headers=headers, + timeout=timeout, + ) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + if not response.content: + return None + data = response.json() + if isinstance(data, dict) and 'result' in data: + return data.get('result') + return data + + +def encode_connect_envelope(message): + payload = json.dumps(message or {}, separators=(',', ':')).encode('utf-8') + return struct.pack('>BI', 0, len(payload)) + payload + + +def decode_connect_envelopes(data): + if not data: + return [] + if not isinstance(data, bytes): + data = data.encode('utf-8') + messages = [] + offset = 0 + while offset + 5 <= len(data): + flags, length = struct.unpack('>BI', data[offset:offset + 5]) + offset += 5 + if length > MAX_CONNECT_ENVELOPE_BYTES: + raise SandboxError( + 'Sandbox envd stream envelope too large: {0}'.format(length) + ) + if offset + length > len(data): + raise SandboxError('Sandbox envd stream truncated unexpectedly') + payload = data[offset:offset + length] + offset += length + if flags & 2: + if payload: + error = json.loads(payload.decode('utf-8')).get('error') + if error: + raise SandboxError( + error.get('message') or 'Sandbox envd stream failed', + data=error, + ) + continue + if payload: + messages.append(json.loads(payload.decode('utf-8'))) + return messages + + +def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None): + url = sandbox.envd_url() + procedure + headers = envd_headers(sandbox, user, { + 'Content-Type': 'application/connect+json', + 'Keepalive-Ping-Interval': '50', + }) + response = sandbox.client.session.post( + url, + data=encode_connect_envelope(body), + headers=headers, + timeout=timeout, + ) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + + content_type = response.headers.get('Content-Type', '') + if 'application/connect+json' in content_type: + return decode_connect_envelopes(response.content) + + if not response.content: + return [] + data = response.json() + if isinstance(data, dict) and 'result' in data: + data = data.get('result') + if isinstance(data, dict) and 'events' in data: + return data.get('events') + if isinstance(data, dict) and 'event' in data: + return [data] + return data + + +def raw_envd_request(sandbox, method, url, **kwargs): + response = sandbox.client.session.request(method, url, **kwargs) + if response.status_code < 200 or response.status_code >= 300: + raise SandboxError( + 'Sandbox envd request failed with status {0}'.format( + response.status_code), response, response.text, ) + return response diff --git a/qiniu/services/sandbox/errors.py b/qiniu/services/sandbox/errors.py new file mode 100644 index 00000000..3cf5909d --- /dev/null +++ b/qiniu/services/sandbox/errors.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + + +class SandboxError(Exception): + def __init__(self, message, response=None, data=None): + super(SandboxError, self).__init__(message) + self.response = response + self.data = data + self.status_code = getattr(response, 'status_code', None) + + +class TemplateBuildError(SandboxError): + pass + + +class CommandExitError(SandboxError): + def __init__(self, result): + self.result = result + super(CommandExitError, self).__init__( + 'Command exited with code {0}'.format(result.exit_code) + ) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py new file mode 100644 index 00000000..54b39b46 --- /dev/null +++ b/qiniu/services/sandbox/filesystem.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +import time + +from .envd import connect_rpc, envd_headers, raw_envd_request + + +def normalize_entry(entry): + entry = entry or {} + entry_type = entry.get('type') + if entry_type in ('FILE_TYPE_DIRECTORY', 'DIRECTORY', 'dir'): + entry_type = 'dir' + elif entry_type in ('FILE_TYPE_FILE', 'FILE', 'file'): + entry_type = 'file' + if entry_type: + entry['type'] = entry_type + return entry + + +class Filesystem(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def read(self, path, user=None, format='text', **opts): + url = self.sandbox.download_url(path, user=user) + response = raw_envd_request( + self.sandbox, + 'GET', + url, + headers=envd_headers(self.sandbox, user), + ) + if format == 'bytes': + return response.content + return response.content.decode(opts.get('encoding', 'utf-8')) + + def read_text(self, path, user=None, **opts): + return self.read(path, user=user, format='text', **opts) + + readText = read_text + + def write(self, path, data, user=None, **opts): + if not isinstance(data, bytes): + data = str(data).encode(opts.get('encoding', 'utf-8')) + url = self.sandbox.upload_url(path, user=user) + if opts.get('use_octet_stream'): + response = raw_envd_request( + self.sandbox, + 'POST', + url, + data=data, + headers=envd_headers( + self.sandbox, + user, + {'Content-Type': 'application/octet-stream'}, + ), + ) + return self._format_write_response(response) + + boundary = 'qiniu-sandbox-{0}'.format(int(time.time() * 1000)) + response = raw_envd_request( + self.sandbox, + 'POST', + url, + data=self._multipart_body(boundary, path, data), + headers=envd_headers( + self.sandbox, + user, + {'Content-Type': 'multipart/form-data; boundary={0}'.format( + boundary + )}, + ), + ) + return self._format_write_response(response) + + def _format_write_response(self, response): + if not response.content: + return None + data = response.json() + if isinstance(data, list): + return normalize_entry(data[0] if data else {}) + return normalize_entry(data) + + def _multipart_body(self, boundary, filename, data): + safe_filename = str(filename).replace('\\', '\\\\').replace('"', '\\"') + chunks = [ + '--{0}\r\n'.format(boundary).encode('utf-8'), + ( + 'Content-Disposition: form-data; name="file"; ' + 'filename="{0}"\r\n' + ).format(safe_filename).encode('utf-8'), + b'Content-Type: application/octet-stream\r\n\r\n', + data, + b'\r\n', + '--{0}--\r\n'.format(boundary).encode('utf-8'), + ] + return b''.join(chunks) + + def get_info(self, path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Stat', + {'path': path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry')) + + getInfo = get_info + stat = get_info + + def list(self, path, depth=1, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/ListDir', + {'path': path, 'depth': depth}, + user=user, + timeout=timeout, + ) + return [ + normalize_entry(entry) for entry in ( + data or {}).get( + 'entries', [])] + + def exists(self, path, user=None, timeout=None): + try: + self.get_info(path, user=user, timeout=timeout) + return True + except Exception as err: + response = getattr(err, 'response', None) + if response is not None and getattr( + response, 'status_code', None) == 404: + return False + raise + + def make_dir(self, path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/MakeDir', + {'path': path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry')) + + makeDir = make_dir + mkdir = make_dir + + def remove(self, path, user=None, timeout=None): + connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Remove', + {'path': path}, + user=user, + timeout=timeout, + ) + return None + + def rename(self, old_path, new_path, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/Move', + {'source': old_path, 'destination': new_path}, + user=user, + timeout=timeout, + ) + return normalize_entry((data or {}).get('entry')) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py new file mode 100644 index 00000000..2f36984c --- /dev/null +++ b/qiniu/services/sandbox/git.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +from .util import shell_quote + + +class Git(object): + def __init__(self, commands): + self.commands = commands + + def _run_git(self, repo_path, args, **opts): + if repo_path: + opts['cwd'] = repo_path + return self.commands.run('git {0}'.format(' '.join(args)), **opts) + + def clone(self, repo_url, path=None, branch=None, depth=None, **opts): + args = ['clone'] + if depth: + args.extend(['--depth', shell_quote(depth)]) + if branch: + args.extend(['--branch', shell_quote(branch)]) + args.append(shell_quote(repo_url)) + if path: + args.append(shell_quote(path)) + return self._run_git(None, args, **opts) + + def init(self, repo_path, bare=False, initial_branch=None, **opts): + args = ['init'] + if bare: + args.append('--bare') + if initial_branch: + args.extend(['--initial-branch', shell_quote(initial_branch)]) + return self._run_git(repo_path, args, **opts) + + def status(self, repo_path, **opts): + return self._run_git( + repo_path, [ + 'status', '--porcelain=v1', '-b'], **opts) + + def add(self, repo_path, files=None, all=False, **opts): + args = ['add'] + if all: + args.append('--all') + else: + for path in files or ['.']: + args.append(shell_quote(path)) + return self._run_git(repo_path, args, **opts) + + def commit(self, repo_path, message, allow_empty=False, **opts): + args = ['commit', '-m', shell_quote(message)] + if allow_empty: + args.append('--allow-empty') + return self._run_git(repo_path, args, **opts) + + def configure_user(self, repo_path, name, email, **opts): + name_result = self._run_git( + repo_path, + ['config', 'user.name', shell_quote(name)], + **opts + ) + if name_result.exit_code != 0: + return name_result + return self._run_git( + repo_path, + ['config', 'user.email', shell_quote(email)], + **opts + ) + + configureUser = configure_user + + def pull(self, repo_path, remote=None, branch=None, **opts): + args = ['pull'] + if remote: + args.append(shell_quote(remote)) + if branch: + args.append(shell_quote(branch)) + return self._run_git(repo_path, args, **opts) + + def push(self, repo_path, remote=None, branch=None, **opts): + args = ['push'] + if remote: + args.append(shell_quote(remote)) + if branch: + args.append(shell_quote(branch)) + return self._run_git(repo_path, args, **opts) diff --git a/qiniu/services/sandbox/resources.py b/qiniu/services/sandbox/resources.py new file mode 100644 index 00000000..05456ff4 --- /dev/null +++ b/qiniu/services/sandbox/resources.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + + +class GitRepositoryResource(object): + def __init__(self, url, mount_path, authorization_token=None, + repository_type='github_repository'): + self.url = url + self.mount_path = mount_path + self.authorization_token = authorization_token + self.repository_type = repository_type + + def to_dict(self): + data = { + 'type': self.repository_type, + 'url': self.url, + 'mount_path': self.mount_path, + } + if self.authorization_token is not None: + data['authorization_token'] = self.authorization_token + return data + + +class KodoResource(object): + def __init__(self, bucket, mount_path, prefix=None, read_only=None): + self.bucket = bucket + self.mount_path = mount_path + self.prefix = prefix + self.read_only = read_only + + def to_dict(self): + data = { + 'type': 'kodo', + 'bucket': self.bucket, + 'mount_path': self.mount_path, + } + if self.prefix is not None: + data['prefix'] = self.prefix + if self.read_only is not None: + data['read_only'] = self.read_only + return data diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py new file mode 100644 index 00000000..c19fc793 --- /dev/null +++ b/qiniu/services/sandbox/sandbox.py @@ -0,0 +1,312 @@ +# -*- coding: utf-8 -*- +import time + +from .client import SandboxClient +from .commands import Commands +from .constants import DEFAULT_USER, ENVD_PORT, MCP_PORT +from .filesystem import Filesystem +from .git import Git +from .util import ( + append_query, + file_signature, + get_info_value, + utc_timestamp_after, +) + + +class _ConnectDescriptor(object): + def __get__(self, obj, cls): + if obj is None: + def class_connect(sandbox_id, client=None, timeout=15, **opts): + client = client or SandboxClient(**opts) + info = client.connect_sandbox(sandbox_id, timeout=timeout) + sandbox = cls(client=client, info=info, sandbox_id=sandbox_id) + sandbox.refresh_envd_token_if_needed() + return sandbox + return class_connect + + def instance_connect(timeout=15, **opts): + info = obj.client.connect_sandbox( + obj.sandbox_id, timeout=timeout, **opts) + obj.update_info(info) + obj.refresh_envd_token_if_needed() + return obj + return instance_connect + + +class SandboxPaginator(object): + def __init__(self, client=None, **opts): + self.client = client or SandboxClient(**opts) + self.opts = dict(opts) + self.opts.pop('client', None) + self.next_token = opts.get('nextToken') or opts.get('next_token') + self._has_next = True + + @property + def has_next(self): + return bool(self.next_token) or self._has_next + + hasNext = has_next + + def next_items(self, **opts): + request_opts = dict(self.opts) + request_opts.update(opts) + if self.next_token and request_opts.get('nextToken') is None: + request_opts['nextToken'] = self.next_token + data = self.client.list_sandboxes_v2(**request_opts) or {} + items = data if isinstance(data, list) else ( + data.get('items') or data.get('sandboxes') or [] + ) + self.next_token = None if isinstance(data, list) else ( + data.get('nextToken') or data.get('next_token') + ) + self._has_next = bool(self.next_token) + return [Sandbox(client=self.client, info=item) for item in items] + + nextItems = next_items + + +class Sandbox(object): + connect = _ConnectDescriptor() + + def __init__(self, client=None, info=None, sandbox_id=None, sandboxID=None, + envd_url=None, envdAccessToken=None, **client_opts): + self.client = client or SandboxClient(**client_opts) + self.info = info or {} + self.sandbox_id = ( + sandbox_id or sandboxID or + self.info.get('sandboxID') or + self.info.get('sandboxId') or + self.info.get('sandbox_id') or + self.info.get('id') + ) + self.sandboxID = self.sandbox_id + self.template_id = ( + self.info.get('templateID') or + self.info.get('templateId') or + self.info.get('template_id') + ) + self.templateID = self.template_id + self.domain = ( + self.info.get('domain') or + self.info.get('sandboxDomain') or + self.info.get('sandbox_domain') + ) + self.sandbox_domain = self.domain + self.sandboxDomain = self.domain + self.envd_version = get_info_value( + self.info, 'envdVersion', 'envd_version') + self.envdVersion = self.envd_version + self.envd_access_token = ( + envdAccessToken or + get_info_value(self.info, 'envdAccessToken', 'envd_access_token') + ) + self.envdAccessToken = self.envd_access_token + self.traffic_access_token = get_info_value( + self.info, 'trafficAccessToken', 'traffic_access_token' + ) + self.trafficAccessToken = self.traffic_access_token + self._envd_url = envd_url + self.files = Filesystem(self) + self.filesystem = self.files + self.commands = Commands(self) + self.git = Git(self.commands) + + @classmethod + def create(cls, template=None, client=None, timeout=None, metadata=None, + envs=None, secure=True, allow_internet_access=True, mcp=None, + network=None, lifecycle=None, resources=None, injections=None, + **opts): + client_opts = {} + for key in ('endpoint', 'api_url', 'api_key', 'access_token', + 'mac', 'access_key', 'secret_key', 'session'): + if key in opts: + client_opts[key] = opts.pop(key) + client = client or SandboxClient(**client_opts) + info = client.create_sandbox( + template=template, + timeout=timeout, + metadata=metadata, + envs=envs, + secure=secure, + allow_internet_access=allow_internet_access, + mcp=mcp, + network=network, + lifecycle=lifecycle, + resources=resources, + injections=injections, + **opts + ) + sandbox = cls(client=client, info=info) + sandbox.refresh_envd_token_if_needed() + return sandbox + + @classmethod + def list(cls, client=None, **opts): + return SandboxPaginator(client=client, **opts) + + def update_info(self, info): + if not info: + return self + self.info = info + self.sandbox_id = ( + info.get('sandboxID') or info.get('sandboxId') or + info.get('sandbox_id') or self.sandbox_id + ) + self.sandboxID = self.sandbox_id + self.template_id = ( + info.get('templateID') or info.get('templateId') or + info.get('template_id') or self.template_id + ) + self.templateID = self.template_id + self.domain = ( + info.get('domain') or info.get('sandboxDomain') or + info.get('sandbox_domain') or self.domain + ) + self.sandbox_domain = self.domain + self.sandboxDomain = self.domain + self.envd_access_token = ( + get_info_value(info, 'envdAccessToken', 'envd_access_token') or + self.envd_access_token + ) + self.envdAccessToken = self.envd_access_token + self.envd_version = ( + get_info_value(info, 'envdVersion', 'envd_version') or + self.envd_version + ) + self.envdVersion = self.envd_version + return self + + updateInfo = update_info + + def refresh_envd_token_if_needed(self): + if self.envd_access_token or not self.sandbox_id: + return self + try: + self.update_info(self.get_info()) + except Exception: + pass + return self + + refreshEnvdTokenIfNeeded = refresh_envd_token_if_needed + + def kill(self): + return self.client.delete_sandbox(self.sandbox_id) + + def set_timeout(self, timeout): + return self.client.update_sandbox_timeout( + self.sandbox_id, timeout=timeout) + + setTimeout = set_timeout + + def refresh(self, duration=None, **opts): + if duration is not None: + opts['duration'] = duration + return self.client.refresh_sandbox(self.sandbox_id, **opts) + + def pause(self): + return self.client.pause_sandbox(self.sandbox_id) + + beta_pause = pause + betaPause = pause + + def resume(self, **opts): + info = self.client.resume_sandbox(self.sandbox_id, **opts) + self.update_info(info) + return self + + def update_network(self, network): + return self.client.update_sandbox(self.sandbox_id, network=network) + + updateNetwork = update_network + + def get_info(self): + return self.client.get_sandbox(self.sandbox_id) + + getInfo = get_info + + def get_metrics(self, **opts): + return self.client.get_sandbox_metrics(self.sandbox_id, **opts) + + getMetrics = get_metrics + + def get_logs(self, **opts): + return self.client.get_sandbox_logs(self.sandbox_id, **opts) + + getLogs = get_logs + + def get_host(self, port): + if not self.domain: + return '' + return '{0}-{1}.{2}'.format(port, self.sandbox_id, self.domain) + + getHost = get_host + + def envd_url(self): + if self._envd_url: + return self._envd_url + return 'https://{0}'.format(self.get_host(ENVD_PORT)) + + envdUrl = envd_url + + def get_mcp_url(self): + return 'https://{0}/mcp'.format(self.get_host(MCP_PORT)) + + getMcpUrl = get_mcp_url + + def get_mcp_token(self): + return self.traffic_access_token + + getMcpToken = get_mcp_token + + def file_url( + self, + path, + operation, + user=None, + signature_expiration=300, + **opts): + user = user or DEFAULT_USER + query = { + 'path': path, + 'username': user, + } + if self.envd_access_token: + expiration = opts.get('signatureExpiration', signature_expiration) + if expiration < 1000000000: + expiration = utc_timestamp_after(expiration) + query['signature'] = file_signature( + path, + operation, + user, + self.envd_access_token, + expiration, + ) + query['signature_expiration'] = expiration + return self.envd_url() + append_query('/files', query) + + fileUrl = file_url + + def download_url(self, path, **opts): + return self.file_url(path, 'read', **opts) + + downloadUrl = download_url + DownloadURL = download_url + + def upload_url(self, path, **opts): + return self.file_url(path, 'write', **opts) + + uploadUrl = upload_url + UploadURL = upload_url + + def wait_for_ready(self, timeout=60, interval=1): + started = time.time() + while True: + response = self.client.session.get(self.envd_url() + '/health') + if response.status_code >= 200 and response.status_code < 300: + return self + if time.time() - started >= timeout: + raise RuntimeError('Sandbox envd did not become ready') + time.sleep(interval) + + waitForReady = wait_for_ready diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py new file mode 100644 index 00000000..5ea44240 --- /dev/null +++ b/qiniu/services/sandbox/template.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +import json + + +class Template(object): + def __init__(self): + self.build_config = {'steps': []} + + def from_image(self, image, credentials=None): + self.build_config['fromImage'] = image + self.build_config.pop('fromTemplate', None) + if credentials: + self.build_config['fromImageRegistry'] = credentials + return self + + fromImage = from_image + + def from_template(self, template_id): + self.build_config['fromTemplate'] = template_id + self.build_config.pop('fromImage', None) + self.build_config.pop('fromImageRegistry', None) + return self + + fromTemplate = from_template + + def add_step(self, step_type, args, **extra): + step = {'type': step_type, 'args': [str(arg) for arg in args]} + step.update(extra) + self.build_config['steps'].append(step) + return self + + addStep = add_step + + def run_cmd(self, command, user=None): + args = [' && '.join(command) if isinstance( + command, (list, tuple)) else command] + if user: + args.append(user) + return self.add_step('RUN', args) + + run = run_cmd + runCmd = run_cmd + + def copy(self, src, dest, chmod=None, chown=None): + extra = {} + if chmod is not None: + extra['chmod'] = str(chmod) + if chown is not None: + extra['chown'] = chown + return self.add_step('COPY', [src, dest], **extra) + + def set_env(self, key, value): + return self.add_step('ENV', [key, value]) + + setEnv = set_env + + def set_start_cmd(self, command): + self.build_config['startCmd'] = command + return self + + setStartCmd = set_start_cmd + + def set_ready_cmd(self, command): + self.build_config['readyCmd'] = command + return self + + setReadyCmd = set_ready_cmd + + def to_dict(self): + data = dict(self.build_config) + data['steps'] = list(self.build_config.get('steps', [])) + return data + + def to_json(self): + return json.dumps(self.to_dict(), separators=(',', ':')) + + toJSON = to_json diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py new file mode 100644 index 00000000..5b1400f9 --- /dev/null +++ b/qiniu/services/sandbox/util.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +import base64 +import hashlib +import json as std_json +import os +import posixpath +import time + +from qiniu.compat import urlencode, urlparse + +from .constants import DEFAULT_ENDPOINT, DEFAULT_USER + + +def normalize_endpoint(endpoint=None): + endpoint = ( + endpoint or + os.getenv('QINIU_SANDBOX_API_URL') or + os.getenv('QINIU_SANDBOX_ENDPOINT') or + os.getenv('E2B_API_URL') or + DEFAULT_ENDPOINT + ) + return endpoint.rstrip('/') + + +def encode_path(value): + from qiniu.compat import is_py2 + if is_py2: + from urllib import quote + else: + from urllib.parse import quote + return quote(str(value), safe='') + + +def append_query(path, query=None): + query = query or {} + normalized = {} + for key, value in query.items(): + if value is None: + continue + if isinstance(value, (list, tuple)): + value = ','.join([str(item) for item in value]) + normalized[key] = value + if not normalized: + return path + return path + '?' + urlencode(normalized) + + +def json_dumps(data): + return std_json.dumps(data, separators=(',', ':')) + + +def timeout_seconds_from_options(opts): + opts = opts or {} + if opts.get('timeout') is not None: + return opts.get('timeout') + if opts.get('timeout_ms') is not None: + return int((opts.get('timeout_ms') + 999) / 1000) + if opts.get('timeoutMs') is not None: + return int((opts.get('timeoutMs') + 999) / 1000) + return None + + +def basic_auth(user=None): + user = user or DEFAULT_USER + raw = ('{0}:'.format(user)).encode('utf-8') + return 'Basic {0}'.format(base64.b64encode(raw).decode('ascii')) + + +def file_signature(path, operation, user, access_token, expiration): + raw = '{0}:{1}:{2}:{3}:{4}'.format( + path, operation, user, access_token, expiration + ) + return 'v1_{0}'.format(hashlib.sha256(raw.encode('utf-8')).hexdigest()) + + +def file_basename(path): + return posixpath.basename(path.rstrip('/')) or 'file' + + +def shell_quote(value): + try: + from shlex import quote + except ImportError: + from pipes import quote + return quote(str(value)) + + +def utc_timestamp_after(seconds): + return int(time.time()) + int(seconds) + + +def parse_json_response(response): + if response.content in (None, b'', ''): + return None + return response.json() + + +def get_info_value(info, camel_key, snake_key=None): + info = info or {} + if camel_key in info: + return info.get(camel_key) + if snake_key and snake_key in info: + return info.get(snake_key) + return None + + +def parsed_url(url): + return urlparse(url) diff --git a/tests/cases/test_services/test_sandbox/__init__.py b/tests/cases/test_services/test_sandbox/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py new file mode 100644 index 00000000..7a5caeb9 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- +import json + +import pytest +import requests + +try: + from urllib.parse import parse_qs, urlparse +except ImportError: + from urlparse import parse_qs, urlparse + +from qiniu.services.sandbox import ( + DEFAULT_ENDPOINT, + ENVD_PORT, + KodoResource, + Sandbox, + SandboxClient, + SandboxError, + Template, +) + + +class DummyResponse(object): + def __init__(self, status_code=200, body=None): + self.status_code = status_code + self.content = b'' if body is None else json.dumps( + body).encode('utf-8') + self.text = self.content.decode('utf-8') + self.headers = {'Content-Type': 'application/json'} + + def json(self): + if not self.content: + return None + return json.loads(self.content.decode('utf-8')) + + +class RecordingSession(requests.Session): + def __init__(self, responses=None): + super(RecordingSession, self).__init__() + self.responses = list(responses or [DummyResponse(body={})]) + self.requests = [] + + def send(self, request, **kwargs): + self.requests.append(request) + if self.responses: + return self.responses.pop(0) + return DummyResponse(body={}) + + +def body_of(request): + if request.body is None: + return None + if isinstance(request.body, bytes): + return json.loads(request.body.decode('utf-8')) + return json.loads(request.body) + + +def test_client_uses_default_endpoint_and_api_key_headers(): + session = RecordingSession([DummyResponse(201, { + 'sandboxID': 'sbx123', + 'templateID': 'base', + 'domain': 'example.test', + 'envdAccessToken': 'envd-token', + })]) + client = SandboxClient(api_key='api-key', session=session) + + info = client.create_sandbox(template='base', timeout=60, envs={'A': 'B'}) + + assert info['sandboxID'] == 'sbx123' + req = session.requests[0] + assert req.method == 'POST' + assert req.url == DEFAULT_ENDPOINT + '/sandboxes' + assert req.headers['X-API-Key'] == 'api-key' + assert req.headers['Authorization'] == 'Bearer api-key' + assert body_of(req) == { + 'templateID': 'base', + 'timeout': 60, + 'envVars': {'A': 'B'}, + } + + +def test_create_with_kodo_resource_requires_qiniu_credentials(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.create_sandbox( + resources=[ + KodoResource( + bucket='bucket', + mount_path='/mnt/bucket')]) + + assert 'Qiniu AK/SK' in str(err.value) + + +def test_create_with_kodo_resource_uses_qiniu_signature(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient(access_key='ak', secret_key='sk', session=session) + + client.create_sandbox( + resources=[ + KodoResource( + bucket='bucket', + mount_path='/mnt/bucket')]) + + req = session.requests[0] + assert req.headers['Authorization'].startswith('Qiniu ak:') + assert 'X-Qiniu-Date' in req.headers + assert body_of(req)['resources'] == [{ + 'type': 'kodo', + 'bucket': 'bucket', + 'mount_path': '/mnt/bucket', + }] + + +def test_sandbox_create_signature_matches_e2b_style(): + session = RecordingSession([ + DummyResponse(201, { + 'sandboxID': 'sbx123', + 'templateID': 'python', + 'domain': 'example.test', + 'envdAccessToken': 'envd-token', + }) + ]) + client = SandboxClient(api_key='api-key', session=session) + + sandbox = Sandbox.create( + 'python', + timeout=120, + metadata={'app': 'tests'}, + envs={'HELLO': 'world'}, + client=client, + ) + + assert sandbox.sandbox_id == 'sbx123' + assert sandbox.sandboxID == 'sbx123' + assert sandbox.template_id == 'python' + assert sandbox.files is sandbox.filesystem + assert sandbox.commands.sandbox is sandbox + assert body_of(session.requests[0])['templateID'] == 'python' + + +def test_sandbox_instance_lifecycle_methods_call_control_plane(): + session = RecordingSession([ + DummyResponse(204, None), + DummyResponse(204, None), + DummyResponse(200, { + 'sandboxID': 'sbx123', + 'templateID': 'base', + 'envdAccessToken': 'envd-token', + }), + ]) + client = SandboxClient(api_key='api-key', session=session) + sandbox = Sandbox( + client=client, + info={ + 'sandboxID': 'sbx123', + 'templateID': 'base'}) + + assert sandbox.kill() is None + assert sandbox.set_timeout(30) is None + sandbox.connect(timeout=45) + + assert [ + req.method for req in session.requests] == [ + 'DELETE', + 'POST', + 'POST'] + assert session.requests[0].url.endswith('/sandboxes/sbx123') + assert session.requests[1].url.endswith('/sandboxes/sbx123/timeout') + assert body_of(session.requests[1]) == {'timeout': 30} + assert session.requests[2].url.endswith('/sandboxes/sbx123/connect') + assert body_of(session.requests[2]) == {'timeout': 45} + + +def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + + assert sandbox.get_host(ENVD_PORT) == '49983-sbx123.example.test' + assert sandbox.envd_url() == 'https://49983-sbx123.example.test' + + url = sandbox.download_url( + '/tmp/hello.txt', + signature_expiration=1893456000) + parsed = urlparse(url) + query = parse_qs(parsed.query) + + assert parsed.scheme == 'https' + assert parsed.netloc == '49983-sbx123.example.test' + assert parsed.path == '/files' + assert query['path'] == ['/tmp/hello.txt'] + assert query['username'] == ['user'] + assert query['signature_expiration'] == ['1893456000'] + assert query['signature'][0] + + +def test_template_builder_outputs_build_config(): + template = ( + Template() + .from_image('python:3.11') + .run_cmd('pip install qiniu') + .copy('/local/app.py', '/app/app.py') + .set_env('PYTHONUNBUFFERED', '1') + .set_start_cmd('python /app/app.py') + ) + + assert template.to_dict() == { + 'fromImage': 'python:3.11', + 'steps': [ + {'type': 'RUN', 'args': ['pip install qiniu']}, + {'type': 'COPY', 'args': ['/local/app.py', '/app/app.py']}, + {'type': 'ENV', 'args': ['PYTHONUNBUFFERED', '1']}, + ], + 'startCmd': 'python /app/app.py', + } diff --git a/tests/cases/test_services/test_sandbox/test_config.py b/tests/cases/test_services/test_sandbox/test_config.py new file mode 100644 index 00000000..b995bd82 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_config.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +import os + +from qiniu.services.sandbox.config import ( + load_dotenv_if_present, + sandbox_client, +) + + +def test_load_dotenv_if_present_reads_key_values_without_overwriting(tmpdir): + dotenv = tmpdir.join('.env') + dotenv.write( + '\n'.join([ + 'QINIU_SANDBOX_API_KEY=from-file', + 'QINIU_SANDBOX_TEMPLATE="python:3.11"', + 'EXISTING=value-from-file', + '# ignored', + '', + ]) + ) + + old = { + key: os.environ.get(key) + for key in ( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_SANDBOX_TEMPLATE', + 'EXISTING', + ) + } + try: + os.environ.pop('QINIU_SANDBOX_API_KEY', None) + os.environ.pop('QINIU_SANDBOX_TEMPLATE', None) + os.environ['EXISTING'] = 'already-set' + + load_dotenv_if_present(str(dotenv)) + + assert os.environ['QINIU_SANDBOX_API_KEY'] == 'from-file' + assert os.environ['QINIU_SANDBOX_TEMPLATE'] == 'python:3.11' + assert os.environ['EXISTING'] == 'already-set' + finally: + for key, value in old.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def test_sandbox_client_uses_loaded_env(tmpdir): + dotenv = tmpdir.join('.env') + dotenv.write('\n'.join([ + 'QINIU_SANDBOX_API_KEY=api-key', + 'QINIU_SANDBOX_ENDPOINT=https://sandbox.example.test', + 'QINIU_SANDBOX_ACCESS_KEY=ak', + 'QINIU_SANDBOX_SECRET_KEY=sk', + ])) + + old = { + key: os.environ.get(key) + for key in ( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_SANDBOX_ENDPOINT', + 'QINIU_SANDBOX_ACCESS_KEY', + 'QINIU_SANDBOX_SECRET_KEY', + ) + } + try: + for key in old: + os.environ.pop(key, None) + load_dotenv_if_present(str(dotenv)) + + client = sandbox_client() + + assert client.endpoint == 'https://sandbox.example.test' + assert client.api_key == 'api-key' + assert client.mac is not None + finally: + for key, value in old.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py new file mode 100644 index 00000000..c83b5119 --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +import base64 +import json + +from qiniu.services.sandbox import Sandbox, SandboxClient +from qiniu.services.sandbox.envd import ( + decode_connect_envelopes, + encode_connect_envelope, +) + + +class DummyResponse(object): + def __init__(self, status_code=200, body=None, raw=None, + content_type='application/json'): + self.status_code = status_code + self.content = raw if raw is not None else json.dumps( + body or {}).encode('utf-8') + self.text = self.content.decode('utf-8') + self.headers = {'Content-Type': content_type} + + def json(self): + return json.loads(self.content.decode('utf-8')) + + +class EnvdSession(object): + def __init__(self): + self.posts = [] + self.requests = [] + + def post(self, url, data=None, headers=None, timeout=None): + if headers.get('Content-Type') == 'application/connect+json': + decoded = decode_connect_envelopes(data)[0] + else: + decoded = json.loads( + data.decode('utf-8') if isinstance(data, bytes) else data + ) + self.posts.append({ + 'url': url, + 'data': decoded, + 'headers': headers, + 'timeout': timeout, + }) + if url.endswith('/process.Process/Start'): + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'hello\n').decode('ascii'), + 'stderr': base64.b64encode(b'').decode('ascii'), + }}}), + encode_connect_envelope({'event': {'end': {'exitCode': 0}}}), + ]) + return DummyResponse( + raw=raw, + content_type='application/connect+json', + ) + if url.endswith('/filesystem.Filesystem/Stat'): + return DummyResponse( + body={ + 'result': { + 'entry': { + 'name': 'hello.txt', + 'type': 'FILE'}}}) + if url.endswith('/filesystem.Filesystem/ListDir'): + return DummyResponse( + body={'result': {'entries': [{ + 'name': 'hello.txt', + 'type': 'FILE', + }]}}) + return DummyResponse(body={'result': {}}) + + def request(self, method, url, **kwargs): + self.requests.append({'method': method, 'url': url, 'kwargs': kwargs}) + if method == 'GET': + return DummyResponse(raw=b'hello') + return DummyResponse(body={'name': 'hello.txt', 'type': 'FILE'}) + + +def sandbox_with_envd_session(): + session = EnvdSession() + client = SandboxClient(api_key='api-key', session=session) + sandbox = Sandbox(client=client, info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + return sandbox, session + + +def test_commands_run_posts_process_start_and_decodes_events(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.commands.run('echo hello', cwd='/tmp', envs={'A': 'B'}) + + assert result.pid == 12 + assert result.exit_code == 0 + assert result.stdout == 'hello\n' + assert session.posts[0]['url'].endswith('/process.Process/Start') + assert ( + session.posts[0]['headers']['Content-Type'] == + 'application/connect+json' + ) + assert session.posts[0]['headers']['Authorization'] == 'Basic dXNlcjo=' + assert session.posts[0]['headers']['X-Access-Token'] == 'token' + assert session.posts[0]['data']['process'] == { + 'cmd': '/bin/bash', + 'args': ['-l', '-c', 'echo hello'], + 'cwd': '/tmp', + 'envs': {'A': 'B'}, + } + + +def test_filesystem_uses_envd_rpc_and_signed_file_urls(): + sandbox, session = sandbox_with_envd_session() + + assert sandbox.files.read_text('/tmp/hello.txt') == 'hello' + assert sandbox.files.write('/tmp/hello.txt', 'hello')['type'] == 'file' + assert sandbox.files.stat('/tmp/hello.txt')['type'] == 'file' + assert sandbox.files.list( + '/tmp') == [{'name': 'hello.txt', 'type': 'file'}] + + assert session.requests[0]['method'] == 'GET' + assert '/files?' in session.requests[0]['url'] + assert session.requests[1]['method'] == 'POST' + assert session.requests[1]['kwargs']['headers']['Content-Type'].startswith( + 'multipart/form-data; boundary=' + ) + assert b'name="file"' in session.requests[1]['kwargs']['data'] + assert session.posts[0]['url'].endswith('/filesystem.Filesystem/Stat') + assert session.posts[1]['url'].endswith('/filesystem.Filesystem/ListDir') diff --git a/tests/cases/test_services/test_sandbox/test_example_config.py b/tests/cases/test_services/test_sandbox/test_example_config.py new file mode 100644 index 00000000..1f1d579e --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_example_config.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +import os + + +ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..', '..', '..') +) + + +def read_project_file(*parts): + with open(os.path.join(ROOT, *parts), 'r') as f: + return f.read() + + +def test_env_example_only_contains_connection_and_resource_inputs(): + content = read_project_file('.env.example') + + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in content + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in content + assert 'QINIU_SANDBOX_ID' not in content + assert 'QINIU_SANDBOX_KODO_READ_ONLY' not in content + + +def test_examples_handle_runtime_branches_in_code(): + connect = read_project_file('examples', 'sandbox_connect.py') + resources = read_project_file('examples', 'sandbox_resources.py') + integration = read_project_file( + 'tests', + 'cases', + 'test_services', + 'test_sandbox', + 'test_integration.py', + ) + + assert 'QINIU_SANDBOX_ID' not in connect + assert 'QINIU_SANDBOX_KODO_READ_ONLY' not in resources + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in integration + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in integration diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py new file mode 100644 index 00000000..2073773c --- /dev/null +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +import os + +import pytest + +from qiniu.services.sandbox import Sandbox, SandboxClient +from qiniu.services.sandbox.config import load_dotenv_if_present + + +load_dotenv_if_present() + + +def integration_client(): + if not os.getenv('QINIU_SANDBOX_API_KEY'): + pytest.skip('QINIU_SANDBOX_API_KEY is required') + return SandboxClient() + + +def test_create_run_filesystem_and_kill_sandbox(): + sandbox = None + client = integration_client() + template = os.getenv('QINIU_SANDBOX_TEMPLATE', 'base') + try: + sandbox = Sandbox.create( + template, + timeout=300, + metadata={'sdk': 'python', 'test': 'integration'}, + envs={'QINIU_SANDBOX_TEST': '1'}, + client=client, + ) + result = sandbox.commands.run('printf hello') + assert result.exit_code == 0 + assert result.stdout == 'hello' + + sandbox.files.write('/tmp/qiniu-python-sdk.txt', 'hello') + assert sandbox.files.read_text('/tmp/qiniu-python-sdk.txt') == 'hello' + finally: + if sandbox is not None: + sandbox.kill() + + +def test_list_and_connect_existing_sandbox(): + client = integration_client() + page = Sandbox.list(client=client, limit=5).next_items() + assert isinstance(page, list) + if not page: + pytest.skip('No sandbox available to connect') + + connected = Sandbox.connect(page[0].sandbox_id, client=client, timeout=60) + assert connected.sandbox_id == page[0].sandbox_id From 063845834f60fbe3057335e283e4de093e52a509 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 17:59:44 +0800 Subject: [PATCH 02/45] feat(sandbox): align runtime APIs with e2b Add PTY, command streaming, filesystem watcher/write_files, Git helpers and credential authentication APIs for the sandbox module. Expand sandbox unit/integration tests, runnable examples, and the E2B Python SDK differences document including remote Git push coverage. --- docs/sandbox-e2b-python-sdk-differences.md | 182 +++++++++++++++ examples/sandbox_git.py | 97 ++++++++ examples/sandbox_resources.py | 63 +++-- examples/sandbox_runtime.py | 57 +++++ qiniu/__init__.py | 12 +- qiniu/services/sandbox/__init__.py | 17 +- qiniu/services/sandbox/commands.py | 132 +++++++++-- qiniu/services/sandbox/envd.py | 59 ++++- qiniu/services/sandbox/filesystem.py | 110 +++++++++ qiniu/services/sandbox/git.py | 119 ++++++++++ qiniu/services/sandbox/pty.py | 120 ++++++++++ qiniu/services/sandbox/sandbox.py | 2 + .../test_services/test_sandbox/test_client.py | 61 +++++ .../test_services/test_sandbox/test_envd.py | 148 +++++++++++- .../test_sandbox/test_example_config.py | 3 + .../test_sandbox/test_integration.py | 218 +++++++++++++++++- 16 files changed, 1347 insertions(+), 53 deletions(-) create mode 100644 docs/sandbox-e2b-python-sdk-differences.md create mode 100644 examples/sandbox_runtime.py create mode 100644 qiniu/services/sandbox/pty.py diff --git a/docs/sandbox-e2b-python-sdk-differences.md b/docs/sandbox-e2b-python-sdk-differences.md new file mode 100644 index 00000000..18f6e5bc --- /dev/null +++ b/docs/sandbox-e2b-python-sdk-differences.md @@ -0,0 +1,182 @@ +# Qiniu Python SDK Sandbox 与 E2B Python SDK 差异说明 + +本文用于对照当前 `qiniu-python-sdk` sandbox 模块与 E2B Python SDK 的能力边界，帮助评估 API 设计、测试覆盖、示例配置和后端依赖差异。 + +## 对比基准 + +| 项目 | 版本或位置 | 说明 | +| --- | --- | --- | +| Qiniu Python SDK | 当前分支 `sandbox` 的工作区实现 | 以 `qiniu.services.sandbox` 和顶层 `qiniu.Sandbox` 导出为基准 | +| E2B Python SDK | 本地 E2B Python SDK 源码 | 以 `e2b.Sandbox`, `e2b.AsyncSandbox`, `e2b.Template`, `e2b.Volume` 等导出为基准 | +| Qiniu Sandbox 后端 | sandbox OpenAPI spec、envd proto 与当前 `.env` 实测环境 | Qiniu 控制面、envd、Kodo resource 和 injection rules 是 Qiniu 特有产品能力 | + +## 总体定位 + +Qiniu sandbox 模块目前是 `qiniu-python-sdk` 中新增的一组同步 API，面向 Qiniu Sandbox 控制面和 envd，覆盖 sandbox 生命周期、命令、文件、Git、模板、资源挂载、注入规则、配置加载和 examples。 + +E2B Python SDK 是独立的现代 Python SDK，提供 sync/async 双栈、丰富类型导出、Template/Volume/Snapshot/MCP/PTY/watch 等完整产品面。两者使用场景接近，但鉴权模型、后端控制面、运行时依赖、类型系统和部分产品能力并不相同。 + +## 对齐原则 + +本轮 Qiniu Python sandbox API 的目标是尽量贴近 E2B Python SDK 的常用函数签名和对象组织，同时保留 `qiniu-python-sdk` 的兼容约束和 Qiniu 产品语义。 + +- E2B Python SDK 的常用入口尽量保留同名或近似签名，例如 `Sandbox.create(...)`, `Sandbox.connect(...)`, `Sandbox.list(...)`, `sandbox.files`, `sandbox.commands`, `sandbox.pty`, `sandbox.git`, `Template.from_image(...)`。 +- Python 侧同时提供 snake_case 与 camelCase 别名，便于对齐 Python 习惯和已有 Qiniu/JS SDK 使用方式。 +- Qiniu 后端已支持的能力，在 SDK 层实现真实请求，例如 lifecycle、commands、filesystem、Git、template API、Kodo resources、injection rules。 +- Qiniu 特有能力继续显式保留，例如 Qiniu AK/SK 签名、Kodo resource、Git repository resource 和 injection rules。 +- 后端或当前实现没有等价能力的部分不伪装为可用，例如 AsyncSandbox、Volume、snapshot wrapper、template build helper 的完整 E2B 形态。 + +## 主要差异概览 + +| 领域 | Qiniu Python SDK sandbox | E2B Python SDK | 差异影响 | +| --- | --- | --- | --- | +| 包定位 | `qiniu` 包内新增 `qiniu.services.sandbox` 模块 | 独立 `e2b` 包 | Qiniu 需要兼容现有 SDK 导出和依赖策略 | +| Python 兼容 | `setup.py` 仍声明 Python 2.7/3.4-3.7 兼容 | 现代 Python, 使用 `typing`, `typing_extensions`, `httpx` 等 | Qiniu sandbox 实现避免 dataclass、async/await 入口和较新的语法依赖 | +| HTTP 依赖 | 复用 `requests` / `requests.Session` | 使用 `httpx` sync/async client 和 generated API client | 请求封装、超时、异常类型和 async 支持不同 | +| 同步/异步 | 仅同步 API | 同时提供 `Sandbox` 与 `AsyncSandbox`, `Template` 与 `AsyncTemplate`, `Volume` 与 `AsyncVolume` | Qiniu 暂不提供 async 等价入口 | +| 鉴权 | `QINIU_SANDBOX_API_KEY`, `QINIU_SANDBOX_ACCESS_TOKEN`, Qiniu AK/SK `Mac` | `E2B_API_KEY` 与 `ConnectionConfig` | Qiniu 资源挂载和 injection rules 需要 Qiniu AK/SK 签名 | +| 控制面 endpoint | `QINIU_SANDBOX_API_URL`, `QINIU_SANDBOX_ENDPOINT`, `E2B_API_URL` 或默认 Qiniu endpoint | E2B cloud endpoint/domain config | endpoint、header、token 名称和默认值不同 | +| 顶层导出 | `Sandbox`, `SandboxClient`, `Template`, `KodoResource`, `GitRepositoryResource`, `FileType`, `FilesystemEventType`, `PtySize`, `WatchHandle` | `Sandbox`, `AsyncSandbox`, `Template`, `AsyncTemplate`, `Volume`, typed models/errors | Qiniu 当前导出更轻，类型模型更少；`Async*` 与 `Volume` 暂无等价产品面 | +| 类型系统 | 手写 Python 对象和 dict payload | 大量 typed model、paginator、exception、filesystem/git/network 类型 | Qiniu 用户更多接触原始 dict；E2B 类型提示更完整 | + +## Sandbox 生命周期 + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| 创建 sandbox | 支持 `Sandbox.create(template=None, timeout=None, metadata=None, envs=None, secure=True, allow_internet_access=True, mcp=None, network=None, lifecycle=None, resources=None, injections=None, **opts)` | 支持同类签名, 另有 `volume_mounts` | Qiniu 使用 `templateID`, `envVars`, `resources`, `injections` 等控制面字段 | +| 默认 template | 默认使用 Qiniu `DEFAULT_TEMPLATE` | 默认 `base`, MCP 场景可切换到 `mcp-gateway` | Qiniu 不自动改写为 MCP template | +| 连接 sandbox | 支持类方法 `Sandbox.connect(id, ...)` 和实例 `sandbox.connect(...)` | 支持类方法和实例方法 | Qiniu 连接后会刷新 envd token 信息 | +| 列表分页 | 支持 `Sandbox.list(...).next_items()` 和 `nextItems` | 支持 `SandboxPaginator` / `AsyncSandboxPaginator` | Qiniu paginator 较轻量, 返回 `Sandbox` 对象列表 | +| kill | 支持 | 支持 | Qiniu 使用控制面 DELETE | +| set_timeout | 支持 `set_timeout` / `setTimeout` | 支持 | 单位均按秒表达 | +| pause/resume | 支持 `pause`, `betaPause`, `resume` | 支持 pause, 以及 lifecycle/auto-resume 相关能力 | Qiniu `resume` 会更新当前对象信息 | +| refresh | 支持 `refresh` 调用 Qiniu `refreshes` API | E2B 无完全同名核心实例方法 | Qiniu 特有生命周期补充 | +| update_network | 支持 `update_network` / `updateNetwork` | 支持 network update | 请求结构依赖各自控制面 | +| get_info/get_metrics/get_logs | 支持 | E2B 支持 get_info/get_metrics, logs 更多出现在 template/build 等上下文 | Qiniu 暴露 sandbox logs wrapper | +| is_running | 暂无同名方法 | 支持 `is_running()` 通过 envd health 判断 | Qiniu 当前提供 `wait_for_ready()` 轮询 envd health | +| signed file URL | 支持 `download_url`, `upload_url` 和 envd token 签名 | 支持同类签名 URL helper | 签名字段与默认 user 处理保持 Qiniu 实现 | + +## Snapshot 与 MCP + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| create_snapshot/list_snapshots | 暂无 `Sandbox` 实例 wrapper | 支持 `create_snapshot()` 和 `list_snapshots()` | Qiniu Python 当前没有暴露 snapshot paginator | +| MCP create option | `Sandbox.create(..., mcp=...)` 可透传到控制面 | `mcp` 会影响 template 选择并启动 MCP gateway command | Qiniu SDK 不负责自动启动 MCP gateway | +| get_mcp_url | 支持按 50005 端口生成 URL | 支持 | 依赖 sandbox domain | +| get_mcp_token | 返回控制面下发的 `trafficAccessToken` | E2B 会维护 MCP token, 并在 MCP gateway 启动后使用 | Qiniu 当前不读取 `/etc/mcp-gateway/.token`, 也不启动 gateway | + +## Commands 与 PTY + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| run/start | 支持 `commands.run(...)` 和 `commands.start(...)` | 支持 | Qiniu 使用 envd Connect JSON envelope 调用 `/process.Process/Start` | +| background command | `run(..., background=True)` 返回 `CommandHandle` | 支持 `CommandHandle` | Qiniu 普通 `start` 当前仍会聚合 stream 事件；`commands.connect` 和 PTY create/connect 使用流式 envelope 解析 | +| stdout/stderr | 返回聚合后的 `CommandResult.stdout/stderr`, 支持 `on_stdout`/`on_stderr` | 支持输出 handler、stream/handle 等更完整形态 | Qiniu callback 是聚合事件时同步触发 | +| stdin | 支持 `send_stdin` / `sendStdin`, `close_stdin` / `closeStdin` | 支持 | 均依赖 envd process API | +| kill | 支持按 pid 发送 SIGKILL | 支持 | Qiniu `kill` 返回 bool, 404 时返回 `False` | +| 非 0 退出 | 默认不抛错, 传 `throw_on_error=True` 才抛 `CommandExitError` | E2B command handle 默认错误语义更细 | 迁移代码时要注意默认异常行为 | +| command connect | 支持 `commands.connect(pid)` | E2B 支持连接/管理运行中的 command | Qiniu 使用 envd `/process.Process/Connect` | +| PTY | 支持 `sandbox.pty.create/connect/send_stdin/resize/kill` | 支持 `sandbox.pty` | Qiniu 使用 api-spec 中的 PTY Start/Update/SendInput/Connect 能力；当前实测 envd 可创建 PTY, 但 PTY input 返回 501 | + +## Filesystem + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| read | 支持 `read(path, format="text")`, `read_text`, `format="bytes"` | 支持多种读取形态和类型提示 | Qiniu 返回 `str` 或 `bytes`, 类型模型较轻 | +| write 单文件 | 支持 string/bytes, 默认 multipart, 可选 `use_octet_stream=True` | 支持更丰富数据类型和写入信息 | Qiniu 为兼容当前 envd 默认走 multipart | +| write 多文件 | 支持 `write_files` / `writeFiles`, 内部复用单文件 write | 支持 `write_files` | Qiniu 当前是逐文件写入 | +| stat/list/exists/mkdir/remove/rename | 支持 `get_info/stat`, `list`, `exists`, `make_dir/mkdir`, `remove`, `rename` | 支持同类能力 | 返回结构均会归一化为轻量 dict | +| watch_dir | 支持非流式 watcher: `watch_dir`, `WatchHandle.get_new_events()`, `WatchHandle.stop()` | 支持 `watch_dir` 与 watch handle | Qiniu 使用 api-spec 中的 `CreateWatcher/GetWatcherEvents/RemoveWatcher` | +| FileType/EntryInfo 类型 | 导出轻量 `FileType`, `FilesystemEventType`, `FilesystemEvent`, `WatchHandle` | E2B 导出 `FileType`, `EntryInfo`, `WriteInfo` | Qiniu 目前使用字符串常量和轻量对象, 尚无完整 typed model | + +## Git + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| clone/init/status/add/commit | 支持 | 支持 | Qiniu 通过 `commands.run("git ...")` 封装 | +| configure_user | 支持 | 支持类似 config helper | Qiniu 使用 repo path + name/email 签名 | +| pull/push | 支持基础 remote/branch 参数 | 支持并处理更多 auth/upstream 错误 | Qiniu 暂无 typed git exception | +| branch/remote/reset/restore | 支持 remote_add/remote_get/branches/create_branch/checkout_branch/delete_branch/reset/restore | E2B Git 模块更完整 | Qiniu 通过 `git` 命令 wrapper 实现 | +| structured status | 返回 command stdout | E2B 导出 `GitStatus`, `GitBranches`, `GitFileStatus` | Qiniu helper 尚未解析 porcelain 输出 | +| git auth helper | 支持 `dangerously_authenticate(username, password, host="github.com", protocol="https")` 和 `dangerouslyAuthenticate` | E2B 使用 Git credential helper, 另有 auth/upstream 异常和辅助逻辑 | Qiniu 已对齐 credential helper 认证入口；暂未补 typed `GitAuthException`/`GitUpstreamException` | + +## Template + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| Template builder | 支持 `from_image`, `from_template`, `add_step`, `run_cmd/run`, `copy`, `set_env`, `set_start_cmd`, `set_ready_cmd`, `to_dict`, `to_json` | 提供更完整的 `Template` / `AsyncTemplate` builder 和 build workflow | Qiniu builder 是轻量配置生成器 | +| build API | 通过 `SandboxClient.create_template`, `create_template_v2`, `rebuild_template`, `wait_for_build` 等 client 方法调用 | `Template` 对象本身承载 build、logs、tags 等工作流 | Qiniu 的 template 生命周期更偏 client wrapper | +| Dockerfile/devcontainer/context upload | 暂无完整 helper | E2B 支持更完整模板构建上下文能力 | Qiniu 当前只表达基础 steps | +| ReadyCmd helpers | `set_ready_cmd` 接受命令值 | E2B 导出 `wait_for_file`, `wait_for_port`, `wait_for_process`, `wait_for_timeout`, `wait_for_url` | Qiniu 未提供 typed ReadyCmd helper | +| Tags/build logs/status | client wrapper 支持 tags、build status/logs | E2B Template API 更集中 | Qiniu 示例覆盖了模板 list/detail/build/status/logs/tags/delete | + +## Volume 与资源挂载 + +| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| Volume API | 暂无 `Volume` / `AsyncVolume` | 支持完整 persistent Volume 产品 | Qiniu 后端当前使用不同的资源挂载模型 | +| volume_mounts | `Sandbox.create` 不接受 E2B `volume_mounts` 语义 | 支持 mount path 到 Volume/name 的映射 | Qiniu 使用 `resources` 透传资源 | +| Git repository resource | 支持 `GitRepositoryResource(url, mount_path, authorization_token=None, repository_type="github_repository")` | 不适用同名模型 | Qiniu 特有 resource 能力 | +| Kodo resource | 支持 `KodoResource(bucket, mount_path, prefix=None, read_only=None)` | 不适用 | Qiniu 特有能力, 触发 Qiniu AK/SK 签名 | +| injection rules | 支持 list/create/get/update/delete | E2B 无同类 API | Qiniu 特有能力, 使用 Qiniu AK/SK 签名 | + +## 配置与示例 + +Qiniu Python SDK 的 sandbox 配置集中在 `.env` 和 `qiniu.services.sandbox.config`: + +- `QINIU_SANDBOX_API_KEY`: sandbox 控制面 API key。 +- `QINIU_SANDBOX_ENDPOINT` 或 `QINIU_SANDBOX_API_URL`: sandbox 控制面 endpoint。 +- `QINIU_SANDBOX_ACCESS_TOKEN`: 部分 template rebuild 等 access token 场景使用。 +- `QINIU_SANDBOX_ACCESS_KEY` / `QINIU_SANDBOX_SECRET_KEY`: Kodo resource 和 injection rules 等 Qiniu 签名 API 使用。 +- `QINIU_SANDBOX_TEMPLATE`: examples 默认 template。 +- Kodo/Git/injection 示例所需的 bucket、prefix、repository、token 等配置由对应 example 自行判断；缺少时示例会跳过该分支, 不要求额外手动配置测试开关。 + +E2B Python SDK 主要围绕 `E2B_API_KEY` 和 `ConnectionConfig` 组织连接配置；Volume、Template、Sandbox 使用各自的 connection config 和 typed options。 + +## 错误与后端依赖 + +| 项目 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | +| --- | --- | --- | --- | +| 基础错误 | `SandboxError`, `TemplateBuildError`, `CommandExitError` | `SandboxException`, `TimeoutException`, `NotFoundException`, `FileNotFoundException`, `GitAuthException`, `GitUpstreamException`, `TemplateException`, `VolumeException` 等 | Qiniu 当前错误类型较少 | +| HTTP 错误 | 根据 response status 和 message 生成 `SandboxError` | generated client 和 `httpx` 异常映射更细 | Qiniu 目前更多保留原始 response/data | +| envd 依赖 | commands/filesystem/pty 依赖 envd Connect RPC 和 signed file URL | 同样依赖 envd, 并有更多版本检查与 fallback | Qiniu Python 当前版本门控较少；测试和示例按 404/501 等后端能力分支跳过 | +| 后端产品能力 | Kodo resource、injection rules、refresh/logs 是 Qiniu 特有 | Volume、snapshot、async、PTY、watch 是 E2B 完整产品面的一部分 | 差异来自产品边界, 不是单纯 SDK 命名差异 | + +## 测试与示例覆盖 + +Qiniu 当前 sandbox 覆盖包括: + +- `tests/cases/test_services/test_sandbox/`: 覆盖 client、config、envd、example config、sandbox integration 分支；新增覆盖 commands callbacks/connect、PTY、filesystem `write_files`/watcher、Git helper。 +- `examples/sandbox_create.py`: 创建 sandbox 并执行基本命令。 +- `examples/sandbox_lifecycle.py`: lifecycle、timeout、pause/resume 等生命周期能力。 +- `examples/sandbox_connect.py`: list/connect/metrics/logs 等连接和观测能力。 +- `examples/sandbox_git.py`: sandbox 内 Git 操作；配置 `GIT_REPO_URL` 和 Git 凭据时自动执行远端 clone/commit/push, 缺少配置时跳过远端分支。 +- `examples/sandbox_runtime.py`: commands callbacks、filesystem `write_files`/watcher、PTY runtime 交互。 +- `examples/sandbox_templates.py`: template list/detail/build/status/logs/tags/delete。 +- `examples/sandbox_injection_rules.py`: injection rules CRUD, 缺少 Qiniu AK/SK 或测试配置时自动跳过。 +- `examples/sandbox_resources.py`: Git repository resource 与 Kodo resource, 缺少对应配置时自动跳过相关分支。 + +当前实测结果: + +- `python -m pytest tests/cases/test_services/test_sandbox -q`: `26 passed, 1 skipped`。唯一 skipped 是 PTY input, 原因是当前 envd 对 PTY `SendInput` 返回 501。 +- 新增远端 Git push 集成测试 `test_git_remote_push_when_credentials_are_configured`, 在 `.env` 包含 `GIT_REPO_URL` 和 Git 凭据时自动运行；实测已向 `miclle/sandbox-git-demo.git` push `python-sdk-it-*` 分支。 +- `python -m flake8 --show-source --max-line-length=160 ./qiniu`: 通过。 +- `python3 -m compileall -q qiniu tests examples`: 通过。使用系统 Python 3.11 执行, 因为旧 mock server 代码使用 Python 3.10+ `match` 语法。 +- `git diff --check`: 通过。 +- 所有 sandbox 示例均已逐个运行并退出 0: `sandbox_create.py`, `sandbox_lifecycle.py`, `sandbox_connect.py`, `sandbox_git.py`, `sandbox_templates.py`, `sandbox_injection_rules.py`, `sandbox_resources.py`, `sandbox_runtime.py`。 +- `examples/sandbox_git.py` 已在 `GIT_REPO_URL=https://github.com/miclle/sandbox-git-demo.git` 配置下实际 push `python-sdk-example-*` 分支；远端验证可见 `python-sdk-it-*` 和 `python-sdk-example-*` 分支。 +- `sandbox_runtime.py` 实测 commands callbacks、filesystem `write_files`、watcher 成功；PTY create 成功但 PTY input 因 envd 501 被示例跳过。 +- `sandbox_resources.py` 实测 Git repository resource 与 Kodo resource 均成功创建、访问并清理 sandbox；示例中仍保留 404/408/409/429/5xx 的可选资源跳过逻辑, 避免远端资源服务临时不可用时阻塞其他分支。 + +全仓库测试也已尝试按 CI 方式执行。启动 `tests/mock_server/main.py --port 9000` 并设置 `MOCK_SERVER_ADDRESS=http://127.0.0.1:9000` 后, mock-server 相关 HTTP 测试可运行；但当前本地 `.env` 的普通 Kodo/CDN 测试配置不完整或不匹配, 例如 `QINIU_TEST_BUCKET` 与临时映射的 AK/SK 返回 `no such bucket`, 因此 `python -m pytest ./test_qiniu.py tests -q` 仍有大量非 sandbox 旧测试失败。这些失败不来自 sandbox 模块改动。 + +E2B Python SDK 本身有更大范围的 sync/async、Volume、Template、filesystem watch、PTY 等测试和示例。本文仅基于本地源码接口对照, 没有运行 E2B Python SDK 的测试套件。 + +## 后续建议 + +1. 如果 Qiniu Python SDK 需要更贴近 E2B Python SDK, 优先评估是否新增 `AsyncSandbox`。这是 Python 用户迁移时最明显的接口差异。 +2. Commands 后续可继续补普通 command `start` 的真正长连接 streaming handle 和更完整的 output handler 行为。 +3. Filesystem 后续可继续补完整 typed `EntryInfo/WriteInfo`、流式 `WatchDir` 和更高效的批量上传。 +4. Git 后续可把 `status`/`branches` 解析为结构化 `GitStatus`/`GitBranches`/`GitFileStatus`, 并补 typed auth/upstream exception。 +5. Snapshot、MCP gateway、Volume 是否对齐 E2B, 应以 Qiniu 后端产品是否正式开放为准；没有后端能力时继续保持显式缺省。 +6. Template 如需继续追齐 E2B, 可优先补 ReadyCmd helpers、Dockerfile/devcontainer/context upload 和对象化 build workflow。 diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 96b436b2..612e6a2a 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import os +import time + from sandbox_common import cleanup_sandbox, create_sandbox, run_example @@ -17,6 +20,99 @@ def assert_git_ok(step, result): return result +def is_retryable_git_network_error(result): + message = (result.stderr or result.stdout or result.error or '').lower() + return result.exit_code != 0 and ( + 'gnutls' in message or + 'tls connection' in message or + 'unable to access' in message or + 'the remote end hung up unexpectedly' in message + ) + + +def assert_git_network_ok(step, run, attempts=5): + result = None + for attempt in range(attempts): + result = run() + if result.exit_code == 0: + print(step + ':', result.exit_code) + return result + if not is_retryable_git_network_error(result) or attempt == attempts - 1: + return assert_git_ok(step, result) + print('{0}: retry {1}/{2}'.format(step, attempt + 2, attempts)) + time.sleep(attempt + 1) + return assert_git_ok(step, result) + + +def remote_git_config(): + repo_url = os.getenv('GIT_REPO_URL') + username = os.getenv('GIT_USERNAME') + password = os.getenv('GIT_PASSWORD') or os.getenv('GITHUB_TOKEN') + if password and not username: + username = 'x-access-token' + if not repo_url or not username or not password: + return None + return repo_url, username, password + + +def run_remote_push_demo(sandbox): + config = remote_git_config() + if not config: + print('remote git push: skipped, GIT_REPO_URL/Git credentials missing') + return + + repo_url, username, password = config + branch = 'python-sdk-example-{0}'.format(int(time.time() * 1000)) + repo_path = '/tmp/qiniu-python-sdk-git/remote' + + assert_git_ok( + 'git authenticate', + sandbox.git.dangerously_authenticate(username, password), + ) + assert_git_ok( + 'git http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + assert_git_network_ok( + 'git clone remote', + lambda: ( + sandbox.commands.run('rm -rf {0}'.format(repo_path)), + sandbox.git.clone(repo_url, repo_path, depth=1), + )[1], + ) + assert_git_ok( + 'configure remote user', + sandbox.git.configure_user( + repo_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_git_ok( + 'checkout remote branch', + sandbox.git.checkout_branch(repo_path, branch, create=True), + ) + sandbox.files.write( + repo_path + '/python-sdk-example.txt', + 'qiniu-python-sdk example push {0}\n'.format(branch), + ) + assert_git_ok('git add remote', sandbox.git.add(repo_path, all=True)) + assert_git_ok( + 'git commit remote', + sandbox.git.commit(repo_path, 'test: qiniu python sdk example push'), + ) + assert_git_network_ok( + 'git push remote', + lambda: sandbox.git.push( + repo_path, + 'origin', + 'HEAD:refs/heads/{0}'.format(branch), + ), + ) + print('remote git branch:', branch) + + def main(): repo_path = '/tmp/qiniu-python-sdk-git/repo' clone_path = '/tmp/qiniu-python-sdk-git/clone' @@ -42,6 +138,7 @@ def main(): ) assert_git_ok('git clone', sandbox.git.clone(repo_path, clone_path)) print(sandbox.git.status(clone_path).stdout) + run_remote_push_demo(sandbox) finally: cleanup_sandbox(sandbox) diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py index f31c2b59..b403cc84 100644 --- a/examples/sandbox_resources.py +++ b/examples/sandbox_resources.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import print_function -from qiniu.services.sandbox import GitRepositoryResource, KodoResource +from qiniu.services.sandbox import ( + GitRepositoryResource, + KodoResource, + SandboxError, +) from sandbox_common import ( cleanup_sandbox, create_sandbox, @@ -10,6 +14,19 @@ ) +def is_optional_resource_error(err): + status_code = getattr(err, 'status_code', None) + if status_code in (404, 408, 409, 429, 500, 502, 503, 504): + return True + message = str(err).lower() + return ( + 'timeout' in message or + 'timed out' in message or + 'not found' in message or + 'temporarily unavailable' in message + ) + + def run_git_resource_example(): repo_url = env('GIT_REPO_URL') repo_token = env('GITHUB_TOKEN') @@ -21,17 +38,23 @@ def run_git_resource_example(): return mount_path = env('QINIU_SANDBOX_GIT_MOUNT_PATH', '/workspace/repo') - sandbox = create_sandbox( - timeout=300, - metadata={'example': 'sandbox_resources_git'}, - resources=[ - GitRepositoryResource( - url=repo_url, - mount_path=mount_path, - authorization_token=repo_token, - ) - ], - ) + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_git'}, + resources=[ + GitRepositoryResource( + url=repo_url, + mount_path=mount_path, + authorization_token=repo_token, + ) + ], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip GitHub repository resource:', err) + return + raise try: print('Git resource sandbox:', sandbox.sandbox_id) print(sandbox.commands.run('ls -la {0} | head -20'.format( @@ -53,11 +76,17 @@ def run_kodo_resource_example(): mount_path=mount_path, prefix=env('QINIU_SANDBOX_KODO_PREFIX') or None, ) - sandbox = create_sandbox( - timeout=300, - metadata={'example': 'sandbox_resources_kodo'}, - resources=[resource], - ) + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_kodo'}, + resources=[resource], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip Kodo resource:', err) + return + raise try: print('Kodo resource sandbox:', sandbox.sandbox_id) print(sandbox.commands.run('ls -la {0} | head -20'.format( diff --git a/examples/sandbox_runtime.py b/examples/sandbox_runtime.py new file mode 100644 index 00000000..06289bc5 --- /dev/null +++ b/examples/sandbox_runtime.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function +import time + +from qiniu.services.sandbox import PtySize, SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_runtime'}) + watcher = None + try: + stdout = [] + result = sandbox.commands.run( + 'printf runtime', + on_stdout=stdout.append, + ) + print('command stdout:', result.stdout) + print('callback stdout:', ''.join(stdout)) + + sandbox.files.write_files([ + {'path': '/tmp/qiniu-runtime-a.txt', 'data': 'a'}, + {'path': '/tmp/qiniu-runtime-b.txt', 'data': 'b'}, + ]) + print('write_files: ok') + + try: + watcher = sandbox.files.watch_dir('/tmp') + sandbox.files.write('/tmp/qiniu-runtime-watch.txt', 'watch') + time.sleep(1) + for event in watcher.get_new_events(): + print('watch event:', event.name, event.type) + except Exception as err: + print('watch_dir skipped:', err) + finally: + if watcher is not None: + try: + watcher.stop() + except Exception: + pass + + try: + pty = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + print('pty pid:', pty.pid) + sandbox.pty.send_stdin(pty.pid, 'echo pty-ready\n') + sandbox.pty.send_stdin(pty.pid, 'exit\n') + pty_result = pty.wait() + print('pty stdout:', pty_result.stdout) + except SandboxError as err: + print('pty skipped:', err) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 204b215e..35b595cf 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -26,7 +26,17 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient -from .services.sandbox import Sandbox, SandboxClient, Template, KodoResource, GitRepositoryResource +from .services.sandbox import ( + FileType, + FilesystemEventType, + GitRepositoryResource, + KodoResource, + PtySize, + Sandbox, + SandboxClient, + Template, + WatchHandle, +) from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, decode_entry, canonical_mime_header_key diff --git a/qiniu/services/sandbox/__init__.py b/qiniu/services/sandbox/__init__.py index 912edce5..2d7ca086 100644 --- a/qiniu/services/sandbox/__init__.py +++ b/qiniu/services/sandbox/__init__.py @@ -13,6 +13,7 @@ CommandHandle, CommandResult, Commands, + ProcessInfo, ) from .constants import ( DEFAULT_ENDPOINT, @@ -21,8 +22,15 @@ ENVD_PORT, ) from .errors import SandboxError, TemplateBuildError -from .filesystem import Filesystem +from .filesystem import ( + FileType, + Filesystem, + FilesystemEvent, + FilesystemEventType, + WatchHandle, +) from .git import Git +from .pty import Pty, PtySize from .resources import GitRepositoryResource, KodoResource from .sandbox import Sandbox, SandboxPaginator from .template import Template @@ -36,16 +44,23 @@ 'DEFAULT_TEMPLATE', 'DEFAULT_USER', 'ENVD_PORT', + 'FileType', 'Filesystem', + 'FilesystemEvent', + 'FilesystemEventType', 'Git', 'GitRepositoryResource', 'KodoResource', + 'ProcessInfo', + 'Pty', + 'PtySize', 'Sandbox', 'SandboxClient', 'SandboxError', 'SandboxPaginator', 'Template', 'TemplateBuildError', + 'WatchHandle', 'env', 'load_dotenv_if_present', 'required_env', diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index b8754281..63e9774f 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -2,7 +2,28 @@ import base64 from .envd import connect_rpc, connect_stream_rpc -from .errors import CommandExitError +from .errors import CommandExitError, SandboxError + + +class ProcessInfo(object): + def __init__(self, pid=None, tag=None, cmd=None, args=None, envs=None, + cwd=None): + self.pid = pid + self.tag = tag + self.cmd = cmd + self.args = args or [] + self.envs = envs or {} + self.cwd = cwd + + def to_dict(self): + return { + 'pid': self.pid, + 'tag': self.tag, + 'cmd': self.cmd, + 'args': self.args, + 'envs': self.envs, + 'cwd': self.cwd, + } class CommandResult(object): @@ -37,7 +58,8 @@ def _decode_bytes(value): return str(value) -def command_result_from_events(events): +def command_result_from_events(events, on_stdout=None, on_stderr=None, + on_pty=None): pid = 0 stdout = '' stderr = '' @@ -51,8 +73,18 @@ def command_result_from_events(events): if start: pid = start.get('pid') or pid if data: - stdout += _decode_bytes(data.get('stdout')) - stderr += _decode_bytes(data.get('stderr')) + stdout_chunk = _decode_bytes(data.get('stdout')) + stderr_chunk = _decode_bytes(data.get('stderr')) + pty_chunk = _decode_bytes(data.get('pty')) + if stdout_chunk and on_stdout: + on_stdout(stdout_chunk) + if stderr_chunk and on_stderr: + on_stderr(stderr_chunk) + if pty_chunk and on_pty: + on_pty(pty_chunk) + stdout += stdout_chunk + stderr += stderr_chunk + stdout += pty_chunk if end: exit_code = 0 if end.get( 'exitCode') is None else end.get('exitCode') @@ -61,15 +93,31 @@ def command_result_from_events(events): class CommandHandle(object): - def __init__(self, commands, result, throw_on_error=False): + def __init__(self, commands, result=None, throw_on_error=False, + events=None): self.commands = commands + result = result or CommandResult() self.result = result self.pid = result.pid self.stdout = result.stdout self.stderr = result.stderr self.throw_on_error = throw_on_error - - def wait(self): + self._events = events + + def wait(self, on_stdout=None, on_stderr=None): + if self._events is not None: + result = command_result_from_events( + self._events, + on_stdout=on_stdout, + on_stderr=on_stderr, + ) + if not result.pid: + result.pid = self.pid + self.result = result + self.pid = result.pid + self.stdout = result.stdout + self.stderr = result.stderr + self._events = None if self.throw_on_error and self.result.exit_code: raise CommandExitError(self.result) return self.result @@ -84,7 +132,8 @@ def __init__(self, sandbox): def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, tag=None, background=False, throw_on_error=False, - timeout=None, **opts): + timeout=None, request_timeout=None, on_stdout=None, + on_stderr=None, **opts): handle = self.start( cmd, cwd=cwd, @@ -93,13 +142,16 @@ def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, stdin=stdin, tag=tag, throw_on_error=throw_on_error, - timeout=timeout, + timeout=request_timeout if request_timeout is not None else timeout, + on_stdout=on_stdout, + on_stderr=on_stderr, **opts ) return handle if background else handle.wait() def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, - tag=None, throw_on_error=False, timeout=None, **opts): + tag=None, throw_on_error=False, timeout=None, on_stdout=None, + on_stderr=None, **opts): process = { 'cmd': '/bin/bash', 'args': ['-l', '-c', cmd], @@ -121,7 +173,11 @@ def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, user=user, timeout=timeout, ) - result = command_result_from_events(events) + result = command_result_from_events( + events, + on_stdout=on_stdout, + on_stderr=on_stderr, + ) return CommandHandle(self, result, throw_on_error=throw_on_error) def list(self, user=None, timeout=None): @@ -135,16 +191,39 @@ def list(self, user=None, timeout=None): result = [] for process in processes: config = process.get('config') or {} - result.append({ - 'pid': process.get('pid'), - 'tag': process.get('tag'), - 'cmd': config.get('cmd'), - 'args': config.get('args'), - 'envs': config.get('envs'), - 'cwd': config.get('cwd'), - }) + result.append(ProcessInfo( + pid=process.get('pid'), + tag=process.get('tag'), + cmd=config.get('cmd'), + args=config.get('args'), + envs=config.get('envs'), + cwd=config.get('cwd'), + )) return result + def connect(self, pid, tag=None, user=None, timeout=None, + throw_on_error=False): + selector = {'pid': pid} + if tag is not None: + selector = {'tag': tag} + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Connect', + {'process': {'selector': selector}}, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + first_event = next(events) + result = command_result_from_events([first_event]) + return CommandHandle( + self, + result, + events=events, + throw_on_error=throw_on_error, + ) + def send_stdin(self, pid, data, user=None, timeout=None): if not isinstance(data, bytes): data = str(data).encode('utf-8') @@ -163,8 +242,13 @@ def close_stdin(self, pid, user=None, timeout=None): closeStdin = close_stdin def kill(self, pid, user=None, timeout=None): - connect_rpc(self.sandbox, '/process.Process/SendSignal', { - 'process': {'selector': {'pid': pid}}, - 'signal': 'SIGNAL_SIGKILL', - }, user=user, timeout=timeout) - return None + try: + connect_rpc(self.sandbox, '/process.Process/SendSignal', { + 'process': {'selector': {'pid': pid}}, + 'signal': 'SIGNAL_SIGKILL', + }, user=user, timeout=timeout) + return True + except SandboxError as err: + if err.status_code == 404: + return False + raise diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index 6b685de8..e24f7bcf 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -79,23 +79,68 @@ def decode_connect_envelopes(data): return messages -def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None): +def iter_connect_envelopes(chunks): + buffer = b'' + for chunk in chunks: + if not chunk: + continue + if not isinstance(chunk, bytes): + chunk = chunk.encode('utf-8') + buffer += chunk + while len(buffer) >= 5: + flags, length = struct.unpack('>BI', buffer[:5]) + if length > MAX_CONNECT_ENVELOPE_BYTES: + raise SandboxError( + 'Sandbox envd stream envelope too large: {0}'.format( + length) + ) + if len(buffer) < 5 + length: + break + payload = buffer[5:5 + length] + buffer = buffer[5 + length:] + if flags & 2: + if payload: + error = json.loads(payload.decode('utf-8')).get('error') + if error: + raise SandboxError( + error.get('message') or + 'Sandbox envd stream failed', + data=error, + ) + continue + if payload: + yield json.loads(payload.decode('utf-8')) + if buffer: + raise SandboxError('Sandbox envd stream truncated unexpectedly') + + +def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, + stream=False): url = sandbox.envd_url() + procedure headers = envd_headers(sandbox, user, { 'Content-Type': 'application/connect+json', 'Keepalive-Ping-Interval': '50', }) - response = sandbox.client.session.post( - url, - data=encode_connect_envelope(body), - headers=headers, - timeout=timeout, - ) + request_opts = { + 'data': encode_connect_envelope(body), + 'headers': headers, + 'timeout': timeout, + } + if stream: + request_opts['stream'] = True + try: + response = sandbox.client.session.post(url, **request_opts) + except TypeError: + request_opts.pop('stream', None) + response = sandbox.client.session.post(url, **request_opts) if response.status_code < 200 or response.status_code >= 300: raise SandboxError( 'Sandbox envd request failed with status {0}'.format( response.status_code), response, response.text, ) + if stream and hasattr(response, 'iter_content'): + return iter_connect_envelopes(response.iter_content(chunk_size=8192)) + content_type = response.headers.get('Content-Type', '') if 'application/connect+json' in content_type: return decode_connect_envelopes(response.content) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 54b39b46..18354fbc 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -1,9 +1,49 @@ # -*- coding: utf-8 -*- import time +from .errors import SandboxError from .envd import connect_rpc, envd_headers, raw_envd_request +class FileType(object): + FILE = 'file' + DIR = 'dir' + DIRECTORY = DIR + + +class FilesystemEventType(object): + CREATE = 'create' + WRITE = 'write' + REMOVE = 'remove' + RENAME = 'rename' + CHMOD = 'chmod' + + +class FilesystemEvent(object): + def __init__(self, name=None, type=None): + self.name = name + self.type = type + + def to_dict(self): + return {'name': self.name, 'type': self.type} + + +def normalize_event_type(event_type): + mapping = { + 'EVENT_TYPE_CREATE': FilesystemEventType.CREATE, + 'EVENT_TYPE_WRITE': FilesystemEventType.WRITE, + 'EVENT_TYPE_REMOVE': FilesystemEventType.REMOVE, + 'EVENT_TYPE_RENAME': FilesystemEventType.RENAME, + 'EVENT_TYPE_CHMOD': FilesystemEventType.CHMOD, + 1: FilesystemEventType.CREATE, + 2: FilesystemEventType.WRITE, + 3: FilesystemEventType.REMOVE, + 4: FilesystemEventType.RENAME, + 5: FilesystemEventType.CHMOD, + } + return mapping.get(event_type, event_type) + + def normalize_entry(entry): entry = entry or {} entry_type = entry.get('type') @@ -16,6 +56,47 @@ def normalize_entry(entry): return entry +class WatchHandle(object): + def __init__(self, filesystem, watcher_id): + self.filesystem = filesystem + self.watcher_id = watcher_id + self.watcherID = watcher_id + self._closed = False + + def get_new_events(self, user=None, timeout=None): + if self._closed: + raise SandboxError('The watcher is already stopped') + data = connect_rpc( + self.filesystem.sandbox, + '/filesystem.Filesystem/GetWatcherEvents', + {'watcherId': self.watcher_id}, + user=user, + timeout=timeout, + ) + events = [] + for event in (data or {}).get('events', []): + events.append(FilesystemEvent( + name=event.get('name'), + type=normalize_event_type(event.get('type')), + )) + return events + + getNewEvents = get_new_events + + def stop(self, user=None, timeout=None): + if self._closed: + return None + connect_rpc( + self.filesystem.sandbox, + '/filesystem.Filesystem/RemoveWatcher', + {'watcherId': self.watcher_id}, + user=user, + timeout=timeout, + ) + self._closed = True + return None + + class Filesystem(object): def __init__(self, sandbox): self.sandbox = sandbox @@ -94,6 +175,19 @@ def _multipart_body(self, boundary, filename, data): ] return b''.join(chunks) + def write_files(self, files, user=None, **opts): + result = [] + for item in files or []: + if isinstance(item, dict): + path = item.get('path') + data = item.get('data', item.get('content', '')) + else: + path, data = item + result.append(self.write(path, data, user=user, **opts)) + return result + + writeFiles = write_files + def get_info(self, path, user=None, timeout=None): data = connect_rpc( self.sandbox, @@ -163,3 +257,19 @@ def rename(self, old_path, new_path, user=None, timeout=None): timeout=timeout, ) return normalize_entry((data or {}).get('entry')) + + def watch_dir(self, path, recursive=False, user=None, timeout=None): + data = connect_rpc( + self.sandbox, + '/filesystem.Filesystem/CreateWatcher', + {'path': path, 'recursive': recursive}, + user=user, + timeout=timeout, + ) + watcher_id = ( + (data or {}).get('watcherId') or + (data or {}).get('watcher_id') + ) + return WatchHandle(self, watcher_id) + + watchDir = watch_dir diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 2f36984c..ebfbbaeb 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -81,3 +81,122 @@ def push(self, repo_path, remote=None, branch=None, **opts): if branch: args.append(shell_quote(branch)) return self._run_git(repo_path, args, **opts) + + def dangerously_authenticate( + self, + username, + password, + host='github.com', + protocol='https', + **opts): + if not username: + raise ValueError('username is required') + if not password: + raise ValueError('password is required') + + result = self._run_git( + None, + ['config', '--global', 'credential.helper', 'store'], + **opts + ) + if result.exit_code != 0: + return result + + credential = ( + 'protocol={0}\n' + 'host={1}\n' + 'username={2}\n' + 'password={3}\n\n' + ).format(protocol, host, username, password) + return self.commands.run( + 'printf {0} | git credential approve'.format( + shell_quote(credential)), + **opts + ) + + dangerouslyAuthenticate = dangerously_authenticate + + def remote_add(self, repo_path, name, url, **opts): + return self._run_git(repo_path, [ + 'remote', + 'add', + shell_quote(name), + shell_quote(url), + ], **opts) + + remoteAdd = remote_add + + def remote_get(self, repo_path, name='origin', **opts): + return self._run_git(repo_path, [ + 'remote', + 'get-url', + shell_quote(name), + ], **opts) + + remoteGet = remote_get + + def branches(self, repo_path, **opts): + return self._run_git(repo_path, ['branch', '--list'], **opts) + + def create_branch(self, repo_path, name, start_point=None, **opts): + args = ['branch', shell_quote(name)] + if start_point: + args.append(shell_quote(start_point)) + return self._run_git(repo_path, args, **opts) + + createBranch = create_branch + + def checkout_branch(self, repo_path, name, create=False, **opts): + args = ['checkout'] + if create: + args.append('-b') + args.append(shell_quote(name)) + return self._run_git(repo_path, args, **opts) + + checkoutBranch = checkout_branch + + def delete_branch(self, repo_path, name, force=True, **opts): + return self._run_git(repo_path, [ + 'branch', + '-D' if force else '-d', + shell_quote(name), + ], **opts) + + deleteBranch = delete_branch + + def reset(self, repo_path, target=None, mode=None, **opts): + args = ['reset'] + mode = mode or opts.get('reset_type') or opts.get('resetType') + if mode: + args.append('--{0}'.format(mode)) + if target: + args.append(shell_quote(target)) + return self._run_git(repo_path, args, **opts) + + def restore(self, repo_path, paths=None, staged=False, source=None, **opts): + args = ['restore'] + if staged: + args.append('--staged') + if source: + args.extend(['--source', shell_quote(source)]) + for path in paths or opts.get('files') or ['.']: + args.append(shell_quote(path)) + return self._run_git(repo_path, args, **opts) + + def set_config(self, repo_path, key, value, global_config=False, **opts): + args = ['config'] + if global_config: + args.append('--global') + args.extend([shell_quote(key), shell_quote(value)]) + return self._run_git(repo_path, args, **opts) + + setConfig = set_config + + def get_config(self, repo_path, key, global_config=False, **opts): + args = ['config'] + if global_config: + args.append('--global') + args.extend(['--get', shell_quote(key)]) + return self._run_git(repo_path, args, **opts) + + getConfig = get_config diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py new file mode 100644 index 00000000..abbbe711 --- /dev/null +++ b/qiniu/services/sandbox/pty.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +import base64 + +from .commands import CommandHandle, command_result_from_events +from .envd import connect_rpc, connect_stream_rpc +from .errors import SandboxError + + +class PtySize(object): + def __init__(self, rows=24, cols=80): + self.rows = rows + self.cols = cols + + def to_dict(self): + return {'rows': self.rows, 'cols': self.cols} + + +def _normalize_size(size=None, rows=None, cols=None): + if size is None: + return {'rows': rows or 24, 'cols': cols or 80} + if hasattr(size, 'to_dict'): + data = size.to_dict() + elif isinstance(size, dict): + data = size + else: + data = { + 'rows': getattr(size, 'rows', None), + 'cols': getattr(size, 'cols', None), + } + return { + 'rows': data.get('rows') or rows or 24, + 'cols': data.get('cols') or cols or 80, + } + + +class Pty(object): + def __init__(self, sandbox): + self.sandbox = sandbox + + def create(self, size=None, user=None, cwd=None, envs=None, timeout=None, + rows=None, cols=None, **opts): + envs = dict(envs or {}) + envs.setdefault('TERM', 'xterm-256color') + envs.setdefault('LANG', 'C.UTF-8') + envs.setdefault('LC_ALL', 'C.UTF-8') + process = { + 'cmd': '/bin/bash', + 'args': ['-i', '-l'], + 'envs': envs, + } + if cwd: + process['cwd'] = cwd + body = { + 'process': process, + 'pty': {'size': _normalize_size(size, rows=rows, cols=cols)}, + } + if opts.get('tag'): + body['tag'] = opts.get('tag') + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Start', + body, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + first_event = next(events) + result = command_result_from_events([first_event]) + return CommandHandle(self, result, events=events) + + def connect(self, pid, user=None, timeout=None, throw_on_error=False): + events = connect_stream_rpc( + self.sandbox, + '/process.Process/Connect', + {'process': {'selector': {'pid': pid}}}, + user=user, + timeout=timeout, + stream=True, + ) + events = iter(events) + first_event = next(events) + result = command_result_from_events([first_event]) + return CommandHandle( + self, + result, + events=events, + throw_on_error=throw_on_error, + ) + + def send_stdin(self, pid, data, user=None, timeout=None): + if not isinstance(data, bytes): + data = str(data).encode('utf-8') + return connect_rpc(self.sandbox, '/process.Process/SendInput', { + 'process': {'selector': {'pid': pid}}, + 'input': {'pty': base64.b64encode(data).decode('ascii')}, + }, user=user, timeout=timeout) + + sendStdin = send_stdin + send_input = send_stdin + sendInput = send_stdin + + def resize(self, pid, size=None, user=None, timeout=None, rows=None, + cols=None): + return connect_rpc(self.sandbox, '/process.Process/Update', { + 'process': {'selector': {'pid': pid}}, + 'pty': {'size': _normalize_size(size, rows=rows, cols=cols)}, + }, user=user, timeout=timeout) + + def kill(self, pid, user=None, timeout=None): + try: + connect_rpc(self.sandbox, '/process.Process/SendSignal', { + 'process': {'selector': {'pid': pid}}, + 'signal': 'SIGNAL_SIGKILL', + }, user=user, timeout=timeout) + return True + except SandboxError as err: + if err.status_code == 404: + return False + raise diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index c19fc793..5393fe48 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -6,6 +6,7 @@ from .constants import DEFAULT_USER, ENVD_PORT, MCP_PORT from .filesystem import Filesystem from .git import Git +from .pty import Pty from .util import ( append_query, file_signature, @@ -110,6 +111,7 @@ def __init__(self, client=None, info=None, sandbox_id=None, sandboxID=None, self.files = Filesystem(self) self.filesystem = self.files self.commands = Commands(self) + self.pty = Pty(self) self.git = Git(self.commands) @classmethod diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 7a5caeb9..27a50339 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -12,6 +12,7 @@ from qiniu.services.sandbox import ( DEFAULT_ENDPOINT, ENVD_PORT, + Git, KodoResource, Sandbox, SandboxClient, @@ -217,3 +218,63 @@ def test_template_builder_outputs_build_config(): ], 'startCmd': 'python /app/app.py', } + + +class RecordingCommands(object): + def __init__(self): + self.calls = [] + + def run(self, cmd, **opts): + self.calls.append((cmd, opts)) + return type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin https://github.com/qiniu/repo.git\n', + 'stderr': '', + })() + + +def test_git_helpers_align_with_e2b_method_names(): + commands = RecordingCommands() + git = Git(commands) + + git.remote_add('/repo', 'origin', 'https://github.com/qiniu/repo.git') + git.remote_get('/repo', 'origin') + git.branches('/repo') + git.create_branch('/repo', 'feature') + git.checkout_branch('/repo', 'main') + git.delete_branch('/repo', 'old') + git.reset('/repo', 'HEAD~1', mode='hard') + git.restore('/repo', paths=['a.txt', 'b.txt']) + git.set_config('/repo', 'user.name', 'tester') + git.get_config('/repo', 'user.name') + + assert commands.calls[0][0] == ( + 'git remote add origin https://github.com/qiniu/repo.git') + assert commands.calls[1][0] == 'git remote get-url origin' + assert commands.calls[2][0] == 'git branch --list' + assert commands.calls[3][0] == 'git branch feature' + assert commands.calls[4][0] == 'git checkout main' + assert commands.calls[5][0] == 'git branch -D old' + assert commands.calls[6][0] == "git reset --hard 'HEAD~1'" + assert commands.calls[7][0] == 'git restore a.txt b.txt' + assert commands.calls[8][0] == 'git config user.name tester' + assert commands.calls[9][0] == 'git config --get user.name' + + +def test_git_dangerously_authenticate_aligns_with_e2b(): + commands = RecordingCommands() + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-token', + host='github.com', + protocol='https', + ) + + assert commands.calls[0][0] == 'git config --global credential.helper store' + assert commands.calls[1][0] == ( + "printf 'protocol=https\nhost=github.com\n" + "username=git-user\npassword=secret-token\n\n' | " + 'git credential approve' + ) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index c83b5119..1c39fb40 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -2,10 +2,17 @@ import base64 import json -from qiniu.services.sandbox import Sandbox, SandboxClient +from qiniu.services.sandbox import ( + FileType, + FilesystemEventType, + PtySize, + Sandbox, + SandboxClient, +) from qiniu.services.sandbox.envd import ( decode_connect_envelopes, encode_connect_envelope, + iter_connect_envelopes, ) @@ -41,10 +48,12 @@ def post(self, url, data=None, headers=None, timeout=None): 'timeout': timeout, }) if url.endswith('/process.Process/Start'): + decoded = self.posts[-1]['data'] + output_key = 'pty' if decoded.get('pty') else 'stdout' raw = b''.join([ encode_connect_envelope({'event': {'start': {'pid': 12}}}), encode_connect_envelope({'event': {'data': { - 'stdout': base64.b64encode(b'hello\n').decode('ascii'), + output_key: base64.b64encode(b'hello\n').decode('ascii'), 'stderr': base64.b64encode(b'').decode('ascii'), }}}), encode_connect_envelope({'event': {'end': {'exitCode': 0}}}), @@ -53,6 +62,19 @@ def post(self, url, data=None, headers=None, timeout=None): raw=raw, content_type='application/connect+json', ) + if url.endswith('/process.Process/Connect'): + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'connected\n').decode( + 'ascii'), + }}}), + encode_connect_envelope({'event': {'end': {'exitCode': 0}}}), + ]) + return DummyResponse( + raw=raw, + content_type='application/connect+json', + ) if url.endswith('/filesystem.Filesystem/Stat'): return DummyResponse( body={ @@ -66,6 +88,13 @@ def post(self, url, data=None, headers=None, timeout=None): 'name': 'hello.txt', 'type': 'FILE', }]}}) + if url.endswith('/filesystem.Filesystem/CreateWatcher'): + return DummyResponse(body={'result': {'watcherId': 'watch-1'}}) + if url.endswith('/filesystem.Filesystem/GetWatcherEvents'): + return DummyResponse(body={'result': {'events': [{ + 'name': 'hello.txt', + 'type': 'EVENT_TYPE_WRITE', + }]}}) return DummyResponse(body={'result': {}}) def request(self, method, url, **kwargs): @@ -86,6 +115,26 @@ def sandbox_with_envd_session(): return sandbox, session +def test_iter_connect_envelopes_decodes_chunked_stream_frames(): + raw = b''.join([ + encode_connect_envelope({'event': {'start': {'pid': 12}}}), + encode_connect_envelope({'event': {'data': { + 'stdout': base64.b64encode(b'hello').decode('ascii'), + }}}), + ]) + + assert list(iter_connect_envelopes([ + raw[:3], + raw[3:9], + raw[9:], + ])) == [ + {'event': {'start': {'pid': 12}}}, + {'event': {'data': { + 'stdout': base64.b64encode(b'hello').decode('ascii'), + }}}, + ] + + def test_commands_run_posts_process_start_and_decodes_events(): sandbox, session = sandbox_with_envd_session() @@ -109,6 +158,67 @@ def test_commands_run_posts_process_start_and_decodes_events(): } +def test_commands_connect_returns_handle_for_running_process(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.commands.connect(12) + result = handle.wait() + + assert result.pid == 12 + assert result.stdout == 'connected\n' + assert session.posts[0]['url'].endswith('/process.Process/Connect') + assert session.posts[0]['data'] == { + 'process': {'selector': {'pid': 12}}, + } + + +def test_commands_run_supports_e2b_callbacks_and_request_timeout(): + sandbox, session = sandbox_with_envd_session() + stdout = [] + stderr = [] + + result = sandbox.commands.run( + 'echo hello', + on_stdout=stdout.append, + on_stderr=stderr.append, + request_timeout=7, + ) + + assert result.stdout == 'hello\n' + assert stdout == ['hello\n'] + assert stderr == [] + assert session.posts[0]['timeout'] == 7 + + +def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.pty.create(PtySize(rows=24, cols=80), cwd='/workspace') + sandbox.pty.send_stdin(handle.pid, 'ls\n') + sandbox.pty.resize(handle.pid, {'rows': 30, 'cols': 100}) + connected = sandbox.pty.connect(handle.pid) + assert sandbox.pty.kill(handle.pid) is True + + assert handle.wait().stdout == 'hello\n' + assert connected.wait().stdout == 'connected\n' + assert session.posts[0]['url'].endswith('/process.Process/Start') + assert session.posts[0]['data']['process']['args'] == ['-i', '-l'] + assert session.posts[0]['data']['process']['cwd'] == '/workspace' + assert session.posts[0]['data']['process']['envs']['TERM'] == ( + 'xterm-256color') + assert session.posts[0]['data']['pty'] == { + 'size': {'rows': 24, 'cols': 80}, + } + assert session.posts[1]['url'].endswith('/process.Process/SendInput') + assert session.posts[1]['data']['input']['pty'] + assert session.posts[2]['url'].endswith('/process.Process/Update') + assert session.posts[2]['data']['pty'] == { + 'size': {'rows': 30, 'cols': 100}, + } + assert session.posts[3]['url'].endswith('/process.Process/Connect') + assert session.posts[4]['url'].endswith('/process.Process/SendSignal') + + def test_filesystem_uses_envd_rpc_and_signed_file_urls(): sandbox, session = sandbox_with_envd_session() @@ -127,3 +237,37 @@ def test_filesystem_uses_envd_rpc_and_signed_file_urls(): assert b'name="file"' in session.requests[1]['kwargs']['data'] assert session.posts[0]['url'].endswith('/filesystem.Filesystem/Stat') assert session.posts[1]['url'].endswith('/filesystem.Filesystem/ListDir') + + +def test_filesystem_write_files_accepts_e2b_style_file_list(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.files.write_files([ + {'path': '/tmp/a.txt', 'data': 'a'}, + {'path': '/tmp/b.txt', 'data': b'b'}, + ]) + + assert [entry['type'] for entry in result] == ['file', 'file'] + assert session.requests[0]['method'] == 'POST' + assert session.requests[1]['method'] == 'POST' + + +def test_filesystem_watch_dir_returns_e2b_style_watch_handle(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.files.watch_dir('/tmp', recursive=True) + events = handle.get_new_events() + handle.stop() + + assert handle.watcher_id == 'watch-1' + assert events[0].name == 'hello.txt' + assert events[0].type == FilesystemEventType.WRITE + assert FileType.FILE == 'file' + assert session.posts[0]['url'].endswith( + '/filesystem.Filesystem/CreateWatcher') + assert session.posts[0]['data'] == {'path': '/tmp', 'recursive': True} + assert session.posts[1]['url'].endswith( + '/filesystem.Filesystem/GetWatcherEvents') + assert session.posts[1]['data'] == {'watcherId': 'watch-1'} + assert session.posts[2]['url'].endswith( + '/filesystem.Filesystem/RemoveWatcher') diff --git a/tests/cases/test_services/test_sandbox/test_example_config.py b/tests/cases/test_services/test_sandbox/test_example_config.py index 1f1d579e..8844c87f 100644 --- a/tests/cases/test_services/test_sandbox/test_example_config.py +++ b/tests/cases/test_services/test_sandbox/test_example_config.py @@ -24,6 +24,7 @@ def test_env_example_only_contains_connection_and_resource_inputs(): def test_examples_handle_runtime_branches_in_code(): connect = read_project_file('examples', 'sandbox_connect.py') resources = read_project_file('examples', 'sandbox_resources.py') + runtime = read_project_file('examples', 'sandbox_runtime.py') integration = read_project_file( 'tests', 'cases', @@ -34,5 +35,7 @@ def test_examples_handle_runtime_branches_in_code(): assert 'QINIU_SANDBOX_ID' not in connect assert 'QINIU_SANDBOX_KODO_READ_ONLY' not in resources + assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in runtime + assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in runtime assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in integration assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in integration diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py index 2073773c..5287195b 100644 --- a/tests/cases/test_services/test_sandbox/test_integration.py +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- import os +import time import pytest -from qiniu.services.sandbox import Sandbox, SandboxClient +from qiniu.services.sandbox import PtySize, Sandbox, SandboxClient, SandboxError from qiniu.services.sandbox.config import load_dotenv_if_present @@ -16,6 +17,68 @@ def integration_client(): return SandboxClient() +def is_unsupported_runtime_error(err): + response = getattr(err, 'response', None) + status_code = getattr(response, 'status_code', None) + if status_code in (404, 501): + return True + message = str(err).lower() + return ( + 'unimplemented' in message or + 'not implemented' in message or + 'not found' in message + ) + + +def create_integration_sandbox(test_name): + return Sandbox.create( + os.getenv('QINIU_SANDBOX_TEMPLATE', 'base'), + timeout=300, + metadata={'sdk': 'python', 'test': test_name}, + envs={'QINIU_SANDBOX_TEST': '1'}, + client=integration_client(), + ) + + +def assert_command_ok(step, result): + assert result.exit_code == 0, '{0} failed: {1}'.format( + step, result.stderr or result.stdout or result.error) + return result + + +def is_retryable_git_network_error(result): + message = (result.stderr or result.stdout or result.error or '').lower() + return result.exit_code != 0 and ( + 'gnutls' in message or + 'tls connection' in message or + 'unable to access' in message or + 'the remote end hung up unexpectedly' in message + ) + + +def assert_git_network_ok(step, run, attempts=5): + result = None + for attempt in range(attempts): + result = run() + if result.exit_code == 0: + return result + if not is_retryable_git_network_error(result) or attempt == attempts - 1: + return assert_command_ok(step, result) + time.sleep(attempt + 1) + return assert_command_ok(step, result) + + +def remote_git_credentials(): + repo_url = os.getenv('GIT_REPO_URL') + username = os.getenv('GIT_USERNAME') + password = os.getenv('GIT_PASSWORD') or os.getenv('GITHUB_TOKEN') + if password and not username: + username = 'x-access-token' + if not repo_url or not username or not password: + pytest.skip('GIT_REPO_URL and Git credentials are required') + return repo_url, username, password + + def test_create_run_filesystem_and_kill_sandbox(): sandbox = None client = integration_client() @@ -39,6 +102,159 @@ def test_create_run_filesystem_and_kill_sandbox(): sandbox.kill() +def test_git_remote_push_when_credentials_are_configured(): + repo_url, username, password = remote_git_credentials() + sandbox = None + branch = 'python-sdk-it-{0}'.format(int(time.time() * 1000)) + repo_path = '/tmp/qiniu-python-sdk-remote-git' + try: + sandbox = create_integration_sandbox('remote_git') + assert_command_ok( + 'git authenticate', + sandbox.git.dangerously_authenticate(username, password), + ) + assert_command_ok( + 'git http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + assert_git_network_ok( + 'git clone', + lambda: ( + sandbox.commands.run('rm -rf {0}'.format(repo_path)), + sandbox.git.clone(repo_url, repo_path, depth=1), + )[1], + ) + assert_command_ok( + 'configure user', + sandbox.git.configure_user( + repo_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_command_ok( + 'checkout branch', + sandbox.git.checkout_branch(repo_path, branch, create=True), + ) + sandbox.files.write( + repo_path + '/python-sdk-integration.txt', + 'qiniu-python-sdk remote push {0}\n'.format(branch), + ) + assert_command_ok('git add', sandbox.git.add(repo_path, all=True)) + assert_command_ok( + 'git commit', + sandbox.git.commit(repo_path, 'test: qiniu python sdk remote push'), + ) + assert_git_network_ok( + 'git push', + lambda: sandbox.git.push( + repo_path, + 'origin', + 'HEAD:refs/heads/{0}'.format(branch), + ), + ) + finally: + if sandbox is not None: + sandbox.kill() + + +def test_runtime_commands_filesystem_and_git_helpers(): + sandbox = None + try: + sandbox = create_integration_sandbox('runtime') + + stdout = [] + result = sandbox.commands.run( + 'printf runtime', + on_stdout=stdout.append, + ) + assert result.exit_code == 0 + assert result.stdout == 'runtime' + assert stdout == ['runtime'] + + sandbox.files.write_files([ + {'path': '/tmp/qiniu-python-sdk-a.txt', 'data': 'a'}, + {'path': '/tmp/qiniu-python-sdk-b.txt', 'data': 'b'}, + ]) + assert sandbox.files.read_text('/tmp/qiniu-python-sdk-a.txt') == 'a' + assert sandbox.files.read_text('/tmp/qiniu-python-sdk-b.txt') == 'b' + + repo_path = '/tmp/qiniu-python-sdk-runtime-repo' + sandbox.commands.run('rm -rf {0} && mkdir -p {0}'.format(repo_path)) + assert sandbox.git.init(repo_path).exit_code == 0 + assert sandbox.git.set_config( + repo_path, 'user.name', 'Sandbox Demo').exit_code == 0 + assert sandbox.git.set_config( + repo_path, 'user.email', 'sandbox-demo@example.com').exit_code == 0 + sandbox.files.write(repo_path + '/README.md', '# runtime\n') + assert sandbox.git.add(repo_path, all=True).exit_code == 0 + assert sandbox.git.commit( + repo_path, 'feat: runtime', allow_empty=True).exit_code == 0 + assert sandbox.git.create_branch(repo_path, 'feature').exit_code == 0 + assert sandbox.git.checkout_branch(repo_path, 'feature').exit_code == 0 + assert sandbox.git.branches(repo_path).exit_code == 0 + assert sandbox.git.get_config(repo_path, 'user.name').exit_code == 0 + assert sandbox.git.reset(repo_path, 'HEAD', mode='hard').exit_code == 0 + assert sandbox.git.restore(repo_path, paths=['README.md']).exit_code == 0 + finally: + if sandbox is not None: + sandbox.kill() + + +def test_filesystem_watch_dir_when_envd_supports_it(): + sandbox = None + watcher = None + try: + sandbox = create_integration_sandbox('watch_dir') + try: + watcher = sandbox.files.watch_dir('/tmp') + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd does not support filesystem watcher') + raise + + sandbox.files.write('/tmp/qiniu-python-sdk-watch.txt', 'watch') + events = [] + for _ in range(5): + events = watcher.get_new_events() + if events: + break + time.sleep(1) + assert isinstance(events, list) + assert all(hasattr(event, 'name') for event in events) + finally: + if watcher is not None: + watcher.stop() + if sandbox is not None: + sandbox.kill() + + +def test_pty_when_envd_supports_it(): + sandbox = None + try: + sandbox = create_integration_sandbox('pty') + try: + handle = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd does not support pty') + raise + + try: + sandbox.pty.send_stdin(handle.pid, 'echo qiniu-pty-ready\n') + sandbox.pty.send_stdin(handle.pid, 'exit\n') + except SandboxError as err: + if is_unsupported_runtime_error(err): + pytest.skip('envd can create pty but does not support input') + raise + result = handle.wait() + assert 'qiniu-pty-ready' in result.stdout + finally: + if sandbox is not None: + sandbox.kill() + + def test_list_and_connect_existing_sandbox(): client = integration_client() page = Sandbox.list(client=client, limit=5).next_items() From 1ade2ea1b2affc53d879f09f834a3941a00328f4 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 18:01:10 +0800 Subject: [PATCH 03/45] chore(sandbox): remove differences doc from commit Keep the sandbox runtime, test, and example changes tracked while removing the generated E2B differences document from the submitted branch. --- docs/sandbox-e2b-python-sdk-differences.md | 182 --------------------- 1 file changed, 182 deletions(-) delete mode 100644 docs/sandbox-e2b-python-sdk-differences.md diff --git a/docs/sandbox-e2b-python-sdk-differences.md b/docs/sandbox-e2b-python-sdk-differences.md deleted file mode 100644 index 18f6e5bc..00000000 --- a/docs/sandbox-e2b-python-sdk-differences.md +++ /dev/null @@ -1,182 +0,0 @@ -# Qiniu Python SDK Sandbox 与 E2B Python SDK 差异说明 - -本文用于对照当前 `qiniu-python-sdk` sandbox 模块与 E2B Python SDK 的能力边界，帮助评估 API 设计、测试覆盖、示例配置和后端依赖差异。 - -## 对比基准 - -| 项目 | 版本或位置 | 说明 | -| --- | --- | --- | -| Qiniu Python SDK | 当前分支 `sandbox` 的工作区实现 | 以 `qiniu.services.sandbox` 和顶层 `qiniu.Sandbox` 导出为基准 | -| E2B Python SDK | 本地 E2B Python SDK 源码 | 以 `e2b.Sandbox`, `e2b.AsyncSandbox`, `e2b.Template`, `e2b.Volume` 等导出为基准 | -| Qiniu Sandbox 后端 | sandbox OpenAPI spec、envd proto 与当前 `.env` 实测环境 | Qiniu 控制面、envd、Kodo resource 和 injection rules 是 Qiniu 特有产品能力 | - -## 总体定位 - -Qiniu sandbox 模块目前是 `qiniu-python-sdk` 中新增的一组同步 API，面向 Qiniu Sandbox 控制面和 envd，覆盖 sandbox 生命周期、命令、文件、Git、模板、资源挂载、注入规则、配置加载和 examples。 - -E2B Python SDK 是独立的现代 Python SDK，提供 sync/async 双栈、丰富类型导出、Template/Volume/Snapshot/MCP/PTY/watch 等完整产品面。两者使用场景接近，但鉴权模型、后端控制面、运行时依赖、类型系统和部分产品能力并不相同。 - -## 对齐原则 - -本轮 Qiniu Python sandbox API 的目标是尽量贴近 E2B Python SDK 的常用函数签名和对象组织，同时保留 `qiniu-python-sdk` 的兼容约束和 Qiniu 产品语义。 - -- E2B Python SDK 的常用入口尽量保留同名或近似签名，例如 `Sandbox.create(...)`, `Sandbox.connect(...)`, `Sandbox.list(...)`, `sandbox.files`, `sandbox.commands`, `sandbox.pty`, `sandbox.git`, `Template.from_image(...)`。 -- Python 侧同时提供 snake_case 与 camelCase 别名，便于对齐 Python 习惯和已有 Qiniu/JS SDK 使用方式。 -- Qiniu 后端已支持的能力，在 SDK 层实现真实请求，例如 lifecycle、commands、filesystem、Git、template API、Kodo resources、injection rules。 -- Qiniu 特有能力继续显式保留，例如 Qiniu AK/SK 签名、Kodo resource、Git repository resource 和 injection rules。 -- 后端或当前实现没有等价能力的部分不伪装为可用，例如 AsyncSandbox、Volume、snapshot wrapper、template build helper 的完整 E2B 形态。 - -## 主要差异概览 - -| 领域 | Qiniu Python SDK sandbox | E2B Python SDK | 差异影响 | -| --- | --- | --- | --- | -| 包定位 | `qiniu` 包内新增 `qiniu.services.sandbox` 模块 | 独立 `e2b` 包 | Qiniu 需要兼容现有 SDK 导出和依赖策略 | -| Python 兼容 | `setup.py` 仍声明 Python 2.7/3.4-3.7 兼容 | 现代 Python, 使用 `typing`, `typing_extensions`, `httpx` 等 | Qiniu sandbox 实现避免 dataclass、async/await 入口和较新的语法依赖 | -| HTTP 依赖 | 复用 `requests` / `requests.Session` | 使用 `httpx` sync/async client 和 generated API client | 请求封装、超时、异常类型和 async 支持不同 | -| 同步/异步 | 仅同步 API | 同时提供 `Sandbox` 与 `AsyncSandbox`, `Template` 与 `AsyncTemplate`, `Volume` 与 `AsyncVolume` | Qiniu 暂不提供 async 等价入口 | -| 鉴权 | `QINIU_SANDBOX_API_KEY`, `QINIU_SANDBOX_ACCESS_TOKEN`, Qiniu AK/SK `Mac` | `E2B_API_KEY` 与 `ConnectionConfig` | Qiniu 资源挂载和 injection rules 需要 Qiniu AK/SK 签名 | -| 控制面 endpoint | `QINIU_SANDBOX_API_URL`, `QINIU_SANDBOX_ENDPOINT`, `E2B_API_URL` 或默认 Qiniu endpoint | E2B cloud endpoint/domain config | endpoint、header、token 名称和默认值不同 | -| 顶层导出 | `Sandbox`, `SandboxClient`, `Template`, `KodoResource`, `GitRepositoryResource`, `FileType`, `FilesystemEventType`, `PtySize`, `WatchHandle` | `Sandbox`, `AsyncSandbox`, `Template`, `AsyncTemplate`, `Volume`, typed models/errors | Qiniu 当前导出更轻，类型模型更少；`Async*` 与 `Volume` 暂无等价产品面 | -| 类型系统 | 手写 Python 对象和 dict payload | 大量 typed model、paginator、exception、filesystem/git/network 类型 | Qiniu 用户更多接触原始 dict；E2B 类型提示更完整 | - -## Sandbox 生命周期 - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| 创建 sandbox | 支持 `Sandbox.create(template=None, timeout=None, metadata=None, envs=None, secure=True, allow_internet_access=True, mcp=None, network=None, lifecycle=None, resources=None, injections=None, **opts)` | 支持同类签名, 另有 `volume_mounts` | Qiniu 使用 `templateID`, `envVars`, `resources`, `injections` 等控制面字段 | -| 默认 template | 默认使用 Qiniu `DEFAULT_TEMPLATE` | 默认 `base`, MCP 场景可切换到 `mcp-gateway` | Qiniu 不自动改写为 MCP template | -| 连接 sandbox | 支持类方法 `Sandbox.connect(id, ...)` 和实例 `sandbox.connect(...)` | 支持类方法和实例方法 | Qiniu 连接后会刷新 envd token 信息 | -| 列表分页 | 支持 `Sandbox.list(...).next_items()` 和 `nextItems` | 支持 `SandboxPaginator` / `AsyncSandboxPaginator` | Qiniu paginator 较轻量, 返回 `Sandbox` 对象列表 | -| kill | 支持 | 支持 | Qiniu 使用控制面 DELETE | -| set_timeout | 支持 `set_timeout` / `setTimeout` | 支持 | 单位均按秒表达 | -| pause/resume | 支持 `pause`, `betaPause`, `resume` | 支持 pause, 以及 lifecycle/auto-resume 相关能力 | Qiniu `resume` 会更新当前对象信息 | -| refresh | 支持 `refresh` 调用 Qiniu `refreshes` API | E2B 无完全同名核心实例方法 | Qiniu 特有生命周期补充 | -| update_network | 支持 `update_network` / `updateNetwork` | 支持 network update | 请求结构依赖各自控制面 | -| get_info/get_metrics/get_logs | 支持 | E2B 支持 get_info/get_metrics, logs 更多出现在 template/build 等上下文 | Qiniu 暴露 sandbox logs wrapper | -| is_running | 暂无同名方法 | 支持 `is_running()` 通过 envd health 判断 | Qiniu 当前提供 `wait_for_ready()` 轮询 envd health | -| signed file URL | 支持 `download_url`, `upload_url` 和 envd token 签名 | 支持同类签名 URL helper | 签名字段与默认 user 处理保持 Qiniu 实现 | - -## Snapshot 与 MCP - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| create_snapshot/list_snapshots | 暂无 `Sandbox` 实例 wrapper | 支持 `create_snapshot()` 和 `list_snapshots()` | Qiniu Python 当前没有暴露 snapshot paginator | -| MCP create option | `Sandbox.create(..., mcp=...)` 可透传到控制面 | `mcp` 会影响 template 选择并启动 MCP gateway command | Qiniu SDK 不负责自动启动 MCP gateway | -| get_mcp_url | 支持按 50005 端口生成 URL | 支持 | 依赖 sandbox domain | -| get_mcp_token | 返回控制面下发的 `trafficAccessToken` | E2B 会维护 MCP token, 并在 MCP gateway 启动后使用 | Qiniu 当前不读取 `/etc/mcp-gateway/.token`, 也不启动 gateway | - -## Commands 与 PTY - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| run/start | 支持 `commands.run(...)` 和 `commands.start(...)` | 支持 | Qiniu 使用 envd Connect JSON envelope 调用 `/process.Process/Start` | -| background command | `run(..., background=True)` 返回 `CommandHandle` | 支持 `CommandHandle` | Qiniu 普通 `start` 当前仍会聚合 stream 事件；`commands.connect` 和 PTY create/connect 使用流式 envelope 解析 | -| stdout/stderr | 返回聚合后的 `CommandResult.stdout/stderr`, 支持 `on_stdout`/`on_stderr` | 支持输出 handler、stream/handle 等更完整形态 | Qiniu callback 是聚合事件时同步触发 | -| stdin | 支持 `send_stdin` / `sendStdin`, `close_stdin` / `closeStdin` | 支持 | 均依赖 envd process API | -| kill | 支持按 pid 发送 SIGKILL | 支持 | Qiniu `kill` 返回 bool, 404 时返回 `False` | -| 非 0 退出 | 默认不抛错, 传 `throw_on_error=True` 才抛 `CommandExitError` | E2B command handle 默认错误语义更细 | 迁移代码时要注意默认异常行为 | -| command connect | 支持 `commands.connect(pid)` | E2B 支持连接/管理运行中的 command | Qiniu 使用 envd `/process.Process/Connect` | -| PTY | 支持 `sandbox.pty.create/connect/send_stdin/resize/kill` | 支持 `sandbox.pty` | Qiniu 使用 api-spec 中的 PTY Start/Update/SendInput/Connect 能力；当前实测 envd 可创建 PTY, 但 PTY input 返回 501 | - -## Filesystem - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| read | 支持 `read(path, format="text")`, `read_text`, `format="bytes"` | 支持多种读取形态和类型提示 | Qiniu 返回 `str` 或 `bytes`, 类型模型较轻 | -| write 单文件 | 支持 string/bytes, 默认 multipart, 可选 `use_octet_stream=True` | 支持更丰富数据类型和写入信息 | Qiniu 为兼容当前 envd 默认走 multipart | -| write 多文件 | 支持 `write_files` / `writeFiles`, 内部复用单文件 write | 支持 `write_files` | Qiniu 当前是逐文件写入 | -| stat/list/exists/mkdir/remove/rename | 支持 `get_info/stat`, `list`, `exists`, `make_dir/mkdir`, `remove`, `rename` | 支持同类能力 | 返回结构均会归一化为轻量 dict | -| watch_dir | 支持非流式 watcher: `watch_dir`, `WatchHandle.get_new_events()`, `WatchHandle.stop()` | 支持 `watch_dir` 与 watch handle | Qiniu 使用 api-spec 中的 `CreateWatcher/GetWatcherEvents/RemoveWatcher` | -| FileType/EntryInfo 类型 | 导出轻量 `FileType`, `FilesystemEventType`, `FilesystemEvent`, `WatchHandle` | E2B 导出 `FileType`, `EntryInfo`, `WriteInfo` | Qiniu 目前使用字符串常量和轻量对象, 尚无完整 typed model | - -## Git - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| clone/init/status/add/commit | 支持 | 支持 | Qiniu 通过 `commands.run("git ...")` 封装 | -| configure_user | 支持 | 支持类似 config helper | Qiniu 使用 repo path + name/email 签名 | -| pull/push | 支持基础 remote/branch 参数 | 支持并处理更多 auth/upstream 错误 | Qiniu 暂无 typed git exception | -| branch/remote/reset/restore | 支持 remote_add/remote_get/branches/create_branch/checkout_branch/delete_branch/reset/restore | E2B Git 模块更完整 | Qiniu 通过 `git` 命令 wrapper 实现 | -| structured status | 返回 command stdout | E2B 导出 `GitStatus`, `GitBranches`, `GitFileStatus` | Qiniu helper 尚未解析 porcelain 输出 | -| git auth helper | 支持 `dangerously_authenticate(username, password, host="github.com", protocol="https")` 和 `dangerouslyAuthenticate` | E2B 使用 Git credential helper, 另有 auth/upstream 异常和辅助逻辑 | Qiniu 已对齐 credential helper 认证入口；暂未补 typed `GitAuthException`/`GitUpstreamException` | - -## Template - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| Template builder | 支持 `from_image`, `from_template`, `add_step`, `run_cmd/run`, `copy`, `set_env`, `set_start_cmd`, `set_ready_cmd`, `to_dict`, `to_json` | 提供更完整的 `Template` / `AsyncTemplate` builder 和 build workflow | Qiniu builder 是轻量配置生成器 | -| build API | 通过 `SandboxClient.create_template`, `create_template_v2`, `rebuild_template`, `wait_for_build` 等 client 方法调用 | `Template` 对象本身承载 build、logs、tags 等工作流 | Qiniu 的 template 生命周期更偏 client wrapper | -| Dockerfile/devcontainer/context upload | 暂无完整 helper | E2B 支持更完整模板构建上下文能力 | Qiniu 当前只表达基础 steps | -| ReadyCmd helpers | `set_ready_cmd` 接受命令值 | E2B 导出 `wait_for_file`, `wait_for_port`, `wait_for_process`, `wait_for_timeout`, `wait_for_url` | Qiniu 未提供 typed ReadyCmd helper | -| Tags/build logs/status | client wrapper 支持 tags、build status/logs | E2B Template API 更集中 | Qiniu 示例覆盖了模板 list/detail/build/status/logs/tags/delete | - -## Volume 与资源挂载 - -| 能力 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| Volume API | 暂无 `Volume` / `AsyncVolume` | 支持完整 persistent Volume 产品 | Qiniu 后端当前使用不同的资源挂载模型 | -| volume_mounts | `Sandbox.create` 不接受 E2B `volume_mounts` 语义 | 支持 mount path 到 Volume/name 的映射 | Qiniu 使用 `resources` 透传资源 | -| Git repository resource | 支持 `GitRepositoryResource(url, mount_path, authorization_token=None, repository_type="github_repository")` | 不适用同名模型 | Qiniu 特有 resource 能力 | -| Kodo resource | 支持 `KodoResource(bucket, mount_path, prefix=None, read_only=None)` | 不适用 | Qiniu 特有能力, 触发 Qiniu AK/SK 签名 | -| injection rules | 支持 list/create/get/update/delete | E2B 无同类 API | Qiniu 特有能力, 使用 Qiniu AK/SK 签名 | - -## 配置与示例 - -Qiniu Python SDK 的 sandbox 配置集中在 `.env` 和 `qiniu.services.sandbox.config`: - -- `QINIU_SANDBOX_API_KEY`: sandbox 控制面 API key。 -- `QINIU_SANDBOX_ENDPOINT` 或 `QINIU_SANDBOX_API_URL`: sandbox 控制面 endpoint。 -- `QINIU_SANDBOX_ACCESS_TOKEN`: 部分 template rebuild 等 access token 场景使用。 -- `QINIU_SANDBOX_ACCESS_KEY` / `QINIU_SANDBOX_SECRET_KEY`: Kodo resource 和 injection rules 等 Qiniu 签名 API 使用。 -- `QINIU_SANDBOX_TEMPLATE`: examples 默认 template。 -- Kodo/Git/injection 示例所需的 bucket、prefix、repository、token 等配置由对应 example 自行判断；缺少时示例会跳过该分支, 不要求额外手动配置测试开关。 - -E2B Python SDK 主要围绕 `E2B_API_KEY` 和 `ConnectionConfig` 组织连接配置；Volume、Template、Sandbox 使用各自的 connection config 和 typed options。 - -## 错误与后端依赖 - -| 项目 | Qiniu 当前状态 | E2B Python SDK 状态 | 说明 | -| --- | --- | --- | --- | -| 基础错误 | `SandboxError`, `TemplateBuildError`, `CommandExitError` | `SandboxException`, `TimeoutException`, `NotFoundException`, `FileNotFoundException`, `GitAuthException`, `GitUpstreamException`, `TemplateException`, `VolumeException` 等 | Qiniu 当前错误类型较少 | -| HTTP 错误 | 根据 response status 和 message 生成 `SandboxError` | generated client 和 `httpx` 异常映射更细 | Qiniu 目前更多保留原始 response/data | -| envd 依赖 | commands/filesystem/pty 依赖 envd Connect RPC 和 signed file URL | 同样依赖 envd, 并有更多版本检查与 fallback | Qiniu Python 当前版本门控较少；测试和示例按 404/501 等后端能力分支跳过 | -| 后端产品能力 | Kodo resource、injection rules、refresh/logs 是 Qiniu 特有 | Volume、snapshot、async、PTY、watch 是 E2B 完整产品面的一部分 | 差异来自产品边界, 不是单纯 SDK 命名差异 | - -## 测试与示例覆盖 - -Qiniu 当前 sandbox 覆盖包括: - -- `tests/cases/test_services/test_sandbox/`: 覆盖 client、config、envd、example config、sandbox integration 分支；新增覆盖 commands callbacks/connect、PTY、filesystem `write_files`/watcher、Git helper。 -- `examples/sandbox_create.py`: 创建 sandbox 并执行基本命令。 -- `examples/sandbox_lifecycle.py`: lifecycle、timeout、pause/resume 等生命周期能力。 -- `examples/sandbox_connect.py`: list/connect/metrics/logs 等连接和观测能力。 -- `examples/sandbox_git.py`: sandbox 内 Git 操作；配置 `GIT_REPO_URL` 和 Git 凭据时自动执行远端 clone/commit/push, 缺少配置时跳过远端分支。 -- `examples/sandbox_runtime.py`: commands callbacks、filesystem `write_files`/watcher、PTY runtime 交互。 -- `examples/sandbox_templates.py`: template list/detail/build/status/logs/tags/delete。 -- `examples/sandbox_injection_rules.py`: injection rules CRUD, 缺少 Qiniu AK/SK 或测试配置时自动跳过。 -- `examples/sandbox_resources.py`: Git repository resource 与 Kodo resource, 缺少对应配置时自动跳过相关分支。 - -当前实测结果: - -- `python -m pytest tests/cases/test_services/test_sandbox -q`: `26 passed, 1 skipped`。唯一 skipped 是 PTY input, 原因是当前 envd 对 PTY `SendInput` 返回 501。 -- 新增远端 Git push 集成测试 `test_git_remote_push_when_credentials_are_configured`, 在 `.env` 包含 `GIT_REPO_URL` 和 Git 凭据时自动运行；实测已向 `miclle/sandbox-git-demo.git` push `python-sdk-it-*` 分支。 -- `python -m flake8 --show-source --max-line-length=160 ./qiniu`: 通过。 -- `python3 -m compileall -q qiniu tests examples`: 通过。使用系统 Python 3.11 执行, 因为旧 mock server 代码使用 Python 3.10+ `match` 语法。 -- `git diff --check`: 通过。 -- 所有 sandbox 示例均已逐个运行并退出 0: `sandbox_create.py`, `sandbox_lifecycle.py`, `sandbox_connect.py`, `sandbox_git.py`, `sandbox_templates.py`, `sandbox_injection_rules.py`, `sandbox_resources.py`, `sandbox_runtime.py`。 -- `examples/sandbox_git.py` 已在 `GIT_REPO_URL=https://github.com/miclle/sandbox-git-demo.git` 配置下实际 push `python-sdk-example-*` 分支；远端验证可见 `python-sdk-it-*` 和 `python-sdk-example-*` 分支。 -- `sandbox_runtime.py` 实测 commands callbacks、filesystem `write_files`、watcher 成功；PTY create 成功但 PTY input 因 envd 501 被示例跳过。 -- `sandbox_resources.py` 实测 Git repository resource 与 Kodo resource 均成功创建、访问并清理 sandbox；示例中仍保留 404/408/409/429/5xx 的可选资源跳过逻辑, 避免远端资源服务临时不可用时阻塞其他分支。 - -全仓库测试也已尝试按 CI 方式执行。启动 `tests/mock_server/main.py --port 9000` 并设置 `MOCK_SERVER_ADDRESS=http://127.0.0.1:9000` 后, mock-server 相关 HTTP 测试可运行；但当前本地 `.env` 的普通 Kodo/CDN 测试配置不完整或不匹配, 例如 `QINIU_TEST_BUCKET` 与临时映射的 AK/SK 返回 `no such bucket`, 因此 `python -m pytest ./test_qiniu.py tests -q` 仍有大量非 sandbox 旧测试失败。这些失败不来自 sandbox 模块改动。 - -E2B Python SDK 本身有更大范围的 sync/async、Volume、Template、filesystem watch、PTY 等测试和示例。本文仅基于本地源码接口对照, 没有运行 E2B Python SDK 的测试套件。 - -## 后续建议 - -1. 如果 Qiniu Python SDK 需要更贴近 E2B Python SDK, 优先评估是否新增 `AsyncSandbox`。这是 Python 用户迁移时最明显的接口差异。 -2. Commands 后续可继续补普通 command `start` 的真正长连接 streaming handle 和更完整的 output handler 行为。 -3. Filesystem 后续可继续补完整 typed `EntryInfo/WriteInfo`、流式 `WatchDir` 和更高效的批量上传。 -4. Git 后续可把 `status`/`branches` 解析为结构化 `GitStatus`/`GitBranches`/`GitFileStatus`, 并补 typed auth/upstream exception。 -5. Snapshot、MCP gateway、Volume 是否对齐 E2B, 应以 Qiniu 后端产品是否正式开放为准；没有后端能力时继续保持显式缺省。 -6. Template 如需继续追齐 E2B, 可优先补 ReadyCmd helpers、Dockerfile/devcontainer/context upload 和对象化 build workflow。 From eb025fccf03f16034fb1708ed44534c3f162ad89 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 18:09:47 +0800 Subject: [PATCH 04/45] fix(sandbox): honor background command handles Return command handles from a streaming start path so background commands do not wait for process completion. Bound wait_for_ready health probes with per-request timeouts and cover both review fixes with tests. --- qiniu/services/sandbox/commands.py | 33 ++++++++++++++++--- qiniu/services/sandbox/sandbox.py | 12 +++++-- .../test_services/test_sandbox/test_client.py | 26 +++++++++++++++ .../test_services/test_sandbox/test_envd.py | 19 ++++++++++- 4 files changed, 82 insertions(+), 8 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 63e9774f..bb21e37a 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -94,27 +94,37 @@ def command_result_from_events(events, on_stdout=None, on_stderr=None, class CommandHandle(object): def __init__(self, commands, result=None, throw_on_error=False, - events=None): + events=None, on_stdout=None, on_stderr=None): self.commands = commands result = result or CommandResult() self.result = result self.pid = result.pid + self.exit_code = result.exit_code + self.exitCode = result.exit_code self.stdout = result.stdout self.stderr = result.stderr self.throw_on_error = throw_on_error self._events = events + self._on_stdout = on_stdout + self._on_stderr = on_stderr def wait(self, on_stdout=None, on_stderr=None): if self._events is not None: result = command_result_from_events( self._events, - on_stdout=on_stdout, - on_stderr=on_stderr, + on_stdout=on_stdout or self._on_stdout, + on_stderr=on_stderr or self._on_stderr, ) if not result.pid: result.pid = self.pid + result.stdout = self.result.stdout + result.stdout + result.stderr = self.result.stderr + result.stderr + if not result.error: + result.error = self.result.error self.result = result self.pid = result.pid + self.exit_code = result.exit_code + self.exitCode = result.exit_code self.stdout = result.stdout self.stderr = result.stderr self._events = None @@ -172,13 +182,26 @@ def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, body, user=user, timeout=timeout, + stream=True, ) + events = iter(events) + try: + first_event = next(events) + except StopIteration: + first_event = None result = command_result_from_events( - events, + [first_event] if first_event else [], + on_stdout=on_stdout, + on_stderr=on_stderr, + ) + return CommandHandle( + self, + result, + throw_on_error=throw_on_error, + events=events, on_stdout=on_stdout, on_stderr=on_stderr, ) - return CommandHandle(self, result, throw_on_error=throw_on_error) def list(self, user=None, timeout=None): data = connect_rpc( diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 5393fe48..28cb7230 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -304,10 +304,18 @@ def upload_url(self, path, **opts): def wait_for_ready(self, timeout=60, interval=1): started = time.time() while True: - response = self.client.session.get(self.envd_url() + '/health') + elapsed = time.time() - started + remaining = None if timeout is None else max(timeout - elapsed, 0) + request_timeout = interval + if remaining is not None: + request_timeout = min(interval, remaining) + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) if response.status_code >= 200 and response.status_code < 300: return self - if time.time() - started >= timeout: + if timeout is not None and time.time() - started >= timeout: raise RuntimeError('Sandbox envd did not become ready') time.sleep(interval) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 27a50339..5519f802 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -47,6 +47,16 @@ def send(self, request, **kwargs): return self.responses.pop(0) return DummyResponse(body={}) + def get(self, url, **kwargs): + self.requests.append(type('Request', (object,), { + 'method': 'GET', + 'url': url, + 'kwargs': kwargs, + })()) + if self.responses: + return self.responses.pop(0) + return DummyResponse(body={}) + def body_of(request): if request.body is None: @@ -199,6 +209,22 @@ def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): assert query['signature'][0] +def test_wait_for_ready_passes_request_timeout_to_health_check(): + session = RecordingSession([DummyResponse(200, {})]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=10, interval=2) + + assert session.requests[0].url == sandbox.envd_url() + '/health' + assert session.requests[0].kwargs['timeout'] == 2 + + def test_template_builder_outputs_build_config(): template = ( Template() diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 1c39fb40..43f2a9d0 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -28,13 +28,17 @@ def __init__(self, status_code=200, body=None, raw=None, def json(self): return json.loads(self.content.decode('utf-8')) + def iter_content(self, chunk_size=8192): + del chunk_size + yield self.content + class EnvdSession(object): def __init__(self): self.posts = [] self.requests = [] - def post(self, url, data=None, headers=None, timeout=None): + def post(self, url, data=None, headers=None, timeout=None, stream=False): if headers.get('Content-Type') == 'application/connect+json': decoded = decode_connect_envelopes(data)[0] else: @@ -46,6 +50,7 @@ def post(self, url, data=None, headers=None, timeout=None): 'data': decoded, 'headers': headers, 'timeout': timeout, + 'stream': stream, }) if url.endswith('/process.Process/Start'): decoded = self.posts[-1]['data'] @@ -190,6 +195,18 @@ def test_commands_run_supports_e2b_callbacks_and_request_timeout(): assert session.posts[0]['timeout'] == 7 +def test_commands_run_background_returns_before_command_finishes(): + sandbox, session = sandbox_with_envd_session() + + handle = sandbox.commands.run('sleep 30', background=True) + + assert session.posts[0]['stream'] is True + assert handle.pid == 12 + assert handle.exit_code == -1 + assert handle.stdout == '' + assert handle.wait().stdout == 'hello\n' + + def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): sandbox, session = sandbox_with_envd_session() From 31628681af50e227c7e16afb732c793b8508fbe4 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 19:44:46 +0800 Subject: [PATCH 05/45] feat(sandbox): align more e2b python APIs Add E2B-style filesystem info objects, stream reads, IO writes, Git structured status and branches, Git typed exceptions, ReadyCmd helpers, is_running, and compatible Git config signatures. Keep Volume, Snapshot, and MCP gateway behavior out of scope. --- examples/sandbox_git.py | 3 +- qiniu/__init__.py | 14 + qiniu/services/sandbox/__init__.py | 40 +- qiniu/services/sandbox/errors.py | 16 + qiniu/services/sandbox/filesystem.py | 127 +++++- qiniu/services/sandbox/git.py | 360 +++++++++++++++++- qiniu/services/sandbox/sandbox.py | 13 + qiniu/services/sandbox/template.py | 46 ++- .../test_services/test_sandbox/test_client.py | 159 +++++++- .../test_services/test_sandbox/test_envd.py | 28 +- .../test_sandbox/test_integration.py | 4 +- 11 files changed, 766 insertions(+), 44 deletions(-) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 612e6a2a..6970a2b1 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -137,7 +137,8 @@ def main(): sandbox.git.commit(repo_path, 'feat: initial commit'), ) assert_git_ok('git clone', sandbox.git.clone(repo_path, clone_path)) - print(sandbox.git.status(clone_path).stdout) + status = sandbox.git.status(clone_path) + print('clone status clean:', status.is_clean) run_remote_push_demo(sandbox) finally: cleanup_sandbox(sandbox) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 35b595cf..a2090ab7 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -27,15 +27,29 @@ from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient from .services.sandbox import ( + EntryInfo, FileType, + GitAuthException, + GitBranches, + GitFileStatus, FilesystemEventType, GitRepositoryResource, + GitStatus, + GitUpstreamException, KodoResource, PtySize, + ReadyCmd, Sandbox, SandboxClient, Template, WatchHandle, + WriteEntry, + WriteInfo, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, ) from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token diff --git a/qiniu/services/sandbox/__init__.py b/qiniu/services/sandbox/__init__.py index 2d7ca086..7ce0f9d8 100644 --- a/qiniu/services/sandbox/__init__.py +++ b/qiniu/services/sandbox/__init__.py @@ -21,19 +21,37 @@ DEFAULT_USER, ENVD_PORT, ) -from .errors import SandboxError, TemplateBuildError +from .errors import ( + FileNotFoundException, + GitAuthException, + GitUpstreamException, + InvalidArgumentException, + SandboxError, + TemplateBuildError, +) from .filesystem import ( + EntryInfo, FileType, Filesystem, FilesystemEvent, FilesystemEventType, WatchHandle, + WriteEntry, + WriteInfo, ) -from .git import Git +from .git import Git, GitBranches, GitFileStatus, GitStatus from .pty import Pty, PtySize from .resources import GitRepositoryResource, KodoResource from .sandbox import Sandbox, SandboxPaginator -from .template import Template +from .template import ( + ReadyCmd, + Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) __all__ = [ 'CommandExitError', @@ -44,16 +62,25 @@ 'DEFAULT_TEMPLATE', 'DEFAULT_USER', 'ENVD_PORT', + 'EntryInfo', 'FileType', + 'FileNotFoundException', 'Filesystem', 'FilesystemEvent', 'FilesystemEventType', 'Git', + 'GitAuthException', + 'GitBranches', + 'GitFileStatus', 'GitRepositoryResource', + 'GitStatus', + 'GitUpstreamException', + 'InvalidArgumentException', 'KodoResource', 'ProcessInfo', 'Pty', 'PtySize', + 'ReadyCmd', 'Sandbox', 'SandboxClient', 'SandboxError', @@ -61,9 +88,16 @@ 'Template', 'TemplateBuildError', 'WatchHandle', + 'WriteEntry', + 'WriteInfo', 'env', 'load_dotenv_if_present', 'required_env', 'sandbox_client', 'sandbox_template', + 'wait_for_file', + 'wait_for_port', + 'wait_for_process', + 'wait_for_timeout', + 'wait_for_url', ] diff --git a/qiniu/services/sandbox/errors.py b/qiniu/services/sandbox/errors.py index 3cf5909d..1635fabe 100644 --- a/qiniu/services/sandbox/errors.py +++ b/qiniu/services/sandbox/errors.py @@ -19,3 +19,19 @@ def __init__(self, result): super(CommandExitError, self).__init__( 'Command exited with code {0}'.format(result.exit_code) ) + + +class InvalidArgumentException(SandboxError): + pass + + +class FileNotFoundException(SandboxError): + pass + + +class GitAuthException(SandboxError): + pass + + +class GitUpstreamException(SandboxError): + pass diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 18354fbc..1245236d 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import time +from io import IOBase, TextIOBase -from .errors import SandboxError +from .errors import InvalidArgumentException, SandboxError from .envd import connect_rpc, envd_headers, raw_envd_request @@ -28,6 +29,63 @@ def to_dict(self): return {'name': self.name, 'type': self.type} +class WriteEntry(dict): + def __init__(self, path=None, data=None, **kwargs): + dict.__init__(self, path=path, data=data, **kwargs) + self.path = path + self.data = data + + +class WriteInfo(object): + def __init__(self, name=None, type=None, path=None): + self.name = name + self.type = type + self.path = path + + def to_dict(self): + return { + 'name': self.name, + 'type': self.type, + 'path': self.path, + } + + def __getitem__(self, key): + return self.to_dict()[key] + + def __eq__(self, other): + if isinstance(other, dict): + data = self.to_dict() + return all(data.get(key) == value for key, value in other.items()) + return object.__eq__(self, other) + + +class EntryInfo(WriteInfo): + def __init__(self, name=None, type=None, path=None, size=None, mode=None, + permissions=None, owner=None, group=None, modified_time=None, + symlink_target=None): + WriteInfo.__init__(self, name=name, type=type, path=path) + self.size = size + self.mode = mode + self.permissions = permissions + self.owner = owner + self.group = group + self.modified_time = modified_time + self.symlink_target = symlink_target + + def to_dict(self): + data = WriteInfo.to_dict(self) + data.update({ + 'size': self.size, + 'mode': self.mode, + 'permissions': self.permissions, + 'owner': self.owner, + 'group': self.group, + 'modified_time': self.modified_time, + 'symlink_target': self.symlink_target, + }) + return data + + def normalize_event_type(event_type): mapping = { 'EVENT_TYPE_CREATE': FilesystemEventType.CREATE, @@ -44,16 +102,49 @@ def normalize_event_type(event_type): return mapping.get(event_type, event_type) -def normalize_entry(entry): +def normalize_entry_type(entry_type): + if entry_type in ('FILE_TYPE_DIRECTORY', 'DIRECTORY', 'dir'): + return FileType.DIR + if entry_type in ('FILE_TYPE_FILE', 'FILE', 'file'): + return FileType.FILE + return entry_type + + +def normalize_entry(entry, extended=False): entry = entry or {} entry_type = entry.get('type') - if entry_type in ('FILE_TYPE_DIRECTORY', 'DIRECTORY', 'dir'): - entry_type = 'dir' - elif entry_type in ('FILE_TYPE_FILE', 'FILE', 'file'): - entry_type = 'file' - if entry_type: - entry['type'] = entry_type - return entry + entry_type = normalize_entry_type(entry_type) + info_cls = EntryInfo if extended else WriteInfo + return info_cls( + name=entry.get('name'), + type=entry_type, + path=entry.get('path'), + size=entry.get('size'), + mode=entry.get('mode'), + permissions=entry.get('permissions'), + owner=entry.get('owner'), + group=entry.get('group'), + modified_time=entry.get('modifiedTime') or entry.get('modified_time'), + symlink_target=entry.get('symlinkTarget') or entry.get( + 'symlink_target'), + ) if extended else info_cls( + name=entry.get('name'), + type=entry_type, + path=entry.get('path'), + ) + + +def to_upload_body(data, encoding='utf-8'): + if isinstance(data, bytes): + return data + if isinstance(data, str): + return data.encode(encoding) + if isinstance(data, TextIOBase): + return data.read().encode(encoding) + if isinstance(data, IOBase): + return data.read() + raise InvalidArgumentException( + 'Unsupported data type for filesystem write: {0}'.format(type(data))) class WatchHandle(object): @@ -109,8 +200,13 @@ def read(self, path, user=None, format='text', **opts): url, headers=envd_headers(self.sandbox, user), ) + if format == 'stream': + if hasattr(response, 'iter_content'): + return response.iter_content(chunk_size=opts.get( + 'chunk_size', 8192)) + return iter([response.content]) if format == 'bytes': - return response.content + return bytearray(response.content) return response.content.decode(opts.get('encoding', 'utf-8')) def read_text(self, path, user=None, **opts): @@ -119,8 +215,7 @@ def read_text(self, path, user=None, **opts): readText = read_text def write(self, path, data, user=None, **opts): - if not isinstance(data, bytes): - data = str(data).encode(opts.get('encoding', 'utf-8')) + data = to_upload_body(data, opts.get('encoding', 'utf-8')) url = self.sandbox.upload_url(path, user=user) if opts.get('use_octet_stream'): response = raw_envd_request( @@ -196,7 +291,7 @@ def get_info(self, path, user=None, timeout=None): user=user, timeout=timeout, ) - return normalize_entry((data or {}).get('entry')) + return normalize_entry((data or {}).get('entry'), extended=True) getInfo = get_info stat = get_info @@ -210,7 +305,7 @@ def list(self, path, depth=1, user=None, timeout=None): timeout=timeout, ) return [ - normalize_entry(entry) for entry in ( + normalize_entry(entry, extended=True) for entry in ( data or {}).get( 'entries', [])] @@ -233,7 +328,7 @@ def make_dir(self, path, user=None, timeout=None): user=user, timeout=timeout, ) - return normalize_entry((data or {}).get('entry')) + return normalize_entry((data or {}).get('entry'), extended=True) makeDir = make_dir mkdir = make_dir @@ -256,7 +351,7 @@ def rename(self, old_path, new_path, user=None, timeout=None): user=user, timeout=timeout, ) - return normalize_entry((data or {}).get('entry')) + return normalize_entry((data or {}).get('entry'), extended=True) def watch_dir(self, path, recursive=False, user=None, timeout=None): data = connect_rpc( diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index ebfbbaeb..9d3156fa 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,7 +1,253 @@ # -*- coding: utf-8 -*- +from .errors import GitAuthException, GitUpstreamException from .util import shell_quote +class GitFileStatus(object): + def __init__(self, name, status, index_status, working_tree_status, + staged, renamed_from=None): + self.name = name + self.status = status + self.index_status = index_status + self.working_tree_status = working_tree_status + self.staged = staged + self.renamed_from = renamed_from + + +class GitStatus(object): + def __init__(self, current_branch=None, upstream=None, ahead=0, behind=0, + detached=False, file_status=None): + self.current_branch = current_branch + self.upstream = upstream + self.ahead = ahead + self.behind = behind + self.detached = detached + self.file_status = file_status or [] + + @property + def is_clean(self): + return len(self.file_status) == 0 + + @property + def has_changes(self): + return len(self.file_status) > 0 + + @property + def has_staged(self): + return any(item.staged for item in self.file_status) + + @property + def has_untracked(self): + return any(item.status == 'untracked' for item in self.file_status) + + @property + def has_conflicts(self): + return any(item.status == 'conflict' for item in self.file_status) + + @property + def total_count(self): + return len(self.file_status) + + @property + def staged_count(self): + return sum(1 for item in self.file_status if item.staged) + + @property + def unstaged_count(self): + return sum(1 for item in self.file_status if not item.staged) + + @property + def untracked_count(self): + return sum(1 for item in self.file_status + if item.status == 'untracked') + + @property + def conflict_count(self): + return sum(1 for item in self.file_status + if item.status == 'conflict') + + +class GitBranches(object): + def __init__(self, branches=None, current_branch=None): + self.branches = branches or [] + self.current_branch = current_branch + + +def _parse_ahead_behind(segment): + ahead = 0 + behind = 0 + if not segment: + return ahead, behind + if 'ahead' in segment: + try: + ahead = int(segment.split('ahead')[1].split(',')[0].strip()) + except Exception: + ahead = 0 + if 'behind' in segment: + try: + behind = int(segment.split('behind')[1].split(',')[0].strip()) + except Exception: + behind = 0 + return ahead, behind + + +def _normalize_branch_name(name): + if name.startswith('HEAD (detached at '): + return name.replace('HEAD (detached at ', '').rstrip(')') + return ( + name.replace('HEAD (no branch)', 'HEAD') + .replace('No commits yet on ', '') + .replace('Initial commit on ', '') + ) + + +def _derive_status(index_status, working_status): + statuses = set([index_status, working_status]) + if 'U' in statuses: + return 'conflict' + if 'R' in statuses: + return 'renamed' + if 'C' in statuses: + return 'copied' + if 'D' in statuses: + return 'deleted' + if 'A' in statuses: + return 'added' + if 'M' in statuses: + return 'modified' + if 'T' in statuses: + return 'typechange' + if '?' in statuses: + return 'untracked' + return 'unknown' + + +def parse_git_status(output): + lines = [line.rstrip() for line in (output or '').split('\n') + if line.strip()] + current_branch = None + upstream = None + ahead = 0 + behind = 0 + detached = False + file_status = [] + + if not lines: + return GitStatus(file_status=file_status) + + branch_line = lines[0] + if branch_line.startswith('## '): + branch_info = branch_line[3:] + ahead_start = branch_info.find(' [') + branch_part = branch_info if ahead_start == -1 else branch_info[ + :ahead_start] + ahead_part = None if ahead_start == -1 else branch_info[ + ahead_start + 2:-1] + normalized = _normalize_branch_name(branch_part) + is_detached = branch_part.startswith('HEAD (detached at ') or ( + 'detached' in branch_part) + if is_detached or normalized.startswith('HEAD'): + detached = True + elif '...' in normalized: + current_branch, upstream = normalized.split('...', 1) + current_branch = current_branch or None + upstream = upstream or None + else: + current_branch = normalized or None + ahead, behind = _parse_ahead_behind(ahead_part) + + for line in lines[1:]: + if line.startswith('?? '): + file_status.append(GitFileStatus( + name=line[3:], + status='untracked', + index_status='?', + working_tree_status='?', + staged=False, + )) + continue + if len(line) < 3: + continue + index_status = line[0] + working_status = line[1] + path = line[3:] + renamed_from = None + name = path + if ' -> ' in path: + renamed_from, name = path.split(' -> ', 1) + file_status.append(GitFileStatus( + name=name, + status=_derive_status(index_status, working_status), + index_status=index_status, + working_tree_status=working_status, + staged=index_status not in (' ', '?'), + renamed_from=renamed_from, + )) + + return GitStatus( + current_branch=current_branch, + upstream=upstream, + ahead=ahead, + behind=behind, + detached=detached, + file_status=file_status, + ) + + +def parse_git_branches(output): + branches = [] + current_branch = None + for line in [line.strip() for line in (output or '').split('\n') + if line.strip()]: + if '\t' in line: + name, marker = line.split('\t', 1) + marker = marker.strip() + else: + marker = '*' if line.startswith('* ') else '' + name = line[2:] if line.startswith('* ') else line + branches.append(name) + if marker == '*': + current_branch = name + return GitBranches(branches=branches, current_branch=current_branch) + + +def _is_auth_failure(result): + message = '{0}\n{1}\n{2}'.format( + getattr(result, 'stderr', ''), + getattr(result, 'stdout', ''), + getattr(result, 'error', ''), + ).lower() + snippets = ( + 'authentication failed', + 'terminal prompts disabled', + 'could not read username', + 'invalid username or password', + 'access denied', + 'permission denied', + 'not authorized', + ) + return any(snippet in message for snippet in snippets) + + +def _is_missing_upstream(result): + message = '{0}\n{1}\n{2}'.format( + getattr(result, 'stderr', ''), + getattr(result, 'stdout', ''), + getattr(result, 'error', ''), + ).lower() + snippets = ( + 'has no upstream branch', + 'no upstream branch', + 'no upstream configured', + 'no tracking information for the current branch', + 'no tracking information', + 'set the remote as upstream', + 'set the upstream branch', + 'please specify which branch you want to merge with', + ) + return any(snippet in message for snippet in snippets) + + class Git(object): def __init__(self, commands): self.commands = commands @@ -31,9 +277,10 @@ def init(self, repo_path, bare=False, initial_branch=None, **opts): return self._run_git(repo_path, args, **opts) def status(self, repo_path, **opts): - return self._run_git( + result = self._run_git( repo_path, [ - 'status', '--porcelain=v1', '-b'], **opts) + 'status', '--porcelain=1', '-b'], **opts) + return parse_git_status(result.stdout) def add(self, repo_path, files=None, all=False, **opts): args = ['add'] @@ -44,8 +291,16 @@ def add(self, repo_path, files=None, all=False, **opts): args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) - def commit(self, repo_path, message, allow_empty=False, **opts): - args = ['commit', '-m', shell_quote(message)] + def commit(self, repo_path, message, author_name=None, author_email=None, + allow_empty=False, **opts): + args = [] + if author_name: + args.extend(['-c', shell_quote('user.name={0}'.format( + author_name))]) + if author_email: + args.extend(['-c', shell_quote('user.email={0}'.format( + author_email))]) + args.extend(['commit', '-m', shell_quote(message)]) if allow_empty: args.append('--allow-empty') return self._run_git(repo_path, args, **opts) @@ -66,21 +321,47 @@ def configure_user(self, repo_path, name, email, **opts): configureUser = configure_user - def pull(self, repo_path, remote=None, branch=None, **opts): + def pull(self, repo_path, remote=None, branch=None, username=None, + password=None, **opts): + if password and not username: + raise GitAuthException( + 'Git pull requires username when password is provided') args = ['pull'] if remote: args.append(shell_quote(remote)) if branch: args.append(shell_quote(branch)) - return self._run_git(repo_path, args, **opts) - - def push(self, repo_path, remote=None, branch=None, **opts): + result = self._run_git(repo_path, args, **opts) + if result.exit_code: + if _is_auth_failure(result): + raise GitAuthException( + 'Git pull requires credentials for private repositories.') + if _is_missing_upstream(result): + raise GitUpstreamException( + 'Git pull failed because no upstream branch is configured.') + return result + + def push(self, repo_path, remote=None, branch=None, set_upstream=True, + username=None, password=None, **opts): + if password and not username: + raise GitAuthException( + 'Git push requires username when password is provided') args = ['push'] + if set_upstream and remote: + args.append('--set-upstream') if remote: args.append(shell_quote(remote)) if branch: args.append(shell_quote(branch)) - return self._run_git(repo_path, args, **opts) + result = self._run_git(repo_path, args, **opts) + if result.exit_code: + if _is_auth_failure(result): + raise GitAuthException( + 'Git push requires credentials for private repositories.') + if _is_missing_upstream(result): + raise GitUpstreamException( + 'Git push failed because no upstream branch is configured.') + return result def dangerously_authenticate( self, @@ -136,10 +417,15 @@ def remote_get(self, repo_path, name='origin', **opts): remoteGet = remote_get def branches(self, repo_path, **opts): - return self._run_git(repo_path, ['branch', '--list'], **opts) + result = self._run_git( + repo_path, + ['branch', shell_quote('--format=%(refname:short)\t%(HEAD)')], + **opts + ) + return parse_git_branches(result.stdout) def create_branch(self, repo_path, name, start_point=None, **opts): - args = ['branch', shell_quote(name)] + args = ['checkout', '-b', shell_quote(name)] if start_point: args.append(shell_quote(start_point)) return self._run_git(repo_path, args, **opts) @@ -183,20 +469,60 @@ def restore(self, repo_path, paths=None, staged=False, source=None, **opts): args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) - def set_config(self, repo_path, key, value, global_config=False, **opts): + def set_config(self, *args, **opts): + global_config = opts.pop('global_config', False) + scope = opts.pop('scope', None) + path = opts.pop('path', None) + if len(args) == 3: + repo_path, key, value = args + args_list = ['config'] + if global_config: + args_list.append('--global') + args_list.extend([shell_quote(key), shell_quote(value)]) + return self._run_git(repo_path, args_list, **opts) + if len(args) != 2: + raise TypeError('set_config expects key and value') + key, value = args + scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] - if global_config: - args.append('--global') + if scope_flag: + args.append(scope_flag) args.extend([shell_quote(key), shell_quote(value)]) return self._run_git(repo_path, args, **opts) setConfig = set_config - def get_config(self, repo_path, key, global_config=False, **opts): + def get_config(self, *args, **opts): + global_config = opts.pop('global_config', False) + scope = opts.pop('scope', None) + path = opts.pop('path', None) + if len(args) == 2: + repo_path, key = args + args_list = ['config'] + if global_config: + args_list.append('--global') + args_list.extend(['--get', shell_quote(key)]) + return self._run_git(repo_path, args_list, **opts) + if len(args) != 1: + raise TypeError('get_config expects key') + key = args[0] + scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] - if global_config: - args.append('--global') + if scope_flag: + args.append(scope_flag) args.extend(['--get', shell_quote(key)]) return self._run_git(repo_path, args, **opts) getConfig = get_config + + def _resolve_config_scope(self, scope=None, path=None): + scope_name = (scope or 'global').strip().lower() + if scope_name not in ('global', 'local', 'system'): + raise ValueError('Git config scope must be global, local, or system') + if scope_name == 'local': + if not path: + raise ValueError('Repository path is required for local scope') + return '--local', path + if scope_name == 'system': + return '--system', None + return '--global', None diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 28cb7230..d17a6769 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -320,3 +320,16 @@ def wait_for_ready(self, timeout=60, interval=1): time.sleep(interval) waitForReady = wait_for_ready + + def is_running(self, request_timeout=None): + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) + if response.status_code == 502: + return False + if response.status_code >= 200 and response.status_code < 300: + return True + return False + + isRunning = is_running diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index 5ea44240..b4d431b2 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -2,6 +2,46 @@ import json +class ReadyCmd(object): + def __init__(self, cmd): + self._cmd = cmd + + def get_cmd(self): + return self._cmd + + +def wait_for_port(port): + return ReadyCmd('ss -tuln | grep :{0}'.format(port)) + + +def wait_for_url(url, status_code=200): + return ReadyCmd( + 'curl -s -o /dev/null -w "%{{http_code}}" {0} | grep -q "{1}"'.format( + url, + status_code, + ) + ) + + +def wait_for_process(process_name): + return ReadyCmd('pgrep {0} > /dev/null'.format(process_name)) + + +def wait_for_file(filename): + return ReadyCmd('[ -f {0} ]'.format(filename)) + + +def wait_for_timeout(timeout): + seconds = max(1, int(timeout) // 1000) + return ReadyCmd('sleep {0}'.format(seconds)) + + +def _ready_cmd_value(command): + if hasattr(command, 'get_cmd'): + return command.get_cmd() + return command + + class Template(object): def __init__(self): self.build_config = {'steps': []} @@ -54,14 +94,16 @@ def set_env(self, key, value): setEnv = set_env - def set_start_cmd(self, command): + def set_start_cmd(self, command, ready_cmd=None): self.build_config['startCmd'] = command + if ready_cmd is not None: + self.build_config['readyCmd'] = _ready_cmd_value(ready_cmd) return self setStartCmd = set_start_cmd def set_ready_cmd(self, command): - self.build_config['readyCmd'] = command + self.build_config['readyCmd'] = _ready_cmd_value(command) return self setReadyCmd = set_ready_cmd diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 5519f802..e6bbe77c 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -12,12 +12,22 @@ from qiniu.services.sandbox import ( DEFAULT_ENDPOINT, ENVD_PORT, + GitAuthException, + GitBranches, + GitStatus, Git, + EntryInfo, KodoResource, + ReadyCmd, Sandbox, SandboxClient, SandboxError, Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, ) @@ -35,6 +45,17 @@ def json(self): return json.loads(self.content.decode('utf-8')) +class ErrorResponse(object): + def __init__(self, status_code=502): + self.status_code = status_code + self.content = b'' + self.text = '' + self.headers = {} + + def json(self): + return None + + class RecordingSession(requests.Session): def __init__(self, responses=None): super(RecordingSession, self).__init__() @@ -225,6 +246,24 @@ def test_wait_for_ready_passes_request_timeout_to_health_check(): assert session.requests[0].kwargs['timeout'] == 2 +def test_is_running_matches_e2b_health_check_semantics(): + running_session = RecordingSession([DummyResponse(200, {})]) + running = Sandbox(client=SandboxClient( + api_key='api-key', + session=running_session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + stopped_session = RecordingSession([ErrorResponse(502)]) + stopped = Sandbox(client=SandboxClient( + api_key='api-key', + session=stopped_session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + assert running.is_running(request_timeout=3) is True + assert running_session.requests[0].kwargs['timeout'] == 3 + assert stopped.is_running() is False + + def test_template_builder_outputs_build_config(): template = ( Template() @@ -246,17 +285,49 @@ def test_template_builder_outputs_build_config(): } +def test_template_ready_cmd_helpers_align_with_e2b(): + ready = wait_for_port(8000) + assert isinstance(ready, ReadyCmd) + assert ready.get_cmd() == 'ss -tuln | grep :8000' + assert wait_for_url( + 'http://localhost:3000/health', + status_code=204, + ).get_cmd() == ( + 'curl -s -o /dev/null -w "%{http_code}" ' + 'http://localhost:3000/health | grep -q "204"' + ) + assert wait_for_process('nginx').get_cmd() == 'pgrep nginx > /dev/null' + assert wait_for_file('/tmp/ready').get_cmd() == '[ -f /tmp/ready ]' + assert wait_for_timeout(500).get_cmd() == 'sleep 1' + + template = ( + Template() + .from_image('python:3.11') + .set_start_cmd('python app.py', wait_for_port(8000)) + ) + + assert template.to_dict()['startCmd'] == 'python app.py' + assert template.to_dict()['readyCmd'] == 'ss -tuln | grep :8000' + + class RecordingCommands(object): def __init__(self): self.calls = [] + self.results = [] def run(self, cmd, **opts): self.calls.append((cmd, opts)) - return type('Result', (object,), { + result = self.results.pop(0) if self.results else type( + 'Result', (object,), { 'exit_code': 0, 'stdout': 'origin https://github.com/qiniu/repo.git\n', 'stderr': '', + 'error': '', })() + if result.exit_code and opts.get('throw_on_error'): + from qiniu.services.sandbox import CommandExitError + raise CommandExitError(result) + return result def test_git_helpers_align_with_e2b_method_names(): @@ -277,8 +348,9 @@ def test_git_helpers_align_with_e2b_method_names(): assert commands.calls[0][0] == ( 'git remote add origin https://github.com/qiniu/repo.git') assert commands.calls[1][0] == 'git remote get-url origin' - assert commands.calls[2][0] == 'git branch --list' - assert commands.calls[3][0] == 'git branch feature' + assert commands.calls[2][0] == ( + "git branch '--format=%(refname:short)\t%(HEAD)'") + assert commands.calls[3][0] == 'git checkout -b feature' assert commands.calls[4][0] == 'git checkout main' assert commands.calls[5][0] == 'git branch -D old' assert commands.calls[6][0] == "git reset --hard 'HEAD~1'" @@ -304,3 +376,84 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): "username=git-user\npassword=secret-token\n\n' | " 'git credential approve' ) + + +def test_git_status_and_branches_return_structured_e2b_types(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': ( + '## main...origin/main [ahead 2, behind 1]\n' + ' M changed.txt\n' + 'A staged.txt\n' + '?? new.txt\n' + ), + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'main\t*\nfeature\t\n', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + status = git.status('/repo') + branches = git.branches('/repo') + + assert isinstance(status, GitStatus) + assert status.current_branch == 'main' + assert status.upstream == 'origin/main' + assert status.ahead == 2 + assert status.behind == 1 + assert status.has_changes is True + assert status.has_staged is True + assert status.has_untracked is True + assert status.file_status[0].name == 'changed.txt' + assert status.file_status[0].status == 'modified' + assert isinstance(branches, GitBranches) + assert branches.branches == ['main', 'feature'] + assert branches.current_branch == 'main' + + +def test_git_push_maps_auth_failure_to_e2b_exception(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push('/repo') + + +def test_git_helpers_accept_e2b_style_signatures(): + commands = RecordingCommands() + git = Git(commands) + + git.create_branch('/repo', 'feature') + git.commit( + '/repo', + 'feat: demo', + author_name='Demo User', + author_email='demo@example.com', + allow_empty=True, + ) + git.set_config('user.name', 'Demo User', scope='local', path='/repo') + git.get_config('user.name', scope='local', path='/repo') + + assert commands.calls[0][0] == 'git checkout -b feature' + assert commands.calls[1][0] == ( + "git -c 'user.name=Demo User' -c user.email=demo@example.com " + "commit -m 'feat: demo' --allow-empty" + ) + assert commands.calls[2][0] == "git config --local user.name 'Demo User'" + assert commands.calls[2][1]['cwd'] == '/repo' + assert commands.calls[3][0] == 'git config --local --get user.name' + assert commands.calls[3][1]['cwd'] == '/repo' diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 43f2a9d0..66256855 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- import base64 +from io import BytesIO import json from qiniu.services.sandbox import ( + EntryInfo, FileType, FilesystemEventType, PtySize, @@ -106,7 +108,11 @@ def request(self, method, url, **kwargs): self.requests.append({'method': method, 'url': url, 'kwargs': kwargs}) if method == 'GET': return DummyResponse(raw=b'hello') - return DummyResponse(body={'name': 'hello.txt', 'type': 'FILE'}) + return DummyResponse(body=[{ + 'name': 'hello.txt', + 'type': 'FILE', + 'path': '/tmp/hello.txt', + }]) def sandbox_with_envd_session(): @@ -256,6 +262,26 @@ def test_filesystem_uses_envd_rpc_and_signed_file_urls(): assert session.posts[1]['url'].endswith('/filesystem.Filesystem/ListDir') +def test_filesystem_returns_e2b_style_entry_objects_and_streams(): + sandbox, session = sandbox_with_envd_session() + + stream = sandbox.files.read('/tmp/hello.txt', format='stream') + written = sandbox.files.write('/tmp/hello.txt', BytesIO(b'hello')) + info = sandbox.files.stat('/tmp/hello.txt') + entries = sandbox.files.list('/tmp') + + assert b''.join(stream) == b'hello' + assert written.name == 'hello.txt' + assert written.path == '/tmp/hello.txt' + assert written.type == FileType.FILE + assert isinstance(info, EntryInfo) + assert info.name == 'hello.txt' + assert info.type == FileType.FILE + assert entries[0].name == 'hello.txt' + assert entries[0].type == FileType.FILE + assert b'hello' in session.requests[1]['kwargs']['data'] + + def test_filesystem_write_files_accepts_e2b_style_file_list(): sandbox, session = sandbox_with_envd_session() diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py index 5287195b..1059f107 100644 --- a/tests/cases/test_services/test_sandbox/test_integration.py +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -193,7 +193,9 @@ def test_runtime_commands_filesystem_and_git_helpers(): repo_path, 'feat: runtime', allow_empty=True).exit_code == 0 assert sandbox.git.create_branch(repo_path, 'feature').exit_code == 0 assert sandbox.git.checkout_branch(repo_path, 'feature').exit_code == 0 - assert sandbox.git.branches(repo_path).exit_code == 0 + branches = sandbox.git.branches(repo_path) + assert 'feature' in branches.branches + assert branches.current_branch == 'feature' assert sandbox.git.get_config(repo_path, 'user.name').exit_code == 0 assert sandbox.git.reset(repo_path, 'HEAD', mode='hard').exit_code == 0 assert sandbox.git.restore(repo_path, paths=['README.md']).exit_code == 0 From 361e2d372d52c3b55abccb73c677cd9dd1215cdb Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:00:03 +0800 Subject: [PATCH 06/45] fix(sandbox): honor git credentials and file timeouts Use temporary remote credential URLs for authenticated push and pull, restoring the original remote after success or failure. Forward request_timeout/timeout through filesystem file transfers and cover the behavior with sandbox tests. --- qiniu/services/sandbox/filesystem.py | 8 + qiniu/services/sandbox/git.py | 163 +++++++++++++++--- .../test_services/test_sandbox/test_client.py | 162 ++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 17 ++ 4 files changed, 321 insertions(+), 29 deletions(-) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 1245236d..923b174c 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -192,6 +192,11 @@ class Filesystem(object): def __init__(self, sandbox): self.sandbox = sandbox + def _request_timeout(self, opts): + if opts.get('request_timeout') is not None: + return opts.get('request_timeout') + return opts.get('timeout') + def read(self, path, user=None, format='text', **opts): url = self.sandbox.download_url(path, user=user) response = raw_envd_request( @@ -199,6 +204,7 @@ def read(self, path, user=None, format='text', **opts): 'GET', url, headers=envd_headers(self.sandbox, user), + timeout=self._request_timeout(opts), ) if format == 'stream': if hasattr(response, 'iter_content'): @@ -228,6 +234,7 @@ def write(self, path, data, user=None, **opts): user, {'Content-Type': 'application/octet-stream'}, ), + timeout=self._request_timeout(opts), ) return self._format_write_response(response) @@ -244,6 +251,7 @@ def write(self, path, data, user=None, **opts): boundary )}, ), + timeout=self._request_timeout(opts), ) return self._format_write_response(response) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 9d3156fa..ae12e9db 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,5 +1,14 @@ # -*- coding: utf-8 -*- -from .errors import GitAuthException, GitUpstreamException +try: + from urllib.parse import urlparse, urlunparse +except ImportError: + from urlparse import urlparse, urlunparse + +from .errors import ( + GitAuthException, + GitUpstreamException, + InvalidArgumentException, +) from .util import shell_quote @@ -248,6 +257,22 @@ def _is_missing_upstream(result): return any(snippet in message for snippet in snippets) +def _with_credentials(url, username, password): + if not username and not password: + return url + if not username or not password: + raise InvalidArgumentException( + 'Both username and password are required when using Git ' + 'credentials.') + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https'): + raise InvalidArgumentException( + 'Only http(s) Git URLs support username/password credentials.') + return urlunparse(parsed._replace( + netloc='{0}:{1}@{2}'.format(username, password, parsed.netloc) + )) + + class Git(object): def __init__(self, commands): self.commands = commands @@ -257,6 +282,76 @@ def _run_git(self, repo_path, args, **opts): opts['cwd'] = repo_path return self.commands.run('git {0}'.format(' '.join(args)), **opts) + def _get_remote_url(self, repo_path, remote, **opts): + result = self._run_git( + repo_path, + ['remote', 'get-url', shell_quote(remote)], + **opts + ) + url = (getattr(result, 'stdout', '') or '').strip() + if not url: + raise InvalidArgumentException( + 'Remote "{0}" URL not found in repository.'.format(remote)) + return url + + def _resolve_remote_name(self, repo_path, remote=None, **opts): + if remote: + return remote + result = self._run_git(repo_path, ['remote'], **opts) + remotes = [ + line.strip() + for line in (getattr(result, 'stdout', '') or '').splitlines() + if line.strip() + ] + if len(remotes) == 1: + return remotes[0] + raise InvalidArgumentException( + 'Remote is required when using username/password and the ' + 'repository has multiple remotes.') + + def _with_remote_credentials(self, repo_path, remote, username, password, + operation, **opts): + original_url = self._get_remote_url(repo_path, remote, **opts) + credential_url = _with_credentials(original_url, username, password) + set_result = self._run_git(repo_path, [ + 'remote', + 'set-url', + shell_quote(remote), + shell_quote(credential_url), + ], **opts) + if getattr(set_result, 'exit_code', 0): + return set_result + + operation_error = None + result = None + try: + result = operation() + except Exception as err: + operation_error = err + finally: + restore_result = self._run_git(repo_path, [ + 'remote', + 'set-url', + shell_quote(remote), + shell_quote(original_url), + ], **opts) + if operation_error: + raise operation_error + if getattr(restore_result, 'exit_code', 0): + return restore_result + return result + + def _raise_known_result_error(self, result, operation): + if result.exit_code: + if _is_auth_failure(result): + raise GitAuthException( + 'Git {0} requires credentials for private repositories.' + .format(operation)) + if _is_missing_upstream(result): + raise GitUpstreamException( + 'Git {0} failed because no upstream branch is configured.' + .format(operation)) + def clone(self, repo_url, path=None, branch=None, depth=None, **opts): args = ['clone'] if depth: @@ -326,19 +421,30 @@ def pull(self, repo_path, remote=None, branch=None, username=None, if password and not username: raise GitAuthException( 'Git pull requires username when password is provided') + remote_name = None + if username and password: + remote_name = self._resolve_remote_name(repo_path, remote, **opts) + args = ['pull'] - if remote: - args.append(shell_quote(remote)) + target_remote = remote_name or remote + if target_remote: + args.append(shell_quote(target_remote)) if branch: args.append(shell_quote(branch)) + if username and password: + result = self._with_remote_credentials( + repo_path, + remote_name, + username, + password, + lambda: self._run_git(repo_path, args, **opts), + **opts + ) + self._raise_known_result_error(result, 'pull') + return result + result = self._run_git(repo_path, args, **opts) - if result.exit_code: - if _is_auth_failure(result): - raise GitAuthException( - 'Git pull requires credentials for private repositories.') - if _is_missing_upstream(result): - raise GitUpstreamException( - 'Git pull failed because no upstream branch is configured.') + self._raise_known_result_error(result, 'pull') return result def push(self, repo_path, remote=None, branch=None, set_upstream=True, @@ -346,21 +452,32 @@ def push(self, repo_path, remote=None, branch=None, set_upstream=True, if password and not username: raise GitAuthException( 'Git push requires username when password is provided') + remote_name = None + if username and password: + remote_name = self._resolve_remote_name(repo_path, remote, **opts) + args = ['push'] - if set_upstream and remote: + target_remote = remote_name or remote + if set_upstream and target_remote: args.append('--set-upstream') - if remote: - args.append(shell_quote(remote)) + if target_remote: + args.append(shell_quote(target_remote)) if branch: args.append(shell_quote(branch)) + if username and password: + result = self._with_remote_credentials( + repo_path, + remote_name, + username, + password, + lambda: self._run_git(repo_path, args, **opts), + **opts + ) + self._raise_known_result_error(result, 'push') + return result + result = self._run_git(repo_path, args, **opts) - if result.exit_code: - if _is_auth_failure(result): - raise GitAuthException( - 'Git push requires credentials for private repositories.') - if _is_missing_upstream(result): - raise GitUpstreamException( - 'Git push failed because no upstream branch is configured.') + self._raise_known_result_error(result, 'push') return result def dangerously_authenticate( @@ -459,7 +576,8 @@ def reset(self, repo_path, target=None, mode=None, **opts): args.append(shell_quote(target)) return self._run_git(repo_path, args, **opts) - def restore(self, repo_path, paths=None, staged=False, source=None, **opts): + def restore(self, repo_path, paths=None, staged=False, source=None, + **opts): args = ['restore'] if staged: args.append('--staged') @@ -518,7 +636,8 @@ def get_config(self, *args, **opts): def _resolve_config_scope(self, scope=None, path=None): scope_name = (scope or 'global').strip().lower() if scope_name not in ('global', 'local', 'system'): - raise ValueError('Git config scope must be global, local, or system') + raise ValueError( + 'Git config scope must be global, local, or system') if scope_name == 'local': if not path: raise ValueError('Repository path is required for local scope') diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index e6bbe77c..47c67e09 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -16,7 +16,6 @@ GitBranches, GitStatus, Git, - EntryInfo, KodoResource, ReadyCmd, Sandbox, @@ -319,11 +318,11 @@ def run(self, cmd, **opts): self.calls.append((cmd, opts)) result = self.results.pop(0) if self.results else type( 'Result', (object,), { - 'exit_code': 0, - 'stdout': 'origin https://github.com/qiniu/repo.git\n', - 'stderr': '', - 'error': '', - })() + 'exit_code': 0, + 'stdout': 'origin https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })() if result.exit_code and opts.get('throw_on_error'): from qiniu.services.sandbox import CommandExitError raise CommandExitError(result) @@ -370,7 +369,8 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): protocol='https', ) - assert commands.calls[0][0] == 'git config --global credential.helper store' + assert commands.calls[0][0] == ( + 'git config --global credential.helper store') assert commands.calls[1][0] == ( "printf 'protocol=https\nhost=github.com\n" "username=git-user\npassword=secret-token\n\n' | " @@ -433,6 +433,154 @@ def test_git_push_maps_auth_failure_to_e2b_exception(): git.push('/repo') +def test_git_push_with_credentials_sets_remote_url_temporarily(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + git.push( + '/repo', + remote='origin', + branch='main', + username='git-user', + password='secret-token', + request_timeout=7, + ) + + assert commands.calls[0][0] == 'git remote get-url origin' + assert commands.calls[1][0] == ( + 'git remote set-url origin ' + 'https://git-user:secret-token@github.com/qiniu/repo.git' + ) + assert commands.calls[2][0] == 'git push --set-upstream origin main' + assert commands.calls[3][0] == ( + 'git remote set-url origin https://github.com/qiniu/repo.git' + ) + assert commands.calls[2][1]['request_timeout'] == 7 + + +def test_git_pull_with_credentials_resolves_single_remote(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + ) + + assert commands.calls[0][0] == 'git remote' + assert commands.calls[2][0] == ( + 'git remote set-url origin ' + 'https://git-user:secret-token@github.com/qiniu/repo.git' + ) + assert commands.calls[3][0] == 'git pull origin main' + assert commands.calls[4][0] == ( + 'git remote set-url origin https://github.com/qiniu/repo.git' + ) + + +def test_git_push_with_credentials_restores_remote_on_auth_failure(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push( + '/repo', + remote='origin', + username='git-user', + password='bad-token', + ) + + assert commands.calls[-1][0] == ( + 'git remote set-url origin https://github.com/qiniu/repo.git' + ) + + def test_git_helpers_accept_e2b_style_signatures(): commands = RecordingCommands() git = Git(commands) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 66256855..1e920531 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -282,6 +282,23 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): assert b'hello' in session.requests[1]['kwargs']['data'] +def test_filesystem_read_write_pass_request_timeout_to_file_requests(): + sandbox, session = sandbox_with_envd_session() + + sandbox.files.read_text('/tmp/hello.txt', timeout=3, request_timeout=7) + sandbox.files.write('/tmp/hello.txt', 'hello', timeout=5) + sandbox.files.write( + '/tmp/octet.txt', + b'hello', + use_octet_stream=True, + request_timeout=11, + ) + + assert session.requests[0]['kwargs']['timeout'] == 7 + assert session.requests[1]['kwargs']['timeout'] == 5 + assert session.requests[2]['kwargs']['timeout'] == 11 + + def test_filesystem_write_files_accepts_e2b_style_file_list(): sandbox, session = sandbox_with_envd_session() From e0deaca159d0c5886ca2984860086cd129276293 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:19:15 +0800 Subject: [PATCH 07/45] fix(sandbox): address PR feedback Handle empty envd streams, tighten sandbox exception handling, refresh traffic tokens, and prevent stale paginator tokens. Add regression coverage for reviewer feedback and keep external Git TLS failures from failing integration runs. --- examples/sandbox_runtime.py | 4 +- qiniu/services/sandbox/client.py | 4 +- qiniu/services/sandbox/commands.py | 12 ++- qiniu/services/sandbox/config.py | 3 +- qiniu/services/sandbox/filesystem.py | 2 +- qiniu/services/sandbox/pty.py | 10 ++- qiniu/services/sandbox/sandbox.py | 32 ++++++-- .../test_services/test_sandbox/test_client.py | 75 ++++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 35 +++++++++ .../test_sandbox/test_integration.py | 24 ++++-- 10 files changed, 176 insertions(+), 25 deletions(-) diff --git a/examples/sandbox_runtime.py b/examples/sandbox_runtime.py index 06289bc5..1a211cfc 100644 --- a/examples/sandbox_runtime.py +++ b/examples/sandbox_runtime.py @@ -37,8 +37,8 @@ def main(): if watcher is not None: try: watcher.stop() - except Exception: - pass + except SandboxError as err: + print('failed to stop watcher:', err) try: pty = sandbox.pty.create(PtySize(rows=24, cols=80), timeout=30) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 7e9e0db4..37c5ad8e 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import os +import time import requests @@ -171,7 +172,7 @@ def _request(self, method, path, params=None, body=_UNSET, response_data = None try: response_data = response.json() - except Exception: + except ValueError: response_data = getattr(response, 'text', None) message = 'Sandbox API request failed with status {0}'.format( response.status_code @@ -516,7 +517,6 @@ def delete_injection_rule(self, rule_id): deleteInjectionRule = delete_injection_rule def wait_for_build(self, template_id, build_id, interval=1, timeout=60): - import time start = time.time() while True: info = self.get_template_build_status(template_id, build_id) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index bb21e37a..da7bc0e2 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import base64 +import binascii from .envd import connect_rpc, connect_stream_rpc from .errors import CommandExitError, SandboxError @@ -53,7 +54,7 @@ def _decode_bytes(value): if isinstance(value, str): try: return base64.b64decode(value).decode('utf-8') - except Exception: + except (binascii.Error, TypeError): return value return str(value) @@ -152,7 +153,9 @@ def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, stdin=stdin, tag=tag, throw_on_error=throw_on_error, - timeout=request_timeout if request_timeout is not None else timeout, + timeout=( + request_timeout if request_timeout is not None else timeout + ), on_stdout=on_stdout, on_stderr=on_stderr, **opts @@ -238,7 +241,10 @@ def connect(self, pid, tag=None, user=None, timeout=None, stream=True, ) events = iter(events) - first_event = next(events) + try: + first_event = next(events) + except StopIteration: + first_event = None result = command_result_from_events([first_event]) return CommandHandle( self, diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py index 3818a47f..f6ba9572 100644 --- a/qiniu/services/sandbox/config.py +++ b/qiniu/services/sandbox/config.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import io import os from .client import SandboxClient @@ -13,7 +14,7 @@ def load_dotenv_if_present(*paths): for path in paths: if not path or not os.path.exists(path): continue - with open(path, 'r') as f: + with io.open(path, 'r', encoding='utf-8') as f: for raw_line in f: line = raw_line.strip() if not line or line.startswith('#') or '=' not in line: diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 923b174c..686458b9 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -321,7 +321,7 @@ def exists(self, path, user=None, timeout=None): try: self.get_info(path, user=user, timeout=timeout) return True - except Exception as err: + except SandboxError as err: response = getattr(err, 'response', None) if response is not None and getattr( response, 'status_code', None) == 404: diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py index abbbe711..17e5191e 100644 --- a/qiniu/services/sandbox/pty.py +++ b/qiniu/services/sandbox/pty.py @@ -65,7 +65,10 @@ def create(self, size=None, user=None, cwd=None, envs=None, timeout=None, stream=True, ) events = iter(events) - first_event = next(events) + try: + first_event = next(events) + except StopIteration: + first_event = None result = command_result_from_events([first_event]) return CommandHandle(self, result, events=events) @@ -79,7 +82,10 @@ def connect(self, pid, user=None, timeout=None, throw_on_error=False): stream=True, ) events = iter(events) - first_event = next(events) + try: + first_event = next(events) + except StopIteration: + first_event = None result = command_result_from_events([first_event]) return CommandHandle( self, diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index d17a6769..01ae3941 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- import time +import requests + from .client import SandboxClient from .commands import Commands from .constants import DEFAULT_USER, ENVD_PORT, MCP_PORT +from .errors import SandboxError from .filesystem import Filesystem from .git import Git from .pty import Pty @@ -41,6 +44,8 @@ def __init__(self, client=None, **opts): self.opts = dict(opts) self.opts.pop('client', None) self.next_token = opts.get('nextToken') or opts.get('next_token') + self.opts.pop('nextToken', None) + self.opts.pop('next_token', None) self._has_next = True @property @@ -172,6 +177,14 @@ def update_info(self, info): self.envd_access_token ) self.envdAccessToken = self.envd_access_token + self.traffic_access_token = ( + get_info_value( + info, + 'trafficAccessToken', + 'traffic_access_token', + ) or self.traffic_access_token + ) + self.trafficAccessToken = self.traffic_access_token self.envd_version = ( get_info_value(info, 'envdVersion', 'envd_version') or self.envd_version @@ -186,7 +199,7 @@ def refresh_envd_token_if_needed(self): return self try: self.update_info(self.get_info()) - except Exception: + except SandboxError: pass return self @@ -309,14 +322,17 @@ def wait_for_ready(self, timeout=60, interval=1): request_timeout = interval if remaining is not None: request_timeout = min(interval, remaining) - response = self.client.session.get( - self.envd_url() + '/health', - timeout=request_timeout, - ) - if response.status_code >= 200 and response.status_code < 300: - return self + try: + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) + if response.status_code >= 200 and response.status_code < 300: + return self + except requests.RequestException: + pass if timeout is not None and time.time() - started >= timeout: - raise RuntimeError('Sandbox envd did not become ready') + raise SandboxError('Sandbox envd did not become ready') time.sleep(interval) waitForReady = wait_for_ready diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 47c67e09..873308a2 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -21,6 +21,7 @@ Sandbox, SandboxClient, SandboxError, + SandboxPaginator, Template, wait_for_file, wait_for_port, @@ -74,7 +75,10 @@ def get(self, url, **kwargs): 'kwargs': kwargs, })()) if self.responses: - return self.responses.pop(0) + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response return DummyResponse(body={}) @@ -245,6 +249,75 @@ def test_wait_for_ready_passes_request_timeout_to_health_check(): assert session.requests[0].kwargs['timeout'] == 2 +def test_wait_for_ready_ignores_startup_request_errors_until_ready(): + session = RecordingSession([ + requests.exceptions.ConnectionError('envd is starting'), + DummyResponse(200, {}), + ]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=10, interval=0) + + assert len(session.requests) == 2 + + +def test_wait_for_ready_raises_sandbox_error_on_timeout(): + session = RecordingSession([ + requests.exceptions.ConnectionError('envd is starting'), + ]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + with pytest.raises(SandboxError): + sandbox.wait_for_ready(timeout=0, interval=0) + + +def test_update_info_refreshes_traffic_access_token(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'trafficAccessToken': 'old-token', + }) + + sandbox.update_info({'trafficAccessToken': 'new-token'}) + + assert sandbox.traffic_access_token == 'new-token' + assert sandbox.trafficAccessToken == 'new-token' + + +class RecordingSandboxListClient(object): + def __init__(self): + self.calls = [] + + def list_sandboxes_v2(self, **opts): + self.calls.append(opts) + if len(self.calls) == 1: + return {'items': [], 'nextToken': 'next-page'} + return {'items': []} + + +def test_sandbox_paginator_does_not_reuse_initial_next_token(): + client = RecordingSandboxListClient() + paginator = SandboxPaginator(client=client, next_token='saved-page') + + paginator.next_items() + paginator.next_items() + + assert client.calls[0]['nextToken'] == 'saved-page' + assert client.calls[1]['nextToken'] == 'next-page' + + def test_is_running_matches_e2b_health_check_semantics(): running_session = RecordingSession([DummyResponse(200, {})]) running = Sandbox(client=SandboxClient( diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 1e920531..203f8057 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -39,6 +39,7 @@ class EnvdSession(object): def __init__(self): self.posts = [] self.requests = [] + self.empty_stream_paths = set() def post(self, url, data=None, headers=None, timeout=None, stream=False): if headers.get('Content-Type') == 'application/connect+json': @@ -54,6 +55,11 @@ def post(self, url, data=None, headers=None, timeout=None, stream=False): 'timeout': timeout, 'stream': stream, }) + if any(url.endswith(path) for path in self.empty_stream_paths): + return DummyResponse( + raw=b'', + content_type='application/connect+json', + ) if url.endswith('/process.Process/Start'): decoded = self.posts[-1]['data'] output_key = 'pty' if decoded.get('pty') else 'stdout' @@ -183,6 +189,18 @@ def test_commands_connect_returns_handle_for_running_process(): } +def test_commands_connect_handles_empty_event_stream(): + sandbox, session = sandbox_with_envd_session() + session.empty_stream_paths.add('/process.Process/Connect') + + handle = sandbox.commands.connect(12) + result = handle.wait() + + assert result.pid == 0 + assert result.exit_code == -1 + assert result.stdout == '' + + def test_commands_run_supports_e2b_callbacks_and_request_timeout(): sandbox, session = sandbox_with_envd_session() stdout = [] @@ -242,6 +260,23 @@ def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): assert session.posts[4]['url'].endswith('/process.Process/SendSignal') +def test_pty_create_and_connect_handle_empty_event_streams(): + sandbox, session = sandbox_with_envd_session() + session.empty_stream_paths.add('/process.Process/Start') + + handle = sandbox.pty.create(PtySize(rows=24, cols=80)) + result = handle.wait() + + assert result.pid == 0 + assert result.exit_code == -1 + assert result.stdout == '' + + session.empty_stream_paths = set(['/process.Process/Connect']) + connected = sandbox.pty.connect(12) + + assert connected.wait().stdout == '' + + def test_filesystem_uses_envd_rpc_and_signed_file_urls(): sandbox, session = sandbox_with_envd_session() diff --git a/tests/cases/test_services/test_sandbox/test_integration.py b/tests/cases/test_services/test_sandbox/test_integration.py index 1059f107..f4a647f2 100644 --- a/tests/cases/test_services/test_sandbox/test_integration.py +++ b/tests/cases/test_services/test_sandbox/test_integration.py @@ -4,7 +4,12 @@ import pytest -from qiniu.services.sandbox import PtySize, Sandbox, SandboxClient, SandboxError +from qiniu.services.sandbox import ( + PtySize, + Sandbox, + SandboxClient, + SandboxError, +) from qiniu.services.sandbox.config import load_dotenv_if_present @@ -51,7 +56,7 @@ def is_retryable_git_network_error(result): return result.exit_code != 0 and ( 'gnutls' in message or 'tls connection' in message or - 'unable to access' in message or + 'connection was non-properly terminated' in message or 'the remote end hung up unexpectedly' in message ) @@ -62,8 +67,11 @@ def assert_git_network_ok(step, run, attempts=5): result = run() if result.exit_code == 0: return result - if not is_retryable_git_network_error(result) or attempt == attempts - 1: + if not is_retryable_git_network_error(result): return assert_command_ok(step, result) + if attempt == attempts - 1: + pytest.skip('{0} skipped due to external Git network error: {1}' + .format(step, result.stderr or result.stdout)) time.sleep(attempt + 1) return assert_command_ok(step, result) @@ -144,7 +152,10 @@ def test_git_remote_push_when_credentials_are_configured(): assert_command_ok('git add', sandbox.git.add(repo_path, all=True)) assert_command_ok( 'git commit', - sandbox.git.commit(repo_path, 'test: qiniu python sdk remote push'), + sandbox.git.commit( + repo_path, + 'test: qiniu python sdk remote push', + ), ) assert_git_network_ok( 'git push', @@ -198,7 +209,10 @@ def test_runtime_commands_filesystem_and_git_helpers(): assert branches.current_branch == 'feature' assert sandbox.git.get_config(repo_path, 'user.name').exit_code == 0 assert sandbox.git.reset(repo_path, 'HEAD', mode='hard').exit_code == 0 - assert sandbox.git.restore(repo_path, paths=['README.md']).exit_code == 0 + assert sandbox.git.restore( + repo_path, + paths=['README.md'], + ).exit_code == 0 finally: if sandbox is not None: sandbox.kill() From edb32dd9b44ede34c7da0b1ee140e9eea4ab4f5e Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:28:53 +0800 Subject: [PATCH 08/45] fix(sandbox): harden reviewer feedback paths Encode Git credentials safely, handle wait/read/stdin edge cases, and serialize metrics IDs consistently. Add regression tests for the second PR feedback round. --- qiniu/services/sandbox/client.py | 2 +- qiniu/services/sandbox/commands.py | 7 ++++-- qiniu/services/sandbox/filesystem.py | 2 ++ qiniu/services/sandbox/git.py | 16 ++++++++++--- qiniu/services/sandbox/pty.py | 5 +++- qiniu/services/sandbox/sandbox.py | 4 ++-- .../test_services/test_sandbox/test_client.py | 23 +++++++++++++----- .../test_services/test_sandbox/test_envd.py | 24 ++++++++++++++++++- 8 files changed, 67 insertions(+), 16 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 37c5ad8e..67c26fba 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -337,7 +337,7 @@ def get_sandboxes_metrics(self, sandbox_ids): 'GET', '/sandboxes/metrics', params={ - 'sandbox_ids': ids}) + 'sandbox_ids': ','.join(ids)}) getSandboxesMetrics = get_sandboxes_metrics diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index da7bc0e2..1521abbf 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -54,7 +54,7 @@ def _decode_bytes(value): if isinstance(value, str): try: return base64.b64decode(value).decode('utf-8') - except (binascii.Error, TypeError): + except (binascii.Error, TypeError, ValueError): return value return str(value) @@ -255,7 +255,10 @@ def connect(self, pid, tag=None, user=None, timeout=None, def send_stdin(self, pid, data, user=None, timeout=None): if not isinstance(data, bytes): - data = str(data).encode('utf-8') + if hasattr(data, 'encode'): + data = data.encode('utf-8') + else: + data = str(data).encode('utf-8') return connect_rpc(self.sandbox, '/process.Process/SendInput', { 'process': {'selector': {'pid': pid}}, 'input': {'stdin': base64.b64encode(data).decode('ascii')}, diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 686458b9..bd1c918e 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -199,12 +199,14 @@ def _request_timeout(self, opts): def read(self, path, user=None, format='text', **opts): url = self.sandbox.download_url(path, user=user) + stream_mode = format == 'stream' response = raw_envd_request( self.sandbox, 'GET', url, headers=envd_headers(self.sandbox, user), timeout=self._request_timeout(opts), + stream=stream_mode, ) if format == 'stream': if hasattr(response, 'iter_content'): diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index ae12e9db..308a4e3a 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- try: - from urllib.parse import urlparse, urlunparse + from urllib.parse import quote, urlparse, urlunparse except ImportError: + from urllib import quote from urlparse import urlparse, urlunparse from .errors import ( @@ -268,8 +269,17 @@ def _with_credentials(url, username, password): if parsed.scheme not in ('http', 'https'): raise InvalidArgumentException( 'Only http(s) Git URLs support username/password credentials.') + host = parsed.hostname or '' + if ':' in host and not host.startswith('['): + host = '[{0}]'.format(host) + if parsed.port: + host = '{0}:{1}'.format(host, parsed.port) return urlunparse(parsed._replace( - netloc='{0}:{1}@{2}'.format(username, password, parsed.netloc) + netloc='{0}:{1}@{2}'.format( + quote(str(username), safe=''), + quote(str(password), safe=''), + host, + ) )) @@ -507,7 +517,7 @@ def dangerously_authenticate( 'password={3}\n\n' ).format(protocol, host, username, password) return self.commands.run( - 'printf {0} | git credential approve'.format( + "printf '%s' {0} | git credential approve".format( shell_quote(credential)), **opts ) diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py index 17e5191e..6a0dc17f 100644 --- a/qiniu/services/sandbox/pty.py +++ b/qiniu/services/sandbox/pty.py @@ -96,7 +96,10 @@ def connect(self, pid, user=None, timeout=None, throw_on_error=False): def send_stdin(self, pid, data, user=None, timeout=None): if not isinstance(data, bytes): - data = str(data).encode('utf-8') + if hasattr(data, 'encode'): + data = data.encode('utf-8') + else: + data = str(data).encode('utf-8') return connect_rpc(self.sandbox, '/process.Process/SendInput', { 'process': {'selector': {'pid': pid}}, 'input': {'pty': base64.b64encode(data).decode('ascii')}, diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 01ae3941..d0b8e000 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -317,6 +317,8 @@ def upload_url(self, path, **opts): def wait_for_ready(self, timeout=60, interval=1): started = time.time() while True: + if timeout is not None and time.time() - started >= timeout: + raise SandboxError('Sandbox envd did not become ready') elapsed = time.time() - started remaining = None if timeout is None else max(timeout - elapsed, 0) request_timeout = interval @@ -331,8 +333,6 @@ def wait_for_ready(self, timeout=60, interval=1): return self except requests.RequestException: pass - if timeout is not None and time.time() - started >= timeout: - raise SandboxError('Sandbox envd did not become ready') time.sleep(interval) waitForReady = wait_for_ready diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 873308a2..6da61795 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -336,6 +336,16 @@ def test_is_running_matches_e2b_health_check_semantics(): assert stopped.is_running() is False +def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(['sbx1', 'sbx2']) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx1,sbx2'] + + def test_template_builder_outputs_build_config(): template = ( Template() @@ -445,7 +455,7 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): assert commands.calls[0][0] == ( 'git config --global credential.helper store') assert commands.calls[1][0] == ( - "printf 'protocol=https\nhost=github.com\n" + "printf '%s' 'protocol=https\nhost=github.com\n" "username=git-user\npassword=secret-token\n\n' | " 'git credential approve' ) @@ -511,7 +521,7 @@ def test_git_push_with_credentials_sets_remote_url_temporarily(): commands.results = [ type('Result', (object,), { 'exit_code': 0, - 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stdout': 'https://old:old-token@github.com/qiniu/repo.git\n', 'stderr': '', 'error': '', })(), @@ -540,19 +550,20 @@ def test_git_push_with_credentials_sets_remote_url_temporarily(): '/repo', remote='origin', branch='main', - username='git-user', - password='secret-token', + username='git:user', + password='secret:%@token', request_timeout=7, ) assert commands.calls[0][0] == 'git remote get-url origin' assert commands.calls[1][0] == ( 'git remote set-url origin ' - 'https://git-user:secret-token@github.com/qiniu/repo.git' + 'https://git%3Auser:secret%3A%25%40token@github.com/qiniu/repo.git' ) assert commands.calls[2][0] == 'git push --set-upstream origin main' assert commands.calls[3][0] == ( - 'git remote set-url origin https://github.com/qiniu/repo.git' + 'git remote set-url origin ' + 'https://old:old-token@github.com/qiniu/repo.git' ) assert commands.calls[2][1]['request_timeout'] == 7 diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 203f8057..7ebfad07 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -16,6 +16,7 @@ encode_connect_envelope, iter_connect_envelopes, ) +from qiniu.services.sandbox.commands import command_result_from_events class DummyResponse(object): @@ -152,6 +153,14 @@ def test_iter_connect_envelopes_decodes_chunked_stream_frames(): ] +def test_command_event_decode_keeps_non_utf8_base64_text(): + result = command_result_from_events([{ + 'event': {'data': {'stdout': '////'}}, + }]) + + assert result.stdout == '////' + + def test_commands_run_posts_process_start_and_decodes_events(): sandbox, session = sandbox_with_envd_session() @@ -201,6 +210,15 @@ def test_commands_connect_handles_empty_event_stream(): assert result.stdout == '' +def test_commands_send_stdin_encodes_unicode_text(): + sandbox, session = sandbox_with_envd_session() + + sandbox.commands.send_stdin(12, u'你好') + + raw = session.posts[0]['data']['input']['stdin'] + assert base64.b64decode(raw).decode('utf-8') == u'你好' + + def test_commands_run_supports_e2b_callbacks_and_request_timeout(): sandbox, session = sandbox_with_envd_session() stdout = [] @@ -235,7 +253,7 @@ def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): sandbox, session = sandbox_with_envd_session() handle = sandbox.pty.create(PtySize(rows=24, cols=80), cwd='/workspace') - sandbox.pty.send_stdin(handle.pid, 'ls\n') + sandbox.pty.send_stdin(handle.pid, u'你好\n') sandbox.pty.resize(handle.pid, {'rows': 30, 'cols': 100}) connected = sandbox.pty.connect(handle.pid) assert sandbox.pty.kill(handle.pid) is True @@ -252,6 +270,9 @@ def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): } assert session.posts[1]['url'].endswith('/process.Process/SendInput') assert session.posts[1]['data']['input']['pty'] + assert base64.b64decode( + session.posts[1]['data']['input']['pty'] + ).decode('utf-8') == u'你好\n' assert session.posts[2]['url'].endswith('/process.Process/Update') assert session.posts[2]['data']['pty'] == { 'size': {'rows': 30, 'cols': 100}, @@ -314,6 +335,7 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): assert info.type == FileType.FILE assert entries[0].name == 'hello.txt' assert entries[0].type == FileType.FILE + assert session.requests[0]['kwargs']['stream'] is True assert b'hello' in session.requests[1]['kwargs']['data'] From d1f007d4da6b3a4c8720c0160a29dea4ecbb6a0b Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:39:49 +0800 Subject: [PATCH 09/45] fix(sandbox): tighten credential and compat paths Avoid credential exposure in git authentication, report remote restore failures, use Python 2 compatible text handling, and set a default client timeout. Cover the reviewer feedback with focused sandbox regression tests. --- qiniu/services/sandbox/client.py | 2 +- qiniu/services/sandbox/commands.py | 10 +- qiniu/services/sandbox/filesystem.py | 4 +- qiniu/services/sandbox/git.py | 44 ++++++- .../test_services/test_sandbox/test_client.py | 113 +++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 18 ++- 6 files changed, 173 insertions(+), 18 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 67c26fba..53b7b1b9 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -124,7 +124,7 @@ def __init__(self, endpoint=None, api_url=None, api_key=None, if self.mac is None and access_key and secret_key: self.mac = QiniuMacAuth(access_key, secret_key) self.session = session or requests.Session() - self.timeout = timeout + self.timeout = timeout if timeout is not None else 30 def _headers(self, auth_type=None): headers = {'Content-Type': 'application/json'} diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 1521abbf..a3a7da30 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -2,6 +2,8 @@ import base64 import binascii +from qiniu.compat import basestring + from .envd import connect_rpc, connect_stream_rpc from .errors import CommandExitError, SandboxError @@ -50,11 +52,11 @@ def _decode_bytes(value): if value is None: return '' if isinstance(value, list): - return bytearray(value).decode('utf-8') - if isinstance(value, str): + return bytearray(value).decode('utf-8', 'replace') + if isinstance(value, basestring): try: - return base64.b64decode(value).decode('utf-8') - except (binascii.Error, TypeError, ValueError): + return base64.b64decode(value).decode('utf-8', 'replace') + except (binascii.Error, TypeError): return value return str(value) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index bd1c918e..acd4675e 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -2,6 +2,8 @@ import time from io import IOBase, TextIOBase +from qiniu.compat import basestring + from .errors import InvalidArgumentException, SandboxError from .envd import connect_rpc, envd_headers, raw_envd_request @@ -137,7 +139,7 @@ def normalize_entry(entry, extended=False): def to_upload_body(data, encoding='utf-8'): if isinstance(data, bytes): return data - if isinstance(data, str): + if isinstance(data, basestring): return data.encode(encoding) if isinstance(data, TextIOBase): return data.read().encode(encoding) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 308a4e3a..cb1811a3 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import time + try: from urllib.parse import quote, urlparse, urlunparse except ImportError: @@ -9,6 +11,7 @@ GitAuthException, GitUpstreamException, InvalidArgumentException, + SandboxError, ) from .util import shell_quote @@ -334,6 +337,7 @@ def _with_remote_credentials(self, repo_path, remote, username, password, operation_error = None result = None + restore_result = None try: result = operation() except Exception as err: @@ -345,10 +349,21 @@ def _with_remote_credentials(self, repo_path, remote, username, password, shell_quote(remote), shell_quote(original_url), ], **opts) + if getattr(restore_result, 'exit_code', 0): + message = ( + 'Failed to restore original remote URL. Credentials may be ' + 'leaked in .git/config: {0}' + ).format( + getattr(restore_result, 'stderr', '') or + getattr(restore_result, 'stdout', '') or + getattr(restore_result, 'error', '') + ) + if operation_error: + message += '; original operation error: {0}'.format( + operation_error) + raise SandboxError(message) if operation_error: raise operation_error - if getattr(restore_result, 'exit_code', 0): - return restore_result return result def _raise_known_result_error(self, result, operation): @@ -516,11 +531,30 @@ def dangerously_authenticate( 'username={2}\n' 'password={3}\n\n' ).format(protocol, host, username, password) - return self.commands.run( - "printf '%s' {0} | git credential approve".format( - shell_quote(credential)), + sandbox = getattr(self.commands, 'sandbox', None) + filesystem = getattr(sandbox, 'files', None) + if filesystem is not None: + path = '/tmp/qiniu-git-credential-{0}'.format( + int(time.time() * 1000)) + filesystem.write(path, credential) + quoted_path = shell_quote(path) + script = ( + 'trap "rm -f {0}" EXIT; ' + 'git credential approve < {0}' + ).format(quoted_path) + return self.commands.run( + 'sh -c {0}'.format(shell_quote(script)), + **opts + ) + handle = self.commands.run( + 'git credential approve', + stdin=True, + background=True, **opts ) + self.commands.send_stdin(handle.pid, credential) + self.commands.close_stdin(handle.pid) + return handle.wait() dangerouslyAuthenticate = dangerously_authenticate diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 6da61795..1f063753 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -175,6 +175,18 @@ def test_sandbox_create_signature_matches_e2b_style(): assert body_of(session.requests[0])['templateID'] == 'python' +def test_client_uses_default_http_timeout(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + custom = SandboxClient( + api_key='api-key', + session=RecordingSession(), + timeout=12, + ) + + assert client.timeout == 30 + assert custom.timeout == 12 + + def test_sandbox_instance_lifecycle_methods_call_control_plane(): session = RecordingSession([ DummyResponse(204, None), @@ -401,6 +413,7 @@ def run(self, cmd, **opts): self.calls.append((cmd, opts)) result = self.results.pop(0) if self.results else type( 'Result', (object,), { + 'pid': 12, 'exit_code': 0, 'stdout': 'origin https://github.com/qiniu/repo.git\n', 'stderr': '', @@ -409,8 +422,21 @@ def run(self, cmd, **opts): if result.exit_code and opts.get('throw_on_error'): from qiniu.services.sandbox import CommandExitError raise CommandExitError(result) + if opts.get('background'): + return type('Handle', (object,), { + 'pid': getattr(result, 'pid', 12), + 'wait': lambda self: result, + })() return result + def send_stdin(self, pid, data): + self.calls.append(('send_stdin', {'pid': pid, 'data': data})) + return None + + def close_stdin(self, pid): + self.calls.append(('close_stdin', {'pid': pid})) + return None + def test_git_helpers_align_with_e2b_method_names(): commands = RecordingCommands() @@ -454,11 +480,48 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): assert commands.calls[0][0] == ( 'git config --global credential.helper store') - assert commands.calls[1][0] == ( - "printf '%s' 'protocol=https\nhost=github.com\n" - "username=git-user\npassword=secret-token\n\n' | " - 'git credential approve' + assert commands.calls[1][0] == 'git credential approve' + assert commands.calls[1][1]['stdin'] is True + assert commands.calls[1][1]['background'] is True + assert commands.calls[2] == ('send_stdin', { + 'pid': 12, + 'data': ( + 'protocol=https\nhost=github.com\n' + 'username=git-user\npassword=secret-token\n\n' + ), + }) + assert commands.calls[3] == ('close_stdin', {'pid': 12}) + + +def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): + class FakeFiles(object): + def __init__(self): + self.writes = [] + + def write(self, path, data, **opts): + self.writes.append((path, data, opts)) + return None + + commands = RecordingCommands() + files = FakeFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-%-token', + host='github.com', + protocol='https', + ) + + assert files.writes[0][1] == ( + 'protocol=https\nhost=github.com\n' + 'username=git-user\npassword=secret-%-token\n\n' ) + assert files.writes[0][2] == {} + assert commands.calls[1][0].startswith('sh -c ') + assert 'git credential approve' in commands.calls[1][0] + assert 'secret-%-token' not in commands.calls[1][0] def test_git_status_and_branches_return_structured_e2b_types(): @@ -665,6 +728,48 @@ def test_git_push_with_credentials_restores_remote_on_auth_failure(): ) +def test_git_push_reports_restore_failure_after_credentials_are_injected(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'config locked', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError) as err: + git.push( + '/repo', + remote='origin', + username='git-user', + password='bad-token', + ) + + assert 'Credentials may be leaked' in str(err.value) + assert 'config locked' in str(err.value) + + def test_git_helpers_accept_e2b_style_signatures(): commands = RecordingCommands() git = Git(commands) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 7ebfad07..07191efe 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -153,12 +153,24 @@ def test_iter_connect_envelopes_decodes_chunked_stream_frames(): ] -def test_command_event_decode_keeps_non_utf8_base64_text(): +def test_command_event_decode_handles_base64_and_non_utf8_output(): result = command_result_from_events([{ - 'event': {'data': {'stdout': '////'}}, + 'event': {'data': { + 'stdout': base64.b64encode(b'YWJj').decode('ascii'), + 'stderr': '////', + }}, }]) - assert result.stdout == '////' + assert result.stdout == 'YWJj' + assert result.stderr == u'\ufffd\ufffd\ufffd' + + +def test_filesystem_write_accepts_unicode_text(): + sandbox, session = sandbox_with_envd_session() + + sandbox.files.write('/tmp/unicode.txt', u'你好') + + assert u'你好'.encode('utf-8') in session.requests[0]['kwargs']['data'] def test_commands_run_posts_process_start_and_decodes_events(): From d93e9144053a77f11c90f57e7ee8bb6aa45b7263 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:50:49 +0800 Subject: [PATCH 10/45] fix(sandbox): address latest review feedback --- examples/sandbox_git.py | 4 +- qiniu/services/sandbox/envd.py | 48 ++++++++++++------- qiniu/services/sandbox/filesystem.py | 27 +---------- qiniu/services/sandbox/git.py | 15 +++++- qiniu/services/sandbox/sandbox.py | 11 +++-- qiniu/services/sandbox/template.py | 10 ++-- .../test_services/test_sandbox/test_client.py | 44 +++++++++++++++++ .../test_services/test_sandbox/test_envd.py | 39 +++++++++++++-- 8 files changed, 141 insertions(+), 57 deletions(-) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 6970a2b1..2a6923f2 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -37,7 +37,9 @@ def assert_git_network_ok(step, run, attempts=5): if result.exit_code == 0: print(step + ':', result.exit_code) return result - if not is_retryable_git_network_error(result) or attempt == attempts - 1: + if ( + not is_retryable_git_network_error(result) or + attempt == attempts - 1): return assert_git_ok(step, result) print('{0}: retry {1}/{2}'.format(step, attempt + 2, attempts)) time.sleep(attempt + 1) diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index e24f7bcf..0f5ebe64 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -66,19 +66,39 @@ def decode_connect_envelopes(data): payload = data[offset:offset + length] offset += length if flags & 2: - if payload: - error = json.loads(payload.decode('utf-8')).get('error') - if error: - raise SandboxError( - error.get('message') or 'Sandbox envd stream failed', - data=error, - ) - continue + if _is_connect_end(payload): + continue + _raise_connect_error(payload) if payload: messages.append(json.loads(payload.decode('utf-8'))) return messages +def _raise_connect_error(payload): + error = None + message = 'Sandbox envd stream failed' + if payload: + try: + data = json.loads(payload.decode('utf-8')) + if isinstance(data, dict): + error = data.get('error') + if isinstance(error, dict) and error.get('message'): + message = error.get('message') + except (TypeError, ValueError): + pass + raise SandboxError(message, data=error) + + +def _is_connect_end(payload): + if not payload: + return False + try: + data = json.loads(payload.decode('utf-8')) + except (TypeError, ValueError): + return False + return data == {} + + def iter_connect_envelopes(chunks): buffer = b'' for chunk in chunks: @@ -99,15 +119,9 @@ def iter_connect_envelopes(chunks): payload = buffer[5:5 + length] buffer = buffer[5 + length:] if flags & 2: - if payload: - error = json.loads(payload.decode('utf-8')).get('error') - if error: - raise SandboxError( - error.get('message') or - 'Sandbox envd stream failed', - data=error, - ) - continue + if _is_connect_end(payload): + continue + _raise_connect_error(payload) if payload: yield json.loads(payload.decode('utf-8')) if buffer: diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index acd4675e..875b5b2b 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import time from io import IOBase, TextIOBase from qiniu.compat import basestring @@ -242,19 +241,12 @@ def write(self, path, data, user=None, **opts): ) return self._format_write_response(response) - boundary = 'qiniu-sandbox-{0}'.format(int(time.time() * 1000)) response = raw_envd_request( self.sandbox, 'POST', url, - data=self._multipart_body(boundary, path, data), - headers=envd_headers( - self.sandbox, - user, - {'Content-Type': 'multipart/form-data; boundary={0}'.format( - boundary - )}, - ), + files={'file': (path, data)}, + headers=envd_headers(self.sandbox, user), timeout=self._request_timeout(opts), ) return self._format_write_response(response) @@ -267,21 +259,6 @@ def _format_write_response(self, response): return normalize_entry(data[0] if data else {}) return normalize_entry(data) - def _multipart_body(self, boundary, filename, data): - safe_filename = str(filename).replace('\\', '\\\\').replace('"', '\\"') - chunks = [ - '--{0}\r\n'.format(boundary).encode('utf-8'), - ( - 'Content-Disposition: form-data; name="file"; ' - 'filename="{0}"\r\n' - ).format(safe_filename).encode('utf-8'), - b'Content-Type: application/octet-stream\r\n\r\n', - data, - b'\r\n', - '--{0}--\r\n'.format(boundary).encode('utf-8'), - ] - return b''.join(chunks) - def write_files(self, files, user=None, **opts): result = [] for item in files or []: diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index cb1811a3..76cbcf5a 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import time +from qiniu.compat import basestring + try: from urllib.parse import quote, urlparse, urlunparse except ImportError: @@ -16,6 +18,14 @@ from .util import shell_quote +def _normalize_paths(paths): + if paths is None: + return None + if isinstance(paths, basestring): + return [paths] + return paths + + class GitFileStatus(object): def __init__(self, name, status, index_status, working_tree_status, staged, renamed_from=None): @@ -407,7 +417,7 @@ def add(self, repo_path, files=None, all=False, **opts): if all: args.append('--all') else: - for path in files or ['.']: + for path in _normalize_paths(files) or ['.']: args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) @@ -627,7 +637,8 @@ def restore(self, repo_path, paths=None, staged=False, source=None, args.append('--staged') if source: args.extend(['--source', shell_quote(source)]) - for path in paths or opts.get('files') or ['.']: + paths = paths if paths is not None else opts.get('files') + for path in _normalize_paths(paths) or ['.']: args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index d0b8e000..42874d94 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -338,10 +338,13 @@ def wait_for_ready(self, timeout=60, interval=1): waitForReady = wait_for_ready def is_running(self, request_timeout=None): - response = self.client.session.get( - self.envd_url() + '/health', - timeout=request_timeout, - ) + try: + response = self.client.session.get( + self.envd_url() + '/health', + timeout=request_timeout, + ) + except requests.RequestException: + return False if response.status_code == 502: return False if response.status_code >= 200 and response.status_code < 300: diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index b4d431b2..a1d466af 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import json +from .util import shell_quote + class ReadyCmd(object): def __init__(self, cmd): @@ -11,24 +13,26 @@ def get_cmd(self): def wait_for_port(port): + port = int(port) return ReadyCmd('ss -tuln | grep :{0}'.format(port)) def wait_for_url(url, status_code=200): + status_code = int(status_code) return ReadyCmd( 'curl -s -o /dev/null -w "%{{http_code}}" {0} | grep -q "{1}"'.format( - url, + shell_quote(url), status_code, ) ) def wait_for_process(process_name): - return ReadyCmd('pgrep {0} > /dev/null'.format(process_name)) + return ReadyCmd('pgrep {0} > /dev/null'.format(shell_quote(process_name))) def wait_for_file(filename): - return ReadyCmd('[ -f {0} ]'.format(filename)) + return ReadyCmd('[ -f {0} ]'.format(shell_quote(filename))) def wait_for_timeout(timeout): diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 1f063753..61a9d685 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -348,6 +348,17 @@ def test_is_running_matches_e2b_health_check_semantics(): assert stopped.is_running() is False +def test_is_running_returns_false_for_envd_request_errors(): + session = RecordingSession([requests.Timeout('timed out')]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={'sandboxID': 'sbx123', 'domain': 'example.test'}) + + assert sandbox.is_running(request_timeout=1) is False + assert session.requests[0].kwargs['timeout'] == 1 + + def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): session = RecordingSession([DummyResponse(200, {'metrics': []})]) client = SandboxClient(api_key='api-key', session=session) @@ -404,6 +415,26 @@ def test_template_ready_cmd_helpers_align_with_e2b(): assert template.to_dict()['readyCmd'] == 'ss -tuln | grep :8000' +def test_template_ready_cmd_helpers_quote_shell_inputs(): + assert wait_for_url( + 'http://localhost:3000/health; touch /tmp/pwn', + status_code='204', + ).get_cmd() == ( + 'curl -s -o /dev/null -w "%{http_code}" ' + "'http://localhost:3000/health; touch /tmp/pwn' | grep -q \"204\"" + ) + assert wait_for_process('nginx; touch /tmp/pwn').get_cmd() == ( + "pgrep 'nginx; touch /tmp/pwn' > /dev/null" + ) + assert wait_for_file('/tmp/ready; touch /tmp/pwn').get_cmd() == ( + "[ -f '/tmp/ready; touch /tmp/pwn' ]" + ) + with pytest.raises(ValueError): + wait_for_port('8000; touch /tmp/pwn') + with pytest.raises(ValueError): + wait_for_url('http://localhost:3000', status_code='200; true') + + class RecordingCommands(object): def __init__(self): self.calls = [] @@ -467,6 +498,19 @@ def test_git_helpers_align_with_e2b_method_names(): assert commands.calls[9][0] == 'git config --get user.name' +def test_git_add_and_restore_accept_single_string_path(): + commands = RecordingCommands() + git = Git(commands) + + git.add('/repo', files='README.md') + git.restore('/repo', paths='README.md') + git.restore('/repo', files='setup.py') + + assert commands.calls[0][0] == 'git add README.md' + assert commands.calls[1][0] == 'git restore README.md' + assert commands.calls[2][0] == 'git restore setup.py' + + def test_git_dangerously_authenticate_aligns_with_e2b(): commands = RecordingCommands() git = Git(commands) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 07191efe..44de6d0e 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -2,6 +2,7 @@ import base64 from io import BytesIO import json +import struct from qiniu.services.sandbox import ( EntryInfo, @@ -10,6 +11,7 @@ PtySize, Sandbox, SandboxClient, + SandboxError, ) from qiniu.services.sandbox.envd import ( decode_connect_envelopes, @@ -165,12 +167,35 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): assert result.stderr == u'\ufffd\ufffd\ufffd' +def test_connect_error_envelopes_raise_default_sandbox_error(): + empty_error = struct.pack('>BI', 2, 0) + missing_message = ( + struct.pack('>BI', 2, len(b'{"error":{}}')) + b'{"error":{}}' + ) + + for data in (empty_error, missing_message): + try: + decode_connect_envelopes(data) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream failed' + + try: + list(iter_connect_envelopes([empty_error])) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream failed' + + def test_filesystem_write_accepts_unicode_text(): sandbox, session = sandbox_with_envd_session() sandbox.files.write('/tmp/unicode.txt', u'你好') - assert u'你好'.encode('utf-8') in session.requests[0]['kwargs']['data'] + assert session.requests[0]['kwargs']['files']['file'] == ( + '/tmp/unicode.txt', + u'你好'.encode('utf-8'), + ) def test_commands_run_posts_process_start_and_decodes_events(): @@ -322,10 +347,11 @@ def test_filesystem_uses_envd_rpc_and_signed_file_urls(): assert session.requests[0]['method'] == 'GET' assert '/files?' in session.requests[0]['url'] assert session.requests[1]['method'] == 'POST' - assert session.requests[1]['kwargs']['headers']['Content-Type'].startswith( - 'multipart/form-data; boundary=' + assert 'Content-Type' not in session.requests[1]['kwargs']['headers'] + assert session.requests[1]['kwargs']['files']['file'] == ( + '/tmp/hello.txt', + b'hello', ) - assert b'name="file"' in session.requests[1]['kwargs']['data'] assert session.posts[0]['url'].endswith('/filesystem.Filesystem/Stat') assert session.posts[1]['url'].endswith('/filesystem.Filesystem/ListDir') @@ -348,7 +374,10 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): assert entries[0].name == 'hello.txt' assert entries[0].type == FileType.FILE assert session.requests[0]['kwargs']['stream'] is True - assert b'hello' in session.requests[1]['kwargs']['data'] + assert session.requests[1]['kwargs']['files']['file'] == ( + '/tmp/hello.txt', + b'hello', + ) def test_filesystem_read_write_pass_request_timeout_to_file_requests(): From bdba67575eea3ff7f5586899722d058736c5d6b6 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 20:58:44 +0800 Subject: [PATCH 11/45] fix(sandbox): harden latest review paths --- qiniu/services/sandbox/filesystem.py | 3 +- qiniu/services/sandbox/git.py | 13 +++++++ qiniu/services/sandbox/sandbox.py | 4 +-- .../test_services/test_sandbox/test_client.py | 35 ++++++++++++++++--- .../test_services/test_sandbox/test_envd.py | 6 ++-- 5 files changed, 51 insertions(+), 10 deletions(-) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 875b5b2b..3d8ee0b6 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -5,6 +5,7 @@ from .errors import InvalidArgumentException, SandboxError from .envd import connect_rpc, envd_headers, raw_envd_request +from .util import file_basename class FileType(object): @@ -245,7 +246,7 @@ def write(self, path, data, user=None, **opts): self.sandbox, 'POST', url, - files={'file': (path, data)}, + files={'file': (file_basename(path), data)}, headers=envd_headers(self.sandbox, user), timeout=self._request_timeout(opts), ) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 76cbcf5a..4daeb5c7 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -26,6 +26,9 @@ def _normalize_paths(paths): return paths +RESET_MODES = set(['soft', 'mixed', 'hard', 'merge', 'keep']) + + class GitFileStatus(object): def __init__(self, name, status, index_status, working_tree_status, staged, renamed_from=None): @@ -548,6 +551,12 @@ def dangerously_authenticate( int(time.time() * 1000)) filesystem.write(path, credential) quoted_path = shell_quote(path) + chmod_result = self.commands.run( + 'chmod 600 {0}'.format(quoted_path), + **opts + ) + if chmod_result.exit_code != 0: + return chmod_result script = ( 'trap "rm -f {0}" EXIT; ' 'git credential approve < {0}' @@ -625,6 +634,10 @@ def reset(self, repo_path, target=None, mode=None, **opts): args = ['reset'] mode = mode or opts.get('reset_type') or opts.get('resetType') if mode: + if mode not in RESET_MODES: + raise InvalidArgumentException( + 'Unsupported git reset mode: {0}'.format(mode) + ) args.append('--{0}'.format(mode)) if target: args.append(shell_quote(target)) diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 42874d94..e5ceb66c 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -321,9 +321,9 @@ def wait_for_ready(self, timeout=60, interval=1): raise SandboxError('Sandbox envd did not become ready') elapsed = time.time() - started remaining = None if timeout is None else max(timeout - elapsed, 0) - request_timeout = interval + request_timeout = 5 if remaining is not None: - request_timeout = min(interval, remaining) + request_timeout = min(5, remaining) try: response = self.client.session.get( self.envd_url() + '/health', diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 61a9d685..8d090ec6 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -16,6 +16,7 @@ GitBranches, GitStatus, Git, + InvalidArgumentException, KodoResource, ReadyCmd, Sandbox, @@ -258,7 +259,22 @@ def test_wait_for_ready_passes_request_timeout_to_health_check(): sandbox.wait_for_ready(timeout=10, interval=2) assert session.requests[0].url == sandbox.envd_url() + '/health' - assert session.requests[0].kwargs['timeout'] == 2 + assert session.requests[0].kwargs['timeout'] == 5 + + +def test_wait_for_ready_caps_request_timeout_to_remaining_timeout(): + session = RecordingSession([DummyResponse(200, {})]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + + sandbox.wait_for_ready(timeout=3, interval=2) + + assert session.requests[0].kwargs['timeout'] <= 3 def test_wait_for_ready_ignores_startup_request_errors_until_ready(): @@ -511,6 +527,16 @@ def test_git_add_and_restore_accept_single_string_path(): assert commands.calls[2][0] == 'git restore setup.py' +def test_git_reset_rejects_unsupported_mode(): + commands = RecordingCommands() + git = Git(commands) + + with pytest.raises(InvalidArgumentException): + git.reset('/repo', 'HEAD', mode='hard; touch /tmp/pwn') + + assert commands.calls == [] + + def test_git_dangerously_authenticate_aligns_with_e2b(): commands = RecordingCommands() git = Git(commands) @@ -563,9 +589,10 @@ def write(self, path, data, **opts): 'username=git-user\npassword=secret-%-token\n\n' ) assert files.writes[0][2] == {} - assert commands.calls[1][0].startswith('sh -c ') - assert 'git credential approve' in commands.calls[1][0] - assert 'secret-%-token' not in commands.calls[1][0] + assert commands.calls[1][0].startswith('chmod 600 /tmp/') + assert commands.calls[2][0].startswith('sh -c ') + assert 'git credential approve' in commands.calls[2][0] + assert 'secret-%-token' not in commands.calls[2][0] def test_git_status_and_branches_return_structured_e2b_types(): diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 44de6d0e..68d2fdc0 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -193,7 +193,7 @@ def test_filesystem_write_accepts_unicode_text(): sandbox.files.write('/tmp/unicode.txt', u'你好') assert session.requests[0]['kwargs']['files']['file'] == ( - '/tmp/unicode.txt', + 'unicode.txt', u'你好'.encode('utf-8'), ) @@ -349,7 +349,7 @@ def test_filesystem_uses_envd_rpc_and_signed_file_urls(): assert session.requests[1]['method'] == 'POST' assert 'Content-Type' not in session.requests[1]['kwargs']['headers'] assert session.requests[1]['kwargs']['files']['file'] == ( - '/tmp/hello.txt', + 'hello.txt', b'hello', ) assert session.posts[0]['url'].endswith('/filesystem.Filesystem/Stat') @@ -375,7 +375,7 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): assert entries[0].type == FileType.FILE assert session.requests[0]['kwargs']['stream'] is True assert session.requests[1]['kwargs']['files']['file'] == ( - '/tmp/hello.txt', + 'hello.txt', b'hello', ) From e3f513d4ae21048d1068fd0bc9e57746b11926cb Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:04:29 +0800 Subject: [PATCH 12/45] fix(sandbox): close latest review gaps --- qiniu/services/sandbox/config.py | 12 ++++ qiniu/services/sandbox/envd.py | 66 +++++++++++-------- qiniu/services/sandbox/filesystem.py | 11 ++-- qiniu/services/sandbox/git.py | 40 +++++++---- .../test_services/test_sandbox/test_client.py | 44 +++++++++++++ .../test_services/test_sandbox/test_config.py | 10 +++ .../test_services/test_sandbox/test_envd.py | 30 +++++++++ 7 files changed, 167 insertions(+), 46 deletions(-) diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py index f6ba9572..bf01b1d1 100644 --- a/qiniu/services/sandbox/config.py +++ b/qiniu/services/sandbox/config.py @@ -2,6 +2,8 @@ import io import os +from qiniu.compat import is_py2, str as text_type + from .client import SandboxClient @@ -29,9 +31,19 @@ def load_dotenv_if_present(*paths): ): value = value[1:-1] if key and key not in os.environ: + key, value = _native_env_pair(key, value) os.environ[key] = value +def _native_env_pair(key, value): + if is_py2: + if isinstance(key, text_type): + key = key.encode('utf-8') + if isinstance(value, text_type): + value = value.encode('utf-8') + return key, value + + def env(key, fallback=None): return os.getenv(key) or fallback diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index 0f5ebe64..aa6a81da 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -99,33 +99,40 @@ def _is_connect_end(payload): return data == {} -def iter_connect_envelopes(chunks): - buffer = b'' - for chunk in chunks: - if not chunk: - continue - if not isinstance(chunk, bytes): - chunk = chunk.encode('utf-8') - buffer += chunk - while len(buffer) >= 5: - flags, length = struct.unpack('>BI', buffer[:5]) - if length > MAX_CONNECT_ENVELOPE_BYTES: - raise SandboxError( - 'Sandbox envd stream envelope too large: {0}'.format( - length) - ) - if len(buffer) < 5 + length: - break - payload = buffer[5:5 + length] - buffer = buffer[5 + length:] - if flags & 2: - if _is_connect_end(payload): - continue - _raise_connect_error(payload) - if payload: - yield json.loads(payload.decode('utf-8')) - if buffer: - raise SandboxError('Sandbox envd stream truncated unexpectedly') +def iter_connect_envelopes(chunks, response=None): + try: + buffer = b'' + for chunk in chunks: + if not chunk: + continue + if not isinstance(chunk, bytes): + chunk = chunk.encode('utf-8') + buffer += chunk + while len(buffer) >= 5: + flags, length = struct.unpack('>BI', buffer[:5]) + if length > MAX_CONNECT_ENVELOPE_BYTES: + raise SandboxError( + 'Sandbox envd stream envelope too large: {0}'.format( + length) + ) + if len(buffer) < 5 + length: + break + payload = buffer[5:5 + length] + buffer = buffer[5 + length:] + if flags & 2: + if _is_connect_end(payload): + continue + _raise_connect_error(payload) + if payload: + yield json.loads(payload.decode('utf-8')) + if buffer: + raise SandboxError('Sandbox envd stream truncated unexpectedly') + finally: + if response is not None: + try: + response.close() + except Exception: + pass def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, @@ -153,7 +160,10 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, response.status_code), response, response.text, ) if stream and hasattr(response, 'iter_content'): - return iter_connect_envelopes(response.iter_content(chunk_size=8192)) + return iter_connect_envelopes( + response.iter_content(chunk_size=8192), + response=response, + ) content_type = response.headers.get('Content-Type', '') if 'application/connect+json' in content_type: diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 3d8ee0b6..84f321c4 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -141,10 +141,13 @@ def to_upload_body(data, encoding='utf-8'): return data if isinstance(data, basestring): return data.encode(encoding) - if isinstance(data, TextIOBase): - return data.read().encode(encoding) - if isinstance(data, IOBase): - return data.read() + if isinstance(data, (TextIOBase, IOBase)) or hasattr(data, 'read'): + content = data.read() + if isinstance(content, bytes): + return content + if isinstance(content, basestring): + return content.encode(encoding) + return content raise InvalidArgumentException( 'Unsupported data type for filesystem write: {0}'.format(type(data))) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 4daeb5c7..bd0e738f 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -29,6 +29,13 @@ def _normalize_paths(paths): RESET_MODES = set(['soft', 'mixed', 'hard', 'merge', 'keep']) +def _remove_credential_file(filesystem, path): + try: + filesystem.remove(path) + except Exception: + pass + + class GitFileStatus(object): def __init__(self, name, status, index_status, working_tree_status, staged, renamed_from=None): @@ -551,20 +558,25 @@ def dangerously_authenticate( int(time.time() * 1000)) filesystem.write(path, credential) quoted_path = shell_quote(path) - chmod_result = self.commands.run( - 'chmod 600 {0}'.format(quoted_path), - **opts - ) - if chmod_result.exit_code != 0: - return chmod_result - script = ( - 'trap "rm -f {0}" EXIT; ' - 'git credential approve < {0}' - ).format(quoted_path) - return self.commands.run( - 'sh -c {0}'.format(shell_quote(script)), - **opts - ) + try: + chmod_result = self.commands.run( + 'chmod 600 {0}'.format(quoted_path), + **opts + ) + if chmod_result.exit_code != 0: + _remove_credential_file(filesystem, path) + return chmod_result + script = ( + 'trap "rm -f {0}" EXIT; ' + 'git credential approve < {0}' + ).format(quoted_path) + return self.commands.run( + 'sh -c {0}'.format(shell_quote(script)), + **opts + ) + except Exception: + _remove_credential_file(filesystem, path) + raise handle = self.commands.run( 'git credential approve', stdin=True, diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 8d090ec6..a31f59b3 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -595,6 +595,50 @@ def write(self, path, data, **opts): assert 'secret-%-token' not in commands.calls[2][0] +def test_git_dangerously_authenticate_removes_temp_file_on_chmod_failure(): + class FakeFiles(object): + def __init__(self): + self.writes = [] + self.removes = [] + + def write(self, path, data, **opts): + self.writes.append((path, data, opts)) + return None + + def remove(self, path, **opts): + self.removes.append((path, opts)) + return None + + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'pid': 12, + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'pid': 12, + 'exit_code': 1, + 'stdout': '', + 'stderr': 'chmod failed', + 'error': 'chmod failed', + })(), + ] + files = FakeFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + git = Git(commands) + + result = git.dangerously_authenticate( + username='git-user', + password='secret-token', + ) + + assert result.exit_code == 1 + assert files.removes == [(files.writes[0][0], {})] + + def test_git_status_and_branches_return_structured_e2b_types(): commands = RecordingCommands() commands.results = [ diff --git a/tests/cases/test_services/test_sandbox/test_config.py b/tests/cases/test_services/test_sandbox/test_config.py index b995bd82..af794315 100644 --- a/tests/cases/test_services/test_sandbox/test_config.py +++ b/tests/cases/test_services/test_sandbox/test_config.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os +import qiniu.services.sandbox.config as sandbox_config from qiniu.services.sandbox.config import ( load_dotenv_if_present, sandbox_client, @@ -79,3 +80,12 @@ def test_sandbox_client_uses_loaded_env(tmpdir): os.environ.pop(key, None) else: os.environ[key] = value + + +def test_native_env_pair_encodes_unicode_on_python2(monkeypatch): + monkeypatch.setattr(sandbox_config, 'is_py2', True) + + key, value = sandbox_config._native_env_pair(u'KEY', u'值') + + assert key == b'KEY' + assert value == u'值'.encode('utf-8') diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 68d2fdc0..ecad44c0 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -29,6 +29,7 @@ def __init__(self, status_code=200, body=None, raw=None, body or {}).encode('utf-8') self.text = self.content.decode('utf-8') self.headers = {'Content-Type': content_type} + self.closed = False def json(self): return json.loads(self.content.decode('utf-8')) @@ -37,6 +38,9 @@ def iter_content(self, chunk_size=8192): del chunk_size yield self.content + def close(self): + self.closed = True + class EnvdSession(object): def __init__(self): @@ -155,6 +159,17 @@ def test_iter_connect_envelopes_decodes_chunked_stream_frames(): ] +def test_iter_connect_envelopes_closes_stream_response(): + raw = encode_connect_envelope({'event': {'start': {'pid': 12}}}) + response = DummyResponse(raw=raw, content_type='application/connect+json') + + assert list(iter_connect_envelopes( + response.iter_content(chunk_size=8192), + response=response, + )) == [{'event': {'start': {'pid': 12}}}] + assert response.closed is True + + def test_command_event_decode_handles_base64_and_non_utf8_output(): result = command_result_from_events([{ 'event': {'data': { @@ -198,6 +213,21 @@ def test_filesystem_write_accepts_unicode_text(): ) +def test_filesystem_write_accepts_duck_typed_file_like_objects(): + class FileLike(object): + def read(self): + return b'hello' + + sandbox, session = sandbox_with_envd_session() + + sandbox.files.write('/tmp/file-like.txt', FileLike()) + + assert session.requests[0]['kwargs']['files']['file'] == ( + 'file-like.txt', + b'hello', + ) + + def test_commands_run_posts_process_start_and_decodes_events(): sandbox, session = sandbox_with_envd_session() From 083ffdeadb8e6e54424a145ad2e910b28e199494 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:11:37 +0800 Subject: [PATCH 13/45] fix(sandbox): sign saved injection sandbox creates --- qiniu/services/sandbox/client.py | 18 +++++++++-- .../test_services/test_sandbox/test_client.py | 30 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 53b7b1b9..851c6e23 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -57,6 +57,18 @@ def _has_kodo_resource(resources): return False +def _has_saved_injection_rule(injections): + for injection in injections or []: + data = _normalize_injection(injection) + if not isinstance(data, dict): + continue + if data.get('type') == 'id': + return True + if data.get('ruleID') or data.get('rule_id') or data.get('id'): + return True + return False + + def _normalize_sandbox_create_options(template=None, **opts): body = {'templateID': ( opts.pop('templateID', None) or @@ -204,8 +216,10 @@ def list_sandboxes_v2(self, **opts): def create_sandbox(self, template=None, **opts): body = _normalize_sandbox_create_options(template, **opts) - auth_type = 'qiniu' if _has_kodo_resource( - opts.get('resources')) else None + auth_type = 'qiniu' if ( + _has_kodo_resource(body.get('resources')) or + _has_saved_injection_rule(body.get('injections')) + ) else None return self._request( 'POST', '/sandboxes', diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index a31f59b3..0aad9d2d 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -149,6 +149,36 @@ def test_create_with_kodo_resource_uses_qiniu_signature(): }] +def test_create_with_saved_injection_rule_requires_qiniu_credentials(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.create_sandbox(injections=[{ + 'type': 'id', + 'ruleID': 'rule-1', + }]) + + assert 'Qiniu AK/SK' in str(err.value) + + +def test_create_with_saved_injection_rule_uses_qiniu_signature(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient(access_key='ak', secret_key='sk', session=session) + + client.create_sandbox(injections=[{ + 'type': 'id', + 'ruleId': 'rule-1', + }]) + + req = session.requests[0] + assert req.headers['Authorization'].startswith('Qiniu ak:') + assert body_of(req)['injections'] == [{ + 'type': 'id', + 'ruleID': 'rule-1', + }] + + def test_sandbox_create_signature_matches_e2b_style(): session = RecordingSession([ DummyResponse(201, { From c8ec04d45d756a178832aa214a1aba486b558722 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:13:30 +0800 Subject: [PATCH 14/45] fix(sandbox): clean up latest review nits --- examples/sandbox_git.py | 10 ++++++---- qiniu/services/sandbox/client.py | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 2a6923f2..6b866bd7 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -76,12 +76,14 @@ def run_remote_push_demo(sandbox): sandbox.git.set_config( None, 'http.version', 'HTTP/1.1', global_config=True), ) + + def _clone_with_cleanup(): + sandbox.commands.run('rm -rf {0}'.format(repo_path)) + return sandbox.git.clone(repo_url, repo_path, depth=1) + assert_git_network_ok( 'git clone remote', - lambda: ( - sandbox.commands.run('rm -rf {0}'.format(repo_path)), - sandbox.git.clone(repo_url, repo_path, depth=1), - )[1], + _clone_with_cleanup, ) assert_git_ok( 'configure remote user', diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 851c6e23..97adc9cf 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -83,7 +83,8 @@ def _normalize_sandbox_create_options(template=None, **opts): 'secure', 'network', 'metadata', - 'mcp'): + 'mcp', + ): if opts.get(key) is not None: body[key] = opts.get(key) if opts.get('auto_pause') is not None: From 774d1b54eebc05e21be63c01ab374f1a8f7361c7 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:23:19 +0800 Subject: [PATCH 15/45] fix(sandbox): address expanded review feedback --- qiniu/services/sandbox/client.py | 11 +- qiniu/services/sandbox/commands.py | 13 ++- qiniu/services/sandbox/config.py | 24 +++- qiniu/services/sandbox/filesystem.py | 17 ++- qiniu/services/sandbox/git.py | 54 ++++++--- qiniu/services/sandbox/sandbox.py | 18 ++- qiniu/services/sandbox/template.py | 2 +- qiniu/services/sandbox/util.py | 11 -- .../test_services/test_sandbox/test_client.py | 108 +++++++++++++++++- .../test_services/test_sandbox/test_config.py | 8 ++ .../test_services/test_sandbox/test_envd.py | 11 +- 11 files changed, 215 insertions(+), 62 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 97adc9cf..978f69cf 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -180,7 +180,10 @@ def _request(self, method, path, params=None, body=_UNSET, auth=auth, ) prepared = self.session.prepare_request(request) - response = self.session.send(prepared, timeout=self.timeout) + try: + response = self.session.send(prepared, timeout=self.timeout) + except requests.RequestException as err: + raise SandboxError('Sandbox API request failed: {0}'.format(err)) if response.status_code < 200 or response.status_code >= 300: response_data = None try: @@ -273,9 +276,7 @@ def resume_sandbox(self, sandbox_id, **opts): resumeSandbox = resume_sandbox - def connect_sandbox(self, sandbox_id, timeout=15, **opts): - if timeout is None: - timeout = opts.pop('timeout', 15) + def connect_sandbox(self, sandbox_id, timeout=15): return self._request( 'POST', '/sandboxes/{0}/connect'.format(encode_path(sandbox_id)), @@ -398,6 +399,8 @@ def get_template(self, template_id, **opts): getTemplate = get_template def delete_template(self, template_id): + if not template_id: + raise SandboxError('template_id is required') return self._request( 'DELETE', '/templates/{0}'.format(encode_path(template_id)), diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index a3a7da30..b00de18f 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -155,9 +155,8 @@ def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, stdin=stdin, tag=tag, throw_on_error=throw_on_error, - timeout=( - request_timeout if request_timeout is not None else timeout - ), + timeout=timeout, + request_timeout=request_timeout, on_stdout=on_stdout, on_stderr=on_stderr, **opts @@ -165,8 +164,8 @@ def run(self, cmd, cwd=None, envs=None, user=None, stdin=False, return handle if background else handle.wait() def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, - tag=None, throw_on_error=False, timeout=None, on_stdout=None, - on_stderr=None, **opts): + tag=None, throw_on_error=False, timeout=None, + request_timeout=None, on_stdout=None, on_stderr=None, **opts): process = { 'cmd': '/bin/bash', 'args': ['-l', '-c', cmd], @@ -181,12 +180,14 @@ def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, } if tag: body['tag'] = tag + if timeout is not None: + body['timeout'] = timeout events = connect_stream_rpc( self.sandbox, '/process.Process/Start', body, user=user, - timeout=timeout, + timeout=request_timeout, stream=True, ) events = iter(events) diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py index bf01b1d1..367c3894 100644 --- a/qiniu/services/sandbox/config.py +++ b/qiniu/services/sandbox/config.py @@ -23,7 +23,7 @@ def load_dotenv_if_present(*paths): continue key, value = line.split('=', 1) key = key.strip() - value = value.strip() + value = _strip_inline_comment(value.strip()).strip() if ( len(value) >= 2 and value[0] == value[-1] and @@ -44,6 +44,28 @@ def _native_env_pair(key, value): return key, value +def _strip_inline_comment(value): + quote = None + escaped = False + for index, char in enumerate(value): + if escaped: + escaped = False + continue + if char == '\\': + escaped = True + continue + if quote: + if char == quote: + quote = None + continue + if char in ('"', "'"): + quote = char + continue + if char == '#' and (index == 0 or value[index - 1].isspace()): + return value[:index] + return value + + def env(key, fallback=None): return os.getenv(key) or fallback diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 84f321c4..8ad1e480 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -142,12 +142,7 @@ def to_upload_body(data, encoding='utf-8'): if isinstance(data, basestring): return data.encode(encoding) if isinstance(data, (TextIOBase, IOBase)) or hasattr(data, 'read'): - content = data.read() - if isinstance(content, bytes): - return content - if isinstance(content, basestring): - return content.encode(encoding) - return content + return data raise InvalidArgumentException( 'Unsupported data type for filesystem write: {0}'.format(type(data))) @@ -202,9 +197,11 @@ def _request_timeout(self, opts): return opts.get('request_timeout') return opts.get('timeout') - def read(self, path, user=None, format='text', **opts): + def read(self, path, user=None, fmt=None, **opts): + if fmt is None: + fmt = opts.pop('format', 'text') url = self.sandbox.download_url(path, user=user) - stream_mode = format == 'stream' + stream_mode = fmt == 'stream' response = raw_envd_request( self.sandbox, 'GET', @@ -213,12 +210,12 @@ def read(self, path, user=None, format='text', **opts): timeout=self._request_timeout(opts), stream=stream_mode, ) - if format == 'stream': + if fmt == 'stream': if hasattr(response, 'iter_content'): return response.iter_content(chunk_size=opts.get( 'chunk_size', 8192)) return iter([response.content]) - if format == 'bytes': + if fmt == 'bytes': return bytearray(response.content) return response.content.decode(opts.get('encoding', 'utf-8')) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index bd0e738f..9fce24e9 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -338,6 +338,9 @@ def _resolve_remote_name(self, repo_path, remote=None, **opts): ] if len(remotes) == 1: return remotes[0] + if len(remotes) == 0: + raise InvalidArgumentException( + 'No remotes found in the repository.') raise InvalidArgumentException( 'Remote is required when using username/password and the ' 'repository has multiple remotes.') @@ -386,7 +389,8 @@ def _with_remote_credentials(self, repo_path, remote, username, password, raise operation_error return result - def _raise_known_result_error(self, result, operation): + def _raise_known_result_error( + self, result, operation, throw_on_error=False): if result.exit_code: if _is_auth_failure(result): raise GitAuthException( @@ -396,6 +400,9 @@ def _raise_known_result_error(self, result, operation): raise GitUpstreamException( 'Git {0} failed because no upstream branch is configured.' .format(operation)) + if throw_on_error: + from qiniu.services.sandbox import CommandExitError + raise CommandExitError(result) def clone(self, repo_url, path=None, branch=None, depth=None, **opts): args = ['clone'] @@ -463,6 +470,7 @@ def configure_user(self, repo_path, name, email, **opts): def pull(self, repo_path, remote=None, branch=None, username=None, password=None, **opts): + throw_on_error = opts.pop('throw_on_error', False) if password and not username: raise GitAuthException( 'Git pull requires username when password is provided') @@ -482,18 +490,22 @@ def pull(self, repo_path, remote=None, branch=None, username=None, remote_name, username, password, - lambda: self._run_git(repo_path, args, **opts), + lambda: self._run_git( + repo_path, args, throw_on_error=False, **opts), **opts ) - self._raise_known_result_error(result, 'pull') + self._raise_known_result_error( + result, 'pull', throw_on_error=throw_on_error) return result - result = self._run_git(repo_path, args, **opts) - self._raise_known_result_error(result, 'pull') + result = self._run_git(repo_path, args, throw_on_error=False, **opts) + self._raise_known_result_error( + result, 'pull', throw_on_error=throw_on_error) return result def push(self, repo_path, remote=None, branch=None, set_upstream=True, username=None, password=None, **opts): + throw_on_error = opts.pop('throw_on_error', False) if password and not username: raise GitAuthException( 'Git push requires username when password is provided') @@ -515,14 +527,17 @@ def push(self, repo_path, remote=None, branch=None, set_upstream=True, remote_name, username, password, - lambda: self._run_git(repo_path, args, **opts), + lambda: self._run_git( + repo_path, args, throw_on_error=False, **opts), **opts ) - self._raise_known_result_error(result, 'push') + self._raise_known_result_error( + result, 'push', throw_on_error=throw_on_error) return result - result = self._run_git(repo_path, args, **opts) - self._raise_known_result_error(result, 'push') + result = self._run_git(repo_path, args, throw_on_error=False, **opts) + self._raise_known_result_error( + result, 'push', throw_on_error=throw_on_error) return result def dangerously_authenticate( @@ -570,10 +585,7 @@ def dangerously_authenticate( 'trap "rm -f {0}" EXIT; ' 'git credential approve < {0}' ).format(quoted_path) - return self.commands.run( - 'sh -c {0}'.format(shell_quote(script)), - **opts - ) + return self.commands.run(script, **opts) except Exception: _remove_credential_file(filesystem, path) raise @@ -590,12 +602,26 @@ def dangerously_authenticate( dangerouslyAuthenticate = dangerously_authenticate def remote_add(self, repo_path, name, url, **opts): - return self._run_git(repo_path, [ + fetch = opts.pop('fetch', False) + overwrite = opts.pop('overwrite', False) + if overwrite: + self._run_git(repo_path, [ + 'remote', + 'remove', + shell_quote(name), + ], **opts) + result = self._run_git(repo_path, [ 'remote', 'add', shell_quote(name), shell_quote(url), ], **opts) + if result.exit_code != 0 or not fetch: + return result + return self._run_git(repo_path, [ + 'fetch', + shell_quote(name), + ], **opts) remoteAdd = remote_add diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index e5ceb66c..f9b88b6a 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -18,6 +18,12 @@ ) +try: + from time import monotonic as _monotonic_time +except ImportError: + _monotonic_time = time.time + + class _ConnectDescriptor(object): def __get__(self, obj, cls): if obj is None: @@ -29,9 +35,9 @@ def class_connect(sandbox_id, client=None, timeout=15, **opts): return sandbox return class_connect - def instance_connect(timeout=15, **opts): + def instance_connect(timeout=15): info = obj.client.connect_sandbox( - obj.sandbox_id, timeout=timeout, **opts) + obj.sandbox_id, timeout=timeout) obj.update_info(info) obj.refresh_envd_token_if_needed() return obj @@ -315,15 +321,15 @@ def upload_url(self, path, **opts): UploadURL = upload_url def wait_for_ready(self, timeout=60, interval=1): - started = time.time() + started = _monotonic_time() while True: - if timeout is not None and time.time() - started >= timeout: + if timeout is not None and _monotonic_time() - started >= timeout: raise SandboxError('Sandbox envd did not become ready') - elapsed = time.time() - started + elapsed = _monotonic_time() - started remaining = None if timeout is None else max(timeout - elapsed, 0) request_timeout = 5 if remaining is not None: - request_timeout = min(5, remaining) + request_timeout = max(0.1, min(5, remaining)) try: response = self.client.session.get( self.envd_url() + '/health', diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index a1d466af..9d6b02e3 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -20,7 +20,7 @@ def wait_for_port(port): def wait_for_url(url, status_code=200): status_code = int(status_code) return ReadyCmd( - 'curl -s -o /dev/null -w "%{{http_code}}" {0} | grep -q "{1}"'.format( + '[ "$(curl -s -o /dev/null -w "%{{http_code}}" {0})" = "{1}" ]'.format( shell_quote(url), status_code, ) diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index 5b1400f9..4cfb8660 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -49,17 +49,6 @@ def json_dumps(data): return std_json.dumps(data, separators=(',', ':')) -def timeout_seconds_from_options(opts): - opts = opts or {} - if opts.get('timeout') is not None: - return opts.get('timeout') - if opts.get('timeout_ms') is not None: - return int((opts.get('timeout_ms') + 999) / 1000) - if opts.get('timeoutMs') is not None: - return int((opts.get('timeoutMs') + 999) / 1000) - return None - - def basic_auth(user=None): user = user or DEFAULT_USER raw = ('{0}:'.format(user)).encode('utf-8') diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 0aad9d2d..233d17bc 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -10,6 +10,7 @@ from urlparse import parse_qs, urlparse from qiniu.services.sandbox import ( + CommandExitError, DEFAULT_ENDPOINT, ENVD_PORT, GitAuthException, @@ -66,7 +67,10 @@ def __init__(self, responses=None): def send(self, request, **kwargs): self.requests.append(request) if self.responses: - return self.responses.pop(0) + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response return DummyResponse(body={}) def get(self, url, **kwargs): @@ -218,6 +222,18 @@ def test_client_uses_default_http_timeout(): assert custom.timeout == 12 +def test_client_wraps_request_exceptions_in_sandbox_error(): + client = SandboxClient( + api_key='api-key', + session=RecordingSession([requests.Timeout('timed out')])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert 'Sandbox API request failed' in str(err.value) + assert 'timed out' in str(err.value) + + def test_sandbox_instance_lifecycle_methods_call_control_plane(): session = RecordingSession([ DummyResponse(204, None), @@ -251,6 +267,24 @@ def test_sandbox_instance_lifecycle_methods_call_control_plane(): assert body_of(session.requests[2]) == {'timeout': 45} +def test_connect_sandbox_uses_timeout_body_only(): + session = RecordingSession([DummyResponse(200, {'sandboxID': 'sbx123'})]) + client = SandboxClient(api_key='api-key', session=session) + + client.connect_sandbox('sbx123', timeout=9) + + assert body_of(session.requests[0]) == {'timeout': 9} + + +def test_delete_template_requires_template_id(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.delete_template(None) + + assert 'template_id is required' in str(err.value) + + def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): sandbox = Sandbox(info={ 'sandboxID': 'sbx123', @@ -444,8 +478,8 @@ def test_template_ready_cmd_helpers_align_with_e2b(): 'http://localhost:3000/health', status_code=204, ).get_cmd() == ( - 'curl -s -o /dev/null -w "%{http_code}" ' - 'http://localhost:3000/health | grep -q "204"' + '[ "$(curl -s -o /dev/null -w "%{http_code}" ' + 'http://localhost:3000/health)" = "204" ]' ) assert wait_for_process('nginx').get_cmd() == 'pgrep nginx > /dev/null' assert wait_for_file('/tmp/ready').get_cmd() == '[ -f /tmp/ready ]' @@ -466,8 +500,8 @@ def test_template_ready_cmd_helpers_quote_shell_inputs(): 'http://localhost:3000/health; touch /tmp/pwn', status_code='204', ).get_cmd() == ( - 'curl -s -o /dev/null -w "%{http_code}" ' - "'http://localhost:3000/health; touch /tmp/pwn' | grep -q \"204\"" + '[ "$(curl -s -o /dev/null -w "%{http_code}" ' + '\'http://localhost:3000/health; touch /tmp/pwn\')" = "204" ]' ) assert wait_for_process('nginx; touch /tmp/pwn').get_cmd() == ( "pgrep 'nginx; touch /tmp/pwn' > /dev/null" @@ -544,6 +578,24 @@ def test_git_helpers_align_with_e2b_method_names(): assert commands.calls[9][0] == 'git config --get user.name' +def test_git_remote_add_supports_overwrite_and_fetch_options(): + commands = RecordingCommands() + git = Git(commands) + + git.remote_add( + '/repo', + 'origin', + 'https://github.com/qiniu/repo.git', + overwrite=True, + fetch=True, + ) + + assert commands.calls[0][0] == 'git remote remove origin' + assert commands.calls[1][0] == ( + 'git remote add origin https://github.com/qiniu/repo.git') + assert commands.calls[2][0] == 'git fetch origin' + + def test_git_add_and_restore_accept_single_string_path(): commands = RecordingCommands() git = Git(commands) @@ -620,7 +672,7 @@ def write(self, path, data, **opts): ) assert files.writes[0][2] == {} assert commands.calls[1][0].startswith('chmod 600 /tmp/') - assert commands.calls[2][0].startswith('sh -c ') + assert commands.calls[2][0].startswith('trap "rm -f /tmp/') assert 'git credential approve' in commands.calls[2][0] assert 'secret-%-token' not in commands.calls[2][0] @@ -724,6 +776,50 @@ def test_git_push_maps_auth_failure_to_e2b_exception(): git.push('/repo') +def test_git_credential_remote_requires_existing_remote(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(InvalidArgumentException) as err: + git.push('/repo', username='git-user', password='secret') + + assert 'No remotes found' in str(err.value) + + +def test_git_push_maps_known_errors_before_throw_on_error(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: Authentication failed', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(GitAuthException): + git.push('/repo', throw_on_error=True) + + +def test_git_push_respects_throw_on_error_for_unknown_errors(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'unexpected failure', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(CommandExitError): + git.push('/repo', throw_on_error=True) + + def test_git_push_with_credentials_sets_remote_url_temporarily(): commands = RecordingCommands() commands.results = [ diff --git a/tests/cases/test_services/test_sandbox/test_config.py b/tests/cases/test_services/test_sandbox/test_config.py index af794315..116fa154 100644 --- a/tests/cases/test_services/test_sandbox/test_config.py +++ b/tests/cases/test_services/test_sandbox/test_config.py @@ -14,6 +14,8 @@ def test_load_dotenv_if_present_reads_key_values_without_overwriting(tmpdir): '\n'.join([ 'QINIU_SANDBOX_API_KEY=from-file', 'QINIU_SANDBOX_TEMPLATE="python:3.11"', + 'QINIU_SANDBOX_COMMENTED=value # inline comment', + 'QINIU_SANDBOX_HASH="value # not comment"', 'EXISTING=value-from-file', '# ignored', '', @@ -25,18 +27,24 @@ def test_load_dotenv_if_present_reads_key_values_without_overwriting(tmpdir): for key in ( 'QINIU_SANDBOX_API_KEY', 'QINIU_SANDBOX_TEMPLATE', + 'QINIU_SANDBOX_COMMENTED', + 'QINIU_SANDBOX_HASH', 'EXISTING', ) } try: os.environ.pop('QINIU_SANDBOX_API_KEY', None) os.environ.pop('QINIU_SANDBOX_TEMPLATE', None) + os.environ.pop('QINIU_SANDBOX_COMMENTED', None) + os.environ.pop('QINIU_SANDBOX_HASH', None) os.environ['EXISTING'] = 'already-set' load_dotenv_if_present(str(dotenv)) assert os.environ['QINIU_SANDBOX_API_KEY'] == 'from-file' assert os.environ['QINIU_SANDBOX_TEMPLATE'] == 'python:3.11' + assert os.environ['QINIU_SANDBOX_COMMENTED'] == 'value' + assert os.environ['QINIU_SANDBOX_HASH'] == 'value # not comment' assert os.environ['EXISTING'] == 'already-set' finally: for key, value in old.items(): diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index ecad44c0..5c6fb52a 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -219,12 +219,13 @@ def read(self): return b'hello' sandbox, session = sandbox_with_envd_session() + file_like = FileLike() - sandbox.files.write('/tmp/file-like.txt', FileLike()) + sandbox.files.write('/tmp/file-like.txt', file_like) assert session.requests[0]['kwargs']['files']['file'] == ( 'file-like.txt', - b'hello', + file_like, ) @@ -295,12 +296,14 @@ def test_commands_run_supports_e2b_callbacks_and_request_timeout(): 'echo hello', on_stdout=stdout.append, on_stderr=stderr.append, + timeout=3, request_timeout=7, ) assert result.stdout == 'hello\n' assert stdout == ['hello\n'] assert stderr == [] + assert session.posts[0]['data']['timeout'] == 3 assert session.posts[0]['timeout'] == 7 @@ -404,10 +407,12 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): assert entries[0].name == 'hello.txt' assert entries[0].type == FileType.FILE assert session.requests[0]['kwargs']['stream'] is True + written_data = session.requests[1]['kwargs']['files']['file'][1] assert session.requests[1]['kwargs']['files']['file'] == ( 'hello.txt', - b'hello', + written_data, ) + assert isinstance(written_data, BytesIO) def test_filesystem_read_write_pass_request_timeout_to_file_requests(): From a1d437757ef8f5b0d72183e5283a6ba5bb1b41bf Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:33:41 +0800 Subject: [PATCH 16/45] fix(sandbox): tighten python compatibility feedback --- qiniu/services/sandbox/client.py | 15 ++++++++--- qiniu/services/sandbox/envd.py | 2 ++ qiniu/services/sandbox/filesystem.py | 14 ++++++++-- qiniu/services/sandbox/git.py | 10 ------- qiniu/services/sandbox/sandbox.py | 2 +- qiniu/services/sandbox/util.py | 27 ++++++++++++++----- .../test_services/test_sandbox/test_client.py | 26 ++++++++++++++++++ .../test_services/test_sandbox/test_envd.py | 19 +++++++++++-- 8 files changed, 89 insertions(+), 26 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 978f69cf..8af2c453 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -5,6 +5,7 @@ import requests from qiniu.auth import QiniuMacAuth, QiniuMacRequestsAuth +from qiniu.compat import basestring from .constants import DEFAULT_TEMPLATE from .errors import SandboxError, TemplateBuildError @@ -20,6 +21,12 @@ _UNSET = object() +try: + from time import monotonic as _monotonic_time +except ImportError: + _monotonic_time = time.time + + def _to_dict(value): if hasattr(value, 'to_dict'): return value.to_dict() @@ -197,7 +204,7 @@ def _request(self, method, path, params=None, body=_UNSET, response_data, dict) and response_data.get('message'): message += ': {0}'.format(response_data.get('message')) - elif isinstance(response_data, str) and response_data: + elif isinstance(response_data, basestring) and response_data: message += ': {0}'.format(response_data) raise SandboxError(message, response, response_data) if empty: @@ -339,7 +346,7 @@ def get_sandboxes_metrics(self, sandbox_ids): values = [values] ids = [] for value in values: - if isinstance(value, str): + if isinstance(value, basestring): ids.append(value) elif isinstance(value, dict): ids.append(value.get('sandboxId') or value.get( @@ -535,7 +542,7 @@ def delete_injection_rule(self, rule_id): deleteInjectionRule = delete_injection_rule def wait_for_build(self, template_id, build_id, interval=1, timeout=60): - start = time.time() + start = _monotonic_time() while True: info = self.get_template_build_status(template_id, build_id) if info and info.get('status') in ('ready', 'error'): @@ -551,7 +558,7 @@ def wait_for_build(self, template_id, build_id, interval=1, timeout=60): ) raise TemplateBuildError(message, data=info) return info - if time.time() - start >= timeout: + if _monotonic_time() - start >= timeout: raise SandboxError('Sandbox template build polling timed out') time.sleep(interval) diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index aa6a81da..c3b6582c 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -71,6 +71,8 @@ def decode_connect_envelopes(data): _raise_connect_error(payload) if payload: messages.append(json.loads(payload.decode('utf-8'))) + if offset < len(data): + raise SandboxError('Sandbox envd stream truncated unexpectedly') return messages diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 8ad1e480..6ecf778e 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -147,6 +147,14 @@ def to_upload_body(data, encoding='utf-8'): 'Unsupported data type for filesystem write: {0}'.format(type(data))) +def _response_stream(response, chunk_size=8192): + try: + for chunk in response.iter_content(chunk_size=chunk_size): + yield chunk + finally: + response.close() + + class WatchHandle(object): def __init__(self, filesystem, watcher_id): self.filesystem = filesystem @@ -212,8 +220,10 @@ def read(self, path, user=None, fmt=None, **opts): ) if fmt == 'stream': if hasattr(response, 'iter_content'): - return response.iter_content(chunk_size=opts.get( - 'chunk_size', 8192)) + return _response_stream( + response, + chunk_size=opts.get('chunk_size', 8192), + ) return iter([response.content]) if fmt == 'bytes': return bytearray(response.content) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 9fce24e9..cab4ced4 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -358,13 +358,8 @@ def _with_remote_credentials(self, repo_path, remote, username, password, if getattr(set_result, 'exit_code', 0): return set_result - operation_error = None - result = None - restore_result = None try: result = operation() - except Exception as err: - operation_error = err finally: restore_result = self._run_git(repo_path, [ 'remote', @@ -381,12 +376,7 @@ def _with_remote_credentials(self, repo_path, remote, username, password, getattr(restore_result, 'stdout', '') or getattr(restore_result, 'error', '') ) - if operation_error: - message += '; original operation error: {0}'.format( - operation_error) raise SandboxError(message) - if operation_error: - raise operation_error return result def _raise_known_result_error( diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index f9b88b6a..6da74faa 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -294,7 +294,7 @@ def file_url( } if self.envd_access_token: expiration = opts.get('signatureExpiration', signature_expiration) - if expiration < 1000000000: + if expiration is not None and expiration < 1000000000: expiration = utc_timestamp_after(expiration) query['signature'] = file_signature( path, diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index 4cfb8660..06642993 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -6,7 +6,8 @@ import posixpath import time -from qiniu.compat import urlencode, urlparse +from qiniu.compat import bytes as bytes_type +from qiniu.compat import is_py2, str as text_type, urlencode, urlparse from .constants import DEFAULT_ENDPOINT, DEFAULT_USER @@ -23,12 +24,16 @@ def normalize_endpoint(endpoint=None): def encode_path(value): - from qiniu.compat import is_py2 if is_py2: from urllib import quote + if isinstance(value, text_type): + value = value.encode('utf-8') + else: + value = str(value) else: from urllib.parse import quote - return quote(str(value), safe='') + value = str(value) + return quote(value, safe='') def append_query(path, query=None): @@ -56,10 +61,18 @@ def basic_auth(user=None): def file_signature(path, operation, user, access_token, expiration): - raw = '{0}:{1}:{2}:{3}:{4}'.format( - path, operation, user, access_token, expiration - ) - return 'v1_{0}'.format(hashlib.sha256(raw.encode('utf-8')).hexdigest()) + components = [path, operation, user, access_token, expiration] + raw = b':'.join([_to_utf8_bytes(component) + for component in components]) + return 'v1_{0}'.format(hashlib.sha256(raw).hexdigest()) + + +def _to_utf8_bytes(value): + if isinstance(value, bytes_type): + return value + if isinstance(value, text_type): + return value.encode('utf-8') + return str(value).encode('utf-8') def file_basename(path): diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 233d17bc..70db1dc0 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -31,6 +31,7 @@ wait_for_timeout, wait_for_url, ) +from qiniu.services.sandbox.util import encode_path, file_signature class DummyResponse(object): @@ -119,6 +120,17 @@ def test_client_uses_default_endpoint_and_api_key_headers(): } +def test_util_helpers_encode_unicode_values_safely(): + assert encode_path(u'目录/文件.txt') + assert file_signature( + u'/tmp/文件.txt', + 'read', + u'用户', + u'token', + 1893456000, + ).startswith('v1_') + + def test_create_with_kodo_resource_requires_qiniu_credentials(): client = SandboxClient(api_key='api-key', session=RecordingSession()) @@ -310,6 +322,20 @@ def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): assert query['signature'][0] +def test_file_url_accepts_none_signature_expiration(): + sandbox = Sandbox(info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + 'envdAccessToken': 'token', + }) + + url = sandbox.download_url('/tmp/hello.txt', signatureExpiration=None) + query = parse_qs(urlparse(url).query) + + assert query['signature'][0] + assert 'signature_expiration' not in query + + def test_wait_for_ready_passes_request_timeout_to_health_check(): session = RecordingSession([DummyResponse(200, {})]) sandbox = Sandbox(client=SandboxClient( diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 5c6fb52a..a1c5527e 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -120,12 +120,16 @@ def post(self, url, data=None, headers=None, timeout=None, stream=False): def request(self, method, url, **kwargs): self.requests.append({'method': method, 'url': url, 'kwargs': kwargs}) if method == 'GET': - return DummyResponse(raw=b'hello') - return DummyResponse(body=[{ + response = DummyResponse(raw=b'hello') + self.requests[-1]['response'] = response + return response + response = DummyResponse(body=[{ 'name': 'hello.txt', 'type': 'FILE', 'path': '/tmp/hello.txt', }]) + self.requests[-1]['response'] = response + return response def sandbox_with_envd_session(): @@ -159,6 +163,16 @@ def test_iter_connect_envelopes_decodes_chunked_stream_frames(): ] +def test_decode_connect_envelopes_rejects_trailing_bytes(): + raw = encode_connect_envelope({'event': {'start': {'pid': 12}}}) + b'x' + + try: + decode_connect_envelopes(raw) + assert False, 'expected SandboxError' + except SandboxError as err: + assert str(err) == 'Sandbox envd stream truncated unexpectedly' + + def test_iter_connect_envelopes_closes_stream_response(): raw = encode_connect_envelope({'event': {'start': {'pid': 12}}}) response = DummyResponse(raw=raw, content_type='application/connect+json') @@ -398,6 +412,7 @@ def test_filesystem_returns_e2b_style_entry_objects_and_streams(): entries = sandbox.files.list('/tmp') assert b''.join(stream) == b'hello' + assert session.requests[0]['response'].closed is True assert written.name == 'hello.txt' assert written.path == '/tmp/hello.txt' assert written.type == FileType.FILE From b9a22f65007bbcc8a780c9a4266b02ecac43eb7a Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:41:41 +0800 Subject: [PATCH 17/45] fix(sandbox): harden git credential cleanup --- qiniu/services/sandbox/client.py | 4 ++ qiniu/services/sandbox/commands.py | 2 +- qiniu/services/sandbox/git.py | 29 ++++++---- .../test_services/test_sandbox/test_client.py | 53 +++++++++++++++++++ 4 files changed, 77 insertions(+), 11 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 8af2c453..a9a298f1 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -338,10 +338,14 @@ def get_sandbox_metrics(self, sandbox_id, **opts): getMetrics = get_sandbox_metrics def get_sandboxes_metrics(self, sandbox_ids): + if not sandbox_ids: + raise SandboxError('At least one sandbox ID must be provided') values = sandbox_ids if isinstance(sandbox_ids, dict): values = sandbox_ids.get( 'sandbox_ids') or sandbox_ids.get('sandboxIDs') + if values is None: + raise SandboxError('At least one sandbox ID must be provided') if not isinstance(values, (list, tuple)): values = [values] ids = [] diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index b00de18f..dba2d89e 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -284,6 +284,6 @@ def kill(self, pid, user=None, timeout=None): }, user=user, timeout=timeout) return True except SandboxError as err: - if err.status_code == 404: + if getattr(err, 'status_code', None) == 404: return False raise diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index cab4ced4..5cca954c 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -358,15 +358,19 @@ def _with_remote_credentials(self, repo_path, remote, username, password, if getattr(set_result, 'exit_code', 0): return set_result + operation_error = None try: result = operation() - finally: - restore_result = self._run_git(repo_path, [ - 'remote', - 'set-url', - shell_quote(remote), - shell_quote(original_url), - ], **opts) + except Exception as err: + operation_error = err + result = None + + restore_result = self._run_git(repo_path, [ + 'remote', + 'set-url', + shell_quote(remote), + shell_quote(original_url), + ], **opts) if getattr(restore_result, 'exit_code', 0): message = ( 'Failed to restore original remote URL. Credentials may be ' @@ -376,7 +380,14 @@ def _with_remote_credentials(self, repo_path, remote, username, password, getattr(restore_result, 'stdout', '') or getattr(restore_result, 'error', '') ) + if operation_error is not None: + message = '{0}. Original operation failed with: {1}'.format( + message, + operation_error, + ) raise SandboxError(message) + if operation_error is not None: + raise operation_error return result def _raise_known_result_error( @@ -569,16 +580,14 @@ def dangerously_authenticate( **opts ) if chmod_result.exit_code != 0: - _remove_credential_file(filesystem, path) return chmod_result script = ( 'trap "rm -f {0}" EXIT; ' 'git credential approve < {0}' ).format(quoted_path) return self.commands.run(script, **opts) - except Exception: + finally: _remove_credential_file(filesystem, path) - raise handle = self.commands.run( 'git credential approve', stdin=True, diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 70db1dc0..78c81272 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -475,6 +475,15 @@ def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): assert query['sandbox_ids'] == ['sbx1,sbx2'] +def test_get_sandboxes_metrics_rejects_empty_dict_values(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError): + client.get_sandboxes_metrics({}) + with pytest.raises(SandboxError): + client.get_sandboxes_metrics({'sandboxIDs': None}) + + def test_template_builder_outputs_build_config(): template = ( Template() @@ -675,11 +684,16 @@ def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): class FakeFiles(object): def __init__(self): self.writes = [] + self.removes = [] def write(self, path, data, **opts): self.writes.append((path, data, opts)) return None + def remove(self, path, **opts): + self.removes.append((path, opts)) + return None + commands = RecordingCommands() files = FakeFiles() commands.sandbox = type('Sandbox', (object,), {'files': files})() @@ -701,6 +715,7 @@ def write(self, path, data, **opts): assert commands.calls[2][0].startswith('trap "rm -f /tmp/') assert 'git credential approve' in commands.calls[2][0] assert 'secret-%-token' not in commands.calls[2][0] + assert files.removes == [(files.writes[0][0], {})] def test_git_dangerously_authenticate_removes_temp_file_on_chmod_failure(): @@ -1037,6 +1052,44 @@ def test_git_push_reports_restore_failure_after_credentials_are_injected(): assert 'config locked' in str(err.value) +def test_git_push_reports_restore_failure_after_operation_exception(): + commands = RecordingCommands() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'config locked', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError) as err: + git._with_remote_credentials( + '/repo', + 'origin', + 'git-user', + 'bad-token', + lambda: (_ for _ in ()).throw(SandboxError('rpc timed out')), + ) + + assert 'Credentials may be leaked' in str(err.value) + assert 'config locked' in str(err.value) + assert 'rpc timed out' in str(err.value) + + def test_git_helpers_accept_e2b_style_signatures(): commands = RecordingCommands() git = Git(commands) From 0aeef161018916414b236e1f85282172ed6d5f56 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:46:07 +0800 Subject: [PATCH 18/45] fix(sandbox): handle bytes command output --- qiniu/services/sandbox/client.py | 2 ++ qiniu/services/sandbox/commands.py | 4 +++- .../test_services/test_sandbox/test_client.py | 16 ++++++++++++++++ .../test_services/test_sandbox/test_envd.py | 12 ++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index a9a298f1..b5da4c65 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -45,6 +45,8 @@ def _normalize_injection(injection): data['base_url'] = data.pop('baseUrl') if 'ruleId' in data and 'ruleID' not in data: data['ruleID'] = data.pop('ruleId') + if 'rule_id' in data and 'ruleID' not in data: + data['ruleID'] = data.pop('rule_id') return data diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index dba2d89e..065e7e25 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -53,10 +53,12 @@ def _decode_bytes(value): return '' if isinstance(value, list): return bytearray(value).decode('utf-8', 'replace') - if isinstance(value, basestring): + if isinstance(value, (bytes, basestring)): try: return base64.b64decode(value).decode('utf-8', 'replace') except (binascii.Error, TypeError): + if isinstance(value, bytes): + return value.decode('utf-8', 'replace') return value return str(value) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 78c81272..7b6c90ef 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -195,6 +195,22 @@ def test_create_with_saved_injection_rule_uses_qiniu_signature(): }] +def test_create_with_saved_injection_rule_accepts_snake_case_rule_id(): + session = RecordingSession( + [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) + client = SandboxClient(access_key='ak', secret_key='sk', session=session) + + client.create_sandbox(injections=[{ + 'type': 'id', + 'rule_id': 'rule-1', + }]) + + assert body_of(session.requests[0])['injections'] == [{ + 'type': 'id', + 'ruleID': 'rule-1', + }] + + def test_sandbox_create_signature_matches_e2b_style(): session = RecordingSession([ DummyResponse(201, { diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index a1c5527e..14952b19 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -196,6 +196,18 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): assert result.stderr == u'\ufffd\ufffd\ufffd' +def test_command_event_decode_handles_bytes_values(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': base64.b64encode(b'abc'), + 'stderr': b'plain bytes', + }}, + }]) + + assert result.stdout == 'abc' + assert result.stderr == 'plain bytes' + + def test_connect_error_envelopes_raise_default_sandbox_error(): empty_error = struct.pack('>BI', 2, 0) missing_message = ( From 7674645f3fc46a835fe548a9efb91d0a5fcb5736 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:47:59 +0800 Subject: [PATCH 19/45] fix(sandbox): support api key env fallbacks --- qiniu/services/sandbox/client.py | 10 +++++++++- qiniu/services/sandbox/config.py | 18 +++++++++++++++++- .../test_services/test_sandbox/test_client.py | 17 +++++++++++++++++ .../test_services/test_sandbox/test_config.py | 13 +++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index b5da4c65..9b588b1b 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -131,6 +131,14 @@ def _normalize_list_options(opts): return opts +def _sandbox_api_key_from_env(): + return ( + os.getenv('QINIU_SANDBOX_API_KEY') or + os.getenv('QINIU_API_KEY') or + os.getenv('E2B_API_KEY') + ) + + class SandboxClient(object): def __init__(self, endpoint=None, api_url=None, api_key=None, access_token=None, mac=None, access_key=None, @@ -139,7 +147,7 @@ def __init__(self, endpoint=None, api_url=None, api_key=None, raise SandboxError( 'Both access_key and secret_key must be provided') self.endpoint = normalize_endpoint(endpoint or api_url) - self.api_key = api_key or os.getenv('QINIU_SANDBOX_API_KEY') + self.api_key = api_key or _sandbox_api_key_from_env() self.access_token = access_token or os.getenv( 'QINIU_SANDBOX_ACCESS_TOKEN') self.mac = mac diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py index 367c3894..6d192bb9 100644 --- a/qiniu/services/sandbox/config.py +++ b/qiniu/services/sandbox/config.py @@ -70,6 +70,14 @@ def env(key, fallback=None): return os.getenv(key) or fallback +def first_env(*keys): + for key in keys: + value = os.getenv(key) + if value: + return value + return None + + def required_env(key): value = os.getenv(key) if value: @@ -82,7 +90,15 @@ def sandbox_endpoint(): def sandbox_api_key(): - return required_env('QINIU_SANDBOX_API_KEY') + value = first_env( + 'QINIU_SANDBOX_API_KEY', + 'QINIU_API_KEY', + 'E2B_API_KEY', + ) + if value: + return value + raise RuntimeError( + 'Please set QINIU_SANDBOX_API_KEY, QINIU_API_KEY, or E2B_API_KEY') def sandbox_template(): diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 7b6c90ef..37111753 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -120,6 +120,23 @@ def test_client_uses_default_endpoint_and_api_key_headers(): } +def test_client_reads_documented_api_key_env_fallbacks(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_API_KEY', raising=False) + monkeypatch.delenv('QINIU_API_KEY', raising=False) + monkeypatch.delenv('E2B_API_KEY', raising=False) + + monkeypatch.setenv('QINIU_API_KEY', 'qiniu-api-key') + assert SandboxClient(session=RecordingSession()).api_key == 'qiniu-api-key' + + monkeypatch.delenv('QINIU_API_KEY') + monkeypatch.setenv('E2B_API_KEY', 'e2b-api-key') + assert SandboxClient(session=RecordingSession()).api_key == 'e2b-api-key' + + monkeypatch.setenv('QINIU_SANDBOX_API_KEY', 'sandbox-api-key') + assert SandboxClient(session=RecordingSession()).api_key == ( + 'sandbox-api-key') + + def test_util_helpers_encode_unicode_values_safely(): assert encode_path(u'目录/文件.txt') assert file_signature( diff --git a/tests/cases/test_services/test_sandbox/test_config.py b/tests/cases/test_services/test_sandbox/test_config.py index 116fa154..35f1191c 100644 --- a/tests/cases/test_services/test_sandbox/test_config.py +++ b/tests/cases/test_services/test_sandbox/test_config.py @@ -90,6 +90,19 @@ def test_sandbox_client_uses_loaded_env(tmpdir): os.environ[key] = value +def test_sandbox_client_uses_documented_api_key_env_fallbacks(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_API_KEY', raising=False) + monkeypatch.delenv('QINIU_API_KEY', raising=False) + monkeypatch.delenv('E2B_API_KEY', raising=False) + + monkeypatch.setenv('QINIU_API_KEY', 'qiniu-api-key') + assert sandbox_client().api_key == 'qiniu-api-key' + + monkeypatch.delenv('QINIU_API_KEY') + monkeypatch.setenv('E2B_API_KEY', 'e2b-api-key') + assert sandbox_client().api_key == 'e2b-api-key' + + def test_native_env_pair_encodes_unicode_on_python2(monkeypatch): monkeypatch.setattr(sandbox_config, 'is_py2', True) From 97d39676e1dae503cb7aad3c77f2f7be88b53b14 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:52:36 +0800 Subject: [PATCH 20/45] fix(sandbox): harden latest review paths --- qiniu/services/sandbox/filesystem.py | 7 +++ qiniu/services/sandbox/git.py | 10 ++++- qiniu/services/sandbox/sandbox.py | 8 +++- qiniu/services/sandbox/util.py | 2 + .../test_services/test_sandbox/test_client.py | 43 +++++++++++++++++-- .../test_services/test_sandbox/test_envd.py | 11 +++++ 6 files changed, 75 insertions(+), 6 deletions(-) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 6ecf778e..e6822dc9 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -162,6 +162,13 @@ def __init__(self, filesystem, watcher_id): self.watcherID = watcher_id self._closed = False + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + return False + def get_new_events(self, user=None, timeout=None): if self._closed: raise SandboxError('The watcher is already stopped') diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 5cca954c..2655d307 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -570,7 +570,15 @@ def dangerously_authenticate( sandbox = getattr(self.commands, 'sandbox', None) filesystem = getattr(sandbox, 'files', None) if filesystem is not None: - path = '/tmp/qiniu-git-credential-{0}'.format( + temp_dir = '/tmp/qiniu-git-auth' + prepare_result = self.commands.run( + 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), + **opts + ) + if prepare_result.exit_code != 0: + return prepare_result + path = '{0}/qiniu-git-credential-{1}'.format( + temp_dir, int(time.time() * 1000)) filesystem.write(path, credential) quoted_path = shell_quote(path) diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 6da74faa..2318dfe9 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -30,7 +30,12 @@ def __get__(self, obj, cls): def class_connect(sandbox_id, client=None, timeout=15, **opts): client = client or SandboxClient(**opts) info = client.connect_sandbox(sandbox_id, timeout=timeout) - sandbox = cls(client=client, info=info, sandbox_id=sandbox_id) + sandbox = cls( + client=client, + info=info, + sandbox_id=sandbox_id, + **opts + ) sandbox.refresh_envd_token_if_needed() return sandbox return class_connect @@ -83,6 +88,7 @@ class Sandbox(object): def __init__(self, client=None, info=None, sandbox_id=None, sandboxID=None, envd_url=None, envdAccessToken=None, **client_opts): + envd_url = envd_url or client_opts.pop('envdUrl', None) self.client = client or SandboxClient(**client_opts) self.info = info or {} self.sandbox_id = ( diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index 06642993..f8522604 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -61,6 +61,8 @@ def basic_auth(user=None): def file_signature(path, operation, user, access_token, expiration): + if expiration is None: + expiration = '' components = [path, operation, user, access_token, expiration] raw = b':'.join([_to_utf8_bytes(component) for component in components]) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 37111753..58860526 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -321,6 +321,22 @@ def test_connect_sandbox_uses_timeout_body_only(): assert body_of(session.requests[0]) == {'timeout': 9} +def test_sandbox_connect_forwards_envd_options_to_instance(): + session = RecordingSession([DummyResponse(200, {'sandboxID': 'sbx123'})]) + client = SandboxClient(api_key='api-key', session=session) + + sandbox = Sandbox.connect( + 'sbx123', + client=client, + envd_url='http://envd.local', + envdAccessToken='envd-token', + ) + + assert sandbox.envd_url() == 'http://envd.local' + assert sandbox.envdAccessToken == 'envd-token' + assert body_of(session.requests[0]) == {'timeout': 15} + + def test_delete_template_requires_template_id(): client = SandboxClient(api_key='api-key', session=RecordingSession()) @@ -366,6 +382,13 @@ def test_file_url_accepts_none_signature_expiration(): query = parse_qs(urlparse(url).query) assert query['signature'][0] + assert query['signature'][0] == file_signature( + '/tmp/hello.txt', + 'read', + 'user', + 'token', + '', + ) assert 'signature_expiration' not in query @@ -743,11 +766,16 @@ def remove(self, path, **opts): 'protocol=https\nhost=github.com\n' 'username=git-user\npassword=secret-%-token\n\n' ) + assert files.writes[0][0].startswith( + '/tmp/qiniu-git-auth/qiniu-git-credential-') assert files.writes[0][2] == {} - assert commands.calls[1][0].startswith('chmod 600 /tmp/') - assert commands.calls[2][0].startswith('trap "rm -f /tmp/') - assert 'git credential approve' in commands.calls[2][0] - assert 'secret-%-token' not in commands.calls[2][0] + assert commands.calls[1][0] == 'install -d -m 700 /tmp/qiniu-git-auth' + assert commands.calls[2][0].startswith( + 'chmod 600 /tmp/qiniu-git-auth/qiniu-git-credential-') + assert commands.calls[3][0].startswith( + 'trap "rm -f /tmp/qiniu-git-auth/qiniu-git-credential-') + assert 'git credential approve' in commands.calls[3][0] + assert 'secret-%-token' not in commands.calls[3][0] assert files.removes == [(files.writes[0][0], {})] @@ -774,6 +802,13 @@ def remove(self, path, **opts): 'stderr': '', 'error': '', })(), + type('Result', (object,), { + 'pid': 12, + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), type('Result', (object,), { 'pid': 12, 'exit_code': 1, diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 14952b19..6c918485 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -491,3 +491,14 @@ def test_filesystem_watch_dir_returns_e2b_style_watch_handle(): assert session.posts[1]['data'] == {'watcherId': 'watch-1'} assert session.posts[2]['url'].endswith( '/filesystem.Filesystem/RemoveWatcher') + + +def test_filesystem_watch_dir_handle_supports_context_manager(): + sandbox, session = sandbox_with_envd_session() + + with sandbox.files.watch_dir('/tmp') as handle: + assert handle.watcher_id == 'watch-1' + + assert handle._closed is True + assert session.posts[1]['url'].endswith( + '/filesystem.Filesystem/RemoveWatcher') From 8627b2bb3234f42e5f842f4b38f6d8792c7aaec7 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:54:55 +0800 Subject: [PATCH 21/45] fix(sandbox): wrap envd transport errors --- examples/sandbox_resources.py | 11 +++-- qiniu/services/sandbox/envd.py | 33 ++++++++++----- .../test_services/test_sandbox/test_envd.py | 42 +++++++++++++++++++ 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py index b403cc84..a1a70114 100644 --- a/examples/sandbox_resources.py +++ b/examples/sandbox_resources.py @@ -6,6 +6,7 @@ KodoResource, SandboxError, ) +from qiniu.services.sandbox.util import shell_quote from sandbox_common import ( cleanup_sandbox, create_sandbox, @@ -58,7 +59,7 @@ def run_git_resource_example(): try: print('Git resource sandbox:', sandbox.sandbox_id) print(sandbox.commands.run('ls -la {0} | head -20'.format( - mount_path + shell_quote(mount_path) )).stdout) finally: cleanup_sandbox(sandbox) @@ -90,11 +91,15 @@ def run_kodo_resource_example(): try: print('Kodo resource sandbox:', sandbox.sandbox_id) print(sandbox.commands.run('ls -la {0} | head -20'.format( - mount_path + shell_quote(mount_path) )).stdout) test_path = mount_path + '/qiniu-python-sdk-resource-test.txt' result = sandbox.commands.run( - 'sh -c "echo qiniu-python-sdk > {0} && cat {0}"'.format(test_path) + 'sh -c {0}'.format(shell_quote( + 'echo qiniu-python-sdk > {0} && cat {0}'.format( + shell_quote(test_path) + ) + )) ) if result.exit_code != 0: raise RuntimeError(result.stderr or result.stdout) diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index c3b6582c..a6864ef6 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -2,6 +2,8 @@ import json import struct +import requests + from .constants import DEFAULT_USER from .errors import SandboxError from .util import basic_auth @@ -24,12 +26,15 @@ def envd_headers(sandbox, user=None, extra=None): def connect_rpc(sandbox, procedure, body=None, user=None, timeout=None): url = sandbox.envd_url() + procedure headers = envd_headers(sandbox, user, {'Content-Type': 'application/json'}) - response = sandbox.client.session.post( - url, - data=json.dumps(body or {}, separators=(',', ':')), - headers=headers, - timeout=timeout, - ) + try: + response = sandbox.client.session.post( + url, + data=json.dumps(body or {}, separators=(',', ':')), + headers=headers, + timeout=timeout, + ) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) if response.status_code < 200 or response.status_code >= 300: raise SandboxError( 'Sandbox envd request failed with status {0}'.format( @@ -152,10 +157,13 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, if stream: request_opts['stream'] = True try: - response = sandbox.client.session.post(url, **request_opts) - except TypeError: - request_opts.pop('stream', None) - response = sandbox.client.session.post(url, **request_opts) + try: + response = sandbox.client.session.post(url, **request_opts) + except TypeError: + request_opts.pop('stream', None) + response = sandbox.client.session.post(url, **request_opts) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) if response.status_code < 200 or response.status_code >= 300: raise SandboxError( 'Sandbox envd request failed with status {0}'.format( @@ -184,7 +192,10 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, def raw_envd_request(sandbox, method, url, **kwargs): - response = sandbox.client.session.request(method, url, **kwargs) + try: + response = sandbox.client.session.request(method, url, **kwargs) + except requests.RequestException as err: + raise SandboxError('Sandbox envd request failed: {0}'.format(err)) if response.status_code < 200 or response.status_code >= 300: raise SandboxError( 'Sandbox envd request failed with status {0}'.format( diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 6c918485..67876a3d 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -4,6 +4,9 @@ import json import struct +import pytest +import requests + from qiniu.services.sandbox import ( EntryInfo, FileType, @@ -47,8 +50,12 @@ def __init__(self): self.posts = [] self.requests = [] self.empty_stream_paths = set() + self.post_exception = None + self.request_exception = None def post(self, url, data=None, headers=None, timeout=None, stream=False): + if self.post_exception is not None: + raise self.post_exception if headers.get('Content-Type') == 'application/connect+json': decoded = decode_connect_envelopes(data)[0] else: @@ -118,6 +125,8 @@ def post(self, url, data=None, headers=None, timeout=None, stream=False): return DummyResponse(body={'result': {}}) def request(self, method, url, **kwargs): + if self.request_exception is not None: + raise self.request_exception self.requests.append({'method': method, 'url': url, 'kwargs': kwargs}) if method == 'GET': response = DummyResponse(raw=b'hello') @@ -228,6 +237,39 @@ def test_connect_error_envelopes_raise_default_sandbox_error(): assert str(err) == 'Sandbox envd stream failed' +def test_connect_rpc_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.post_exception = requests.Timeout('timed out') + + with pytest.raises(SandboxError) as err: + sandbox.commands.run('echo hello') + + assert 'Sandbox envd request failed' in str(err.value) + assert 'timed out' in str(err.value) + + +def test_connect_stream_rpc_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.post_exception = requests.ConnectionError('connection reset') + + with pytest.raises(SandboxError) as err: + sandbox.commands.connect(12) + + assert 'Sandbox envd request failed' in str(err.value) + assert 'connection reset' in str(err.value) + + +def test_raw_envd_request_wraps_transport_errors(): + sandbox, session = sandbox_with_envd_session() + session.request_exception = requests.Timeout('timed out') + + with pytest.raises(SandboxError) as err: + sandbox.files.read_text('/tmp/hello.txt') + + assert 'Sandbox envd request failed' in str(err.value) + assert 'timed out' in str(err.value) + + def test_filesystem_write_accepts_unicode_text(): sandbox, session = sandbox_with_envd_session() From 89209eb871899b2c6b36b48f4c6ec94ae4fa8285 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 21:58:53 +0800 Subject: [PATCH 22/45] fix(sandbox): avoid git credential command leaks --- examples/sandbox_git.py | 1 - qiniu/services/sandbox/git.py | 104 +++++++----- .../test_services/test_sandbox/test_client.py | 158 ++++++------------ 3 files changed, 121 insertions(+), 142 deletions(-) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 6b866bd7..0c9336cc 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -43,7 +43,6 @@ def assert_git_network_ok(step, run, attempts=5): return assert_git_ok(step, result) print('{0}: retry {1}/{2}'.format(step, attempt + 2, attempts)) time.sleep(attempt + 1) - return assert_git_ok(step, result) def remote_git_config(): diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 2655d307..86d3cdec 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -306,6 +306,17 @@ def _with_credentials(url, username, password): )) +def _askpass_script(): + return ( + '#!/bin/sh\n' + 'case "$1" in\n' + '*Username*) printf "%s\\n" "$GIT_USERNAME" ;;\n' + '*Password*) printf "%s\\n" "$GIT_PASSWORD" ;;\n' + '*) printf "\\n" ;;\n' + 'esac\n' + ) + + class Git(object): def __init__(self, commands): self.commands = commands @@ -348,44 +359,51 @@ def _resolve_remote_name(self, repo_path, remote=None, **opts): def _with_remote_credentials(self, repo_path, remote, username, password, operation, **opts): original_url = self._get_remote_url(repo_path, remote, **opts) - credential_url = _with_credentials(original_url, username, password) - set_result = self._run_git(repo_path, [ - 'remote', - 'set-url', - shell_quote(remote), - shell_quote(credential_url), - ], **opts) - if getattr(set_result, 'exit_code', 0): - return set_result + _with_credentials(original_url, username, password) + sandbox = getattr(self.commands, 'sandbox', None) + filesystem = getattr(sandbox, 'files', None) + if filesystem is None: + raise SandboxError( + 'Sandbox filesystem is required for credentialed Git ' + 'operations.') + temp_dir = '/tmp/qiniu-git-auth' + prepare_result = self.commands.run( + 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), + **opts + ) + if getattr(prepare_result, 'exit_code', 0): + return prepare_result + askpass_path = '{0}/qiniu-git-askpass-{1}'.format( + temp_dir, + int(time.time() * 1000), + ) + filesystem.write(askpass_path, _askpass_script()) + chmod_result = self.commands.run( + 'chmod 700 {0}'.format(shell_quote(askpass_path)), + **opts + ) + if getattr(chmod_result, 'exit_code', 0): + _remove_credential_file(filesystem, askpass_path) + return chmod_result + + auth_opts = dict(opts) + envs = dict(auth_opts.get('envs') or {}) + envs.update({ + 'GIT_ASKPASS': askpass_path, + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_USERNAME': username, + 'GIT_PASSWORD': password, + }) + auth_opts['envs'] = envs operation_error = None try: - result = operation() + result = operation(auth_opts) except Exception as err: operation_error = err result = None - - restore_result = self._run_git(repo_path, [ - 'remote', - 'set-url', - shell_quote(remote), - shell_quote(original_url), - ], **opts) - if getattr(restore_result, 'exit_code', 0): - message = ( - 'Failed to restore original remote URL. Credentials may be ' - 'leaked in .git/config: {0}' - ).format( - getattr(restore_result, 'stderr', '') or - getattr(restore_result, 'stdout', '') or - getattr(restore_result, 'error', '') - ) - if operation_error is not None: - message = '{0}. Original operation failed with: {1}'.format( - message, - operation_error, - ) - raise SandboxError(message) + finally: + _remove_credential_file(filesystem, askpass_path) if operation_error is not None: raise operation_error return result @@ -491,8 +509,8 @@ def pull(self, repo_path, remote=None, branch=None, username=None, remote_name, username, password, - lambda: self._run_git( - repo_path, args, throw_on_error=False, **opts), + lambda auth_opts: self._run_git( + repo_path, args, throw_on_error=False, **auth_opts), **opts ) self._raise_known_result_error( @@ -528,8 +546,8 @@ def push(self, repo_path, remote=None, branch=None, set_upstream=True, remote_name, username, password, - lambda: self._run_git( - repo_path, args, throw_on_error=False, **opts), + lambda auth_opts: self._run_git( + repo_path, args, throw_on_error=False, **auth_opts), **opts ) self._raise_known_result_error( @@ -549,9 +567,9 @@ def dangerously_authenticate( protocol='https', **opts): if not username: - raise ValueError('username is required') + raise InvalidArgumentException('username is required') if not password: - raise ValueError('password is required') + raise InvalidArgumentException('password is required') result = self._run_git( None, @@ -701,6 +719,11 @@ def restore(self, repo_path, paths=None, staged=False, source=None, return self._run_git(repo_path, args, **opts) def set_config(self, *args, **opts): + """Set a Git config value. + + Supports both set_config(repo_path, key, value, global_config=False) + and set_config(key, value, scope='global', path=None). + """ global_config = opts.pop('global_config', False) scope = opts.pop('scope', None) path = opts.pop('path', None) @@ -724,6 +747,11 @@ def set_config(self, *args, **opts): setConfig = set_config def get_config(self, *args, **opts): + """Get a Git config value. + + Supports both get_config(repo_path, key, global_config=False) and + get_config(key, scope='global', path=None). + """ global_config = opts.pop('global_config', False) scope = opts.pop('scope', None) path = opts.pop('path', None) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 58860526..8b8cc727 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -640,6 +640,26 @@ def close_stdin(self, pid): return None +class RecordingFiles(object): + def __init__(self): + self.writes = [] + self.removes = [] + + def write(self, path, data, **opts): + self.writes.append((path, data, opts)) + return None + + def remove(self, path, **opts): + self.removes.append((path, opts)) + return None + + +def attach_recording_files(commands): + files = RecordingFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + return files + + def test_git_helpers_align_with_e2b_method_names(): commands = RecordingCommands() git = Git(commands) @@ -737,22 +757,8 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): - class FakeFiles(object): - def __init__(self): - self.writes = [] - self.removes = [] - - def write(self, path, data, **opts): - self.writes.append((path, data, opts)) - return None - - def remove(self, path, **opts): - self.removes.append((path, opts)) - return None - commands = RecordingCommands() - files = FakeFiles() - commands.sandbox = type('Sandbox', (object,), {'files': files})() + files = attach_recording_files(commands) git = Git(commands) git.dangerously_authenticate( @@ -780,19 +786,6 @@ def remove(self, path, **opts): def test_git_dangerously_authenticate_removes_temp_file_on_chmod_failure(): - class FakeFiles(object): - def __init__(self): - self.writes = [] - self.removes = [] - - def write(self, path, data, **opts): - self.writes.append((path, data, opts)) - return None - - def remove(self, path, **opts): - self.removes.append((path, opts)) - return None - commands = RecordingCommands() commands.results = [ type('Result', (object,), { @@ -817,8 +810,7 @@ def remove(self, path, **opts): 'error': 'chmod failed', })(), ] - files = FakeFiles() - commands.sandbox = type('Sandbox', (object,), {'files': files})() + files = attach_recording_files(commands) git = Git(commands) result = git.dangerously_authenticate( @@ -929,12 +921,13 @@ def test_git_push_respects_throw_on_error_for_unknown_errors(): git.push('/repo', throw_on_error=True) -def test_git_push_with_credentials_sets_remote_url_temporarily(): +def test_git_push_with_credentials_uses_askpass_without_remote_url_leak(): commands = RecordingCommands() + files = attach_recording_files(commands) commands.results = [ type('Result', (object,), { 'exit_code': 0, - 'stdout': 'https://old:old-token@github.com/qiniu/repo.git\n', + 'stdout': 'https://github.com/qiniu/repo.git\n', 'stderr': '', 'error': '', })(), @@ -969,20 +962,23 @@ def test_git_push_with_credentials_sets_remote_url_temporarily(): ) assert commands.calls[0][0] == 'git remote get-url origin' - assert commands.calls[1][0] == ( - 'git remote set-url origin ' - 'https://git%3Auser:secret%3A%25%40token@github.com/qiniu/repo.git' - ) - assert commands.calls[2][0] == 'git push --set-upstream origin main' - assert commands.calls[3][0] == ( - 'git remote set-url origin ' - 'https://old:old-token@github.com/qiniu/repo.git' - ) - assert commands.calls[2][1]['request_timeout'] == 7 + assert commands.calls[1][0] == 'install -d -m 700 /tmp/qiniu-git-auth' + assert files.writes[0][0].startswith( + '/tmp/qiniu-git-auth/qiniu-git-askpass-') + assert commands.calls[2][0].startswith( + 'chmod 700 /tmp/qiniu-git-auth/qiniu-git-askpass-') + assert commands.calls[3][0] == 'git push --set-upstream origin main' + assert 'secret:%@token' not in commands.calls[3][0] + assert commands.calls[3][1]['request_timeout'] == 7 + assert commands.calls[3][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] + assert commands.calls[3][1]['envs']['GIT_USERNAME'] == 'git:user' + assert commands.calls[3][1]['envs']['GIT_PASSWORD'] == 'secret:%@token' + assert files.removes == [(files.writes[0][0], {})] def test_git_pull_with_credentials_resolves_single_remote(): commands = RecordingCommands() + files = attach_recording_files(commands) commands.results = [ type('Result', (object,), { 'exit_code': 0, @@ -1025,18 +1021,16 @@ def test_git_pull_with_credentials_resolves_single_remote(): ) assert commands.calls[0][0] == 'git remote' - assert commands.calls[2][0] == ( - 'git remote set-url origin ' - 'https://git-user:secret-token@github.com/qiniu/repo.git' - ) - assert commands.calls[3][0] == 'git pull origin main' - assert commands.calls[4][0] == ( - 'git remote set-url origin https://github.com/qiniu/repo.git' - ) + assert commands.calls[2][0] == 'install -d -m 700 /tmp/qiniu-git-auth' + assert commands.calls[4][0] == 'git pull origin main' + assert 'secret-token' not in commands.calls[4][0] + assert commands.calls[4][1]['envs']['GIT_ASKPASS'] == files.writes[0][0] + assert files.removes == [(files.writes[0][0], {})] -def test_git_push_with_credentials_restores_remote_on_auth_failure(): +def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): commands = RecordingCommands() + files = attach_recording_files(commands) commands.results = [ type('Result', (object,), { 'exit_code': 0, @@ -1050,43 +1044,6 @@ def test_git_push_with_credentials_restores_remote_on_auth_failure(): 'stderr': '', 'error': '', })(), - type('Result', (object,), { - 'exit_code': 128, - 'stdout': '', - 'stderr': 'fatal: Authentication failed', - 'error': '', - })(), - type('Result', (object,), { - 'exit_code': 0, - 'stdout': '', - 'stderr': '', - 'error': '', - })(), - ] - git = Git(commands) - - with pytest.raises(GitAuthException): - git.push( - '/repo', - remote='origin', - username='git-user', - password='bad-token', - ) - - assert commands.calls[-1][0] == ( - 'git remote set-url origin https://github.com/qiniu/repo.git' - ) - - -def test_git_push_reports_restore_failure_after_credentials_are_injected(): - commands = RecordingCommands() - commands.results = [ - type('Result', (object,), { - 'exit_code': 0, - 'stdout': 'https://github.com/qiniu/repo.git\n', - 'stderr': '', - 'error': '', - })(), type('Result', (object,), { 'exit_code': 0, 'stdout': '', @@ -1100,15 +1057,15 @@ def test_git_push_reports_restore_failure_after_credentials_are_injected(): 'error': '', })(), type('Result', (object,), { - 'exit_code': 1, + 'exit_code': 0, 'stdout': '', - 'stderr': 'config locked', + 'stderr': '', 'error': '', })(), ] git = Git(commands) - with pytest.raises(SandboxError) as err: + with pytest.raises(GitAuthException): git.push( '/repo', remote='origin', @@ -1116,12 +1073,13 @@ def test_git_push_reports_restore_failure_after_credentials_are_injected(): password='bad-token', ) - assert 'Credentials may be leaked' in str(err.value) - assert 'config locked' in str(err.value) + assert commands.calls[-1][0] == 'git push --set-upstream origin' + assert files.removes == [(files.writes[0][0], {})] -def test_git_push_reports_restore_failure_after_operation_exception(): +def test_git_push_cleans_askpass_after_operation_exception(): commands = RecordingCommands() + files = attach_recording_files(commands) commands.results = [ type('Result', (object,), { 'exit_code': 0, @@ -1135,12 +1093,6 @@ def test_git_push_reports_restore_failure_after_operation_exception(): 'stderr': '', 'error': '', })(), - type('Result', (object,), { - 'exit_code': 1, - 'stdout': '', - 'stderr': 'config locked', - 'error': '', - })(), ] git = Git(commands) @@ -1150,12 +1102,12 @@ def test_git_push_reports_restore_failure_after_operation_exception(): 'origin', 'git-user', 'bad-token', - lambda: (_ for _ in ()).throw(SandboxError('rpc timed out')), + lambda auth_opts: (_ for _ in ()).throw( + SandboxError('rpc timed out')), ) - assert 'Credentials may be leaked' in str(err.value) - assert 'config locked' in str(err.value) assert 'rpc timed out' in str(err.value) + assert files.removes == [(files.writes[0][0], {})] def test_git_helpers_accept_e2b_style_signatures(): From 25e769cc5922281d60086e4004606685c9742657 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:00:55 +0800 Subject: [PATCH 23/45] fix(sandbox): tighten client edge cases --- qiniu/services/sandbox/client.py | 6 ++- qiniu/services/sandbox/commands.py | 7 +++- .../test_services/test_sandbox/test_client.py | 38 +++++++++++++++++++ .../test_services/test_sandbox/test_envd.py | 12 +++++- 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 9b588b1b..b29ec788 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -143,6 +143,8 @@ class SandboxClient(object): def __init__(self, endpoint=None, api_url=None, api_key=None, access_token=None, mac=None, access_key=None, secret_key=None, session=None, timeout=None, **opts): + access_key = access_key or os.getenv('QINIU_SANDBOX_ACCESS_KEY') + secret_key = secret_key or os.getenv('QINIU_SANDBOX_SECRET_KEY') if (access_key and not secret_key) or (secret_key and not access_key): raise SandboxError( 'Both access_key and secret_key must be provided') @@ -215,6 +217,8 @@ def _request(self, method, path, params=None, body=_UNSET, dict) and response_data.get('message'): message += ': {0}'.format(response_data.get('message')) elif isinstance(response_data, basestring) and response_data: + if len(response_data) > 200: + response_data = response_data[:200] + '...' message += ': {0}'.format(response_data) raise SandboxError(message, response, response_data) if empty: @@ -356,7 +360,7 @@ def get_sandboxes_metrics(self, sandbox_ids): 'sandbox_ids') or sandbox_ids.get('sandboxIDs') if values is None: raise SandboxError('At least one sandbox ID must be provided') - if not isinstance(values, (list, tuple)): + if not isinstance(values, (list, tuple, set)): values = [values] ids = [] for value in values: diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 065e7e25..6e5fcdc9 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -55,11 +55,16 @@ def _decode_bytes(value): return bytearray(value).decode('utf-8', 'replace') if isinstance(value, (bytes, basestring)): try: - return base64.b64decode(value).decode('utf-8', 'replace') + decoded = base64.b64decode(value).decode('utf-8', 'replace') except (binascii.Error, TypeError): if isinstance(value, bytes): return value.decode('utf-8', 'replace') return value + if u'\ufffd' in decoded: + if isinstance(value, bytes): + return value.decode('utf-8', 'replace') + return value + return decoded return str(value) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 8b8cc727..09c50390 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -137,6 +137,17 @@ def test_client_reads_documented_api_key_env_fallbacks(monkeypatch): 'sandbox-api-key') +def test_client_reads_qiniu_mac_credentials_from_env(monkeypatch): + monkeypatch.delenv('QINIU_SANDBOX_ACCESS_KEY', raising=False) + monkeypatch.delenv('QINIU_SANDBOX_SECRET_KEY', raising=False) + monkeypatch.setenv('QINIU_SANDBOX_ACCESS_KEY', 'ak') + monkeypatch.setenv('QINIU_SANDBOX_SECRET_KEY', 'sk') + + client = SandboxClient(session=RecordingSession()) + + assert client.mac is not None + + def test_util_helpers_encode_unicode_values_safely(): assert encode_path(u'目录/文件.txt') assert file_signature( @@ -279,6 +290,23 @@ def test_client_wraps_request_exceptions_in_sandbox_error(): assert 'timed out' in str(err.value) +def test_client_truncates_long_text_error_responses(): + class TextErrorResponse(ErrorResponse): + def json(self): + raise ValueError('not json') + + client = SandboxClient( + api_key='api-key', + session=RecordingSession([TextErrorResponse(502)])) + client.session.responses[0].text = '' + ('x' * 300) + '' + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert len(str(err.value)) < 260 + assert str(err.value).endswith('...') + + def test_sandbox_instance_lifecycle_methods_call_control_plane(): session = RecordingSession([ DummyResponse(204, None), @@ -531,6 +559,16 @@ def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): assert query['sandbox_ids'] == ['sbx1,sbx2'] +def test_get_sandboxes_metrics_accepts_set_values(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(set(['sbx1', 'sbx2'])) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert set(query['sandbox_ids'][0].split(',')) == set(['sbx1', 'sbx2']) + + def test_get_sandboxes_metrics_rejects_empty_dict_values(): client = SandboxClient(api_key='api-key', session=RecordingSession()) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 67876a3d..8f5cbab9 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -202,7 +202,17 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): }]) assert result.stdout == 'YWJj' - assert result.stderr == u'\ufffd\ufffd\ufffd' + assert result.stderr == '////' + + +def test_command_event_decode_preserves_plain_strings(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': 'test', + }}, + }]) + + assert result.stdout == 'test' def test_command_event_decode_handles_bytes_values(): From 0898f5f6d6ebfe96bfa1c0da7d0ab3816a00660c Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:04:54 +0800 Subject: [PATCH 24/45] fix(sandbox): close paginated review feedback --- qiniu/services/sandbox/commands.py | 10 ++++------ qiniu/services/sandbox/envd.py | 6 ++++++ qiniu/services/sandbox/git.py | 2 +- qiniu/services/sandbox/sandbox.py | 16 ++++++++++------ .../test_services/test_sandbox/test_client.py | 15 +++++++++++++++ .../test_services/test_sandbox/test_envd.py | 4 +++- 6 files changed, 39 insertions(+), 14 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 6e5fcdc9..bc089bc8 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -2,7 +2,7 @@ import base64 import binascii -from qiniu.compat import basestring +from qiniu.compat import basestring, bytes as bytes_type from .envd import connect_rpc, connect_stream_rpc from .errors import CommandExitError, SandboxError @@ -53,16 +53,14 @@ def _decode_bytes(value): return '' if isinstance(value, list): return bytearray(value).decode('utf-8', 'replace') - if isinstance(value, (bytes, basestring)): + if isinstance(value, bytes_type): + return value.decode('utf-8', 'replace') + if isinstance(value, basestring): try: decoded = base64.b64decode(value).decode('utf-8', 'replace') except (binascii.Error, TypeError): - if isinstance(value, bytes): - return value.decode('utf-8', 'replace') return value if u'\ufffd' in decoded: - if isinstance(value, bytes): - return value.decode('utf-8', 'replace') return value return decoded return str(value) diff --git a/qiniu/services/sandbox/envd.py b/qiniu/services/sandbox/envd.py index a6864ef6..e75acaa0 100644 --- a/qiniu/services/sandbox/envd.py +++ b/qiniu/services/sandbox/envd.py @@ -24,6 +24,8 @@ def envd_headers(sandbox, user=None, extra=None): def connect_rpc(sandbox, procedure, body=None, user=None, timeout=None): + if timeout is None: + timeout = sandbox.client.timeout url = sandbox.envd_url() + procedure headers = envd_headers(sandbox, user, {'Content-Type': 'application/json'}) try: @@ -144,6 +146,8 @@ def iter_connect_envelopes(chunks, response=None): def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, stream=False): + if timeout is None: + timeout = sandbox.client.timeout url = sandbox.envd_url() + procedure headers = envd_headers(sandbox, user, { 'Content-Type': 'application/connect+json', @@ -192,6 +196,8 @@ def connect_stream_rpc(sandbox, procedure, body=None, user=None, timeout=None, def raw_envd_request(sandbox, method, url, **kwargs): + if kwargs.get('timeout') is None: + kwargs['timeout'] = sandbox.client.timeout try: response = sandbox.client.session.request(method, url, **kwargs) except requests.RequestException as err: diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 86d3cdec..cf1192ad 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -399,7 +399,7 @@ def _with_remote_credentials(self, repo_path, remote, username, password, operation_error = None try: result = operation(auth_opts) - except Exception as err: + except BaseException as err: operation_error = err result = None finally: diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 2318dfe9..62e2be13 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -51,12 +51,16 @@ def instance_connect(timeout=15): class SandboxPaginator(object): def __init__(self, client=None, **opts): - self.client = client or SandboxClient(**opts) - self.opts = dict(opts) - self.opts.pop('client', None) - self.next_token = opts.get('nextToken') or opts.get('next_token') - self.opts.pop('nextToken', None) - self.opts.pop('next_token', None) + opts = dict(opts) + client_opts = {} + for key in ('endpoint', 'api_url', 'api_key', 'access_token', + 'mac', 'access_key', 'secret_key', 'session', 'timeout'): + if key in opts: + client_opts[key] = opts.pop(key) + self.client = client or SandboxClient(**client_opts) + self.opts = opts + self.next_token = opts.pop('nextToken', None) or opts.pop( + 'next_token', None) self._has_next = True @property diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 09c50390..fa8814b5 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -520,6 +520,21 @@ def test_sandbox_paginator_does_not_reuse_initial_next_token(): assert client.calls[1]['nextToken'] == 'next-page' +def test_sandbox_paginator_does_not_send_client_credentials_as_filters(): + client = RecordingSandboxListClient() + paginator = SandboxPaginator( + client=client, + api_key='api-key', + access_token='access-token', + endpoint='https://sandbox.example.test', + metadata={'app': 'tests'}, + ) + + paginator.next_items() + + assert client.calls[0] == {'metadata': {'app': 'tests'}} + + def test_is_running_matches_e2b_health_check_semantics(): running_session = RecordingSession([DummyResponse(200, {})]) running = Sandbox(client=SandboxClient( diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 8f5cbab9..92669938 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -223,7 +223,7 @@ def test_command_event_decode_handles_bytes_values(): }}, }]) - assert result.stdout == 'abc' + assert result.stdout == 'YWJj' assert result.stderr == 'plain bytes' @@ -322,6 +322,7 @@ def test_commands_run_posts_process_start_and_decodes_events(): ) assert session.posts[0]['headers']['Authorization'] == 'Basic dXNlcjo=' assert session.posts[0]['headers']['X-Access-Token'] == 'token' + assert session.posts[0]['timeout'] == 30 assert session.posts[0]['data']['process'] == { 'cmd': '/bin/bash', 'args': ['-l', '-c', 'echo hello'], @@ -338,6 +339,7 @@ def test_commands_connect_returns_handle_for_running_process(): assert result.pid == 12 assert result.stdout == 'connected\n' + assert session.posts[0]['timeout'] == 30 assert session.posts[0]['url'].endswith('/process.Process/Connect') assert session.posts[0]['data'] == { 'process': {'selector': {'pid': 12}}, From 850fe56000d4eb24e875af480a56d1f009241efc Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:09:45 +0800 Subject: [PATCH 25/45] fix(sandbox): simplify git config helpers --- qiniu/services/sandbox/git.py | 47 +++---------------- .../test_services/test_sandbox/test_client.py | 10 ++-- 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index cf1192ad..4dea3af1 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -718,25 +718,8 @@ def restore(self, repo_path, paths=None, staged=False, source=None, args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) - def set_config(self, *args, **opts): - """Set a Git config value. - - Supports both set_config(repo_path, key, value, global_config=False) - and set_config(key, value, scope='global', path=None). - """ - global_config = opts.pop('global_config', False) - scope = opts.pop('scope', None) - path = opts.pop('path', None) - if len(args) == 3: - repo_path, key, value = args - args_list = ['config'] - if global_config: - args_list.append('--global') - args_list.extend([shell_quote(key), shell_quote(value)]) - return self._run_git(repo_path, args_list, **opts) - if len(args) != 2: - raise TypeError('set_config expects key and value') - key, value = args + def set_config(self, key, value, scope='global', path=None, **opts): + """Set a Git config value.""" scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] if scope_flag: @@ -746,25 +729,8 @@ def set_config(self, *args, **opts): setConfig = set_config - def get_config(self, *args, **opts): - """Get a Git config value. - - Supports both get_config(repo_path, key, global_config=False) and - get_config(key, scope='global', path=None). - """ - global_config = opts.pop('global_config', False) - scope = opts.pop('scope', None) - path = opts.pop('path', None) - if len(args) == 2: - repo_path, key = args - args_list = ['config'] - if global_config: - args_list.append('--global') - args_list.extend(['--get', shell_quote(key)]) - return self._run_git(repo_path, args_list, **opts) - if len(args) != 1: - raise TypeError('get_config expects key') - key = args[0] + def get_config(self, key, scope='global', path=None, **opts): + """Get a Git config value.""" scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] if scope_flag: @@ -777,11 +743,12 @@ def get_config(self, *args, **opts): def _resolve_config_scope(self, scope=None, path=None): scope_name = (scope or 'global').strip().lower() if scope_name not in ('global', 'local', 'system'): - raise ValueError( + raise InvalidArgumentException( 'Git config scope must be global, local, or system') if scope_name == 'local': if not path: - raise ValueError('Repository path is required for local scope') + raise InvalidArgumentException( + 'Repository path is required for local scope') return '--local', path if scope_name == 'system': return '--system', None diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index fa8814b5..65a49f7c 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -725,8 +725,8 @@ def test_git_helpers_align_with_e2b_method_names(): git.delete_branch('/repo', 'old') git.reset('/repo', 'HEAD~1', mode='hard') git.restore('/repo', paths=['a.txt', 'b.txt']) - git.set_config('/repo', 'user.name', 'tester') - git.get_config('/repo', 'user.name') + git.set_config('user.name', 'tester', scope='local', path='/repo') + git.get_config('user.name', scope='local', path='/repo') assert commands.calls[0][0] == ( 'git remote add origin https://github.com/qiniu/repo.git') @@ -738,8 +738,10 @@ def test_git_helpers_align_with_e2b_method_names(): assert commands.calls[5][0] == 'git branch -D old' assert commands.calls[6][0] == "git reset --hard 'HEAD~1'" assert commands.calls[7][0] == 'git restore a.txt b.txt' - assert commands.calls[8][0] == 'git config user.name tester' - assert commands.calls[9][0] == 'git config --get user.name' + assert commands.calls[8][0] == 'git config --local user.name tester' + assert commands.calls[8][1]['cwd'] == '/repo' + assert commands.calls[9][0] == 'git config --local --get user.name' + assert commands.calls[9][1]['cwd'] == '/repo' def test_git_remote_add_supports_overwrite_and_fetch_options(): From 2d09a65f79f7b9ce9d36d99b3da22279eb515305 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:12:35 +0800 Subject: [PATCH 26/45] fix(sandbox): align command stream timeout --- qiniu/services/sandbox/commands.py | 5 ++++- tests/cases/test_services/test_sandbox/test_envd.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index bc089bc8..56718c3d 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -187,12 +187,15 @@ def start(self, cmd, cwd=None, envs=None, user=None, stdin=False, body['tag'] = tag if timeout is not None: body['timeout'] = timeout + stream_timeout = request_timeout + if stream_timeout is None: + stream_timeout = timeout events = connect_stream_rpc( self.sandbox, '/process.Process/Start', body, user=user, - timeout=request_timeout, + timeout=stream_timeout, stream=True, ) events = iter(events) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 92669938..c63b6100 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -387,6 +387,16 @@ def test_commands_run_supports_e2b_callbacks_and_request_timeout(): assert session.posts[0]['timeout'] == 7 +def test_commands_run_uses_command_timeout_for_stream_request_by_default(): + sandbox, session = sandbox_with_envd_session() + + result = sandbox.commands.run('sleep 60', timeout=120) + + assert result.exit_code == 0 + assert session.posts[0]['data']['timeout'] == 120 + assert session.posts[0]['timeout'] == 120 + + def test_commands_run_background_returns_before_command_finishes(): sandbox, session = sandbox_with_envd_session() From ecb581c8c02c460c01b17a6a0b45b83717dfa33a Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:17:09 +0800 Subject: [PATCH 27/45] fix(sandbox): preserve e2b compatibility edges --- qiniu/services/sandbox/commands.py | 3 +- qiniu/services/sandbox/git.py | 30 +++++++++++++++++++ qiniu/services/sandbox/template.py | 4 +-- .../test_services/test_sandbox/test_client.py | 20 +++++++++++++ .../test_services/test_sandbox/test_envd.py | 2 +- 5 files changed, 55 insertions(+), 4 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 56718c3d..e3ecae3f 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -60,7 +60,8 @@ def _decode_bytes(value): decoded = base64.b64decode(value).decode('utf-8', 'replace') except (binascii.Error, TypeError): return value - if u'\ufffd' in decoded: + if u'\ufffd' in decoded and all( + ch.isalnum() for ch in value): return value return decoded return str(value) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 4dea3af1..04a24048 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -720,6 +720,8 @@ def restore(self, repo_path, paths=None, staged=False, source=None, def set_config(self, key, value, scope='global', path=None, **opts): """Set a Git config value.""" + key, value, scope, path = self._normalize_set_config_args( + key, value, scope, path, opts) scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] if scope_flag: @@ -731,6 +733,8 @@ def set_config(self, key, value, scope='global', path=None, **opts): def get_config(self, key, scope='global', path=None, **opts): """Get a Git config value.""" + key, scope, path = self._normalize_get_config_args( + key, scope, path, opts) scope_flag, repo_path = self._resolve_config_scope(scope, path) args = ['config'] if scope_flag: @@ -740,6 +744,32 @@ def get_config(self, key, scope='global', path=None, **opts): getConfig = get_config + def _normalize_set_config_args(self, key, value, scope, path, opts): + global_config = opts.pop( + 'global_config', opts.pop('globalConfig', False)) + if global_config or self._is_legacy_config_call(scope): + repo_path = key + key = value + value = scope + scope = 'global' if global_config or repo_path is None else 'local' + path = None if scope == 'global' else repo_path + return key, value, scope, path + + def _normalize_get_config_args(self, key, scope, path, opts): + global_config = opts.pop( + 'global_config', opts.pop('globalConfig', False)) + if global_config or self._is_legacy_config_call(scope): + repo_path = key + key = scope + scope = 'global' if global_config or repo_path is None else 'local' + path = None if scope == 'global' else repo_path + return key, scope, path + + def _is_legacy_config_call(self, scope): + if scope is None: + return False + return str(scope).strip().lower() not in ('global', 'local', 'system') + def _resolve_config_scope(self, scope=None, path=None): scope_name = (scope or 'global').strip().lower() if scope_name not in ('global', 'local', 'system'): diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index 9d6b02e3..52b1e3a2 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -76,8 +76,8 @@ def add_step(self, step_type, args, **extra): addStep = add_step def run_cmd(self, command, user=None): - args = [' && '.join(command) if isinstance( - command, (list, tuple)) else command] + args = [str(arg) for arg in command] if isinstance( + command, (list, tuple)) else [command] if user: args.append(user) return self.add_step('RUN', args) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 65a49f7c..6c4fc672 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -598,6 +598,7 @@ def test_template_builder_outputs_build_config(): Template() .from_image('python:3.11') .run_cmd('pip install qiniu') + .run_cmd(['python', '-m', 'pip', 'install', 'pytest']) .copy('/local/app.py', '/app/app.py') .set_env('PYTHONUNBUFFERED', '1') .set_start_cmd('python /app/app.py') @@ -607,6 +608,8 @@ def test_template_builder_outputs_build_config(): 'fromImage': 'python:3.11', 'steps': [ {'type': 'RUN', 'args': ['pip install qiniu']}, + {'type': 'RUN', 'args': [ + 'python', '-m', 'pip', 'install', 'pytest']}, {'type': 'COPY', 'args': ['/local/app.py', '/app/app.py']}, {'type': 'ENV', 'args': ['PYTHONUNBUFFERED', '1']}, ], @@ -1189,3 +1192,20 @@ def test_git_helpers_accept_e2b_style_signatures(): assert commands.calls[2][1]['cwd'] == '/repo' assert commands.calls[3][0] == 'git config --local --get user.name' assert commands.calls[3][1]['cwd'] == '/repo' + + +def test_git_config_helpers_accept_legacy_repo_path_signatures(): + commands = RecordingCommands() + git = Git(commands) + + git.set_config(None, 'http.version', 'HTTP/1.1', global_config=True) + git.set_config('/repo', 'user.name', 'Sandbox Demo') + git.get_config('/repo', 'user.name') + + assert commands.calls[0][0] == "git config --global http.version HTTP/1.1" + assert 'cwd' not in commands.calls[0][1] + assert commands.calls[1][0] == ( + "git config --local user.name 'Sandbox Demo'") + assert commands.calls[1][1]['cwd'] == '/repo' + assert commands.calls[2][0] == 'git config --local --get user.name' + assert commands.calls[2][1]['cwd'] == '/repo' diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index c63b6100..dd9ad834 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -202,7 +202,7 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): }]) assert result.stdout == 'YWJj' - assert result.stderr == '////' + assert result.stderr == u'\ufffd\ufffd\ufffd' def test_command_event_decode_preserves_plain_strings(): From 64853e50c50b2e0ca493aab5d713ff4bf04fdcec Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:21:07 +0800 Subject: [PATCH 28/45] fix(sandbox): address review compatibility edges --- qiniu/services/sandbox/client.py | 21 ++-- qiniu/services/sandbox/commands.py | 6 +- qiniu/services/sandbox/git.py | 10 +- qiniu/services/sandbox/template.py | 5 +- .../test_services/test_sandbox/test_client.py | 100 +++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 4 +- 6 files changed, 127 insertions(+), 19 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index b29ec788..0c7e05c4 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -163,9 +163,11 @@ def _headers(self, auth_type=None): if auth_type == 'qiniu': return headers if auth_type == 'accessToken': - if self.access_token: - headers['Authorization'] = 'Bearer {0}'.format( - self.access_token) + if not self.access_token: + raise SandboxError( + 'access_token is required for this operation') + headers['Authorization'] = 'Bearer {0}'.format( + self.access_token) return headers if self.api_key: headers['X-API-Key'] = self.api_key @@ -212,10 +214,13 @@ def _request(self, method, path, params=None, body=_UNSET, message = 'Sandbox API request failed with status {0}'.format( response.status_code ) - if isinstance( - response_data, - dict) and response_data.get('message'): - message += ': {0}'.format(response_data.get('message')) + if isinstance(response_data, dict): + err_msg = response_data.get( + 'message') or response_data.get('error') + if isinstance(err_msg, dict): + err_msg = err_msg.get('message') or err_msg.get('error') + if err_msg: + message += ': {0}'.format(err_msg) elif isinstance(response_data, basestring) and response_data: if len(response_data) > 200: response_data = response_data[:200] + '...' @@ -358,6 +363,8 @@ def get_sandboxes_metrics(self, sandbox_ids): if isinstance(sandbox_ids, dict): values = sandbox_ids.get( 'sandbox_ids') or sandbox_ids.get('sandboxIDs') + if values is None: + values = [sandbox_ids] if values is None: raise SandboxError('At least one sandbox ID must be provided') if not isinstance(values, (list, tuple, set)): diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index e3ecae3f..14fcb53b 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -57,13 +57,9 @@ def _decode_bytes(value): return value.decode('utf-8', 'replace') if isinstance(value, basestring): try: - decoded = base64.b64decode(value).decode('utf-8', 'replace') + return base64.b64decode(value).decode('utf-8', 'replace') except (binascii.Error, TypeError): return value - if u'\ufffd' in decoded and all( - ch.isalnum() for ch in value): - return value - return decoded return str(value) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 04a24048..72e026a7 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -10,6 +10,7 @@ from urlparse import urlparse, urlunparse from .errors import ( + CommandExitError, GitAuthException, GitUpstreamException, InvalidArgumentException, @@ -327,6 +328,8 @@ def _run_git(self, repo_path, args, **opts): return self.commands.run('git {0}'.format(' '.join(args)), **opts) def _get_remote_url(self, repo_path, remote, **opts): + opts = dict(opts) + opts.pop('background', None) result = self._run_git( repo_path, ['remote', 'get-url', shell_quote(remote)], @@ -341,6 +344,8 @@ def _get_remote_url(self, repo_path, remote, **opts): def _resolve_remote_name(self, repo_path, remote=None, **opts): if remote: return remote + opts = dict(opts) + opts.pop('background', None) result = self._run_git(repo_path, ['remote'], **opts) remotes = [ line.strip() @@ -410,6 +415,8 @@ def _with_remote_credentials(self, repo_path, remote, username, password, def _raise_known_result_error( self, result, operation, throw_on_error=False): + if not hasattr(result, 'exit_code'): + return if result.exit_code: if _is_auth_failure(result): raise GitAuthException( @@ -420,7 +427,6 @@ def _raise_known_result_error( 'Git {0} failed because no upstream branch is configured.' .format(operation)) if throw_on_error: - from qiniu.services.sandbox import CommandExitError raise CommandExitError(result) def clone(self, repo_url, path=None, branch=None, depth=None, **opts): @@ -446,6 +452,8 @@ def status(self, repo_path, **opts): result = self._run_git( repo_path, [ 'status', '--porcelain=1', '-b'], **opts) + if result.exit_code: + raise CommandExitError(result) return parse_git_status(result.stdout) def add(self, repo_path, files=None, all=False, **opts): diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index 52b1e3a2..4730e54f 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -76,8 +76,9 @@ def add_step(self, step_type, args, **extra): addStep = add_step def run_cmd(self, command, user=None): - args = [str(arg) for arg in command] if isinstance( - command, (list, tuple)) else [command] + if isinstance(command, (list, tuple)): + command = ' '.join(shell_quote(arg) for arg in command) + args = [command] if user: args.append(user) return self.add_step('RUN', args) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 6c4fc672..49750c93 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -307,6 +307,28 @@ def json(self): assert str(err.value).endswith('...') +def test_client_includes_nested_error_message_in_api_errors(): + client = SandboxClient( + api_key='api-key', + session=RecordingSession([DummyResponse(400, { + 'error': {'message': 'bad sandbox request'}, + })])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert 'bad sandbox request' in str(err.value) + + +def test_access_token_auth_requires_access_token(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as err: + client.rebuild_template('tmpl123') + + assert 'access_token is required' in str(err.value) + + def test_sandbox_instance_lifecycle_methods_call_control_plane(): session = RecordingSession([ DummyResponse(204, None), @@ -584,6 +606,16 @@ def test_get_sandboxes_metrics_accepts_set_values(): assert set(query['sandbox_ids'][0].split(',')) == set(['sbx1', 'sbx2']) +def test_get_sandboxes_metrics_accepts_single_sandbox_dict(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics({'id': 'sbx123'}) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx123'] + + def test_get_sandboxes_metrics_rejects_empty_dict_values(): client = SandboxClient(api_key='api-key', session=RecordingSession()) @@ -608,8 +640,7 @@ def test_template_builder_outputs_build_config(): 'fromImage': 'python:3.11', 'steps': [ {'type': 'RUN', 'args': ['pip install qiniu']}, - {'type': 'RUN', 'args': [ - 'python', '-m', 'pip', 'install', 'pytest']}, + {'type': 'RUN', 'args': ['python -m pip install pytest']}, {'type': 'COPY', 'args': ['/local/app.py', '/app/app.py']}, {'type': 'ENV', 'args': ['PYTHONUNBUFFERED', '1']}, ], @@ -921,6 +952,20 @@ def test_git_status_and_branches_return_structured_e2b_types(): assert branches.current_branch == 'main' +def test_git_status_raises_when_git_command_fails(): + commands = RecordingCommands() + commands.results = [type('Result', (object,), { + 'exit_code': 128, + 'stdout': '', + 'stderr': 'fatal: not a git repository', + 'error': '', + })()] + git = Git(commands) + + with pytest.raises(CommandExitError): + git.status('/not-a-repo') + + def test_git_push_maps_auth_failure_to_e2b_exception(): commands = RecordingCommands() commands.results = [type('Result', (object,), { @@ -1086,6 +1131,57 @@ def test_git_pull_with_credentials_resolves_single_remote(): assert files.removes == [(files.writes[0][0], {})] +def test_git_credential_helpers_ignore_background_when_reading_remote(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + background=True, + ) + + assert 'background' not in commands.calls[0][1] + assert 'background' not in commands.calls[1][1] + assert commands.calls[4][1]['background'] is True + assert files.removes == [(files.writes[0][0], {})] + + def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): commands = RecordingCommands() files = attach_recording_files(commands) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index dd9ad834..e2e102f0 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -208,11 +208,11 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): def test_command_event_decode_preserves_plain_strings(): result = command_result_from_events([{ 'event': {'data': { - 'stdout': 'test', + 'stdout': 'plain text!', }}, }]) - assert result.stdout == 'test' + assert result.stdout == 'plain text!' def test_command_event_decode_handles_bytes_values(): From b50de526eea4262062b232bcd2f0b372d3fbbf5c Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:23:18 +0800 Subject: [PATCH 29/45] fix(sandbox): clarify option and credential validation --- qiniu/services/sandbox/client.py | 26 ++++++++++++------ qiniu/services/sandbox/git.py | 27 ++++++++++++------- .../test_services/test_sandbox/test_client.py | 16 +++++++++++ 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 0c7e05c4..49ec82f7 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -98,14 +98,13 @@ def _normalize_sandbox_create_options(template=None, **opts): body[key] = opts.get(key) if opts.get('auto_pause') is not None: body['autoPause'] = opts.get('auto_pause') - if opts.get('allow_internet_access') is not None: - body['allow_internet_access'] = opts.get('allow_internet_access') - if opts.get('allowInternetAccess') is not None: - body['allow_internet_access'] = opts.get('allowInternetAccess') - if opts.get('envs') is not None: - body['envVars'] = opts.get('envs') - if opts.get('envVars') is not None: - body['envVars'] = opts.get('envVars') + allow_internet_access = _single_alias_value( + opts, 'allow_internet_access', 'allowInternetAccess') + if allow_internet_access is not None: + body['allow_internet_access'] = allow_internet_access + envs = _single_alias_value(opts, 'envs', 'envVars') + if envs is not None: + body['envVars'] = envs if opts.get('lifecycle') is not None: lifecycle = opts.get('lifecycle') or {} on_timeout = lifecycle.get('on_timeout') or lifecycle.get('onTimeout') @@ -119,6 +118,17 @@ def _normalize_sandbox_create_options(template=None, **opts): return body +def _single_alias_value(opts, *names): + present = [name for name in names if opts.get(name) is not None] + if len(present) > 1: + raise SandboxError( + 'Conflicting sandbox create options: {0}'.format( + ', '.join(present))) + if present: + return opts.get(present[0]) + return None + + def _normalize_list_options(opts): opts = dict(opts or {}) query = opts.pop('query', None) or {} diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 72e026a7..ee009949 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -283,16 +283,9 @@ def _is_missing_upstream(result): def _with_credentials(url, username, password): - if not username and not password: + parsed = _validate_git_url_credentials(url, username, password) + if parsed is None: return url - if not username or not password: - raise InvalidArgumentException( - 'Both username and password are required when using Git ' - 'credentials.') - parsed = urlparse(url) - if parsed.scheme not in ('http', 'https'): - raise InvalidArgumentException( - 'Only http(s) Git URLs support username/password credentials.') host = parsed.hostname or '' if ':' in host and not host.startswith('['): host = '[{0}]'.format(host) @@ -307,6 +300,20 @@ def _with_credentials(url, username, password): )) +def _validate_git_url_credentials(url, username, password): + if not username and not password: + return None + if not username or not password: + raise InvalidArgumentException( + 'Both username and password are required when using Git ' + 'credentials.') + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https'): + raise InvalidArgumentException( + 'Only http(s) Git URLs support username/password credentials.') + return parsed + + def _askpass_script(): return ( '#!/bin/sh\n' @@ -364,7 +371,7 @@ def _resolve_remote_name(self, repo_path, remote=None, **opts): def _with_remote_credentials(self, repo_path, remote, username, password, operation, **opts): original_url = self._get_remote_url(repo_path, remote, **opts) - _with_credentials(original_url, username, password) + _validate_git_url_credentials(original_url, username, password) sandbox = getattr(self.commands, 'sandbox', None) filesystem = getattr(sandbox, 'files', None) if filesystem is None: diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 49750c93..0f6701be 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -120,6 +120,22 @@ def test_client_uses_default_endpoint_and_api_key_headers(): } +def test_create_sandbox_rejects_conflicting_option_aliases(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError) as env_err: + client.create_sandbox(envs={'A': 'B'}, envVars={'A': 'C'}) + with pytest.raises(SandboxError) as network_err: + client.create_sandbox( + allow_internet_access=True, + allowInternetAccess=False, + ) + + assert 'envs, envVars' in str(env_err.value) + assert 'allow_internet_access, allowInternetAccess' in str( + network_err.value) + + def test_client_reads_documented_api_key_env_fallbacks(monkeypatch): monkeypatch.delenv('QINIU_SANDBOX_API_KEY', raising=False) monkeypatch.delenv('QINIU_API_KEY', raising=False) From a1e3a11356823d6b901cab2f8149887a2ee42f16 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:26:55 +0800 Subject: [PATCH 30/45] fix(sandbox): harden background credentials and bytes input --- qiniu/services/sandbox/commands.py | 4 +++- qiniu/services/sandbox/filesystem.py | 2 ++ qiniu/services/sandbox/git.py | 20 ++++++++++++++++- qiniu/services/sandbox/pty.py | 4 +++- .../test_services/test_sandbox/test_client.py | 22 ++++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 20 +++++++++++++---- 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 14fcb53b..dd35d5bb 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -262,7 +262,9 @@ def connect(self, pid, tag=None, user=None, timeout=None, ) def send_stdin(self, pid, data, user=None, timeout=None): - if not isinstance(data, bytes): + if isinstance(data, bytearray): + data = bytes(data) + elif not isinstance(data, bytes): if hasattr(data, 'encode'): data = data.encode('utf-8') else: diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index e6822dc9..5004720f 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -137,6 +137,8 @@ def normalize_entry(entry, extended=False): def to_upload_body(data, encoding='utf-8'): + if isinstance(data, bytearray): + return bytes(data) if isinstance(data, bytes): return data if isinstance(data, basestring): diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index ee009949..ab5d2c31 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -31,12 +31,23 @@ def _normalize_paths(paths): def _remove_credential_file(filesystem, path): + if not path: + return try: filesystem.remove(path) except Exception: pass +def _cleanup_after_wait(wait, filesystem, path): + def wrapped_wait(*args, **kwargs): + try: + return wait(*args, **kwargs) + finally: + _remove_credential_file(filesystem, path) + return wrapped_wait + + class GitFileStatus(object): def __init__(self, name, status, index_status, working_tree_status, staged, renamed_from=None): @@ -411,11 +422,16 @@ def _with_remote_credentials(self, repo_path, remote, username, password, operation_error = None try: result = operation(auth_opts) + if opts.get('background') and hasattr(result, 'wait'): + result.wait = _cleanup_after_wait( + result.wait, filesystem, askpass_path) + askpass_path = None except BaseException as err: operation_error = err result = None finally: - _remove_credential_file(filesystem, askpass_path) + if askpass_path: + _remove_credential_file(filesystem, askpass_path) if operation_error is not None: raise operation_error return result @@ -581,6 +597,8 @@ def dangerously_authenticate( host='github.com', protocol='https', **opts): + opts = dict(opts) + opts.pop('background', None) if not username: raise InvalidArgumentException('username is required') if not password: diff --git a/qiniu/services/sandbox/pty.py b/qiniu/services/sandbox/pty.py index 6a0dc17f..500c7a52 100644 --- a/qiniu/services/sandbox/pty.py +++ b/qiniu/services/sandbox/pty.py @@ -95,7 +95,9 @@ def connect(self, pid, user=None, timeout=None, throw_on_error=False): ) def send_stdin(self, pid, data, user=None, timeout=None): - if not isinstance(data, bytes): + if isinstance(data, bytearray): + data = bytes(data) + elif not isinstance(data, bytes): if hasattr(data, 'encode'): data = data.encode('utf-8') else: diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 0f6701be..81825a8b 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -890,6 +890,24 @@ def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): assert files.removes == [(files.writes[0][0], {})] +def test_git_dangerously_authenticate_ignores_background_for_temp_file(): + commands = RecordingCommands() + files = attach_recording_files(commands) + git = Git(commands) + + git.dangerously_authenticate( + username='git-user', + password='secret-token', + background=True, + ) + + assert 'background' not in commands.calls[0][1] + assert 'background' not in commands.calls[1][1] + assert 'background' not in commands.calls[2][1] + assert 'background' not in commands.calls[3][1] + assert files.removes == [(files.writes[0][0], {})] + + def test_git_dangerously_authenticate_removes_temp_file_on_chmod_failure(): commands = RecordingCommands() commands.results = [ @@ -1184,7 +1202,7 @@ def test_git_credential_helpers_ignore_background_when_reading_remote(): ] git = Git(commands) - git.pull( + handle = git.pull( '/repo', branch='main', username='git-user', @@ -1195,6 +1213,8 @@ def test_git_credential_helpers_ignore_background_when_reading_remote(): assert 'background' not in commands.calls[0][1] assert 'background' not in commands.calls[1][1] assert commands.calls[4][1]['background'] is True + assert files.removes == [] + handle.wait() assert files.removes == [(files.writes[0][0], {})] diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index e2e102f0..adaaf150 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -284,11 +284,16 @@ def test_filesystem_write_accepts_unicode_text(): sandbox, session = sandbox_with_envd_session() sandbox.files.write('/tmp/unicode.txt', u'你好') + sandbox.files.write('/tmp/bytes.txt', bytearray(b'abc')) assert session.requests[0]['kwargs']['files']['file'] == ( 'unicode.txt', u'你好'.encode('utf-8'), ) + assert session.requests[1]['kwargs']['files']['file'] == ( + 'bytes.txt', + b'abc', + ) def test_filesystem_write_accepts_duck_typed_file_like_objects(): @@ -362,9 +367,12 @@ def test_commands_send_stdin_encodes_unicode_text(): sandbox, session = sandbox_with_envd_session() sandbox.commands.send_stdin(12, u'你好') + sandbox.commands.send_stdin(12, bytearray(b'abc')) raw = session.posts[0]['data']['input']['stdin'] assert base64.b64decode(raw).decode('utf-8') == u'你好' + raw = session.posts[1]['data']['input']['stdin'] + assert base64.b64decode(raw) == b'abc' def test_commands_run_supports_e2b_callbacks_and_request_timeout(): @@ -414,6 +422,7 @@ def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): handle = sandbox.pty.create(PtySize(rows=24, cols=80), cwd='/workspace') sandbox.pty.send_stdin(handle.pid, u'你好\n') + sandbox.pty.send_stdin(handle.pid, bytearray(b'abc')) sandbox.pty.resize(handle.pid, {'rows': 30, 'cols': 100}) connected = sandbox.pty.connect(handle.pid) assert sandbox.pty.kill(handle.pid) is True @@ -433,12 +442,15 @@ def test_pty_create_send_resize_connect_and_kill_use_process_rpc(): assert base64.b64decode( session.posts[1]['data']['input']['pty'] ).decode('utf-8') == u'你好\n' - assert session.posts[2]['url'].endswith('/process.Process/Update') - assert session.posts[2]['data']['pty'] == { + assert session.posts[2]['url'].endswith('/process.Process/SendInput') + assert base64.b64decode( + session.posts[2]['data']['input']['pty']) == b'abc' + assert session.posts[3]['url'].endswith('/process.Process/Update') + assert session.posts[3]['data']['pty'] == { 'size': {'rows': 30, 'cols': 100}, } - assert session.posts[3]['url'].endswith('/process.Process/Connect') - assert session.posts[4]['url'].endswith('/process.Process/SendSignal') + assert session.posts[4]['url'].endswith('/process.Process/Connect') + assert session.posts[5]['url'].endswith('/process.Process/SendSignal') def test_pty_create_and_connect_handle_empty_event_streams(): From 18579c1231cfbd1448bd0b805251b26625b4eaf5 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:30:27 +0800 Subject: [PATCH 31/45] fix(sandbox): simplify stream decoding and cleanup --- qiniu/services/sandbox/commands.py | 6 +----- qiniu/services/sandbox/git.py | 19 +++++++------------ .../test_services/test_sandbox/test_envd.py | 6 +++--- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index dd35d5bb..619587b5 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import base64 -import binascii from qiniu.compat import basestring, bytes as bytes_type @@ -56,10 +55,7 @@ def _decode_bytes(value): if isinstance(value, bytes_type): return value.decode('utf-8', 'replace') if isinstance(value, basestring): - try: - return base64.b64decode(value).decode('utf-8', 'replace') - except (binascii.Error, TypeError): - return value + return base64.b64decode(value).decode('utf-8', 'replace') return str(value) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index ab5d2c31..9434cf69 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -419,21 +419,16 @@ def _with_remote_credentials(self, repo_path, remote, username, password, }) auth_opts['envs'] = envs - operation_error = None try: result = operation(auth_opts) - if opts.get('background') and hasattr(result, 'wait'): - result.wait = _cleanup_after_wait( - result.wait, filesystem, askpass_path) - askpass_path = None except BaseException as err: - operation_error = err - result = None - finally: - if askpass_path: - _remove_credential_file(filesystem, askpass_path) - if operation_error is not None: - raise operation_error + _remove_credential_file(filesystem, askpass_path) + raise err + if opts.get('background') and hasattr(result, 'wait'): + result.wait = _cleanup_after_wait( + result.wait, filesystem, askpass_path) + else: + _remove_credential_file(filesystem, askpass_path) return result def _raise_known_result_error( diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index adaaf150..ea64cd35 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -205,14 +205,14 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): assert result.stderr == u'\ufffd\ufffd\ufffd' -def test_command_event_decode_preserves_plain_strings(): +def test_command_event_decode_decodes_plain_word_base64(): result = command_result_from_events([{ 'event': {'data': { - 'stdout': 'plain text!', + 'stdout': base64.b64encode(b'text').decode('ascii'), }}, }]) - assert result.stdout == 'plain text!' + assert result.stdout == 'text' def test_command_event_decode_handles_bytes_values(): From 66218bbc1de790fefe61faa352ec050216cb62f4 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:35:13 +0800 Subject: [PATCH 32/45] fix(sandbox): tighten async and timeout edge cases --- qiniu/services/sandbox/client.py | 5 +- qiniu/services/sandbox/git.py | 25 ++++- qiniu/services/sandbox/sandbox.py | 6 +- .../test_services/test_sandbox/test_client.py | 98 +++++++++++++++++++ 4 files changed, 127 insertions(+), 7 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 49ec82f7..18e923fd 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -220,7 +220,10 @@ def _request(self, method, path, params=None, body=_UNSET, try: response_data = response.json() except ValueError: - response_data = getattr(response, 'text', None) + try: + response_data = response.text + except Exception: + response_data = getattr(response, 'content', None) message = 'Sandbox API request failed with status {0}'.format( response.status_code ) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 9434cf69..db842052 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import time +import uuid from qiniu.compat import basestring @@ -390,20 +390,22 @@ def _with_remote_credentials(self, repo_path, remote, username, password, 'Sandbox filesystem is required for credentialed Git ' 'operations.') temp_dir = '/tmp/qiniu-git-auth' + setup_opts = dict(opts) + setup_opts.pop('background', None) prepare_result = self.commands.run( 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), - **opts + **setup_opts ) if getattr(prepare_result, 'exit_code', 0): return prepare_result askpass_path = '{0}/qiniu-git-askpass-{1}'.format( temp_dir, - int(time.time() * 1000), + uuid.uuid4().hex, ) filesystem.write(askpass_path, _askpass_script()) chmod_result = self.commands.run( 'chmod 700 {0}'.format(shell_quote(askpass_path)), - **opts + **setup_opts ) if getattr(chmod_result, 'exit_code', 0): _remove_credential_file(filesystem, askpass_path) @@ -433,6 +435,11 @@ def _with_remote_credentials(self, repo_path, remote, username, password, def _raise_known_result_error( self, result, operation, throw_on_error=False): + if (getattr(result, 'exit_code', None) == -1 and + hasattr(result, 'wait')): + if throw_on_error: + result.wait = self._raise_after_wait(result.wait, operation) + return if not hasattr(result, 'exit_code'): return if result.exit_code: @@ -447,6 +454,14 @@ def _raise_known_result_error( if throw_on_error: raise CommandExitError(result) + def _raise_after_wait(self, wait, operation): + def wrapped_wait(*args, **kwargs): + result = wait(*args, **kwargs) + self._raise_known_result_error( + result, operation, throw_on_error=True) + return result + return wrapped_wait + def clone(self, repo_url, path=None, branch=None, depth=None, **opts): args = ['clone'] if depth: @@ -625,7 +640,7 @@ def dangerously_authenticate( return prepare_result path = '{0}/qiniu-git-credential-{1}'.format( temp_dir, - int(time.time() * 1000)) + uuid.uuid4().hex) filesystem.write(path, credential) quoted_path = shell_quote(path) try: diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 62e2be13..6621e0bb 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -349,7 +349,11 @@ def wait_for_ready(self, timeout=60, interval=1): return self except requests.RequestException: pass - time.sleep(interval) + sleep_time = interval + if remaining is not None: + sleep_time = min(interval, remaining) + if sleep_time > 0: + time.sleep(sleep_time) waitForReady = wait_for_ready diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 81825a8b..d4f9e67c 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -3,6 +3,7 @@ import pytest import requests +import qiniu.services.sandbox.sandbox as sandbox_module try: from urllib.parse import parse_qs, urlparse @@ -323,6 +324,29 @@ def json(self): assert str(err.value).endswith('...') +def test_client_falls_back_to_content_when_error_text_fails(): + class BrokenTextResponse(object): + status_code = 502 + content = b'raw error bytes' + headers = {} + + def json(self): + raise ValueError('not json') + + @property + def text(self): + raise UnicodeDecodeError('utf-8', b'\xff', 0, 1, 'bad') + + client = SandboxClient( + api_key='api-key', + session=RecordingSession([BrokenTextResponse()])) + + with pytest.raises(SandboxError) as err: + client.list_sandboxes() + + assert err.value.data == b'raw error bytes' + + def test_client_includes_nested_error_message_in_api_errors(): client = SandboxClient( api_key='api-key', @@ -489,6 +513,27 @@ def test_wait_for_ready_caps_request_timeout_to_remaining_timeout(): assert session.requests[0].kwargs['timeout'] <= 3 +def test_wait_for_ready_caps_sleep_to_remaining_timeout(monkeypatch): + session = RecordingSession([DummyResponse(503, {})]) + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={ + 'sandboxID': 'sbx123', + 'domain': 'example.test', + }) + times = iter([0, 0, 2.5, 3]) + sleeps = [] + monkeypatch.setattr( + sandbox_module, '_monotonic_time', lambda: next(times)) + monkeypatch.setattr(sandbox_module.time, 'sleep', sleeps.append) + + with pytest.raises(SandboxError): + sandbox.wait_for_ready(timeout=3, interval=2) + + assert sleeps == [0.5] + + def test_wait_for_ready_ignores_startup_request_errors_until_ready(): session = RecordingSession([ requests.exceptions.ConnectionError('envd is starting'), @@ -730,6 +775,7 @@ def run(self, cmd, **opts): if opts.get('background'): return type('Handle', (object,), { 'pid': getattr(result, 'pid', 12), + 'exit_code': -1, 'wait': lambda self: result, })() return result @@ -1218,6 +1264,58 @@ def test_git_credential_helpers_ignore_background_when_reading_remote(): assert files.removes == [(files.writes[0][0], {})] +def test_git_background_throw_on_error_waits_before_raising(): + commands = RecordingCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'origin\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 1, + 'stdout': '', + 'stderr': 'fatal: failed', + 'error': '', + })(), + ] + git = Git(commands) + + handle = git.pull( + '/repo', + branch='main', + username='git-user', + password='secret-token', + background=True, + throw_on_error=True, + ) + + assert files.removes == [] + with pytest.raises(CommandExitError): + handle.wait() + assert files.removes == [(files.writes[0][0], {})] + + def test_git_push_with_credentials_cleans_askpass_on_auth_failure(): commands = RecordingCommands() files = attach_recording_files(commands) From 6af02ec995dcbbf401bda9abd6302535e9a18f24 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:39:52 +0800 Subject: [PATCH 33/45] fix(sandbox): close command git and template edge cases --- qiniu/services/sandbox/commands.py | 10 ++-- qiniu/services/sandbox/filesystem.py | 4 +- qiniu/services/sandbox/git.py | 49 +++++++++---------- qiniu/services/sandbox/template.py | 4 +- qiniu/services/sandbox/util.py | 6 +-- .../test_services/test_sandbox/test_client.py | 34 ++++++++++--- .../test_services/test_sandbox/test_envd.py | 25 +++++++++- 7 files changed, 90 insertions(+), 42 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 619587b5..9e4f9c8b 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import base64 +import binascii from qiniu.compat import basestring, bytes as bytes_type @@ -55,7 +56,10 @@ def _decode_bytes(value): if isinstance(value, bytes_type): return value.decode('utf-8', 'replace') if isinstance(value, basestring): - return base64.b64decode(value).decode('utf-8', 'replace') + try: + return base64.b64decode(value).decode('utf-8', 'replace') + except (binascii.Error, TypeError): + return value return str(value) @@ -87,8 +91,8 @@ def command_result_from_events(events, on_stdout=None, on_stderr=None, stderr += stderr_chunk stdout += pty_chunk if end: - exit_code = 0 if end.get( - 'exitCode') is None else end.get('exitCode') + if end.get('exitCode') is not None: + exit_code = end.get('exitCode') error = end.get('error') or '' return CommandResult(pid, exit_code, stdout, stderr, error) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 5004720f..ffd46d0b 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -143,7 +143,9 @@ def to_upload_body(data, encoding='utf-8'): return data if isinstance(data, basestring): return data.encode(encoding) - if isinstance(data, (TextIOBase, IOBase)) or hasattr(data, 'read'): + if isinstance(data, TextIOBase): + return data.read().encode(encoding) + if isinstance(data, IOBase) or hasattr(data, 'read'): return data raise InvalidArgumentException( 'Unsupported data type for filesystem write: {0}'.format(type(data))) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index db842052..bf1affb1 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -4,10 +4,9 @@ from qiniu.compat import basestring try: - from urllib.parse import quote, urlparse, urlunparse + from urllib.parse import urlparse except ImportError: - from urllib import quote - from urlparse import urlparse, urlunparse + from urlparse import urlparse from .errors import ( CommandExitError, @@ -293,24 +292,6 @@ def _is_missing_upstream(result): return any(snippet in message for snippet in snippets) -def _with_credentials(url, username, password): - parsed = _validate_git_url_credentials(url, username, password) - if parsed is None: - return url - host = parsed.hostname or '' - if ':' in host and not host.startswith('['): - host = '[{0}]'.format(host) - if parsed.port: - host = '{0}:{1}'.format(host, parsed.port) - return urlunparse(parsed._replace( - netloc='{0}:{1}@{2}'.format( - quote(str(username), safe=''), - quote(str(password), safe=''), - host, - ) - )) - - def _validate_git_url_credentials(url, username, password): if not username and not password: return None @@ -494,6 +475,7 @@ def add(self, repo_path, files=None, all=False, **opts): if all: args.append('--all') else: + args.append('--') for path in _normalize_paths(files) or ['.']: args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) @@ -532,7 +514,7 @@ def pull(self, repo_path, remote=None, branch=None, username=None, password=None, **opts): throw_on_error = opts.pop('throw_on_error', False) if password and not username: - raise GitAuthException( + raise InvalidArgumentException( 'Git pull requires username when password is provided') remote_name = None if username and password: @@ -567,7 +549,7 @@ def push(self, repo_path, remote=None, branch=None, set_upstream=True, username=None, password=None, **opts): throw_on_error = opts.pop('throw_on_error', False) if password and not username: - raise GitAuthException( + raise InvalidArgumentException( 'Git push requires username when password is provided') remote_name = None if username and password: @@ -757,12 +739,21 @@ def restore(self, repo_path, paths=None, staged=False, source=None, if source: args.extend(['--source', shell_quote(source)]) paths = paths if paths is not None else opts.get('files') + args.append('--') for path in _normalize_paths(paths) or ['.']: args.append(shell_quote(path)) return self._run_git(repo_path, args, **opts) def set_config(self, key, value, scope='global', path=None, **opts): - """Set a Git config value.""" + """Set a Git config value. + + Preferred signature: + set_config(key, value, scope='global', path=None, **opts) + + For local config, pass scope='local' and path=repo_path. For backward + compatibility this method also accepts the deprecated signature: + set_config(repo_path, key, value, global_config=False, **opts) + """ key, value, scope, path = self._normalize_set_config_args( key, value, scope, path, opts) scope_flag, repo_path = self._resolve_config_scope(scope, path) @@ -775,7 +766,15 @@ def set_config(self, key, value, scope='global', path=None, **opts): setConfig = set_config def get_config(self, key, scope='global', path=None, **opts): - """Get a Git config value.""" + """Get a Git config value. + + Preferred signature: + get_config(key, scope='global', path=None, **opts) + + For local config, pass scope='local' and path=repo_path. For backward + compatibility this method also accepts the deprecated signature: + get_config(repo_path, key, global_config=False, **opts) + """ key, scope, path = self._normalize_get_config_args( key, scope, path, opts) scope_flag, repo_path = self._resolve_config_scope(scope, path) diff --git a/qiniu/services/sandbox/template.py b/qiniu/services/sandbox/template.py index 4730e54f..e719100e 100644 --- a/qiniu/services/sandbox/template.py +++ b/qiniu/services/sandbox/template.py @@ -14,7 +14,9 @@ def get_cmd(self): def wait_for_port(port): port = int(port) - return ReadyCmd('ss -tuln | grep :{0}'.format(port)) + return ReadyCmd( + "ss -tuln | awk '{{print $5}}' | grep -E '(^|:){0}$'".format( + port)) def wait_for_url(url, status_code=200): diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index f8522604..a081c4f7 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -7,7 +7,7 @@ import time from qiniu.compat import bytes as bytes_type -from qiniu.compat import is_py2, str as text_type, urlencode, urlparse +from qiniu.compat import is_py2, str as text_type, urlencode from .constants import DEFAULT_ENDPOINT, DEFAULT_USER @@ -106,7 +106,3 @@ def get_info_value(info, camel_key, snake_key=None): if snake_key and snake_key in info: return info.get(snake_key) return None - - -def parsed_url(url): - return urlparse(url) diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index d4f9e67c..15f3ff4c 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -712,7 +712,8 @@ def test_template_builder_outputs_build_config(): def test_template_ready_cmd_helpers_align_with_e2b(): ready = wait_for_port(8000) assert isinstance(ready, ReadyCmd) - assert ready.get_cmd() == 'ss -tuln | grep :8000' + assert ready.get_cmd() == ( + "ss -tuln | awk '{print $5}' | grep -E '(^|:)8000$'") assert wait_for_url( 'http://localhost:3000/health', status_code=204, @@ -731,7 +732,8 @@ def test_template_ready_cmd_helpers_align_with_e2b(): ) assert template.to_dict()['startCmd'] == 'python app.py' - assert template.to_dict()['readyCmd'] == 'ss -tuln | grep :8000' + assert template.to_dict()['readyCmd'] == ( + "ss -tuln | awk '{print $5}' | grep -E '(^|:)8000$'") def test_template_ready_cmd_helpers_quote_shell_inputs(): @@ -833,7 +835,7 @@ def test_git_helpers_align_with_e2b_method_names(): assert commands.calls[4][0] == 'git checkout main' assert commands.calls[5][0] == 'git branch -D old' assert commands.calls[6][0] == "git reset --hard 'HEAD~1'" - assert commands.calls[7][0] == 'git restore a.txt b.txt' + assert commands.calls[7][0] == 'git restore -- a.txt b.txt' assert commands.calls[8][0] == 'git config --local user.name tester' assert commands.calls[8][1]['cwd'] == '/repo' assert commands.calls[9][0] == 'git config --local --get user.name' @@ -866,9 +868,20 @@ def test_git_add_and_restore_accept_single_string_path(): git.restore('/repo', paths='README.md') git.restore('/repo', files='setup.py') - assert commands.calls[0][0] == 'git add README.md' - assert commands.calls[1][0] == 'git restore README.md' - assert commands.calls[2][0] == 'git restore setup.py' + assert commands.calls[0][0] == 'git add -- README.md' + assert commands.calls[1][0] == 'git restore -- README.md' + assert commands.calls[2][0] == 'git restore -- setup.py' + + +def test_git_add_and_restore_use_path_separator_for_dash_paths(): + commands = RecordingCommands() + git = Git(commands) + + git.add('/repo', files='--intent-to-add') + git.restore('/repo', paths='--worktree') + + assert commands.calls[0][0] == "git add -- --intent-to-add" + assert commands.calls[1][0] == "git restore -- --worktree" def test_git_reset_rejects_unsupported_mode(): @@ -1076,6 +1089,15 @@ def test_git_credential_remote_requires_existing_remote(): assert 'No remotes found' in str(err.value) +def test_git_credentials_require_username_with_password(): + git = Git(RecordingCommands()) + + with pytest.raises(InvalidArgumentException): + git.pull('/repo', password='secret') + with pytest.raises(InvalidArgumentException): + git.push('/repo', password='secret') + + def test_git_push_maps_known_errors_before_throw_on_error(): commands = RecordingCommands() commands.results = [type('Result', (object,), { diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index ea64cd35..3aeca25e 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import base64 -from io import BytesIO +from io import BytesIO, StringIO import json import struct @@ -205,6 +205,16 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): assert result.stderr == u'\ufffd\ufffd\ufffd' +def test_command_event_decode_keeps_invalid_base64_strings(): + result = command_result_from_events([{ + 'event': {'data': { + 'stderr': 'plain text!', + }}, + }]) + + assert result.stderr == 'plain text!' + + def test_command_event_decode_decodes_plain_word_base64(): result = command_result_from_events([{ 'event': {'data': { @@ -215,6 +225,14 @@ def test_command_event_decode_decodes_plain_word_base64(): assert result.stdout == 'text' +def test_command_event_decode_keeps_unknown_exit_code(): + result = command_result_from_events([{ + 'event': {'end': {}}, + }]) + + assert result.exit_code == -1 + + def test_command_event_decode_handles_bytes_values(): result = command_result_from_events([{ 'event': {'data': { @@ -285,6 +303,7 @@ def test_filesystem_write_accepts_unicode_text(): sandbox.files.write('/tmp/unicode.txt', u'你好') sandbox.files.write('/tmp/bytes.txt', bytearray(b'abc')) + sandbox.files.write('/tmp/text-stream.txt', StringIO(u'你好')) assert session.requests[0]['kwargs']['files']['file'] == ( 'unicode.txt', @@ -294,6 +313,10 @@ def test_filesystem_write_accepts_unicode_text(): 'bytes.txt', b'abc', ) + assert session.requests[2]['kwargs']['files']['file'] == ( + 'text-stream.txt', + u'你好'.encode('utf-8'), + ) def test_filesystem_write_accepts_duck_typed_file_like_objects(): From 03049b43ea691f70ada2b3d5bd516081dc479c0f Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:43:10 +0800 Subject: [PATCH 34/45] fix(sandbox): cover readiness and credential cleanup edges --- qiniu/services/sandbox/config.py | 2 +- qiniu/services/sandbox/git.py | 48 +++++++++---------- qiniu/services/sandbox/sandbox.py | 4 +- qiniu/services/sandbox/util.py | 9 +++- .../test_services/test_sandbox/test_client.py | 44 ++++++++++++++++- 5 files changed, 77 insertions(+), 30 deletions(-) diff --git a/qiniu/services/sandbox/config.py b/qiniu/services/sandbox/config.py index 6d192bb9..f96a1033 100644 --- a/qiniu/services/sandbox/config.py +++ b/qiniu/services/sandbox/config.py @@ -30,8 +30,8 @@ def load_dotenv_if_present(*paths): value[0] in ('"', "'") ): value = value[1:-1] + key, value = _native_env_pair(key, value) if key and key not in os.environ: - key, value = _native_env_pair(key, value) os.environ[key] = value diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index bf1affb1..ae066925 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -384,35 +384,35 @@ def _with_remote_credentials(self, repo_path, remote, username, password, uuid.uuid4().hex, ) filesystem.write(askpass_path, _askpass_script()) - chmod_result = self.commands.run( - 'chmod 700 {0}'.format(shell_quote(askpass_path)), - **setup_opts - ) - if getattr(chmod_result, 'exit_code', 0): - _remove_credential_file(filesystem, askpass_path) - return chmod_result - - auth_opts = dict(opts) - envs = dict(auth_opts.get('envs') or {}) - envs.update({ - 'GIT_ASKPASS': askpass_path, - 'GIT_TERMINAL_PROMPT': '0', - 'GIT_USERNAME': username, - 'GIT_PASSWORD': password, - }) - auth_opts['envs'] = envs - try: + chmod_result = self.commands.run( + 'chmod 700 {0}'.format(shell_quote(askpass_path)), + **setup_opts + ) + if getattr(chmod_result, 'exit_code', 0): + _remove_credential_file(filesystem, askpass_path) + return chmod_result + + auth_opts = dict(opts) + envs = dict(auth_opts.get('envs') or {}) + envs.update({ + 'GIT_ASKPASS': askpass_path, + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_USERNAME': username, + 'GIT_PASSWORD': password, + }) + auth_opts['envs'] = envs + result = operation(auth_opts) + if opts.get('background') and hasattr(result, 'wait'): + result.wait = _cleanup_after_wait( + result.wait, filesystem, askpass_path) + else: + _remove_credential_file(filesystem, askpass_path) + return result except BaseException as err: _remove_credential_file(filesystem, askpass_path) raise err - if opts.get('background') and hasattr(result, 'wait'): - result.wait = _cleanup_after_wait( - result.wait, filesystem, askpass_path) - else: - _remove_credential_file(filesystem, askpass_path) - return result def _raise_known_result_error( self, result, operation, throw_on_error=False): diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 6621e0bb..dedefc81 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -333,8 +333,6 @@ def upload_url(self, path, **opts): def wait_for_ready(self, timeout=60, interval=1): started = _monotonic_time() while True: - if timeout is not None and _monotonic_time() - started >= timeout: - raise SandboxError('Sandbox envd did not become ready') elapsed = _monotonic_time() - started remaining = None if timeout is None else max(timeout - elapsed, 0) request_timeout = 5 @@ -349,6 +347,8 @@ def wait_for_ready(self, timeout=60, interval=1): return self except requests.RequestException: pass + if timeout is not None and elapsed >= timeout: + raise SandboxError('Sandbox envd did not become ready') sleep_time = interval if remaining is not None: sleep_time = min(interval, remaining) diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index a081c4f7..cc6ca463 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -86,7 +86,14 @@ def shell_quote(value): from shlex import quote except ImportError: from pipes import quote - return quote(str(value)) + if is_py2: + if isinstance(value, text_type): + value = value.encode('utf-8') + else: + value = str(value) + else: + value = str(value) + return quote(value) def utc_timestamp_after(seconds): diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 15f3ff4c..f81d73c1 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -514,7 +514,10 @@ def test_wait_for_ready_caps_request_timeout_to_remaining_timeout(): def test_wait_for_ready_caps_sleep_to_remaining_timeout(monkeypatch): - session = RecordingSession([DummyResponse(503, {})]) + session = RecordingSession([ + DummyResponse(503, {}), + DummyResponse(503, {}), + ]) sandbox = Sandbox(client=SandboxClient( api_key='api-key', session=session, @@ -522,7 +525,7 @@ def test_wait_for_ready_caps_sleep_to_remaining_timeout(monkeypatch): 'sandboxID': 'sbx123', 'domain': 'example.test', }) - times = iter([0, 0, 2.5, 3]) + times = iter([0, 2.5, 3]) sleeps = [] monkeypatch.setattr( sandbox_module, '_monotonic_time', lambda: next(times)) @@ -566,6 +569,7 @@ def test_wait_for_ready_raises_sandbox_error_on_timeout(): with pytest.raises(SandboxError): sandbox.wait_for_ready(timeout=0, interval=0) + assert len(session.requests) == 1 def test_update_info_refreshes_traffic_access_token(): @@ -1420,6 +1424,42 @@ def test_git_push_cleans_askpass_after_operation_exception(): assert files.removes == [(files.writes[0][0], {})] +def test_git_credentials_clean_askpass_when_chmod_raises(): + class ChmodFailCommands(RecordingCommands): + def run(self, cmd, **opts): + if cmd.startswith('chmod 700 '): + raise SandboxError('chmod rpc failed') + return super(ChmodFailCommands, self).run(cmd, **opts) + + commands = ChmodFailCommands() + files = attach_recording_files(commands) + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError): + git.push( + '/repo', + remote='origin', + username='git-user', + password='secret-token', + ) + + assert files.removes == [(files.writes[0][0], {})] + + def test_git_helpers_accept_e2b_style_signatures(): commands = RecordingCommands() git = Git(commands) From 59d2d3e2c815ef6b524725ff63307b6c2c96c331 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:48:03 +0800 Subject: [PATCH 35/45] fix(sandbox): avoid credential files and harden polling --- qiniu/services/sandbox/client.py | 38 +++--- qiniu/services/sandbox/git.py | 29 ----- qiniu/services/sandbox/sandbox.py | 2 + .../test_services/test_sandbox/test_client.py | 109 +++++++++--------- 4 files changed, 82 insertions(+), 96 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 18e923fd..4a18e720 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -297,6 +297,8 @@ def delete_sandbox(self, sandbox_id): kill = delete_sandbox def pause_sandbox(self, sandbox_id): + if not sandbox_id: + raise SandboxError('sandbox_id is required') return self._request( 'POST', '/sandboxes/{0}/pause'.format(encode_path(sandbox_id)), @@ -307,6 +309,8 @@ def pause_sandbox(self, sandbox_id): pauseSandbox = pause_sandbox def resume_sandbox(self, sandbox_id, **opts): + if not sandbox_id: + raise SandboxError('sandbox_id is required') return self._request( 'POST', '/sandboxes/{0}/resume'.format(encode_path(sandbox_id)), @@ -316,6 +320,8 @@ def resume_sandbox(self, sandbox_id, **opts): resumeSandbox = resume_sandbox def connect_sandbox(self, sandbox_id, timeout=15): + if not sandbox_id: + raise SandboxError('sandbox_id is required') return self._request( 'POST', '/sandboxes/{0}/connect'.format(encode_path(sandbox_id)), @@ -582,20 +588,24 @@ def delete_injection_rule(self, rule_id): def wait_for_build(self, template_id, build_id, interval=1, timeout=60): start = _monotonic_time() while True: - info = self.get_template_build_status(template_id, build_id) - if info and info.get('status') in ('ready', 'error'): - if info.get('status') == 'error': - message = ( - ( - isinstance(info.get('error'), dict) and - info.get('error').get('message') - ) or - info.get('error') or - info.get('message') or - 'Sandbox template build failed' - ) - raise TemplateBuildError(message, data=info) - return info + try: + info = self.get_template_build_status(template_id, build_id) + if info and info.get('status') in ('ready', 'error'): + if info.get('status') == 'error': + message = ( + ( + isinstance(info.get('error'), dict) and + info.get('error').get('message') + ) or + info.get('error') or + info.get('message') or + 'Sandbox template build failed' + ) + raise TemplateBuildError(message, data=info) + return info + except SandboxError as err: + if isinstance(err, TemplateBuildError): + raise if _monotonic_time() - start >= timeout: raise SandboxError('Sandbox template build polling timed out') time.sleep(interval) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index ae066925..4c95b0d0 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -610,35 +610,6 @@ def dangerously_authenticate( 'username={2}\n' 'password={3}\n\n' ).format(protocol, host, username, password) - sandbox = getattr(self.commands, 'sandbox', None) - filesystem = getattr(sandbox, 'files', None) - if filesystem is not None: - temp_dir = '/tmp/qiniu-git-auth' - prepare_result = self.commands.run( - 'install -d -m 700 {0}'.format(shell_quote(temp_dir)), - **opts - ) - if prepare_result.exit_code != 0: - return prepare_result - path = '{0}/qiniu-git-credential-{1}'.format( - temp_dir, - uuid.uuid4().hex) - filesystem.write(path, credential) - quoted_path = shell_quote(path) - try: - chmod_result = self.commands.run( - 'chmod 600 {0}'.format(quoted_path), - **opts - ) - if chmod_result.exit_code != 0: - return chmod_result - script = ( - 'trap "rm -f {0}" EXIT; ' - 'git credential approve < {0}' - ).format(quoted_path) - return self.commands.run(script, **opts) - finally: - _remove_credential_file(filesystem, path) handle = self.commands.run( 'git credential approve', stdin=True, diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index dedefc81..758e8bef 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -276,6 +276,8 @@ def get_host(self, port): def envd_url(self): if self._envd_url: return self._envd_url + if not self.domain: + raise SandboxError('Sandbox domain is not available') return 'https://{0}'.format(self.get_host(ENVD_PORT)) envdUrl = envd_url diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index f81d73c1..6df9e358 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -411,6 +411,17 @@ def test_connect_sandbox_uses_timeout_body_only(): assert body_of(session.requests[0]) == {'timeout': 9} +def test_sandbox_control_methods_require_sandbox_id(): + client = SandboxClient(api_key='api-key', session=RecordingSession()) + + with pytest.raises(SandboxError): + client.pause_sandbox('') + with pytest.raises(SandboxError): + client.resume_sandbox(None) + with pytest.raises(SandboxError): + client.connect_sandbox('') + + def test_sandbox_connect_forwards_envd_options_to_instance(): session = RecordingSession([DummyResponse(200, {'sandboxID': 'sbx123'})]) client = SandboxClient(api_key='api-key', session=session) @@ -461,6 +472,13 @@ def test_sandbox_envd_and_file_urls_are_signed_when_token_is_available(): assert query['signature'][0] +def test_envd_url_requires_domain_without_override(): + sandbox = Sandbox(info={'sandboxID': 'sbx123'}) + + with pytest.raises(SandboxError): + sandbox.envd_url() + + def test_file_url_accepts_none_signature_expiration(): sandbox = Sandbox(info={ 'sandboxID': 'sbx123', @@ -622,6 +640,35 @@ def test_sandbox_paginator_does_not_send_client_credentials_as_filters(): assert client.calls[0] == {'metadata': {'app': 'tests'}} +def test_wait_for_build_retries_transient_sandbox_errors(monkeypatch): + class BuildClient(SandboxClient): + def __init__(self): + super(BuildClient, self).__init__( + api_key='api-key', + session=RecordingSession(), + ) + self.calls = 0 + + def get_template_build_status(self, template_id, build_id, **opts): + del template_id, build_id, opts + self.calls += 1 + if self.calls == 1: + raise SandboxError('temporary gateway error') + return {'status': 'ready', 'templateID': 'tmpl123'} + + monkeypatch.setattr( + 'qiniu.services.sandbox.client.time.sleep', + lambda x: x, + ) + client = BuildClient() + + assert client.wait_for_build('tmpl123', 'build123', interval=0) == { + 'status': 'ready', + 'templateID': 'tmpl123', + } + assert client.calls == 2 + + def test_is_running_matches_e2b_health_check_semantics(): running_session = RecordingSession([DummyResponse(200, {})]) running = Sandbox(client=SandboxClient( @@ -924,7 +971,7 @@ def test_git_dangerously_authenticate_aligns_with_e2b(): assert commands.calls[3] == ('close_stdin', {'pid': 12}) -def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): +def test_git_dangerously_authenticate_uses_stdin_with_real_sandbox(): commands = RecordingCommands() files = attach_recording_files(commands) git = Git(commands) @@ -936,24 +983,16 @@ def test_git_dangerously_authenticate_uses_temp_file_with_real_sandbox(): protocol='https', ) - assert files.writes[0][1] == ( + assert files.writes == [] + assert files.removes == [] + assert commands.calls[1][0] == 'git credential approve' + assert commands.calls[2][1]['data'] == ( 'protocol=https\nhost=github.com\n' 'username=git-user\npassword=secret-%-token\n\n' ) - assert files.writes[0][0].startswith( - '/tmp/qiniu-git-auth/qiniu-git-credential-') - assert files.writes[0][2] == {} - assert commands.calls[1][0] == 'install -d -m 700 /tmp/qiniu-git-auth' - assert commands.calls[2][0].startswith( - 'chmod 600 /tmp/qiniu-git-auth/qiniu-git-credential-') - assert commands.calls[3][0].startswith( - 'trap "rm -f /tmp/qiniu-git-auth/qiniu-git-credential-') - assert 'git credential approve' in commands.calls[3][0] - assert 'secret-%-token' not in commands.calls[3][0] - assert files.removes == [(files.writes[0][0], {})] -def test_git_dangerously_authenticate_ignores_background_for_temp_file(): +def test_git_dangerously_authenticate_ignores_background_for_stdin_flow(): commands = RecordingCommands() files = attach_recording_files(commands) git = Git(commands) @@ -965,47 +1004,11 @@ def test_git_dangerously_authenticate_ignores_background_for_temp_file(): ) assert 'background' not in commands.calls[0][1] - assert 'background' not in commands.calls[1][1] + assert commands.calls[1][1]['background'] is True assert 'background' not in commands.calls[2][1] assert 'background' not in commands.calls[3][1] - assert files.removes == [(files.writes[0][0], {})] - - -def test_git_dangerously_authenticate_removes_temp_file_on_chmod_failure(): - commands = RecordingCommands() - commands.results = [ - type('Result', (object,), { - 'pid': 12, - 'exit_code': 0, - 'stdout': '', - 'stderr': '', - 'error': '', - })(), - type('Result', (object,), { - 'pid': 12, - 'exit_code': 0, - 'stdout': '', - 'stderr': '', - 'error': '', - })(), - type('Result', (object,), { - 'pid': 12, - 'exit_code': 1, - 'stdout': '', - 'stderr': 'chmod failed', - 'error': 'chmod failed', - })(), - ] - files = attach_recording_files(commands) - git = Git(commands) - - result = git.dangerously_authenticate( - username='git-user', - password='secret-token', - ) - - assert result.exit_code == 1 - assert files.removes == [(files.writes[0][0], {})] + assert files.writes == [] + assert files.removes == [] def test_git_status_and_branches_return_structured_e2b_types(): From 563d356daea466d5d024b1a44e96806527644369 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:52:06 +0800 Subject: [PATCH 36/45] fix(sandbox): address review edge cases --- qiniu/services/sandbox/client.py | 19 ++++++--- qiniu/services/sandbox/commands.py | 12 +++++- qiniu/services/sandbox/filesystem.py | 17 +++++++- .../test_services/test_sandbox/test_client.py | 42 ++++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 19 ++++++--- 5 files changed, 94 insertions(+), 15 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 4a18e720..0cd5fe0f 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -5,7 +5,7 @@ import requests from qiniu.auth import QiniuMacAuth, QiniuMacRequestsAuth -from qiniu.compat import basestring +from qiniu.compat import basestring, urlencode from .constants import DEFAULT_TEMPLATE from .errors import SandboxError, TemplateBuildError @@ -133,9 +133,10 @@ def _normalize_list_options(opts): opts = dict(opts or {}) query = opts.pop('query', None) or {} metadata = query.get('metadata') - if metadata: - for key, value in metadata.items(): - opts['metadata[{0}]'.format(key)] = value + if metadata is not None: + opts['metadata'] = metadata + if isinstance(opts.get('metadata'), dict): + opts['metadata'] = urlencode(opts.get('metadata')) if query.get('state') is not None: opts['state'] = query.get('state') return opts @@ -170,6 +171,8 @@ def __init__(self, endpoint=None, api_url=None, api_key=None, def _headers(self, auth_type=None): headers = {'Content-Type': 'application/json'} + if self.api_key: + headers['X-API-Key'] = self.api_key if auth_type == 'qiniu': return headers if auth_type == 'accessToken': @@ -180,7 +183,6 @@ def _headers(self, auth_type=None): self.access_token) return headers if self.api_key: - headers['X-API-Key'] = self.api_key headers['Authorization'] = 'Bearer {0}'.format(self.api_key) elif self.access_token: headers['Authorization'] = 'Bearer {0}'.format(self.access_token) @@ -386,7 +388,12 @@ def get_sandboxes_metrics(self, sandbox_ids): values = [sandbox_ids] if values is None: raise SandboxError('At least one sandbox ID must be provided') - if not isinstance(values, (list, tuple, set)): + if isinstance(values, (list, tuple, set)): + values = list(values) + elif hasattr(values, '__iter__') and not isinstance( + values, (basestring, dict)): + values = list(values) + else: values = [values] ids = [] for value in values: diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 9e4f9c8b..0b2d9c30 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import base64 import binascii +import re from qiniu.compat import basestring, bytes as bytes_type @@ -8,6 +9,9 @@ from .errors import CommandExitError, SandboxError +_BASE64_RE = re.compile(r'^[A-Za-z0-9+/]+={0,2}$') + + class ProcessInfo(object): def __init__(self, pid=None, tag=None, cmd=None, args=None, envs=None, cwd=None): @@ -56,9 +60,13 @@ def _decode_bytes(value): if isinstance(value, bytes_type): return value.decode('utf-8', 'replace') if isinstance(value, basestring): + candidate = value.strip() + if not candidate or len(candidate) % 4 != 0 or not _BASE64_RE.match( + candidate): + return value try: - return base64.b64decode(value).decode('utf-8', 'replace') - except (binascii.Error, TypeError): + return base64.b64decode(candidate).decode('utf-8') + except (binascii.Error, TypeError, UnicodeError): return value return str(value) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index ffd46d0b..21f7677a 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -8,6 +8,21 @@ from .util import file_basename +class _EncodedTextReader(object): + def __init__(self, stream, encoding): + self.stream = stream + self.encoding = encoding + + def read(self, size=-1): + chunk = self.stream.read(size) + if isinstance(chunk, basestring): + return chunk.encode(self.encoding) + return chunk + + def __getattr__(self, name): + return getattr(self.stream, name) + + class FileType(object): FILE = 'file' DIR = 'dir' @@ -144,7 +159,7 @@ def to_upload_body(data, encoding='utf-8'): if isinstance(data, basestring): return data.encode(encoding) if isinstance(data, TextIOBase): - return data.read().encode(encoding) + return _EncodedTextReader(data, encoding) if isinstance(data, IOBase) or hasattr(data, 'read'): return data raise InvalidArgumentException( diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 6df9e358..beb453f5 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -192,7 +192,12 @@ def test_create_with_kodo_resource_requires_qiniu_credentials(): def test_create_with_kodo_resource_uses_qiniu_signature(): session = RecordingSession( [DummyResponse(201, {'sandboxID': 'sbx123', 'templateID': 'base'})]) - client = SandboxClient(access_key='ak', secret_key='sk', session=session) + client = SandboxClient( + api_key='api-key', + access_key='ak', + secret_key='sk', + session=session, + ) client.create_sandbox( resources=[ @@ -201,6 +206,7 @@ def test_create_with_kodo_resource_uses_qiniu_signature(): mount_path='/mnt/bucket')]) req = session.requests[0] + assert req.headers['X-API-Key'] == 'api-key' assert req.headers['Authorization'].startswith('Qiniu ak:') assert 'X-Qiniu-Date' in req.headers assert body_of(req)['resources'] == [{ @@ -640,6 +646,30 @@ def test_sandbox_paginator_does_not_send_client_credentials_as_filters(): assert client.calls[0] == {'metadata': {'app': 'tests'}} +def test_list_sandboxes_v2_serializes_metadata_dict_as_query_string(): + session = RecordingSession([DummyResponse(200, {'items': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes_v2(query={ + 'metadata': {'app': 'tests'}, + 'state': 'running', + }) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == ['app=tests'] + assert query['state'] == ['running'] + + +def test_list_sandboxes_v2_accepts_metadata_string(): + session = RecordingSession([DummyResponse(200, {'items': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.list_sandboxes_v2(metadata='user=abc&app=prod') + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['metadata'] == ['user=abc&app=prod'] + + def test_wait_for_build_retries_transient_sandbox_errors(monkeypatch): class BuildClient(SandboxClient): def __init__(self): @@ -718,6 +748,16 @@ def test_get_sandboxes_metrics_accepts_set_values(): assert set(query['sandbox_ids'][0].split(',')) == set(['sbx1', 'sbx2']) +def test_get_sandboxes_metrics_accepts_generator_values(): + session = RecordingSession([DummyResponse(200, {'metrics': []})]) + client = SandboxClient(api_key='api-key', session=session) + + client.get_sandboxes_metrics(item for item in ['sbx1', 'sbx2']) + + query = parse_qs(urlparse(session.requests[0].url).query) + assert query['sandbox_ids'] == ['sbx1,sbx2'] + + def test_get_sandboxes_metrics_accepts_single_sandbox_dict(): session = RecordingSession([DummyResponse(200, {'metrics': []})]) client = SandboxClient(api_key='api-key', session=session) diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 3aeca25e..e244ae3a 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -202,7 +202,7 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): }]) assert result.stdout == 'YWJj' - assert result.stderr == u'\ufffd\ufffd\ufffd' + assert result.stderr == '////' def test_command_event_decode_keeps_invalid_base64_strings(): @@ -215,6 +215,16 @@ def test_command_event_decode_keeps_invalid_base64_strings(): assert result.stderr == 'plain text!' +def test_command_event_decode_keeps_plain_base64_looking_strings(): + result = command_result_from_events([{ + 'event': {'data': { + 'stdout': 'test', + }}, + }]) + + assert result.stdout == 'test' + + def test_command_event_decode_decodes_plain_word_base64(): result = command_result_from_events([{ 'event': {'data': { @@ -313,10 +323,9 @@ def test_filesystem_write_accepts_unicode_text(): 'bytes.txt', b'abc', ) - assert session.requests[2]['kwargs']['files']['file'] == ( - 'text-stream.txt', - u'你好'.encode('utf-8'), - ) + stream_upload = session.requests[2]['kwargs']['files']['file'] + assert stream_upload[0] == 'text-stream.txt' + assert stream_upload[1].read() == u'你好'.encode('utf-8') def test_filesystem_write_accepts_duck_typed_file_like_objects(): From e4aab28ba87526fe6aeb2e8efd495520f54f850c Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:54:16 +0800 Subject: [PATCH 37/45] fix(sandbox): preserve protocol errors and git status parsing --- qiniu/services/sandbox/client.py | 3 +- qiniu/services/sandbox/commands.py | 12 ++----- qiniu/services/sandbox/git.py | 2 +- .../test_services/test_sandbox/test_client.py | 33 +++++++++++++++++++ .../test_services/test_sandbox/test_envd.py | 8 ++--- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 0cd5fe0f..45747bda 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -611,7 +611,8 @@ def wait_for_build(self, template_id, build_id, interval=1, timeout=60): raise TemplateBuildError(message, data=info) return info except SandboxError as err: - if isinstance(err, TemplateBuildError): + if isinstance(err, TemplateBuildError) or ( + err.status_code is not None and err.status_code < 500): raise if _monotonic_time() - start >= timeout: raise SandboxError('Sandbox template build polling timed out') diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 0b2d9c30..9e4f9c8b 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import base64 import binascii -import re from qiniu.compat import basestring, bytes as bytes_type @@ -9,9 +8,6 @@ from .errors import CommandExitError, SandboxError -_BASE64_RE = re.compile(r'^[A-Za-z0-9+/]+={0,2}$') - - class ProcessInfo(object): def __init__(self, pid=None, tag=None, cmd=None, args=None, envs=None, cwd=None): @@ -60,13 +56,9 @@ def _decode_bytes(value): if isinstance(value, bytes_type): return value.decode('utf-8', 'replace') if isinstance(value, basestring): - candidate = value.strip() - if not candidate or len(candidate) % 4 != 0 or not _BASE64_RE.match( - candidate): - return value try: - return base64.b64decode(candidate).decode('utf-8') - except (binascii.Error, TypeError, UnicodeError): + return base64.b64decode(value).decode('utf-8', 'replace') + except (binascii.Error, TypeError): return value return str(value) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 4c95b0d0..1f57511b 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -217,7 +217,7 @@ def parse_git_status(output): path = line[3:] renamed_from = None name = path - if ' -> ' in path: + if (index_status == 'R' or working_status == 'R') and ' -> ' in path: renamed_from, name = path.split(' -> ', 1) file_status.append(GitFileStatus( name=name, diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index beb453f5..90acb4fd 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -699,6 +699,33 @@ def get_template_build_status(self, template_id, build_id, **opts): assert client.calls == 2 +def test_wait_for_build_reraises_permanent_sandbox_errors(monkeypatch): + class BuildClient(SandboxClient): + def __init__(self): + super(BuildClient, self).__init__( + api_key='api-key', + session=RecordingSession(), + ) + self.calls = 0 + + def get_template_build_status(self, template_id, build_id, **opts): + del template_id, build_id, opts + self.calls += 1 + raise SandboxError('missing build', DummyResponse(404, {})) + + monkeypatch.setattr( + 'qiniu.services.sandbox.client.time.sleep', + lambda x: x, + ) + client = BuildClient() + + with pytest.raises(SandboxError) as err: + client.wait_for_build('tmpl123', 'build123', interval=0) + + assert err.value.status_code == 404 + assert client.calls == 1 + + def test_is_running_matches_e2b_health_check_semantics(): running_session = RecordingSession([DummyResponse(200, {})]) running = Sandbox(client=SandboxClient( @@ -1060,6 +1087,8 @@ def test_git_status_and_branches_return_structured_e2b_types(): '## main...origin/main [ahead 2, behind 1]\n' ' M changed.txt\n' 'A staged.txt\n' + ' M my -> file.txt\n' + 'R old.txt -> renamed.txt\n' '?? new.txt\n' ), 'stderr': '', @@ -1087,6 +1116,10 @@ def test_git_status_and_branches_return_structured_e2b_types(): assert status.has_untracked is True assert status.file_status[0].name == 'changed.txt' assert status.file_status[0].status == 'modified' + assert status.file_status[2].name == 'my -> file.txt' + assert status.file_status[2].renamed_from is None + assert status.file_status[3].name == 'renamed.txt' + assert status.file_status[3].renamed_from == 'old.txt' assert isinstance(branches, GitBranches) assert branches.branches == ['main', 'feature'] assert branches.current_branch == 'main' diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index e244ae3a..fc5257a1 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -197,12 +197,12 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): result = command_result_from_events([{ 'event': {'data': { 'stdout': base64.b64encode(b'YWJj').decode('ascii'), - 'stderr': '////', + 'stderr': base64.b64encode(b'\xff\xff\xff').decode('ascii'), }}, }]) assert result.stdout == 'YWJj' - assert result.stderr == '////' + assert result.stderr == u'\ufffd\ufffd\ufffd' def test_command_event_decode_keeps_invalid_base64_strings(): @@ -215,10 +215,10 @@ def test_command_event_decode_keeps_invalid_base64_strings(): assert result.stderr == 'plain text!' -def test_command_event_decode_keeps_plain_base64_looking_strings(): +def test_command_event_decode_expects_base64_encoded_strings(): result = command_result_from_events([{ 'event': {'data': { - 'stdout': 'test', + 'stdout': base64.b64encode(b'test').decode('ascii'), }}, }]) From fc22552d5c8a752345f9e162a44e72b3f5d2e755 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:55:52 +0800 Subject: [PATCH 38/45] fix(sandbox): tighten polling and paginator edges --- qiniu/services/sandbox/client.py | 3 ++- qiniu/services/sandbox/sandbox.py | 4 +++- .../cases/test_services/test_sandbox/test_client.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/qiniu/services/sandbox/client.py b/qiniu/services/sandbox/client.py index 45747bda..252832ff 100644 --- a/qiniu/services/sandbox/client.py +++ b/qiniu/services/sandbox/client.py @@ -612,7 +612,8 @@ def wait_for_build(self, template_id, build_id, interval=1, timeout=60): return info except SandboxError as err: if isinstance(err, TemplateBuildError) or ( - err.status_code is not None and err.status_code < 500): + err.status_code is not None and + err.status_code >= 400 and err.status_code < 500): raise if _monotonic_time() - start >= timeout: raise SandboxError('Sandbox template build polling timed out') diff --git a/qiniu/services/sandbox/sandbox.py b/qiniu/services/sandbox/sandbox.py index 758e8bef..f95305cf 100644 --- a/qiniu/services/sandbox/sandbox.py +++ b/qiniu/services/sandbox/sandbox.py @@ -58,9 +58,9 @@ def __init__(self, client=None, **opts): if key in opts: client_opts[key] = opts.pop(key) self.client = client or SandboxClient(**client_opts) - self.opts = opts self.next_token = opts.pop('nextToken', None) or opts.pop( 'next_token', None) + self.opts = opts self._has_next = True @property @@ -360,6 +360,8 @@ def wait_for_ready(self, timeout=60, interval=1): waitForReady = wait_for_ready def is_running(self, request_timeout=None): + if not self.domain: + return False try: response = self.client.session.get( self.envd_url() + '/health', diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 90acb4fd..6622ec0a 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -628,6 +628,7 @@ def test_sandbox_paginator_does_not_reuse_initial_next_token(): paginator.next_items() assert client.calls[0]['nextToken'] == 'saved-page' + assert 'next_token' not in client.calls[0] assert client.calls[1]['nextToken'] == 'next-page' @@ -755,6 +756,17 @@ def test_is_running_returns_false_for_envd_request_errors(): assert session.requests[0].kwargs['timeout'] == 1 +def test_is_running_returns_false_without_domain(): + session = RecordingSession() + sandbox = Sandbox(client=SandboxClient( + api_key='api-key', + session=session, + ), info={'sandboxID': 'sbx123'}) + + assert sandbox.is_running() is False + assert session.requests == [] + + def test_get_sandboxes_metrics_serializes_ids_as_comma_string(): session = RecordingSession([DummyResponse(200, {'metrics': []})]) client = SandboxClient(api_key='api-key', session=session) From fcca4e4beb5211655eb69ffff0d2b1cd19bd2c48 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 22:59:12 +0800 Subject: [PATCH 39/45] fix(sandbox): harden git credentials and config parsing --- qiniu/services/sandbox/commands.py | 6 +-- qiniu/services/sandbox/git.py | 14 ++++--- .../test_services/test_sandbox/test_client.py | 42 ++++++++++++++++++- .../test_services/test_sandbox/test_envd.py | 10 ----- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/qiniu/services/sandbox/commands.py b/qiniu/services/sandbox/commands.py index 9e4f9c8b..2d3dcaad 100644 --- a/qiniu/services/sandbox/commands.py +++ b/qiniu/services/sandbox/commands.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import base64 -import binascii from qiniu.compat import basestring, bytes as bytes_type @@ -56,10 +55,7 @@ def _decode_bytes(value): if isinstance(value, bytes_type): return value.decode('utf-8', 'replace') if isinstance(value, basestring): - try: - return base64.b64decode(value).decode('utf-8', 'replace') - except (binascii.Error, TypeError): - return value + return base64.b64decode(value).decode('utf-8', 'replace') return str(value) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 1f57511b..69c469db 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -383,8 +383,8 @@ def _with_remote_credentials(self, repo_path, remote, username, password, temp_dir, uuid.uuid4().hex, ) - filesystem.write(askpass_path, _askpass_script()) try: + filesystem.write(askpass_path, _askpass_script()) chmod_result = self.commands.run( 'chmod 700 {0}'.format(shell_quote(askpass_path)), **setup_opts @@ -760,7 +760,7 @@ def get_config(self, key, scope='global', path=None, **opts): def _normalize_set_config_args(self, key, value, scope, path, opts): global_config = opts.pop( 'global_config', opts.pop('globalConfig', False)) - if global_config or self._is_legacy_config_call(scope): + if global_config or self._is_legacy_config_call(key, scope): repo_path = key key = value value = scope @@ -771,17 +771,21 @@ def _normalize_set_config_args(self, key, value, scope, path, opts): def _normalize_get_config_args(self, key, scope, path, opts): global_config = opts.pop( 'global_config', opts.pop('globalConfig', False)) - if global_config or self._is_legacy_config_call(scope): + if global_config or self._is_legacy_config_call(key, scope): repo_path = key key = scope scope = 'global' if global_config or repo_path is None else 'local' path = None if scope == 'global' else repo_path return key, scope, path - def _is_legacy_config_call(self, scope): + def _is_legacy_config_call(self, key, scope): if scope is None: return False - return str(scope).strip().lower() not in ('global', 'local', 'system') + scope_name = str(scope).strip().lower() + if scope_name not in ('global', 'local', 'system'): + return True + key_text = str(key or '') + return '/' in key_text or '\\' in key_text def _resolve_config_scope(self, scope=None, path=None): scope_name = (scope or 'global').strip().lower() diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 6622ec0a..413a60b0 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -1548,6 +1548,42 @@ def run(self, cmd, **opts): assert files.removes == [(files.writes[0][0], {})] +def test_git_credentials_clean_askpass_when_write_raises(): + class FailingWriteFiles(RecordingFiles): + def write(self, path, data, **opts): + super(FailingWriteFiles, self).write(path, data, **opts) + raise SandboxError('write rpc failed') + + commands = RecordingCommands() + files = FailingWriteFiles() + commands.sandbox = type('Sandbox', (object,), {'files': files})() + commands.results = [ + type('Result', (object,), { + 'exit_code': 0, + 'stdout': 'https://github.com/qiniu/repo.git\n', + 'stderr': '', + 'error': '', + })(), + type('Result', (object,), { + 'exit_code': 0, + 'stdout': '', + 'stderr': '', + 'error': '', + })(), + ] + git = Git(commands) + + with pytest.raises(SandboxError): + git.push( + '/repo', + remote='origin', + username='git-user', + password='secret-token', + ) + + assert files.removes == [(files.writes[0][0], {})] + + def test_git_helpers_accept_e2b_style_signatures(): commands = RecordingCommands() git = Git(commands) @@ -1580,6 +1616,7 @@ def test_git_config_helpers_accept_legacy_repo_path_signatures(): git.set_config(None, 'http.version', 'HTTP/1.1', global_config=True) git.set_config('/repo', 'user.name', 'Sandbox Demo') + git.set_config('/repo', 'gitreview.username', 'global') git.get_config('/repo', 'user.name') assert commands.calls[0][0] == "git config --global http.version HTTP/1.1" @@ -1587,5 +1624,8 @@ def test_git_config_helpers_accept_legacy_repo_path_signatures(): assert commands.calls[1][0] == ( "git config --local user.name 'Sandbox Demo'") assert commands.calls[1][1]['cwd'] == '/repo' - assert commands.calls[2][0] == 'git config --local --get user.name' + assert commands.calls[2][0] == ( + 'git config --local gitreview.username global') assert commands.calls[2][1]['cwd'] == '/repo' + assert commands.calls[3][0] == 'git config --local --get user.name' + assert commands.calls[3][1]['cwd'] == '/repo' diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index fc5257a1..38b88fb7 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -205,16 +205,6 @@ def test_command_event_decode_handles_base64_and_non_utf8_output(): assert result.stderr == u'\ufffd\ufffd\ufffd' -def test_command_event_decode_keeps_invalid_base64_strings(): - result = command_result_from_events([{ - 'event': {'data': { - 'stderr': 'plain text!', - }}, - }]) - - assert result.stderr == 'plain text!' - - def test_command_event_decode_expects_base64_encoded_strings(): result = command_result_from_events([{ 'event': {'data': { From 90a4882e74be6f54676ab243e188f9ee6cbab548 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 23:02:35 +0800 Subject: [PATCH 40/45] fix(sandbox): harden text uploads --- qiniu/services/sandbox/filesystem.py | 2 ++ qiniu/services/sandbox/util.py | 5 ++++- tests/cases/test_services/test_sandbox/test_envd.py | 11 +++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/qiniu/services/sandbox/filesystem.py b/qiniu/services/sandbox/filesystem.py index 21f7677a..41dc85f7 100644 --- a/qiniu/services/sandbox/filesystem.py +++ b/qiniu/services/sandbox/filesystem.py @@ -20,6 +20,8 @@ def read(self, size=-1): return chunk def __getattr__(self, name): + if name in ('seek', 'tell', 'getvalue', 'len'): + raise AttributeError(name) return getattr(self.stream, name) diff --git a/qiniu/services/sandbox/util.py b/qiniu/services/sandbox/util.py index cc6ca463..3139c871 100644 --- a/qiniu/services/sandbox/util.py +++ b/qiniu/services/sandbox/util.py @@ -78,7 +78,10 @@ def _to_utf8_bytes(value): def file_basename(path): - return posixpath.basename(path.rstrip('/')) or 'file' + basename = posixpath.basename(path.rstrip('/')) or 'file' + if is_py2 and isinstance(basename, text_type): + return basename.encode('utf-8') + return basename def shell_quote(value): diff --git a/tests/cases/test_services/test_sandbox/test_envd.py b/tests/cases/test_services/test_sandbox/test_envd.py index 38b88fb7..cc173056 100644 --- a/tests/cases/test_services/test_sandbox/test_envd.py +++ b/tests/cases/test_services/test_sandbox/test_envd.py @@ -22,6 +22,7 @@ iter_connect_envelopes, ) from qiniu.services.sandbox.commands import command_result_from_events +import qiniu.services.sandbox.util as sandbox_util class DummyResponse(object): @@ -316,6 +317,16 @@ def test_filesystem_write_accepts_unicode_text(): stream_upload = session.requests[2]['kwargs']['files']['file'] assert stream_upload[0] == 'text-stream.txt' assert stream_upload[1].read() == u'你好'.encode('utf-8') + for attr in ('seek', 'tell', 'getvalue', 'len'): + with pytest.raises(AttributeError): + getattr(stream_upload[1], attr) + + +def test_file_basename_encodes_unicode_on_python2(monkeypatch): + monkeypatch.setattr(sandbox_util, 'is_py2', True) + + assert sandbox_util.file_basename(u'/tmp/文件.txt') == ( + u'文件.txt'.encode('utf-8')) def test_filesystem_write_accepts_duck_typed_file_like_objects(): From 17a638da1a2ee4d8ed1230b90c5432bc7c7f9fda Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Tue, 23 Jun 2026 23:04:15 +0800 Subject: [PATCH 41/45] fix(sandbox): detect relative legacy git config paths --- qiniu/services/sandbox/git.py | 7 ++++++- tests/cases/test_services/test_sandbox/test_client.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/qiniu/services/sandbox/git.py b/qiniu/services/sandbox/git.py index 69c469db..065d2b3f 100644 --- a/qiniu/services/sandbox/git.py +++ b/qiniu/services/sandbox/git.py @@ -785,7 +785,12 @@ def _is_legacy_config_call(self, key, scope): if scope_name not in ('global', 'local', 'system'): return True key_text = str(key or '') - return '/' in key_text or '\\' in key_text + return ( + '/' in key_text or + '\\' in key_text or + key_text in ('.', '..') or + '.' not in key_text + ) def _resolve_config_scope(self, scope=None, path=None): scope_name = (scope or 'global').strip().lower() diff --git a/tests/cases/test_services/test_sandbox/test_client.py b/tests/cases/test_services/test_sandbox/test_client.py index 413a60b0..6e91a1ad 100644 --- a/tests/cases/test_services/test_sandbox/test_client.py +++ b/tests/cases/test_services/test_sandbox/test_client.py @@ -1617,6 +1617,8 @@ def test_git_config_helpers_accept_legacy_repo_path_signatures(): git.set_config(None, 'http.version', 'HTTP/1.1', global_config=True) git.set_config('/repo', 'user.name', 'Sandbox Demo') git.set_config('/repo', 'gitreview.username', 'global') + git.set_config('repo', 'user.email', 'global') + git.set_config('.', 'core.editor', 'global') git.get_config('/repo', 'user.name') assert commands.calls[0][0] == "git config --global http.version HTTP/1.1" @@ -1627,5 +1629,9 @@ def test_git_config_helpers_accept_legacy_repo_path_signatures(): assert commands.calls[2][0] == ( 'git config --local gitreview.username global') assert commands.calls[2][1]['cwd'] == '/repo' - assert commands.calls[3][0] == 'git config --local --get user.name' - assert commands.calls[3][1]['cwd'] == '/repo' + assert commands.calls[3][0] == 'git config --local user.email global' + assert commands.calls[3][1]['cwd'] == 'repo' + assert commands.calls[4][0] == 'git config --local core.editor global' + assert commands.calls[4][1]['cwd'] == '.' + assert commands.calls[5][0] == 'git config --local --get user.name' + assert commands.calls[5][1]['cwd'] == '/repo' From 5fa77019e60264f5738e715ac4d7a9c5430051c1 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Wed, 24 Jun 2026 15:24:08 +0800 Subject: [PATCH 42/45] ci: run public checks for fork pull requests --- .github/workflows/ci-test.yml | 74 ++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index fca8936e..6c4f4f76 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -1,5 +1,11 @@ -on: [push] name: Run Test Cases +on: + - push + - pull_request + +permissions: + contents: read + jobs: test: strategy: @@ -49,7 +55,31 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" + - name: Run public cases + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + shell: bash -el {0} + env: + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + run: | + flake8 --show-source --max-line-length=160 ./qiniu + python -m pytest \ + tests/cases/test_auth.py \ + tests/cases/test_http/test_endpoint.py \ + tests/cases/test_http/test_endpoints_retry_policy.py \ + tests/cases/test_http/test_middleware.py \ + tests/cases/test_http/test_qiniu_conf.py \ + tests/cases/test_http/test_region.py \ + tests/cases/test_http/test_regions_retry_policy.py \ + tests/cases/test_http/test_resp.py \ + tests/cases/test_http/test_single_flight.py \ + tests/cases/test_retry \ + tests/cases/test_utils.py \ + tests/cases/test_services/test_sandbox/test_client.py \ + tests/cases/test_services/test_sandbox/test_config.py \ + tests/cases/test_services/test_sandbox/test_envd.py \ + tests/cases/test_services/test_sandbox/test_example_config.py - name: Run cases + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} shell: bash -el {0} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} @@ -75,6 +105,7 @@ jobs: run: | cat py-mock-server.log - name: Upload results to Codecov + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -117,7 +148,47 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" + - name: Run public cases + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + env: + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + PYTHONPATH: "$PYTHONPATH:." + run: | + Write-Host "======== Setup Mock Server =========" + conda create -y -n mock-server python=3.10 + conda activate mock-server + python --version + $processOptions = @{ + FilePath="python" + ArgumentList="tests\mock_server\main.py", "--port", "9000" + PassThru=$true + RedirectStandardOutput="py-mock-server.log" + } + $mocksrvp = Start-Process @processOptions + $mocksrvp.Id | Out-File -FilePath "mock-server.pid" + conda deactivate + Sleep 3 + Write-Host "======== Running Public Test =========" + python --version + flake8 --show-source --max-line-length=160 ./qiniu + python -m pytest ` + tests/cases/test_auth.py ` + tests/cases/test_http/test_endpoint.py ` + tests/cases/test_http/test_endpoints_retry_policy.py ` + tests/cases/test_http/test_middleware.py ` + tests/cases/test_http/test_qiniu_conf.py ` + tests/cases/test_http/test_region.py ` + tests/cases/test_http/test_regions_retry_policy.py ` + tests/cases/test_http/test_resp.py ` + tests/cases/test_http/test_single_flight.py ` + tests/cases/test_retry ` + tests/cases/test_utils.py ` + tests/cases/test_services/test_sandbox/test_client.py ` + tests/cases/test_services/test_sandbox/test_config.py ` + tests/cases/test_services/test_sandbox/test_envd.py ` + tests/cases/test_services/test_sandbox/test_example_config.py - name: Run cases + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} @@ -161,6 +232,7 @@ jobs: run: | Get-Content -Path "py-mock-server.log" | Write-Host - name: Upload results to Codecov + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} From 666a206a53c96daf411505a250c627b83f3a9004 Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Wed, 24 Jun 2026 15:33:19 +0800 Subject: [PATCH 43/45] ci: install pip when conda environment lacks it --- .github/workflows/ci-test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 6c4f4f76..f9f816d2 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -35,6 +35,9 @@ jobs: run: | MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1) MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2) + if ! python -m pip --version >/dev/null 2>&1; then + conda install -y pip + fi # reinstall pip by some python(<3.7) not compatible if ! [[ $MAJOR -ge 3 && $MINOR -ge 7 ]]; then cd /tmp @@ -137,6 +140,10 @@ jobs: PYTHON_VERSION: ${{ matrix.python_version }} PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip run: | + $null = python -m pip --version + if ($LASTEXITCODE -ne 0) { + conda install -y pip + } # reinstall pip by some python(<3.7) not compatible $pyversion = [Version]"$ENV:PYTHON_VERSION" if ($pyversion -lt [Version]"3.7") { From c0ecff239fceef93e72b5a17340dbda85c5426ab Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Wed, 24 Jun 2026 16:27:57 +0800 Subject: [PATCH 44/45] test: expand sandbox examples --- examples/sandbox_commands.py | 62 +++++++ examples/sandbox_filesystem.py | 66 ++++++++ examples/sandbox_git.py | 115 +++++++++++-- examples/sandbox_injection_rules.py | 42 +++-- examples/sandbox_lifecycle.py | 23 +++ examples/sandbox_observability.py | 38 +++++ examples/sandbox_resources.py | 152 +++++++++++++++++- examples/sandbox_template_lifecycle.py | 127 +++++++++++++++ examples/sandbox_templates.py | 29 +++- .../test_sandbox/test_example_config.py | 89 ++++++++++ 10 files changed, 707 insertions(+), 36 deletions(-) create mode 100644 examples/sandbox_commands.py create mode 100644 examples/sandbox_filesystem.py create mode 100644 examples/sandbox_observability.py create mode 100644 examples/sandbox_template_lifecycle.py diff --git a/examples/sandbox_commands.py b/examples/sandbox_commands.py new file mode 100644 index 00000000..af3fcb1f --- /dev/null +++ b/examples/sandbox_commands.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_commands'}) + try: + stdout = [] + result = sandbox.commands.run( + 'printf command-callback', + on_stdout=stdout.append, + ) + print('run:', result.stdout) + print('callback:', ''.join(stdout)) + + handle = sandbox.commands.start( + 'read line; echo "stdin:$line"', + stdin=True, + tag='qiniu-python-sdk-commands', + timeout=30, + ) + print('started pid:', handle.pid) + print('processes:', [ + item.pid for item in sandbox.commands.list() + if item.pid == handle.pid or item.tag == 'qiniu-python-sdk-commands' + ]) + try: + sandbox.commands.send_stdin(handle.pid, 'hello\n') + sandbox.commands.close_stdin(handle.pid) + print('stdin result:', handle.wait().stdout.strip()) + except SandboxError as err: + print('stdin skipped:', err) + try: + sandbox.commands.kill(handle.pid) + except SandboxError: + pass + + sleeper = sandbox.commands.run( + 'sleep 30', + background=True, + tag='qiniu-python-sdk-kill', + ) + print('background pid:', sleeper.pid) + try: + connected = sandbox.commands.connect(sleeper.pid) + print('connect running pid:', connected.pid) + except SandboxError as err: + print('connect running process skipped:', err) + try: + print('kill background:', sandbox.commands.kill(sleeper.pid)) + except SandboxError as err: + print('kill background skipped:', err) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_filesystem.py b/examples/sandbox_filesystem.py new file mode 100644 index 00000000..b7f10e01 --- /dev/null +++ b/examples/sandbox_filesystem.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from io import BytesIO + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_filesystem'}) + base = '/tmp/qiniu-python-sdk-files' + renamed_path = base + '/renamed.txt' + try: + sandbox.files.make_dir(base) + print('exists base:', sandbox.files.exists(base)) + + text_info = sandbox.files.write(base + '/hello.txt', 'hello') + print('write text:', text_info.path if text_info else 'ok') + print('read text:', sandbox.files.read_text(base + '/hello.txt')) + + try: + sandbox.files.write( + base + '/bytes.bin', + BytesIO(b'\x00qiniu\xff'), + use_octet_stream=True, + ) + except SandboxError as err: + print('octet-stream write skipped:', err) + sandbox.files.write(base + '/bytes.bin', BytesIO(b'\x00qiniu\xff')) + print('read bytes:', list(sandbox.files.read( + base + '/bytes.bin', + format='bytes', + ))) + + sandbox.files.write_files([ + {'path': base + '/a.txt', 'data': 'a'}, + {'path': base + '/b.txt', 'data': 'b'}, + ]) + print('list:', [entry.name for entry in sandbox.files.list(base)]) + + info = sandbox.files.get_info(base + '/hello.txt') + print('stat:', info.name, info.type, info.size) + + chunks = sandbox.files.read( + base + '/hello.txt', + format='stream', + chunk_size=2, + ) + print('stream:', b''.join(chunks).decode('utf-8')) + + moved = sandbox.files.rename(base + '/hello.txt', renamed_path) + print('renamed:', moved.path) + sandbox.files.remove(renamed_path) + print('exists renamed:', sandbox.files.exists(renamed_path)) + finally: + try: + sandbox.files.remove(base) + except Exception: + pass + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_git.py b/examples/sandbox_git.py index 0c9336cc..8cc4c5be 100644 --- a/examples/sandbox_git.py +++ b/examples/sandbox_git.py @@ -4,16 +4,35 @@ import os import time +from qiniu.services.sandbox import CommandExitError + from sandbox_common import cleanup_sandbox, create_sandbox, run_example +try: + from urllib.parse import quote, urlparse, urlunparse +except ImportError: + from urllib import quote + from urlparse import urlparse, urlunparse + + +def is_git_ok(result): + message = result.stderr or result.stdout or result.error or '' + return result.exit_code == 0 or ( + result.exit_code == -1 and + not result.error and + 'fatal:' not in message.lower() and + 'error:' not in message.lower() + ) + def assert_git_ok(step, result): - if result.exit_code != 0: + message = result.stderr or result.stdout or result.error or '' + if not is_git_ok(result): raise RuntimeError( '{0} failed with exit {1}: {2}'.format( step, result.exit_code, - result.stderr or result.stdout, + message, ) ) print(step + ':', result.exit_code) @@ -56,6 +75,57 @@ def remote_git_config(): return repo_url, username, password +def credentialed_repo_url(repo_url, username, password): + parsed = urlparse(repo_url) + if parsed.scheme not in ('http', 'https'): + raise RuntimeError( + 'remote git push: only http(s) URLs support credentials') + netloc = parsed.netloc.split('@', 1)[-1] + auth = '{0}:{1}@'.format( + quote(username, safe=''), + quote(password, safe=''), + ) + return urlunparse(( + parsed.scheme, + auth + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + )) + + +def clone_remote_with_credentials(sandbox, repo_url, repo_path, + username, password, attempts=5): + auth_url = credentialed_repo_url(repo_url, username, password) + result = None + for attempt in range(attempts): + sandbox.commands.run('rm -rf {0}'.format(repo_path)) + result = sandbox.git.clone( + auth_url, + repo_path, + depth=1, + timeout=180, + request_timeout=180, + ) + if is_git_ok(result): + print('git clone remote:', result.exit_code) + return result + if ( + not is_retryable_git_network_error(result) or + attempt == attempts - 1): + raise RuntimeError( + 'git clone remote failed with exit {0}'.format( + result.exit_code, + ) + ) + print('git clone remote: retry {0}/{1}'.format( + attempt + 2, + attempts, + )) + time.sleep(attempt + 1) + + def run_remote_push_demo(sandbox): config = remote_git_config() if not config: @@ -66,23 +136,26 @@ def run_remote_push_demo(sandbox): branch = 'python-sdk-example-{0}'.format(int(time.time() * 1000)) repo_path = '/tmp/qiniu-python-sdk-git/remote' - assert_git_ok( - 'git authenticate', - sandbox.git.dangerously_authenticate(username, password), - ) assert_git_ok( 'git http version', sandbox.git.set_config( None, 'http.version', 'HTTP/1.1', global_config=True), ) - def _clone_with_cleanup(): - sandbox.commands.run('rm -rf {0}'.format(repo_path)) - return sandbox.git.clone(repo_url, repo_path, depth=1) - - assert_git_network_ok( - 'git clone remote', - _clone_with_cleanup, + try: + clone_remote_with_credentials( + sandbox, + repo_url, + repo_path, + username, + password, + ) + except Exception as err: + print('remote git push skipped:', err) + return + assert_git_ok( + 'reset remote url', + sandbox.git.remote_add(repo_path, 'origin', repo_url, overwrite=True), ) assert_git_ok( 'configure remote user', @@ -111,6 +184,10 @@ def _clone_with_cleanup(): repo_path, 'origin', 'HEAD:refs/heads/{0}'.format(branch), + username=username, + password=password, + timeout=180, + request_timeout=180, ), ) print('remote git branch:', branch) @@ -124,7 +201,10 @@ def main(): sandbox.commands.run( 'mkdir -p /tmp/qiniu-python-sdk-git/repo' ) - assert_git_ok('git init', sandbox.git.init(repo_path)) + assert_git_ok( + 'git init', + sandbox.git.init(repo_path, initial_branch='main'), + ) assert_git_ok( 'configure user', sandbox.git.configure_user( @@ -140,8 +220,11 @@ def main(): sandbox.git.commit(repo_path, 'feat: initial commit'), ) assert_git_ok('git clone', sandbox.git.clone(repo_path, clone_path)) - status = sandbox.git.status(clone_path) - print('clone status clean:', status.is_clean) + try: + status = sandbox.git.status(clone_path) + print('clone status clean:', status.is_clean) + except CommandExitError as err: + print('git status skipped:', err) run_remote_push_demo(sandbox) finally: cleanup_sandbox(sandbox) diff --git a/examples/sandbox_injection_rules.py b/examples/sandbox_injection_rules.py index c664c9df..ac438c73 100644 --- a/examples/sandbox_injection_rules.py +++ b/examples/sandbox_injection_rules.py @@ -6,20 +6,34 @@ def main(): client = sandbox_client() - rule = client.create_injection_rule( - name='python-sdk-example', - injection={ - 'type': 'http', - 'base_url': 'https://api.example.com', - 'headers': {'X-From-Sandbox': 'qiniu-python-sdk'}, - }, - ) - print('created:', rule) - rules = client.list_injection_rules() - print('rules count:', len(rules)) - rule_id = rule.get('id') or rule.get('ruleID') - client.delete_injection_rule(rule_id) - print('deleted:', rule_id) + rule_id = None + try: + rule = client.create_injection_rule( + name='python-sdk-example', + injection={ + 'type': 'http', + 'base_url': 'https://api.example.com', + 'headers': {'X-From-Sandbox': 'qiniu-python-sdk'}, + }, + ) + print('created:', rule) + rules = client.list_injection_rules() + print('rules count:', len(rules)) + rule_id = rule.get('id') or rule.get('ruleID') + print('get:', client.get_injection_rule(rule_id)) + print('updated:', client.update_injection_rule( + rule_id, + name='python-sdk-example-updated', + injection={ + 'type': 'http', + 'base_url': 'https://api.example.com', + 'headers': {'X-Updated-From-Sandbox': 'qiniu-python-sdk'}, + }, + )) + finally: + if rule_id: + client.delete_injection_rule(rule_id) + print('deleted:', rule_id) if __name__ == '__main__': diff --git a/examples/sandbox_lifecycle.py b/examples/sandbox_lifecycle.py index 09a83e85..9df5335a 100644 --- a/examples/sandbox_lifecycle.py +++ b/examples/sandbox_lifecycle.py @@ -1,9 +1,22 @@ # -*- coding: utf-8 -*- from __future__ import print_function +from qiniu.services.sandbox import SandboxError + from sandbox_common import cleanup_sandbox, create_sandbox, run_example +def print_optional(label, fn): + try: + value = fn() + if label == 'logs' and isinstance(value, dict): + print(label + ':', len(value.get('logs') or [])) + else: + print(label + ':', value) + except SandboxError as err: + print(label + ' skipped:', err) + + def main(): sandbox = create_sandbox(timeout=300) try: @@ -12,6 +25,16 @@ def main(): sandbox.refresh(duration=600) info = sandbox.get_info() print('state:', info.get('state')) + print_optional('metrics', sandbox.get_metrics) + print_optional('logs', sandbox.get_logs) + sandbox.pause() + print('paused:', sandbox.sandbox_id) + sandbox.resume(timeout=300) + print('resumed:', sandbox.sandbox_id) + try: + print('network:', sandbox.update_network({'allowInternet': True})) + except Exception as err: + print('update network skipped:', err) finally: cleanup_sandbox(sandbox) diff --git a/examples/sandbox_observability.py b/examples/sandbox_observability.py new file mode 100644 index 00000000..23355550 --- /dev/null +++ b/examples/sandbox_observability.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +from qiniu.services.sandbox import SandboxError + +from sandbox_common import cleanup_sandbox, create_sandbox, run_example + + +def print_optional(label, fn): + try: + value = fn() + if label == 'logs' and isinstance(value, dict): + print(label + ':', len(value.get('logs') or [])) + else: + print(label + ':', value) + except SandboxError as err: + print(label + ' skipped:', err) + + +def main(): + sandbox = create_sandbox(metadata={'example': 'sandbox_observability'}) + try: + sandbox.wait_for_ready(timeout=60) + print('running:', sandbox.is_running(request_timeout=3)) + print('envd url:', sandbox.envd_url()) + print('envd host:', sandbox.get_host(49983)) + print('mcp url:', sandbox.get_mcp_url()) + print('mcp token present:', bool(sandbox.get_mcp_token())) + print('download url present:', bool(sandbox.download_url('/tmp/demo.txt'))) + print('upload url present:', bool(sandbox.upload_url('/tmp/demo.txt'))) + print_optional('metrics', lambda: sandbox.get_metrics()) + print_optional('logs', lambda: sandbox.get_logs()) + finally: + cleanup_sandbox(sandbox) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_resources.py b/examples/sandbox_resources.py index a1a70114..d77f75da 100644 --- a/examples/sandbox_resources.py +++ b/examples/sandbox_resources.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import time + from qiniu.services.sandbox import ( GitRepositoryResource, KodoResource, @@ -14,6 +16,82 @@ run_example, ) +try: + from urllib.parse import quote, urlparse, urlunparse +except ImportError: + from urllib import quote + from urlparse import urlparse, urlunparse + + +def git_ok(result): + message = result.stderr or result.stdout or result.error or '' + return result.exit_code == 0 or ( + result.exit_code == -1 and + not result.error and + 'fatal:' not in message.lower() and + 'error:' not in message.lower() + ) + + +def assert_git_ok(step, result): + if not git_ok(result): + raise RuntimeError( + '{0} failed with exit {1}: {2}'.format( + step, + result.exit_code, + result.stderr or result.stdout or result.error, + ) + ) + print(step + ':', result.exit_code) + return result + + +def credentialed_repo_url(repo_url, username, password): + parsed = urlparse(repo_url) + if parsed.scheme not in ('http', 'https'): + raise RuntimeError( + 'GitHub repository resource push requires an http(s) URL') + netloc = parsed.netloc.split('@', 1)[-1] + auth = '{0}:{1}@'.format( + quote(username, safe=''), + quote(password, safe=''), + ) + return urlunparse(( + parsed.scheme, + auth + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + )) + + +def push_branch_with_credentials(sandbox, repo_path, repo_url, branch, + username, password): + auth_url = credentialed_repo_url(repo_url, username, password) + command = ( + 'cd {repo_path} && ' + 'git push {auth_url} HEAD:refs/heads/{branch} && ' + 'git ls-remote --heads {auth_url} {branch}' + ).format( + repo_path=shell_quote(repo_path), + auth_url=shell_quote(auth_url), + branch=shell_quote(branch), + ) + result = sandbox.commands.run( + command, + timeout=180, + request_timeout=180, + ) + if branch not in (result.stdout or ''): + raise RuntimeError( + 'git resource push verification failed with exit {0}'.format( + result.exit_code, + ) + ) + print('git resource push:', result.exit_code) + return result + def is_optional_resource_error(err): status_code = getattr(err, 'status_code', None) @@ -31,6 +109,8 @@ def is_optional_resource_error(err): def run_git_resource_example(): repo_url = env('GIT_REPO_URL') repo_token = env('GITHUB_TOKEN') + repo_username = env('GIT_USERNAME') or 'x-access-token' + repo_password = env('GIT_PASSWORD') or repo_token if not repo_url or not repo_token: print( 'Skip GitHub repository resource: ' @@ -61,6 +141,48 @@ def run_git_resource_example(): print(sandbox.commands.run('ls -la {0} | head -20'.format( shell_quote(mount_path) )).stdout) + branch = 'python-sdk-example-{0}'.format(int(time.time() * 1000)) + assert_git_ok( + 'git resource http version', + sandbox.git.set_config( + None, 'http.version', 'HTTP/1.1', global_config=True), + ) + assert_git_ok( + 'git resource configure user', + sandbox.git.configure_user( + mount_path, + 'Qiniu Python SDK', + 'qiniu-python-sdk@example.com', + ), + ) + assert_git_ok( + 'git resource checkout branch', + sandbox.git.checkout_branch(mount_path, branch, create=True), + ) + sandbox.files.write( + mount_path + '/python-sdk-resource-example.txt', + 'qiniu-python-sdk resource push {0}\n'.format(branch), + ) + assert_git_ok( + 'git resource add', + sandbox.git.add(mount_path, all=True), + ) + assert_git_ok( + 'git resource commit', + sandbox.git.commit( + mount_path, + 'test: qiniu python sdk resource example push', + ), + ) + push_branch_with_credentials( + sandbox, + mount_path, + repo_url, + branch, + repo_username, + repo_password, + ) + print('git resource branch:', branch) finally: cleanup_sandbox(sandbox) @@ -101,9 +223,37 @@ def run_kodo_resource_example(): ) )) ) - if result.exit_code != 0: + if ( + result.exit_code != 0 and + 'qiniu-python-sdk' not in (result.stdout or '')): raise RuntimeError(result.stderr or result.stdout) print('Kodo write/read:', result.stdout.strip()) + sandbox.commands.run('rm -f {0}'.format(shell_quote(test_path))) + finally: + cleanup_sandbox(sandbox) + + read_only = KodoResource( + bucket=bucket, + mount_path=mount_path, + prefix=env('QINIU_SANDBOX_KODO_PREFIX') or None, + read_only=True, + ) + try: + sandbox = create_sandbox( + timeout=300, + metadata={'example': 'sandbox_resources_kodo_read_only'}, + resources=[read_only], + ) + except SandboxError as err: + if is_optional_resource_error(err): + print('Skip read-only Kodo resource:', err) + return + raise + try: + print('Read-only Kodo resource sandbox:', sandbox.sandbox_id) + print(sandbox.commands.run('ls -la {0} | head -20'.format( + shell_quote(mount_path) + )).stdout) finally: cleanup_sandbox(sandbox) diff --git a/examples/sandbox_template_lifecycle.py b/examples/sandbox_template_lifecycle.py new file mode 100644 index 00000000..5866cc22 --- /dev/null +++ b/examples/sandbox_template_lifecycle.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function + +import time + +from qiniu.services.sandbox import ( + SandboxError, + Template, + wait_for_file, + wait_for_port, + wait_for_process, + wait_for_timeout, + wait_for_url, +) +from sandbox_common import run_example, sandbox_client + + +def pick(data, *keys): + for key in keys: + if isinstance(data, dict) and data.get(key) is not None: + return data.get(key) + return None + + +def main(): + client = sandbox_client() + name = 'qiniu-python-sdk-example-{0}'.format(int(time.time() * 1000)) + target = '{0}:v1'.format(name) + template_id = None + + builder_demo = ( + Template() + .from_template('base') + .copy('README.md', '/tmp/README.md', chmod='0644') + .set_ready_cmd(wait_for_file('/tmp/README.md')) + ) + print('builder demo:', builder_demo.to_dict()) + + config_demo = ( + Template() + .from_image('python:3.11') + .run_cmd(['python', '--version']) + .set_env('PYTHONUNBUFFERED', '1') + .set_start_cmd( + 'python -m http.server 8000', + ready_cmd=wait_for_port(8000), + ) + .set_ready_cmd(wait_for_timeout(1000)) + .to_dict() + ) + print('config demo:', config_demo) + print('ready helpers:', [ + wait_for_file('/tmp/README.md').get_cmd(), + wait_for_process('python').get_cmd(), + wait_for_url('http://127.0.0.1:8000').get_cmd(), + ]) + + try: + created = client.create_template(name=target) + template_id = pick( + created, + 'templateID', + 'templateId', + 'template_id', + 'id', + ) or name + build_id = pick(created, 'buildID', 'buildId', 'build_id') + print('created:', template_id) + + print('default templates:', len(client.list_default_templates())) + print('templates:', len(client.list_templates())) + print('get:', client.get_template(template_id)) + + updated = client.update_template( + template_id, + description='Created by qiniu-python-sdk example', + ) + print('updated:', updated) + + if build_id: + print('build status:', client.get_template_build_status( + template_id, + build_id, + )) + print('build logs:', client.get_template_build_logs( + template_id, + build_id, + )) + client.start_template_build( + template_id, + build_id, + fromTemplate='base', + ) + print('build started:', build_id) + final_build = client.wait_for_build( + template_id, + build_id, + interval=3, + timeout=120, + ) + print( + 'wait build:', + final_build.get('status'), + 'logs:', + len(final_build.get('logs') or []), + ) + + client.assign_template_tags( + target=target, + tags=['example'], + ) + print('tagged:', template_id) + client.delete_template_tags(name=name, tags=['example']) + print('untagged:', template_id) + except SandboxError as err: + print('template lifecycle skipped:', err) + finally: + if template_id: + try: + client.delete_template(template_id) + print('deleted:', template_id) + except SandboxError as err: + print('delete template skipped:', err) + + +if __name__ == '__main__': + run_example(main) diff --git a/examples/sandbox_templates.py b/examples/sandbox_templates.py index 3f88d0bc..aabaef46 100644 --- a/examples/sandbox_templates.py +++ b/examples/sandbox_templates.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- from __future__ import print_function +import time + +from qiniu.services.sandbox import SandboxError from qiniu.services.sandbox import Template from sandbox_common import run_example, sandbox_client def main(): client = sandbox_client() + template_id = None template = ( Template() .from_image('python:3.11') @@ -14,11 +18,26 @@ def main(): .set_env('PYTHONUNBUFFERED', '1') ) - created = client.create_template( - name='qiniu-python-sdk-example', - buildConfig=template.to_dict(), - ) - print('template:', created) + try: + created = client.create_template( + name='qiniu-python-sdk-example-{0}'.format( + int(time.time() * 1000), + ), + buildConfig=template.to_dict(), + ) + template_id = ( + created.get('templateID') or + created.get('templateId') or + created.get('id') + ) + print('template:', created) + finally: + if template_id: + try: + client.delete_template(template_id) + print('deleted:', template_id) + except SandboxError as err: + print('delete template skipped:', err) if __name__ == '__main__': diff --git a/tests/cases/test_services/test_sandbox/test_example_config.py b/tests/cases/test_services/test_sandbox/test_example_config.py index 8844c87f..a222e8a1 100644 --- a/tests/cases/test_services/test_sandbox/test_example_config.py +++ b/tests/cases/test_services/test_sandbox/test_example_config.py @@ -39,3 +39,92 @@ def test_examples_handle_runtime_branches_in_code(): assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in runtime assert 'QINIU_SANDBOX_RUN_INTEGRATION' not in integration assert 'QINIU_SANDBOX_TEST_TIMEOUT' not in integration + + +def test_examples_cover_primary_sandbox_surfaces(): + expected_examples = [ + 'sandbox_commands.py', + 'sandbox_connect.py', + 'sandbox_create.py', + 'sandbox_filesystem.py', + 'sandbox_git.py', + 'sandbox_injection_rules.py', + 'sandbox_lifecycle.py', + 'sandbox_observability.py', + 'sandbox_resources.py', + 'sandbox_runtime.py', + 'sandbox_template_lifecycle.py', + 'sandbox_templates.py', + ] + for name in expected_examples: + assert os.path.exists(os.path.join(ROOT, 'examples', name)) + + checks = { + 'sandbox_commands.py': [ + 'commands.start', + 'commands.list', + 'commands.send_stdin', + 'commands.close_stdin', + 'commands.kill', + 'commands.connect', + ], + 'sandbox_filesystem.py': [ + 'files.make_dir', + 'files.write(', + 'files.read(', + 'files.write_files', + 'files.get_info', + 'files.list', + 'files.rename', + 'files.remove', + ], + 'sandbox_observability.py': [ + 'wait_for_ready', + 'is_running', + 'get_host', + 'get_mcp_url', + 'get_mcp_token', + 'download_url', + 'upload_url', + 'get_metrics', + 'get_logs', + ], + 'sandbox_template_lifecycle.py': [ + 'list_default_templates', + 'list_templates', + 'get_template', + 'update_template', + 'assign_template_tags', + 'delete_template_tags', + 'get_template_build_status', + 'get_template_build_logs', + 'start_template_build', + 'wait_for_build', + 'delete_template', + ], + 'sandbox_injection_rules.py': [ + 'create_injection_rule', + 'list_injection_rules', + 'get_injection_rule', + 'update_injection_rule', + 'delete_injection_rule', + ], + 'sandbox_lifecycle.py': [ + 'set_timeout', + 'refresh', + 'pause', + 'resume', + 'update_network', + ], + 'sandbox_resources.py': [ + 'GitRepositoryResource', + 'push_branch_with_credentials', + 'KodoResource', + 'read_only=True', + ], + } + + for name, snippets in checks.items(): + content = read_project_file('examples', name) + for snippet in snippets: + assert snippet in content From c198b560d49edef8023ecb7f28962e83d3ad2fcf Mon Sep 17 00:00:00 2001 From: Miclle Zheng Date: Wed, 24 Jun 2026 16:49:26 +0800 Subject: [PATCH 45/45] ci: ignore windows mock server cleanup failures --- .github/workflows/ci-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f9f816d2..8237820c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -226,6 +226,7 @@ jobs: python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml - name: Post Setup mock server if: ${{ always() }} + continue-on-error: true run: | Try { $mocksrvpid = Get-Content -Path "mock-server.pid" @@ -236,6 +237,7 @@ jobs: } - name: Print mock server log if: ${{ failure() }} + continue-on-error: true run: | Get-Content -Path "py-mock-server.log" | Write-Host - name: Upload results to Codecov