Alibaba Cloud Model Studio:OpenAI compatible - Batch

Before starting a batch, you can use batch-test-model to perform an end-to-end test, including: validating input data, creating a task, querying task results, and downloading result files. Note:

• The test file must meet the input file format requirements. Also, it must not exceed 1 MB in size and contain no more than 100 lines.

• Concurrency limit: Up to 2 parallel tasks.

• Resource usage: The test model will not perform inference, so it does not incur model inference fees.

Perform the following steps:

1. Prepare the test file

   • Download the sample file test_model.jsonl that contains request information. Make sure it is in the same directory as the Python script below

   • Sample content: The model parameter is set to batch-test-model, and the url is set to the /v1/chat/ds-test endpoint.

     {"custom_id":"1","method":"POST","url":"/v1/chat/ds-test","body":{"model":"batch-test-model","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello! How can I help you?"}]}}
     {"custom_id":"2","method":"POST","url":"/v1/chat/ds-test","body":{"model":"batch-test-model","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]}}

2. Run the script

   • Execute this Python script

     Edit file paths or other parameters as needed.

     import os
     from pathlib import Path
     from openai import OpenAI
     import time
     
     # Initialize the client
     client = OpenAI(
         # If you haven't configured environment variables, you can replace the line below with api_key="sk-xxx" using your Alibaba Cloud Model Studio API Key, but it's not recommended to hardcode API Keys directly in your code in production environments to reduce the risk of API key leakage.
         api_key=os.getenv("DASHSCOPE_API_KEY"),
         base_url="https://dashscope-intl.aliyuncs.com/compatible-mode/v1"  # Alibaba Cloud Model Studio service base_url
     )
     
     def upload_file(file_path):
         print(f"Uploading JSONL file containing request information...")
         file_object = client.files.create(file=Path(file_path), purpose="batch")
         print(f"File uploaded successfully. File ID obtained: {file_object.id}\n")
         return file_object.id
     
     def create_batch_job(input_file_id):
         print(f"Creating a batch based on the file ID...")
         # Note: The endpoint parameter value here must be consistent with the url field in the input file. For the test model (batch-test-model), fill in /v1/chat/ds-test, for other models fill in the `/v1/chat/completions` endpoint
         batch = client.batches.create(input_file_id=input_file_id, endpoint="/v1/chat/ds-test", completion_window="24h")
         print(f"batch creation completed. batch ID obtained: {batch.id}\n")
         return batch.id
     
     def check_job_status(batch_id):
         print(f"Checking batch status...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Batch status: {batch.status}\n")
         return batch.status
     
     def get_output_id(batch_id):
         print(f"Getting the output file ID for successfully executed requests in the batch...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Output file ID: {batch.output_file_id}\n")
         return batch.output_file_id
     
     def get_error_id(batch_id):
         print(f"Getting the output file ID for failed requests in the batch...")
         batch = client.batches.retrieve(batch_id=batch_id)
         print(f"Error file ID: {batch.error_file_id}\n")
         return batch.error_file_id
     
     def download_results(output_file_id, output_file_path):
         print(f"Printing and downloading the successful request results of the batch...")
         content = client.files.content(output_file_id)
         # Print part of the content for testing
         print(f"Printing the first 1000 characters of the successful request results: {content.text[:1000]}...\n")
         # Save the result file locally
         content.write_to_file(output_file_path)
         print(f"Complete output results have been saved to the local output file result.jsonl\n")
     
     def download_errors(error_file_id, error_file_path):
         print(f"Printing and downloading the failed request information of the batch...")
         content = client.files.content(error_file_id)
         # Print part of the content for testing
         print(f"Printing the first 1000 characters of the failed request information: {content.text[:1000]}...\n")
         # Save the error information file locally
         content.write_to_file(error_file_path)
         print(f"Complete failed request information has been saved to the local error file error.jsonl\n")
     
     def main():
         # File paths
         input_file_path = "test_model.jsonl"  # Can be replaced with your input file path
         output_file_path = "result.jsonl"  # Can be replaced with your output file path
         error_file_path = "error.jsonl"  # Can be replaced with your error file path
         try:
             # Step 1: Upload the JSONL file containing request information to get the input file ID
             input_file_id = upload_file(input_file_path)
             # Step 2: Create a Batch based on the input file ID
             batch_id = create_batch_job(input_file_id)
             # Step 3: Check the Batch status until it ends
             status = ""
             while status not in ["completed", "failed", "expired", "cancelled"]:
                 status = check_job_status(batch_id)
                 print(f"Waiting for task completion...")
                 time.sleep(10)  # Wait 10 seconds before checking the status again
             # If the task fails, print the error message and exit
             if status == "failed":
                 batch = client.batches.retrieve(batch_id)
                 print(f"batch failed. Error message: {batch.errors}\n")
                 print(f"See error code documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
                 return
             # Step 4: Download results: If the output file ID is not empty, print the first 1000 characters of the successful request results and download the complete successful request results to the local output file;
             # If the error file ID is not empty, print the first 1000 characters of the failed request information and download the complete failed request information to the local error file.
             output_file_id = get_output_id(batch_id)
             if output_file_id:
                 download_results(output_file_id, output_file_path)
             error_file_id = get_error_id(batch_id)
             if error_file_id:
                 download_errors(error_file_id, error_file_path)
                 print(f"See error code documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
         except Exception as e:
             print(f"An error occurred: {e}")
             print(f"See error code documentation: https://www.alibabacloud.com/help/en/model-studio/developer-reference/error-code")
     
     if __name__ == "__main__":
         main()

3. Verify the test results

   • The task status is completed

   • result.jsonl contains the fixed response {"content":"This is a test result."}

     {"id":"a2b1ae25-21f4-4d9a-8634-99a29926486c","custom_id":"1","response":{"status_code":200,"request_id":"a2b1ae25-21f4-4d9a-8634-99a29926486c","body":{"created":1743562621,"usage":{"completion_tokens":6,"prompt_tokens":20,"total_tokens":26},"model":"batch-test-model","id":"chatcmpl-bca7295b-67c3-4b1f-8239-d78323bb669f","choices":[{"finish_reason":"stop","index":0,"message":{"content":"This is a test result."}}],"object":"chat.completion"}},"error":null}
     {"id":"39b74f09-a902-434f-b9ea-2aaaeebc59e0","custom_id":"2","response":{"status_code":200,"request_id":"39b74f09-a902-434f-b9ea-2aaaeebc59e0","body":{"created":1743562621,"usage":{"completion_tokens":6,"prompt_tokens":20,"total_tokens":26},"model":"batch-test-model","id":"chatcmpl-1e32a8ba-2b69-4dc4-be42-e2897eac9e84","choices":[{"finish_reason":"stop","index":0,"message":{"content":"This is a test result."}}],"object":"chat.completion"}},"error":null}

     In case of errors, see Error messages for solutions.

After the test, follow these steps to execute a batch.

1. Prepare an input file according to the input file format requirements. In the file, set the model parameter to a supported model, and set the url field to: /v1/chat/completions

2. Replace the endpoint parameter in the Python script above to match the url in the input file

3. Run the script and wait for the task to complete. If the task is successful, an output result file result.jsonl will be generated in the same directory

   If the task fails, the program will exit and print the error message.

   If an error file ID is returned, the error file error.jsonl will be generated in the same directory for troubleshooting.

   Exceptions that occur during the process will be caught and error messages will be printed.

Set the url field in the file and the endpoint parameter in your code to /v1/chat/completions.

Single-line request example:

{"custom_id":"1","method":"POST","url":"/v1/chat/completions","body":{"model":"qwen-max","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]}}

Multi-line request example:

{"custom_id":"1","method":"POST","url":"/v1/chat/completions","body":{"model":"qwen-max","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hi, how can I help you?"}]}}
{"custom_id":"2","method":"POST","url":"/v1/chat/completions","body":{"model":"qwen-max","messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is 2+2?"}]}}

Request parameters

Sample response

See the Sample response for creating a batch.

Response parameters

See the response parameters for creating a batch.

Use the returned output_file_id and error_file_id to download result files.

Request parameters

Sample response

{
  "object": "list",
  "data": [
    {
      "id": "batch_xxx",
      "object": "batch",
      "endpoint": "/v1/chat/completions",
      "errors": null,
      "input_file_id": "file-batch-xxx",
      "completion_window": "24h",
      "status": "completed",
      "output_file_id": "file-batch_output-xxx",
      "error_file_id": null,
      "created_at": 1722234109,
      "in_progress_at": 1722234109,
      "expires_at": null,
      "finalizing_at": 1722234165,
      "completed_at": 1722234165,
      "failed_at": null,
      "expired_at": null,
      "cancelling_at": null,
      "cancelled_at": null,
      "request_counts": {
        "total": 100,
        "completed": 95,
        "failed": 5
      },
      "metadata": {}
    },
    { ... }
  ],
  "first_id": "batch_xxx",
  "last_id": "batch_xxx",
  "has_more": true
}

Response parameters

Request parameters

Sample response

See the sample response for creating a batch.

Response parameters

See the response parameters for creating a batch.