devops: separate wheels for OS/drivers (microsoft#119)

Author: mxschmitt

.github/workflows/ci.yml

@@ -31,6 +31,7 @@ jobs:
     - name: Test Sync generation script
       run: bash buildbots/test-sync-generation.sh
   build:
+    name: Build
     timeout-minutes: 30
     strategy:
       fail-fast: false
@@ -87,3 +88,29 @@ jobs:
         path: junit/test-results-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.browser }}.xml
       # Use always() to always run this step to publish test results when there are test failures
       if: ${{ always() }}
+  test-package-installations:
+    name: Test package installations
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - uses: microsoft/playwright-github-action@v1
+    - name: Set up Node.js
+      uses: actions/setup-node@v1
+      with:
+        node-version: 12.x
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r local-requirements.txt
+        pip install -e .
+    - name: Build driver
+      run: python build_driver.py
+    - name: Build package
+      run: python build_package.py
+    - name: Test package installation
+      run: bash buildbots/test-package-installations.sh

MANIFEST.in

@@ -1 +1,5 @@
-recursive-include playwright/drivers *
+include playwright/drivers/browsers.json
+include playwright/*.py
+include README.md
+include SECURITY.md
+include LICENSE

build_driver.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import gzip
 import os
 import re
 import shutil
@@ -33,17 +32,8 @@
 if (driver_path / "out").exists():
     shutil.rmtree(driver_path / "out")
 
-subprocess.run("npm i", cwd=driver_path, shell=True)
-subprocess.run("npm run bake", cwd=driver_path, shell=True)
-
-for driver in ["driver-linux", "driver-macos", "driver-win.exe"]:
-    if (package_path / driver).exists():
-        os.remove((package_path / driver))
-
-    in_path = driver_path / "out" / driver
-    out_path = drivers_path / (driver + ".gz")
-    with open(in_path, "rb") as f_in, gzip.open(out_path, "wb") as f_out:
-        shutil.copyfileobj(f_in, f_out)
+subprocess.check_call("npm i", cwd=driver_path, shell=True)
+subprocess.check_call("npm run bake", cwd=driver_path, shell=True)
 
 node_modules_playwright = driver_path / "node_modules" / "playwright"
 

build_package.py

@@ -12,8 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import glob
+import os
 import shutil
 import subprocess
+import zipfile
 
 from playwright.path_utils import get_file_dirname
 
@@ -28,4 +31,24 @@
 if _egg_dir.exists():
     shutil.rmtree(_egg_dir)
 
-subprocess.run("python setup.py sdist bdist_wheel", shell=True)
+subprocess.check_call("python setup.py sdist bdist_wheel", shell=True)
+
+base_wheel_location = glob.glob("dist/*.whl")[0]
+without_platform = base_wheel_location[:-7]
+
+pack_wheel_drivers = [
+    ("driver-linux", "manylinux1_x86_64.whl"),
+    ("driver-macos", "macosx_10_13_x86_64.whl"),
+    ("driver-win.exe", "win_amd64.whl"),
+]
+
+for driver, wheel in pack_wheel_drivers:
+    wheel_location = without_platform + wheel
+    shutil.copy(base_wheel_location, wheel_location)
+    from_location = f"driver/out/{driver}"
+    to_location = f"playwright/drivers/{driver}"
+    with zipfile.ZipFile(wheel_location, "a") as zipf:
+        zipf.write(from_location, to_location)
+    # for local development
+    shutil.copy(from_location, to_location)
+os.remove(base_wheel_location)

buildbots/assets/stub.py

@@ -0,0 +1,9 @@
+from playwright import sync_playwright
+
+with sync_playwright() as p:
+    for browser_type in [p.chromium, p.firefox, p.webkit]:
+        browser = browser_type.launch()
+        page = browser.newPage()
+        page.setContent("<h1>Test 123</h1>")
+        page.screenshot(path=f"{browser_type.name}.png")
+        browser.close()

buildbots/test-package-installations.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+tmpdir=$(mktemp -d)
+base_dir=$(pwd)
+set -e
+
+# Cleanup to ensure we start fresh
+echo "Deleting driver and browsers from base installation"
+rm -rf driver
+rm -rf playwright
+rm -rf ~/.cache/ms-playwright
+
+cp buildbots/assets/stub.py "$tmpdir/main.py"
+
+cd $tmpdir
+echo "Creating virtual environment"
+virtualenv env
+source env/bin/activate
+echo "Installing Playwright Python via Wheel"
+pip install "$(echo $base_dir/dist/playwright*manylinux1*.whl)"
+echo "Installing browsers"
+python -m playwright install
+echo "Running basic tests"
+python "main.py"
+cd -
+
+test -f "$tmpdir/chromium.png"
+test -f "$tmpdir/firefox.png"
+test -f "$tmpdir/webkit.png"
+echo "Passed package installation tests successfully"

driver/main.js

@@ -18,7 +18,7 @@ const path = require('path');
 
 (async() => {
   if (process.argv.includes('install')) {
-    await require('playwright/lib/install/installer').installBrowsersWithProgressBar(path.join(path.dirname(process.argv[0]), 'drivers'));
+    await require('playwright/lib/install/installer').installBrowsersWithProgressBar(path.dirname(process.argv[0]));
     return;
   }
 

playwright/main.py

@@ -13,10 +13,6 @@
 # limitations under the License.
 
 import asyncio
-import gzip
-import os
-import shutil
-import stat
 import subprocess
 import sys
 from typing import Any
@@ -25,7 +21,7 @@
 
 from playwright.async_api import Playwright as AsyncPlaywright
 from playwright.connection import Connection
-from playwright.helper import Error, not_installed_error
+from playwright.helper import Error
 from playwright.object_factory import create_remote_object
 from playwright.path_utils import get_file_dirname
 from playwright.playwright import Playwright
@@ -47,15 +43,7 @@ def compute_driver_name() -> str:
 async def run_driver_async() -> Connection:
     package_path = get_file_dirname()
     driver_name = compute_driver_name()
-    driver_executable = package_path / driver_name
-    archive_name = package_path / "drivers" / (driver_name + ".gz")
-
-    if not driver_executable.exists() or os.path.getmtime(
-        driver_executable
-    ) < os.path.getmtime(archive_name):
-        raise not_installed_error(
-            "Playwright requires additional post-installation step to be made."
-        )
+    driver_executable = package_path / "drivers" / driver_name
 
     proc = await asyncio.create_subprocess_exec(
         str(driver_executable),
@@ -127,24 +115,8 @@ def main() -> None:
         return
     package_path = get_file_dirname()
     driver_name = compute_driver_name()
-    driver_executable = package_path / driver_name
-    archive_name = package_path / "drivers" / (driver_name + ".gz")
-
-    if not driver_executable.exists() or os.path.getmtime(
-        driver_executable
-    ) < os.path.getmtime(archive_name):
-        print(f"Extracting {archive_name} into {driver_executable}...")
-        with gzip.open(archive_name, "rb") as f_in, open(
-            driver_executable, "wb"
-        ) as f_out:
-            shutil.copyfileobj(f_in, f_out)
-
-        st = os.stat(driver_executable)
-        if st.st_mode & stat.S_IEXEC == 0:
-            print(f"Making {driver_executable} executable...")
-            os.chmod(driver_executable, st.st_mode | stat.S_IEXEC)
-
+    driver_executable = package_path / "drivers" / driver_name
     print("Installing the browsers...")
-    subprocess.run(f"{driver_executable} install", shell=True)
+    subprocess.check_call(f"{driver_executable} install", shell=True)
 
     print("Playwright is now ready for use")