Add Yocto support for PYNQ (#803)

* Remove hard dependencies from PYNQ

All but numpy are now soft dependencies to make it easier to install
PYNQ in restricted environments.

* Support Cross-compilation for PYNQ

Update all of the various build scripts to correctly handle any
cross compilation flags.

* Don't do anything fancy if notebooks_dir is explicitly empty

* Add a filesystem layer to meta-pynq Yocto layer

Author: PeterOgden

pynq/lib/_pynq/_audio/Makefile

@@ -1,6 +1,3 @@
-CC = gcc
-CPP = g++
-
 ifeq ($(BOARD), Pynq-Z1)
 C_SRC += audio_direct.cpp
 OBJSC += audio_direct.o
@@ -14,6 +11,6 @@ OBJSC += audio_adau1761.o \
 endif
 
 all:	
-	$(CC) -fPIC -c -g3 -ggdb $(C_SRC)
-	$(CPP) -g3 -ggdb -shared -fPIC -rdynamic $(OBJSC) -Wl,--start-group $(LIBS) -Wl,--end-group -o libaudio.so
+	$(CC) -fPIC -c -g3 -ggdb $(C_SRC) $(CFLAGS)
+	$(CXX) -g3 -ggdb -shared -fPIC -rdynamic $(OBJSC) -Wl,--start-group $(LIBS) -Wl,--end-group -o libaudio.so $(CFLAGS) $(LDFLAGS)
 	rm *.o

pynq/lib/_pynq/_displayport/Makefile

@@ -2,9 +2,9 @@ CC ?= gcc
 CXX ?= g++
 
 OBJSC = displayport.o 
-INC = -I/usr/include/libdrm
+INC = -I${PYNQ_BUILD_ROOT}/usr/include/libdrm
 
 all:	
-	$(CXX) -fPIC $(INC) -c -g -std=c++11 displayport.cpp
-	$(CXX) -shared -fPIC -rdynamic $(OBJSC) -ldrm -o libdisplayport.so
+	$(CXX) -fPIC $(INC) -c -g -std=c++11 displayport.cpp $(CFLAGS)
+	$(CXX) -shared -fPIC -rdynamic $(OBJSC) -ldrm -o libdisplayport.so $(CFLAGS) $(LDFLAGS)
 	rm *.o

pynq/lib/_pynq/embeddedsw_lib.mk

@@ -9,7 +9,7 @@ EMBEDDEDSW_DIR ?= embeddedsw
 # INC - include directory with the -I prefix
 
 # ARCH := $(shell uname -p)
-ARCH := $(shell uname -p)
+PYNQ_BUILD_ARCH ?= $(shell uname -p)
 ESW_SRC := $(filter-out %_g.c, $(foreach lib, $(ESW_LIBS), $(wildcard $(EMBEDDEDSW_DIR)/XilinxProcessorIPLib/drivers/$(lib)/src/*.c)))
 ESW_INC := $(patsubst %, -I$(EMBEDDEDSW_DIR)/XilinxProcessorIPLib/drivers/%/src, $(ESW_LIBS))
 OS_INC := -I$(EMBEDDEDSW_DIR)/lib/bsp/standalone/src/common -I$(EMBEDDEDSW_DIR)/lib/bsp/standalone/src/arm/common/gcc -I$(EMBEDDEDSW_DIR)/lib/bsp/standalone/src/arm/common
@@ -22,10 +22,10 @@ COMMON_INC := -Icommon
 COMMON_INC_aarch64 := -Icommon/aarch64
 COMMON_INC_armv7l := -Icommon/armv7l
 
-ALL_SRC := $(SRC) $(COMMON_SRC) $(COMMON_SRC_$(ARCH)) $(ESW_SRC)
-ALL_INC := $(INC) $(COMMON_INC) $(COMMON_INC_$(ARCH)) $(ESW_INC) $(OS_INC) $(OS_INC_$(ARCH))
+ALL_SRC := $(SRC) $(COMMON_SRC) $(COMMON_SRC_$(PYNQ_BUILD_ARCH)) $(ESW_SRC)
+ALL_INC := $(INC) $(COMMON_INC) $(COMMON_INC_$(PYNQ_BUILD_ARCH)) $(ESW_INC) $(OS_INC) $(OS_INC_$(PYNQ_BUILD_ARCH))
 
 all: $(LIB_NAME)
 
 $(LIB_NAME): $(EMBEDDEDSW_DIR)
-	gcc -o $(LIB_NAME) -shared -fPIC $(ALL_INC) $(ALL_SRC)
+	$(CC) -o $(LIB_NAME) -shared -fPIC $(ALL_INC) $(ALL_SRC) $(CFLAGS) $(LDFLAGS)

pynq/lib/arduino/arduino_lcd18.py

@@ -31,7 +31,6 @@
 from math import ceil
 import asyncio
 import os
-from PIL import Image
 from numpy import array
 from pynq import Xlnk
 from . import Arduino
@@ -201,6 +200,8 @@ def display_async(self, img_path, x_pos=0, y_pos=127,
         None
 
         """
+        from PIL import Image
+
         if x_pos not in range(160):
             raise ValueError("Valid x_pos is 0 - 159.")
         if y_pos not in range(128):

pynq/lib/logictools/boolean_generator.py

@@ -31,8 +31,6 @@
 import re
 from copy import deepcopy
 from collections import OrderedDict
-from pyeda.inter import exprvar
-from pyeda.inter import expr2truthtable
 from pynq import Register
 from .constants import *
 from .logictools_controller import LogicToolsController
@@ -195,6 +193,12 @@ def setup(self, expressions, frequency_mhz=DEFAULT_CLOCK_FREQUENCY_MHZ):
             The frequency of the captured samples, in MHz.
 
         """
+        try:
+            from pyeda.inter import exprvar
+            from pyeda.inter import expr2truthtable
+        except ImportError:
+            raise ImportError("Using Logictools requires pyeda")
+
         if not MIN_CLOCK_FREQUENCY_MHZ <= frequency_mhz <= \
                 MAX_CLOCK_FREQUENCY_MHZ:
             raise ValueError("Clock frequency out of range "

pynq/lib/logictools/fsm_generator.py

@@ -32,8 +32,6 @@
 from copy import deepcopy
 from math import ceil, log
 import numpy as np
-import pygraphviz as pgv
-from IPython.display import Image, display
 from .constants import *
 from .logictools_controller import LogicToolsController
 from .trace_analyzer import TraceAnalyzer
@@ -1009,6 +1007,9 @@ def show_state_diagram(self, file_name='fsm_spec.png', save_png=False):
             Whether to save the PNG showing the state diagram.
 
         """
+        import pygraphviz as pgv
+        from IPython.display import Image, display
+
         if self.logictools_controller.status[
                 self.__class__.__name__] == 'RESET':
             raise ValueError(

pynq/lib/logictools/waveform.py

@@ -36,8 +36,6 @@
 import base64
 from xml.dom import minidom
 import numpy as np
-import IPython.core.display
-import IPython.display
 from .constants import *
 
 
@@ -224,6 +222,8 @@ def _draw_javascript(data):
         A dump of a Json formatted data.
 
     """
+    import IPython.core.display
+    import IPython.display
     wavedrom_js = 'wavedrom.js'
     wavedromskin_js = 'wavedromskin.js'
 
@@ -277,6 +277,8 @@ def _copy_javascripts():
 
 
 def _draw_phantomjs(data, phantomjs, wavedrom_cli):
+    import IPython.core.display
+    import IPython.display
     """Draw the wavedrom using PhantomJS.
 
     This method requires the PhantomJS to be properly installed on the board.

pynq/lib/pynqmicroblaze/__init__.py

@@ -39,4 +39,8 @@
 from .compile import MicroblazeProgram
 from .rpc import MicroblazeRPC
 from .rpc import MicroblazeLibrary
-from .magic import MicroblazeMagics
+try:
+    __IPYTHON__
+    from .magic import MicroblazeMagics
+except NameError:
+    pass

pynq/lib/wifi.py

@@ -29,7 +29,6 @@
 
 import os
 import subprocess as sproc
-import netifaces
 
 __author__ = "Luca Cerina"
 __copyright__ = "Copyright 2016, NECST Laboratory, Politecnico di Milano"
@@ -65,6 +64,11 @@ def __init__(self, interface='wlan0'):
             The name of the network interface.
 
         """
+        try:
+            import netifaces
+        except ImportError:
+            raise ImportError("netifaces must be installed to configure WiFi")
+
         self.wifi_port = None
         net_device_list = netifaces.interfaces()
 

pynq/pmbus.py

@@ -28,7 +28,6 @@
 #   ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import cffi
-import pandas as pd
 import threading
 import time
 import warnings
@@ -225,7 +224,7 @@
 
 try:
     _ffi.cdef(_c_header)
-    _lib = _ffi.dlopen("libsensors.so")
+    _lib = _ffi.dlopen("libsensors.so.4")
 except Exception as e:
     warnings.warn("Could not initialise libsensors library")
     _lib = None
@@ -469,5 +468,6 @@ def frame(self):
         Sensors* : one column per sensor
 
         """
+        import pandas as pd
         return pd.DataFrame(self._data, columns=self._columns,
                             index=pd.to_datetime(self._times, unit="s"))

sdbuild/boot/meta-pynq/README.md

@@ -0,0 +1,53 @@
+PYNQ and PetaLinux
+==================
+
+The PYNQ environment is primarily designed to run inside of a Ubuntu-based
+filesystem with the kernel being provided by PetaLinux. This is the flow which
+the sdbuild directory supports. It is possible, however, to install PYNQ inside
+of a PetaLinux root filesystem with the following caveats:
+
+ 1) No overlays will be installed
+ 2) Jupyter is not supported - only the PYNQ library
+ 3) Parts of logictools require libraries not available in PetaLinux 
+
+The rest of this guide describes how to configure a PetaLinux project to
+include PYNQ. It is specific to version 2018.2.
+
+Add the required layers
+-----------------------
+
+Run `petalinux-config` in the PetaLinux project and add this directory as a user layer
+
+Add PYNQ to the filesystem target
+---------------------------------
+
+Edit `project-spec/meta-user/recipes-core/images/petalinux-image.bbappend` and
+add the following line  
+
+```
+    IMAGE_INSTALL_append = " python3-pynq"
+```
+
+Next run `petalinux-config -c rootfs` and you will find the option to select
+`python3-pynq` under `user packages`. Select it to add it to the filesystem.
+If the base notebooks are required then the python3-pynq-notebooks package can
+be added as well. 
+
+Note that by default the BOARD environment will not be set in the root filesystem
+and overlays will not be installed. 
+
+The PYNQ bbclass
+----------------
+
+There is also a pynq-package bbclass that is designed to adapt third party
+packages by installing the notebooks into a separate -notebooks package and
+setting the BOARD environment variable during installation. Note that the BOARD
+will need to be changed inside the layer to support the board you are
+targetting.
+
+Build the filesystem
+--------------------
+
+run `petalinux-build` to build all of the components
+run `petalinux-pacakge --boot --u-boot --fpga` to create `BOOT.BIN`
+

sdbuild/boot/meta-pynq/classes/pynq-package.bbclass

@@ -0,0 +1,16 @@
+PYNQ_NOTEBOOK_DIR ?= "/home/root/notebooks"
+
+PYNQ_BOARD ?= ZCU104
+
+do_compile_prepend() {
+export PYNQ_JUPYTER_NOTEBOOKS="${D}${PYNQ_NOTEBOOK_DIR}"
+export BOARD=${PYNQ_BOARD}
+}
+
+do_install_prepend() {
+export PYNQ_JUPYTER_NOTEBOOKS="${D}${PYNQ_NOTEBOOK_DIR}"
+export BOARD=${PYNQ_BOARD}
+}
+
+FILES_${PN}-notebooks = "${PYNQ_NOTEBOOK_DIR}"
+PACKAGES += "${PN}-notebooks"

sdbuild/boot/meta-pynq/recipes-filesystem/python/pl_server_init

@@ -0,0 +1,96 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start daemon at boot time
+# Description:       Enable service provided by daemon.
+### END INIT INFO
+
+dir=""
+cmd="start_pl_server.py"
+user=""
+
+name="pl_server"
+pid_file="/var/run/$name.pid"
+stdout_log="/var/log/$name.log"
+stderr_log="/var/log/$name.err"
+
+get_pid() {
+    cat "$pid_file"
+}
+
+is_running() {
+    [ -f "$pid_file" ] && (ps -o"pid" | grep '^ '`get_pid`'$') > /dev/null 2>&1
+}
+
+case "$1" in
+    start)
+    if is_running; then
+        echo "Already started"
+    else
+        echo "Starting $name"
+        cd "$dir"
+        $cmd >> "$stdout_log" 2>> "$stderr_log" &
+        echo $! > "$pid_file"
+        if ! is_running; then
+            echo "Unable to start, see $stdout_log and $stderr_log"
+            exit 1
+        fi
+    fi
+    ;;
+    stop)
+    if is_running; then
+        echo -n "Stopping $name.."
+        kill `get_pid`
+        for i in 1 2 3 4 5 6 7 8 9 10
+        # for i in `seq 10`
+        do
+            if ! is_running; then
+                break
+            fi
+
+            echo -n "."
+            sleep 1
+        done
+        echo
+
+        if is_running; then
+            echo "Not stopped; may still be shutting down or shutdown may have failed"
+            exit 1
+        else
+            echo "Stopped"
+            if [ -f "$pid_file" ]; then
+                rm "$pid_file"
+            fi
+        fi
+    else
+        echo "Not running"
+    fi
+    ;;
+    restart)
+    $0 stop
+    if is_running; then
+        echo "Unable to stop, will not attempt to start"
+        exit 1
+    fi
+    $0 start
+    ;;
+    status)
+    if is_running; then
+        echo "Running"
+    else
+        echo "Stopped"
+        exit 1
+    fi
+    ;;
+    *)
+    echo "Usage: $0 {start|stop|restart|status}"
+    exit 1
+    ;;
+esac
+
+exit 0
+