Skip to content

Commit ff66830

Browse files
masonrmasonr
committed
add build workflows for fio/iperf3 updates
1 parent 717911b commit ff66830

File tree

3 files changed

+822
-0
lines changed

3 files changed

+822
-0
lines changed

‎.github/workflows/build-fio.yml‎

Lines changed: 360 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,360 @@
1+
name: Build fio Static Binaries
2+
3+
on:
4+
workflow_dispatch:
5+
6+
jobs:
7+
check-release:
8+
runs-on: ubuntu-latest
9+
outputs:
10+
latest-version: ${{ steps.get-version.outputs.version }}
11+
should-build: ${{ steps.check-version.outputs.should-build }}
12+
steps:
13+
- name: Checkout repository
14+
uses: actions/checkout@v4
15+
16+
- name: Get latest fio release
17+
id: get-version
18+
run: |
19+
# Get the latest release tag from GitHub API
20+
LATEST_VERSION=$(curl -s https://api.github.com/repos/axboe/fio/releases/latest | jq -r '.tag_name')
21+
echo "Latest fio version: $LATEST_VERSION"
22+
echo "version=$LATEST_VERSION" >> $GITHUB_OUTPUT
23+
24+
- name: Check if version exists in releases
25+
id: check-version
26+
run: |
27+
VERSION="${{ steps.get-version.outputs.version }}"
28+
29+
# Check if this version already exists in our releases
30+
if gh release view "fio-$VERSION" --repo ${{ github.repository }} >/dev/null 2>&1; then
31+
echo "Version $VERSION already exists in releases"
32+
echo "should-build=false" >> $GITHUB_OUTPUT
33+
else
34+
echo "Version $VERSION does not exist, should build"
35+
echo "should-build=true" >> $GITHUB_OUTPUT
36+
fi
37+
env:
38+
GH_TOKEN: ${{ github.token }}
39+
40+
build:
41+
needs: check-release
42+
if: needs.check-release.outputs.should-build == 'true'
43+
runs-on: ubuntu-latest
44+
strategy:
45+
matrix:
46+
include:
47+
- arch: x64
48+
cross: x86_64-linux-musl
49+
host: x86_64-linux-musl
50+
- arch: x86
51+
cross: i686-linux-musl
52+
host: i686-linux-musl
53+
- arch: aarch64
54+
cross: aarch64-linux-musl
55+
host: aarch64-linux-gnu
56+
- arch: arm
57+
cross: arm-linux-musleabihf
58+
host: arm-linux-gnueabihf
59+
60+
steps:
61+
- name: Checkout repository
62+
uses: actions/checkout@v4
63+
64+
- name: Set up Docker Buildx
65+
uses: docker/setup-buildx-action@v3
66+
67+
- name: Create compilation script
68+
run: |
69+
VERSION="${{ needs.check-release.outputs.latest-version }}"
70+
ARCH="${{ matrix.arch }}"
71+
CROSS="${{ matrix.cross }}"
72+
HOST="${{ matrix.host }}"
73+
74+
# Single script for all architectures using musl cross-compilation
75+
cat > compile-fio.sh << EOF
76+
#!/bin/bash
77+
set -e
78+
79+
# Activate Holy Build Box lib compilation environment
80+
source /hbb/activate
81+
82+
set -x
83+
84+
# remove obsolete CentOS repos
85+
cd /etc/yum.repos.d/
86+
rm -f CentOS-Base.repo CentOS-SCLo-scl-rh.repo CentOS-SCLo-scl.repo CentOS-fasttrack.repo CentOS-x86_64-kernel.repo
87+
88+
yum install -y yum-plugin-ovl # fix for docker overlay fs
89+
yum install -y xz
90+
91+
# download musl cross compilation toolchain
92+
cd ~
93+
curl -L "https://musl.cc/\$CROSS-cross.tgz" -o "\$CROSS-cross.tgz"
94+
tar xf "\$CROSS-cross.tgz"
95+
96+
# download, compile, and install libaio as static library
97+
cd ~
98+
curl -L http://ftp.de.debian.org/debian/pool/main/liba/libaio/libaio_0.3.113.orig.tar.gz -o "libaio.tar.gz"
99+
tar xf libaio.tar.gz
100+
cd libaio-*/src
101+
CC=/root/\$CROSS-cross/bin/\$CROSS-gcc ENABLE_SHARED=0 make prefix=/hbb_exe install
102+
103+
# Activate Holy Build Box exe compilation environment
104+
source /hbb_exe/activate
105+
106+
# download and compile fio
107+
cd ~
108+
curl -L "https://github.com/axboe/fio/archive/\$VERSION.tar.gz" -o "fio.tar.gz"
109+
tar xf fio.tar.gz
110+
cd fio-\${VERSION#fio-}*
111+
CC=/root/\$CROSS-cross/bin/\$CROSS-gcc ./configure --disable-native --build-static
112+
make
113+
114+
# verify no external shared library links
115+
libcheck fio
116+
# copy fio binary to mounted dir
117+
cp fio "/io/fio_\$ARCH"
118+
EOF
119+
120+
chmod +x compile-fio.sh
121+
122+
- name: Compile fio binary
123+
run: |
124+
ARCH="${{ matrix.arch }}"
125+
CROSS="${{ matrix.cross }}"
126+
HOST="${{ matrix.host }}"
127+
128+
# Use musl cross-compilation for all architectures
129+
docker run -t -i --rm -v $(pwd):/io --env ARCH=$ARCH --env CROSS=$CROSS --env HOST=$HOST --env VERSION="${{ needs.check-release.outputs.latest-version }}" phusion/holy-build-box-64:latest bash /io/compile-fio.sh
130+
131+
- name: Verify binary
132+
run: |
133+
ARCH="${{ matrix.arch }}"
134+
ls -la fio_$ARCH
135+
file fio_$ARCH
136+
137+
- name: Upload binary as artifact
138+
uses: actions/upload-artifact@v4
139+
with:
140+
name: fio_${{ matrix.arch }}
141+
path: fio_${{ matrix.arch }}
142+
retention-days: 1
143+
144+
virustotal-scan:
145+
needs: [check-release, build]
146+
if: needs.check-release.outputs.should-build == 'true'
147+
runs-on: ubuntu-latest
148+
outputs:
149+
scan-results: ${{ steps.scan-summary.outputs.results }}
150+
all-clean: ${{ steps.scan-summary.outputs.all-clean }}
151+
steps:
152+
- name: Download all artifacts
153+
uses: actions/download-artifact@v4
154+
with:
155+
path: artifacts
156+
157+
- name: Prepare binaries for scanning
158+
run: |
159+
mkdir -p scan-binaries
160+
find artifacts -name "fio_*" -type f -exec cp {} scan-binaries/ \;
161+
ls -la scan-binaries/
162+
163+
- name: Upload binaries to VirusTotal and scan
164+
id: virustotal-upload
165+
run: |
166+
echo "## VirusTotal Scan Results" > scan_results.md
167+
echo "| Binary | Status | Malicious | Suspicious | Undetected | VirusTotal URL |" >> scan_results.md
168+
echo "|--------|--------|-----------|------------|------------|----------------|" >> scan_results.md
169+
170+
ALL_CLEAN=true
171+
SCAN_DATA=""
172+
173+
for binary in scan-binaries/fio_*; do
174+
filename=$(basename "$binary")
175+
echo "Uploading $filename to VirusTotal..."
176+
177+
# Upload to VirusTotal
178+
upload_response=$(curl -s --request POST \
179+
--url https://www.virustotal.com/api/v3/files \
180+
--header 'accept: application/json' \
181+
--header 'content-type: multipart/form-data' \
182+
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}" \
183+
--form "file=@$binary")
184+
185+
analysis_id=$(echo "$upload_response" | jq -r '.data.id')
186+
echo "Analysis ID for $filename: $analysis_id"
187+
188+
if [ "$analysis_id" = "null" ] || [ -z "$analysis_id" ]; then
189+
echo "Failed to upload $filename to VirusTotal"
190+
echo "| $filename | Upload Failed | N/A | N/A | N/A | N/A |" >> scan_results.md
191+
ALL_CLEAN=false
192+
continue
193+
fi
194+
195+
# Store analysis ID for later retrieval
196+
echo "${filename}:${analysis_id}" >> analysis_ids.txt
197+
done
198+
199+
echo "all_clean_upload=$ALL_CLEAN" >> $GITHUB_OUTPUT
200+
201+
# Wait 2 minutes for scans to complete
202+
echo "Waiting 2 minutes for VirusTotal scans to complete..."
203+
sleep 120
204+
205+
- name: Retrieve VirusTotal scan results
206+
id: get-results
207+
run: |
208+
ALL_CLEAN=true
209+
SCAN_SUMMARY=""
210+
211+
while IFS=':' read -r filename analysis_id; do
212+
echo "Retrieving results for $filename (ID: $analysis_id)..."
213+
214+
# Get scan results
215+
result_response=$(curl -s --request GET \
216+
--url "https://www.virustotal.com/api/v3/analyses/$analysis_id" \
217+
--header 'accept: application/json' \
218+
--header "x-apikey: ${{ secrets.VIRUSTOTAL_API_KEY }}")
219+
220+
status=$(echo "$result_response" | jq -r '.data.attributes.status // "unknown"')
221+
222+
if [ "$status" = "completed" ]; then
223+
malicious=$(echo "$result_response" | jq -r '.data.attributes.stats.malicious // 0')
224+
suspicious=$(echo "$result_response" | jq -r '.data.attributes.stats.suspicious // 0')
225+
undetected=$(echo "$result_response" | jq -r '.data.attributes.stats.undetected // 0')
226+
227+
# Get file hash from the item link
228+
item_link=$(echo "$result_response" | jq -r '.data.links.item // ""')
229+
file_hash=$(echo "$item_link" | sed 's/.*files\///')
230+
vt_url="https://www.virustotal.com/gui/file/$file_hash"
231+
232+
if [ "$malicious" -gt 0 ] || [ "$suspicious" -gt 0 ]; then
233+
status_text="⚠️ FLAGGED"
234+
ALL_CLEAN=false
235+
else
236+
status_text="✅ Clean"
237+
fi
238+
239+
echo "| $filename | $status_text | $malicious | $suspicious | $undetected | [$file_hash]($vt_url) |" >> scan_results.md
240+
241+
else
242+
echo "| $filename | ⏳ Pending | N/A | N/A | N/A | Scan not completed |" >> scan_results.md
243+
ALL_CLEAN=false
244+
fi
245+
246+
done < analysis_ids.txt
247+
248+
echo "all_clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
249+
250+
# Add summary to GitHub step summary
251+
cat scan_results.md >> $GITHUB_STEP_SUMMARY
252+
253+
- name: Create scan summary
254+
id: scan-summary
255+
run: |
256+
RESULTS=$(cat scan_results.md)
257+
ALL_CLEAN="${{ steps.get-results.outputs.all_clean }}"
258+
259+
# Escape newlines for GitHub output
260+
RESULTS_ESCAPED=$(echo "$RESULTS" | sed ':a;N;$!ba;s/\n/\\n/g')
261+
262+
echo "results=$RESULTS_ESCAPED" >> $GITHUB_OUTPUT
263+
echo "all-clean=$ALL_CLEAN" >> $GITHUB_OUTPUT
264+
265+
- name: Upload scan results as artifact
266+
uses: actions/upload-artifact@v4
267+
with:
268+
name: virustotal-scan-results
269+
path: |
270+
scan_results.md
271+
analysis_ids.txt
272+
retention-days: 30
273+
274+
create-release:
275+
needs: [check-release, build, virustotal-scan]
276+
if: needs.check-release.outputs.should-build == 'true' && needs.virustotal-scan.outputs.all-clean == 'true'
277+
runs-on: ubuntu-latest
278+
steps:
279+
- name: Checkout repository
280+
uses: actions/checkout@v4
281+
282+
- name: Download all artifacts
283+
uses: actions/download-artifact@v4
284+
with:
285+
path: artifacts
286+
287+
- name: Prepare release assets
288+
run: |
289+
mkdir -p release-assets
290+
# Move all binaries to release-assets directory
291+
find artifacts -name "fio_*" -type f -exec cp {} release-assets/ \;
292+
ls -la release-assets/
293+
294+
- name: Generate SHA256 checksums
295+
run: |
296+
cd release-assets
297+
sha256sum fio_* > fio-checksums.sha256
298+
cat fio-checksums.sha256
299+
300+
- name: Download VirusTotal scan results
301+
uses: actions/download-artifact@v4
302+
with:
303+
name: virustotal-scan-results
304+
path: vt-results
305+
306+
- name: Create Release
307+
uses: softprops/action-gh-release@v1
308+
with:
309+
tag_name: fio-${{ needs.check-release.outputs.latest-version }}
310+
name: fio ${{ needs.check-release.outputs.latest-version }} Static Binaries
311+
body: |
312+
Static binaries for fio ${{ needs.check-release.outputs.latest-version }}
313+
314+
Built using musl toolchains for maximum compatibility.
315+
316+
**Architectures:**
317+
- `fio_x64` - x86_64 (64-bit)
318+
- `fio_x86` - i686 (32-bit)
319+
- `fio_aarch64` - ARM 64-bit
320+
- `fio_arm` - ARM 32-bit
321+
322+
**Security Verification:**
323+
All binaries have been scanned by VirusTotal and verified clean.
324+
325+
${{ needs.virustotal-scan.outputs.scan-results }}
326+
327+
**Checksum Verification:**
328+
```bash
329+
# Verify checksums
330+
sha256sum -c fio-checksums.sha256
331+
```
332+
333+
For usage in YABS script, place these binaries in the `bin/fio/` directory.
334+
files: |
335+
release-assets/fio_*
336+
release-assets/fio-checksums.sha256
337+
draft: false
338+
prerelease: false
339+
env:
340+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
341+
342+
notify-failure:
343+
needs: [check-release, build, virustotal-scan]
344+
if: always() && needs.check-release.outputs.should-build == 'true' && (failure() || needs.virustotal-scan.outputs.all-clean == 'false')
345+
runs-on: ubuntu-latest
346+
steps:
347+
- name: Send failure notification
348+
run: |
349+
echo "## ⚠️ fio Build Failed" >> $GITHUB_STEP_SUMMARY
350+
echo "Version: ${{ needs.check-release.outputs.latest-version }}" >> $GITHUB_STEP_SUMMARY
351+
352+
if [ "${{ needs.virustotal-scan.outputs.all-clean }}" == "false" ]; then
353+
echo "**Reason:** VirusTotal scan detected issues with one or more binaries" >> $GITHUB_STEP_SUMMARY
354+
echo "**Scan Results:**" >> $GITHUB_STEP_SUMMARY
355+
echo "${{ needs.virustotal-scan.outputs.scan-results }}" >> $GITHUB_STEP_SUMMARY
356+
else
357+
echo "**Reason:** Build compilation failed" >> $GITHUB_STEP_SUMMARY
358+
fi
359+
360+
echo "Check the workflow logs for details: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)