Skip to content

Commit 7a3323f

Browse files
rootclaude
authored and
root
committed
Add FPS reduction pipeline and refactor configuration
- Add s4_fps_reduce.py for reducing landmark data FPS with parallel processing - Add requirements.txt for project dependencies - Refactor conf.py with better organization and TARGET_FPS constant - Update preprocessing scripts to use centralized configuration - Improve function signatures and logging in FPS reduction module 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <[email protected]>
1 parent 643246b commit 7a3323f

File tree

7 files changed

+930
-579
lines changed

7 files changed

+930
-579
lines changed

‎.gitignore‎

Lines changed: 158 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,162 @@
1+
### Python template
2+
# Byte-compiled / optimized / DLL files
13
__pycache__/
2-
*.pyc
3-
*.pyo
4-
.idea/
4+
*.py[cod]
5+
*$py.class
56

6-
/dataset
7+
# C extensions
8+
*.so
79

10+
# Distribution / packaging
11+
.Python
12+
build/
13+
develop-eggs/
14+
dist/
15+
downloads/
16+
eggs/
17+
.eggs/
18+
lib/
19+
lib64/
20+
parts/
21+
sdist/
22+
var/
23+
wheels/
24+
share/python-wheels/
25+
*.egg-info/
26+
.installed.cfg
27+
*.egg
28+
MANIFEST
29+
30+
# PyInstaller
31+
# Usually these files are written by a python script from a template
32+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
33+
*.manifest
34+
*.spec
35+
36+
# Installer logs
37+
pip-log.txt
38+
pip-delete-this-directory.txt
39+
40+
# Unit test / coverage reports
41+
htmlcov/
42+
.tox/
43+
.nox/
44+
.coverage
45+
.coverage.*
46+
.cache
47+
nosetests.xml
48+
coverage.xml
49+
*.cover
50+
*.py,cover
51+
.hypothesis/
52+
.pytest_cache/
53+
cover/
54+
55+
# Translations
56+
*.mo
57+
*.pot
58+
59+
# Django stuff:
60+
*.log
61+
local_settings.py
62+
db.sqlite3
63+
db.sqlite3-journal
64+
65+
# Flask stuff:
66+
instance/
67+
.webassets-cache
68+
69+
# Scrapy stuff:
70+
.scrapy
71+
72+
# Sphinx documentation
73+
docs/_build/
74+
75+
# PyBuilder
76+
.pybuilder/
77+
target/
78+
79+
# Jupyter Notebook
80+
.ipynb_checkpoints
81+
82+
# IPython
83+
profile_default/
84+
ipython_config.py
85+
86+
# pyenv
87+
# For a library or package, you might want to ignore these files since the code is
88+
# intended to run in multiple environments; otherwise, check them in:
89+
# .python-version
90+
91+
# pipenv
92+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
94+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
95+
# install all needed dependencies.
96+
#Pipfile.lock
97+
98+
# poetry
99+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
100+
# This is especially recommended for binary packages to ensure reproducibility, and is more
101+
# commonly ignored for libraries.
102+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
103+
#poetry.lock
104+
105+
# pdm
106+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
107+
#pdm.lock
108+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
109+
# in version control.
110+
# https://pdm.fming.dev/#use-with-ide
111+
.pdm.toml
112+
113+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
114+
__pypackages__/
115+
116+
# Celery stuff
117+
celerybeat-schedule
118+
celerybeat.pid
119+
120+
# SageMath parsed files
121+
*.sage.py
122+
123+
# Environments
124+
.env
125+
.venv
126+
env/
127+
venv/
128+
ENV/
129+
env.bak/
130+
venv.bak/
131+
132+
# Spyder project settings
133+
.spyderproject
134+
.spyproject
135+
136+
# Rope project settings
137+
.ropeproject
138+
139+
# mkdocs documentation
140+
/site
141+
142+
# mypy
143+
.mypy_cache/
144+
.dmypy.json
145+
dmypy.json
146+
147+
# Pyre type checker
148+
.pyre/
149+
150+
# pytype static type analyzer
151+
.pytype/
152+
153+
# Cython debug symbols
154+
cython_debug/
155+
156+
# PyCharm
157+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
158+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
159+
# and can be added to the global gitignore or merged into this file. For a more nuclear
160+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
161+
#.idea/
8162

‎conf.py‎

Lines changed: 93 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,93 @@
1-
import os
2-
3-
# YouTube download configuration
4-
YT_CONFIG = {
5-
"format": "worstvideo[height>=720]/bestvideo[height<=480]",
6-
"writesubtitles": False,
7-
"outtmpl": os.path.join(c.VIDEO_DIR, "%(id)s.%(ext)s"),
8-
"nocheckcertificate": True,
9-
"noplaylist": True,
10-
"no-metadata-json": True,
11-
"no-metadata": True,
12-
"concurrent-fragments": 5,
13-
"hls-prefer-ffmpeg": True,
14-
"http-chunk-size": 10485760, # 10MB chunks
15-
"sleep-interval": 0,
16-
"geo-bypass": True,
17-
"limit_rate": "5M",
18-
}
19-
20-
# the number of frames to skip when extracting frames from a video
21-
FRAME_SKIP = 2
22-
23-
# youtube asl dataset
24-
ROOT = os.path.dirname(os.path.abspath(__file__))
25-
ID = "youtube-asl_youtube_asl_video_ids.txt"
26-
27-
VIDEO_DIR = f"{ROOT}/dataset/origin/"
28-
OUTPUT_DIR = f"{ROOT}/dataset/npy/"
29-
CSV_FILE = f"youtube_asl.csv"
30-
31-
TRANSCRIPT_DIR = f"{ROOT}/dataset/transcript/"
32-
MAX_WORKERS = 2
33-
LANGUAGE = [
34-
"en",
35-
"ase",
36-
"en-US",
37-
"en-CA",
38-
"en-GB",
39-
"en-AU",
40-
"en-NZ",
41-
"en-IN",
42-
"en-ZA",
43-
"en-IE",
44-
"en-SG",
45-
"en-PH",
46-
"en-NG",
47-
"en-PK",
48-
"en-JM",
49-
]
50-
51-
# mediapipe landmark indices
52-
HAND_IDX = list(range(21))
53-
POSE_IDX = [11, 12, 13, 14, 23, 24]
54-
FACE_IDX = [
55-
0, 4, 13, 14, 17, 33, 37, 39, 46, 52, 55, 61, 64, 81, 82, 93,
56-
133, 151, 152, 159, 172, 178, 181, 263, 269, 276, 282, 285, 291,
57-
294, 311, 323, 362, 386, 397, 468, 473
58-
]
59-
60-
1+
import os
2+
3+
# =============================================================================
4+
# PROJECT PATHS AND CONFIGURATION
5+
# =============================================================================
6+
7+
# Base paths
8+
ROOT = os.path.dirname(os.path.abspath(__file__))
9+
VIDEO_DIR = f"{ROOT}/dataset/origin/"
10+
NPY_DIR = f"{ROOT}/dataset/npy/"
11+
TRANSCRIPT_DIR = f"{ROOT}/dataset/transcript/"
12+
13+
# Dataset files
14+
ID = "youtube-asl_youtube_asl_video_ids.txt"
15+
CSV_FILE = f"youtube_asl.csv"
16+
17+
# =============================================================================
18+
# PROCESSING CONFIGURATION
19+
# =============================================================================
20+
21+
# Frame processing
22+
FRAME_SKIP = 2 # Number of frames to skip when extracting frames from a video
23+
24+
# Threading
25+
MAX_WORKERS = 4
26+
27+
# FPS reduction
28+
TARGET_FPS = 8.0 # Target FPS for reduced landmark data
29+
30+
# Supported languages
31+
LANGUAGE = [
32+
"en",
33+
"ase",
34+
"en-US",
35+
"en-CA",
36+
"en-GB",
37+
"en-AU",
38+
"en-NZ",
39+
"en-IN",
40+
"en-ZA",
41+
"en-IE",
42+
"en-SG",
43+
"en-PH",
44+
"en-NG",
45+
"en-PK",
46+
"en-JM",
47+
]
48+
49+
# =============================================================================
50+
# YOUTUBE DOWNLOADER CONFIGURATION
51+
# =============================================================================
52+
53+
YT_CONFIG = {
54+
# Video quality and format
55+
"format": "worstvideo[height>=720]/bestvideo[height<=480]",
56+
"writesubtitles": False,
57+
"outtmpl": os.path.join(VIDEO_DIR, "%(id)s.%(ext)s"),
58+
59+
# Connection and security
60+
"nocheckcertificate": True,
61+
"geo-bypass": True,
62+
"limit_rate": "5M",
63+
"http-chunk-size": 10485760, # 10MB chunks
64+
65+
# Playlist and metadata
66+
"noplaylist": True,
67+
"no-metadata-json": True,
68+
"no-metadata": True,
69+
70+
# Performance optimization
71+
"concurrent-fragments": 5,
72+
"hls-prefer-ffmpeg": True,
73+
"sleep-interval": 0,
74+
}
75+
76+
# =============================================================================
77+
# MEDIAPIPE LANDMARK INDICES
78+
# =============================================================================
79+
80+
# Hand landmarks (21 points)
81+
HAND_IDX = list(range(21))
82+
83+
# Pose landmarks (key body points)
84+
POSE_IDX = [11, 12, 13, 14, 23, 24]
85+
86+
# Face landmarks (key facial features)
87+
FACE_IDX = [
88+
0, 4, 13, 14, 17, 33, 37, 39, 46, 52, 55, 61, 64, 81, 82, 93,
89+
133, 151, 152, 159, 172, 178, 181, 263, 269, 276, 282, 285, 291,
90+
294, 311, 323, 362, 386, 397, 468, 473
91+
]
92+
93+

‎requirements.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
yt-dlp
2+
youtube-transcript-api
3+
tqdm
4+
numpy
5+
pandas
6+
opencv-python
7+
mediapipe

0 commit comments

Comments
 (0)