Skip to content

UI Improvements, Upload Constraints & Space Reorganization #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LucasZimm wants to merge 10 commits into
base: main
Choose a base branch
from
Open

UI Improvements, Upload Constraints & Space Reorganization #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
05801db
issue 10
LucasZimm Nov 24, 2025
0f04098
removed required from collection name
LucasZimm Nov 24, 2025
febc6e6
minor changes + issues 12,13
LucasZimm Nov 24, 2025
0246fd5
added timeouts and more
LucasZimm Nov 24, 2025
9ca04c6
typeing
LucasZimm Nov 24, 2025
6caa2a6
Merge branch 'main' of https://github.com/BAMresearch/openbis-upload-…
LucasZimm Nov 25, 2025
1fb63f3
fix for e.g. "1e10" as byte limit
LucasZimm Nov 25, 2025
26b3123
small fixes
LucasZimm Dec 1, 2025
ca8859f
removed file loading timeout and set available parser to corner
LucasZimm Dec 5, 2025
51b9720
style changes
LucasZimm Dec 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions openbis_upload_helper/app/static/css/style.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,4 +394,19 @@ ul#file-list ul li:last-child {

.tooltip-list li:last-child {
border-bottom: none;
}

.card-has-corner {
position: relative;
}

.card-corner {
position: absolute;
right: 12px;
z-index: 3;
background: rgba(255,255,255,0.98);
border-radius: 6px;
padding: 6px 10px;
margin: 8px;
text-align: right;
}
75 changes: 51 additions & 24 deletions openbis_upload_helper/app/templates/homepage.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,34 +26,61 @@
<div class="tab-content mt-3" id="myTabContent">
<div class="tab-pane fade show active" id="checker-content" role="tabpanel" aria-labelledby="checker-tab">
<h5 class="text-center">Export Options</h5>
<form id="upload-form" method="POST" enctype="multipart/form-data">
{% csrf_token %}
<div class="mb-3">
<!--GET-Form Space-->
<form method="get" action="{% url 'homepage' %}" class="mb-3 d-flex align-items-end gap-2">
<input type="hidden" name="space_select" value="1">
<div style="flex:1">
<label for="selectedSpace" class="form-label">Select Space</label>
{% if spaces %}
<select class="form-control" id="selectedSpace" name="selected_space" required>
<option value="" selected disabled>Select Space</option>
<!-- onchange to set space -->
<select class="form-control" id="selectedSpace" name="space" required onchange="this.form.submit()">
<option value="" disabled {% if not selected_space %}selected{% endif %}>Select Space</option>
{% for space in spaces %}
<option value="{{ space }}">{{ space }}</option>
<option value="{{ space }}" {% if selected_space == space %}selected{% endif %}>{{ space }}</option>
{% endfor %}
</select>
{% else %}
<select class="form-control" id="selectedSpace" name="selected_space" disabled>
<select class="form-control" id="selectedSpace" name="space" disabled>
<option value="" selected disabled>No spaces available</option>
</select>
<div class="text-danger mt-2">No spaces are available for selection. Please contact your administrator.</div>
{% endif %}
</div>
<!-- Fallback without JS: Load-Button -->
<noscript>
<div>
<button type="submit" class="btn btn-outline-primary">Load</button>
</div>
</noscript>
</form>
<form id="upload-form" method="POST" enctype="multipart/form-data">
{% csrf_token %}
{% if selected_space %}
<input type="hidden" name="selected_space" value="{{ selected_space }}">
{% endif %}
<div class="mb-3">
<label for="projectName" class="form-label">Project Name</label>
<input type="text" class="form-control" id="projectName" value="{{ project_name }}" name="project_name" placeholder="Project Name" required>
<!-- Inputfied with suggestions 'projects' -->
<input type="text" class="form-control" id="projectName" value="{{ project_name }}" name="project_name" placeholder="Project Name" list="projects-list" required>
<datalist id="projects-list">
{% if projects %}
{% for p in projects %}
<option value="{{ p }}"></option>
{% endfor %}
{% endif %}
</datalist>
</div>

<div class="mb-3">
<label for="collectionName" class="form-label">Collection Name</label>
<input type="text" class="form-control" id="collectionName" value="{{ collection_name }}" name="collection_name" placeholder="Collection Name" required>
<input type="text" class="form-control" id="collectionName" value="{{ collection_name }}" name="collection_name" placeholder="Collection Name" list="collections-list">
<datalist id="collections-list">
{% if collections %}
{% for c in collections %}
<option value="{{ c }}"></option>
{% endfor %}
{% endif %}
</datalist>
</div>

<div id="drop-area">
<p>Drag & Drop<br>
or<br>
Expand All @@ -65,14 +92,26 @@ <h5 class="text-center">Export Options</h5>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script>
<script src="{% static 'js/homepage.js' %}"></script>
<button type="submit" name="upload" class="btn btn-primary">Upload Files</button>
<div class="parser-tooltip-group" style="">
<div class="parser-tooltip-group">
<span class="tooltip-trigger">
<a href="{% url 'homepage' %}?reset=1" class="btn btn-secondary text-center" style="text-align: right;"><i class="bi bi-arrow-clockwise"></i></a>
</span>
<div class="tooltip-list">
<p>Resets Card 2 and 3</p>
</div>
</div>
<div class="parser-tooltip-group card-corner" >
<span class="tooltip-trigger">
<i class="bi bi-info-circle"></i> Available Parsers
</span>
<div class="tooltip-list">
<ul>
{% for parser in parser_choices %}
<li>{{ parser }}</li>
{% endfor %}
</ul>
</div>
</div>
</form>
</div>
</div>
Expand Down Expand Up @@ -109,18 +148,6 @@ <h5 class="text-center">Select Parser</h5>
<button type="submit" name="assign_parsers" class="btn btn-primary">Parse</button>
{% endif %}
</div>
<div class="parser-tooltip-group" style="text-align: right;">
<span class="tooltip-trigger">
<i class="bi bi-info-circle"></i> Available Parsers
</span>
<div class="tooltip-list">
<ul>
{% for parser in parser_choices %}
<li>{{ parser }}</li>
{% endfor %}
</ul>
</div>
</div>
</div>
</form>
</div>
Expand Down
9 changes: 7 additions & 2 deletions openbis_upload_helper/app/templates/login.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,16 @@ <h5 class="text-center">Login to OpenBIS</h5>
{% csrf_token %}
<div class="mb-3">
<label for="username" class="form-label">Username</label>
<input type="text" class="form-control" id="username" name="username" required>
<input type="text" class="form-control" id="username" name="username">
</div>
<div class="mb-3">
<label for="password" class="form-label">Password</label>
<input type="password" class="form-control" id="password" name="password" required>
<input type="password" class="form-control" id="password" name="password">
</div>
<h6 class"text-center">or</h6>
<div class="mb-3">
<label for="personal_access_token" class="form-label">Personal Access Token</label>
<input type="text" class="form-control" id="personal_access_token" name="personal_access_token">
</div>
{% if error %}
<div class="alert alert-danger">{{ error }}</div>
Expand Down
2 changes: 2 additions & 0 deletions openbis_upload_helper/app/utils/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
FilesParser,
decrypt_password,
encrypt_password,
extract_name,
get_openbis_from_cache,
log_results,
preload_context_request,
reorganize_spaces,
)
96 changes: 89 additions & 7 deletions openbis_upload_helper/app/utils/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
import datetime
import os
import re
import shutil
import tarfile
import tempfile
import time
import uuid
import zipfile

from bam_masterdata.logger import log_storage, logger
from cryptography.fernet import Fernet, InvalidToken
from decouple import config as environ
from django.conf import settings
from django.core.cache import cache
from pybis import Openbis
Expand Down Expand Up @@ -86,10 +89,22 @@ def __init__(self, uploaded_files, selected_files):
self.selected_files = selected_files
self.saved_file_names = []
self.temp_dirs = [] # List to keep track of temporary directories
self.size_limit = environ("UPLOAD_SIZE_LIMIT", default=None)
# timeout in seconds (default 300)

def load_files(self):
if not self.uploaded_files:
raise ValueError("No files uploaded.")
# start countdown
self.start_time = time.time()

file_sizes = 0
for uploaded_file in self.uploaded_files:
file_sizes += uploaded_file.size
if self.size_limit and file_sizes > int(float(self.size_limit)):
raise ValueError(
f"Uploaded files exceed the size limit of {int(float(self.size_limit))} bytes."
)

for uploaded_file in self.uploaded_files:
if uploaded_file.name.endswith(".zip"):
Expand All @@ -99,10 +114,9 @@ def load_files(self):
else:
self._process_regular_file(uploaded_file)

if self.saved_file_names:
return self.saved_file_names
else:
raise ValueError("No files uploaded.")
if not self.saved_file_names:
raise ValueError("No files were saved. Processing may have failed.")
return self.saved_file_names

def _process_zip(self, uploaded_file):
tmp_dir = tempfile.mkdtemp()
Expand All @@ -117,8 +131,16 @@ def _process_zip(self, uploaded_file):
if not zip_info.is_dir():
target_path = os.path.join(tmp_dir, zip_info.filename)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with open(target_path, "wb") as out_file:
out_file.write(zip_ref.read(zip_info.filename))
# read zip member in chunks to allow timeout checks
with (
zip_ref.open(zip_info) as src,
open(target_path, "wb") as out_file,
):
while True:
chunk = src.read(8192)
if not chunk:
break
out_file.write(chunk)
if zip_info.filename in self.selected_files:
self.saved_file_names.append((zip_info.filename, target_path))

Expand All @@ -140,7 +162,11 @@ def _process_tar(self, uploaded_file):
target_path = os.path.join(tmp_dir, member.name)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with open(target_path, "wb") as out_file:
out_file.write(extracted_file.read())
while True:
chunk = extracted_file.read(8192)
if not chunk:
break
out_file.write(chunk)
if member.name in self.selected_files:
self.saved_file_names.append((member.name, target_path))

Expand All @@ -158,6 +184,10 @@ def _process_regular_file(self, uploaded_file):
if uploaded_file.name in self.selected_files:
self.saved_file_names.append((uploaded_file.name, target_path))

def antivirus_scan(self):
# Placeholder for antivirus scanning logic
pass


class FilesParser:
def __init__(self, uploaded_files, available_parsers, o):
Expand Down Expand Up @@ -228,3 +258,55 @@ def log_results(request, parsed_files={}, context={}):
context["logs"] = context_logs
request.session["checker_logs"] = context_logs
return context_logs


def extract_name(obj):
if isinstance(obj, dict):
return obj.get("code") or obj.get("identifier") or obj.get("name") or str(obj)
return (
getattr(obj, "code", None)
or getattr(obj, "identifier", None)
or getattr(obj, "name", None)
or str(obj)
)


def reorganize_spaces(spaces: list[str]) -> list[str]:
"""
Reorganizes a list of space names so that:
- BAM_* spaces appear first
- VP.x_* and VP.xx_* spaces follow (case-insensitive)
- all other spaces come last
Pattern examples:
VP.1_NAME, Vp.01_TEST, vp.12_ABC
"""

# 1. BAM_* spaces (case-sensitive as original)
bam_spaces = sorted([s for s in spaces if s.startswith("BAM_")])

# 2. VP.x_* spaces, case-insensitive
vp_pattern = re.compile(r"(?i)(VP\.(\d{1,2}))_", re.IGNORECASE)

vp_groups: dict[str, list[str]] = {}

for s in spaces:
match = vp_pattern.match(s)
if match:
vp_key = match.group(1).upper() # normalize e.g. VP.1 → VP.1, vp.01 → VP.01
vp_groups.setdefault(vp_key, []).append(s)

# Sort by numeric value, not lexicographically (VP.2 < VP.10)
def vp_sort_key(vp_key: str) -> int:
n = int(vp_key.split(".")[1])
return n

vp_spaces: list[str] = []
for vp_key in sorted(vp_groups.keys(), key=vp_sort_key):
vp_spaces.extend(sorted(vp_groups[vp_key]))

# 3. Others (not BAM and not VP)
others = sorted(
[s for s in spaces if not s.startswith("BAM_") and not vp_pattern.match(s)]
)

return bam_spaces + vp_spaces + others
Loading