Skip to content

Commit 6867cfa

Browse files
klemelaklemela
committed
Get jobs in chunks
1 parent fe84ab8 commit 6867cfa

File tree

5 files changed

+200
-4
lines changed

5 files changed

+200
-4
lines changed

‎src/main/java/fi/csc/chipster/sessiondb/SessionDbClient.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,30 @@ public HashMap<UUID, Job> getJobs(UUID sessionId) throws RestException {
328328
return jobMap;
329329
}
330330

331+
/**
332+
* Use list of jobIds to get the jobs
333+
*
334+
* ID list and returned job list have same length. Jobs are saved in the
335+
* returned list to the same index, where the corresponding ID was in the ID
336+
* list. If the job was not found, the returned list will include a null in its
337+
* place.
338+
*
339+
*/
340+
public List<Job> getJobs(UUID sessionId, List<UUID> jobIds) throws RestException {
341+
342+
String jobsListString = RestMethods.postWithObjectResponse(getJobsTarget(sessionId).path("arrayGet"),
343+
jobIds, String.class);
344+
345+
@SuppressWarnings("unchecked")
346+
List<Job> jobs = RestUtils.parseJson(List.class, Job.class, jobsListString);
347+
348+
return jobs;
349+
}
350+
351+
public List<UUID> getJobIds(UUID sessionId) throws RestException {
352+
return RestMethods.getList(getJobsTarget(sessionId).path("ids"), UUID.class);
353+
}
354+
331355
public Job getJob(UUID sessionId, UUID jobId) throws RestException {
332356
return RestMethods.get(getJobTarget(sessionId, jobId), Job.class);
333357
}

‎src/main/java/fi/csc/chipster/sessiondb/resource/SessionDbAdminResource.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,13 @@ public Response getSessions(@NotNull @QueryParam("userId") List<String> userId,
305305
for (Session session : sessions) {
306306

307307
List<Dataset> datasets = SessionDbApi.getDatasets(hibernate.session(), session);
308+
309+
/*
310+
* Full jobs are needed for parameter count, but getting 1k jobs may take more
311+
* than 30 seconds.
312+
* Get the parameter count directly from db or remove it if this becomes a
313+
* problem.
314+
*/
308315
List<Job> jobs = SessionDbApi.getJobs(hibernate.session(), session);
309316

310317
long sessionSize = datasets.stream().map(dataset -> dataset.getFile()).filter(file -> file != null)

‎src/main/java/fi/csc/chipster/sessiondb/resource/SessionDbApi.java

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -363,16 +363,48 @@ public static Job getJob(UUID sessionId, UUID jobId, org.hibernate.Session hiber
363363
return hibernateSession.get(Job.class, new JobIdPair(sessionId, jobId));
364364
}
365365

366+
public static List<UUID> getJobIds(org.hibernate.Session hibernateSession, Session session) {
367+
368+
CriteriaBuilder cb = hibernateSession.getCriteriaBuilder();
369+
CriteriaQuery<Object[]> c = cb.createQuery(Object[].class);
370+
Root<Job> r = c.from(Job.class);
371+
372+
c.select(
373+
r.get("jobIdPair").get("jobId"));
374+
c.where(cb.equal(
375+
r.get("jobIdPair").get("sessionId"),
376+
session.getSessionId()));
377+
378+
List<Object[]> rows = hibernateSession.createQuery(c).getResultList();
379+
380+
List<UUID> jobIds = rows.stream()
381+
.map(row -> (UUID) row[0])
382+
.collect(Collectors.toList());
383+
384+
return jobIds;
385+
}
386+
387+
public static List<Job> getJobs(org.hibernate.Session hibernateSession, Session session, List<UUID> jobIds) {
388+
389+
List<JobIdPair> jobIdPairs = jobIds.stream()
390+
.map(jobId -> new JobIdPair(session.getSessionId(), jobId))
391+
.collect(Collectors.toList());
392+
393+
return hibernateSession
394+
.byMultipleIds(Job.class)
395+
.multiLoad(jobIdPairs);
396+
}
397+
366398
public static List<Job> getJobs(org.hibernate.Session hibernateSession, Session session) {
367399

368400
CriteriaBuilder cb = hibernateSession.getCriteriaBuilder();
369401
CriteriaQuery<Job> c = cb.createQuery(Job.class);
370402
Root<Job> r = c.from(Job.class);
371403
c.select(r);
372404
c.where(cb.equal(r.get("jobIdPair").get("sessionId"), session.getSessionId()));
373-
List<Job> datasets = hibernateSession.createQuery(c).getResultList();
405+
List<Job> jobs = hibernateSession.createQuery(c).getResultList();
374406

375-
return datasets;
407+
return jobs;
376408
}
377409

378410
public void createJob(Job job, UUID sessionId, org.hibernate.Session hibernateSession) {

‎src/main/java/fi/csc/chipster/sessiondb/resource/SessionJobResource.java

Lines changed: 67 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,21 @@ public SessionJobResource(SessionResource sessionResource, UUID id, SessionDbApi
6363
}
6464

6565
// CRUD
66+
67+
/*
68+
* Get individual job
69+
*
70+
* Used heavily after components get a WebSocket notification about Job
71+
* modification.
72+
*
73+
*/
6674
@GET
6775
@Path("{id}")
6876
@RolesAllowed({ Role.CLIENT, Role.SERVER, Role.SESSION_TOKEN }) // don't allow Role.UNAUTHENTICATED
6977
@Produces(MediaType.APPLICATION_JSON)
7078
@Transaction
7179
public Response get(@PathParam("id") UUID jobId, @Context SecurityContext sc) {
7280

73-
// logger.info(sc.getUserPrincipal().getName());
74-
7581
// checks authorization
7682
Session session = sessionResource.getRuleTable().checkSessionReadAuthorization(sc, sessionId, true);
7783

@@ -84,6 +90,63 @@ public Response get(@PathParam("id") UUID jobId, @Context SecurityContext sc) {
8490
return Response.ok(result).build();
8591
}
8692

93+
/*
94+
* Get job IDs in one session
95+
*
96+
* App uses this to request jobs in chunks to avoid exceeding Ingress' 30 second
97+
* timeout.
98+
*
99+
*/
100+
@GET
101+
@Path("ids")
102+
@RolesAllowed({ Role.CLIENT, Role.SERVER, Role.SESSION_TOKEN })
103+
@Produces(MediaType.APPLICATION_JSON)
104+
@Transaction
105+
public Response getIds(@Context SecurityContext sc) {
106+
107+
// checks authorization
108+
Session session = sessionResource.getRuleTable().checkSessionReadAuthorization(sc, sessionId);
109+
110+
List<UUID> result = SessionDbApi.getJobIds(getHibernate().session(), session);
111+
112+
// if nothing is found, just return 200 (OK) and an empty list
113+
return Response.ok(result).build();
114+
}
115+
116+
/*
117+
* Get list of jobs
118+
*
119+
* Used by app to get jobs in chunks to avoid exceeding the Ingress' 30 second
120+
* timeout.
121+
*
122+
* This should be GET, but browsers don't allow having a body in a GET request
123+
*
124+
*/
125+
@POST
126+
@Path("arrayGet")
127+
@RolesAllowed({ Role.CLIENT, Role.SERVER, Role.SESSION_TOKEN }) // don't allow Role.UNAUTHENTICATED
128+
@Consumes(MediaType.APPLICATION_JSON)
129+
@Produces(MediaType.APPLICATION_JSON)
130+
@Transaction
131+
public Response getArray(List<UUID> jobIds, @Context SecurityContext sc) {
132+
133+
if (jobIds == null) {
134+
throw new BadRequestException("no job IDs given");
135+
}
136+
// checks authorization
137+
Session session = sessionResource.getRuleTable().checkSessionReadAuthorization(sc, sessionId, true);
138+
139+
List<Job> jobs = SessionDbApi.getJobs(getHibernate().session(), session, jobIds);
140+
141+
return Response.ok(jobs).build();
142+
}
143+
144+
/*
145+
* Get all jobs of the session in one request
146+
*
147+
* Useful for server components that do not communicate through Ingress and
148+
* don't have to get response in 30 seconds.
149+
*/
87150
@GET
88151
@RolesAllowed({ Role.CLIENT, Role.SERVER, Role.SESSION_TOKEN })
89152
@Produces(MediaType.APPLICATION_JSON)
@@ -305,6 +368,8 @@ public Response delete(@PathParam("id") UUID jobId, @Context SecurityContext sc)
305368

306369
// checks authorization
307370
Session session = sessionResource.getRuleTable().checkSessionReadWriteAuthorization(sc, sessionId);
371+
372+
// we need the whole job object to send the Job state to the WebSocket
308373
Job dbJob = SessionDbApi.getJob(sessionId, jobId, getHibernate().session());
309374

310375
if (dbJob == null || !dbJob.getSessionId().equals(session.getSessionId())) {

‎src/test/java/fi/csc/chipster/sessiondb/SessionJobResourceTest.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import java.io.IOException;
66
import java.util.ArrayList;
7+
import java.util.List;
78
import java.util.UUID;
89

910
import org.apache.logging.log4j.LogManager;
@@ -132,6 +133,55 @@ public static void testGetJob(int expected, UUID sessionId, UUID jobId, SessionD
132133
}
133134
}
134135

136+
@Test
137+
public void getIds() throws RestException {
138+
139+
UUID id1 = user1Client.createJob(sessionId1, RestUtils.getRandomJob());
140+
UUID id2 = user1Client.createJob(sessionId1, RestUtils.getRandomJob());
141+
142+
assertEquals(true, user1Client.getJobIds(sessionId1).contains(id1));
143+
assertEquals(true, user1Client.getJobIds(sessionId1).contains(id2));
144+
145+
// wrong user
146+
147+
testGetJobIds(403, sessionId1, user2Client);
148+
149+
// wrong session
150+
assertEquals(false, user2Client.getJobIds(sessionId2).contains(id1));
151+
}
152+
153+
@Test
154+
public void getJobsWithIds() throws RestException {
155+
156+
UUID id1 = user1Client.createJob(sessionId1, RestUtils.getRandomJob());
157+
UUID id2 = user1Client.createJob(sessionId1, RestUtils.getRandomJob());
158+
ArrayList<UUID> idList = new ArrayList<UUID>() {
159+
{
160+
add(id1);
161+
add(UUID.randomUUID());
162+
add(id2);
163+
}
164+
};
165+
166+
List<Job> jobs = user1Client.getJobs(sessionId1, idList);
167+
168+
assertEquals(3, jobs.size());
169+
170+
assertEquals(id1, jobs.removeFirst().getJobId());
171+
// job doesn't exist, should be null
172+
assertEquals(null, jobs.removeFirst());
173+
assertEquals(id2, jobs.removeFirst().getJobId());
174+
175+
// existing jobId, but wrong session
176+
assertEquals(null, user2Client.getJobs(sessionId2, idList).get(0));
177+
178+
// empty list
179+
assertEquals(true, user1Client.getJobs(sessionId1, new ArrayList<>()).isEmpty());
180+
181+
// wrong user
182+
testGetJobsWithIds(403, sessionId1, idList, user2Client);
183+
}
184+
135185
@Test
136186
public void getAll() throws RestException {
137187

@@ -158,6 +208,24 @@ public static void testGetJobs(int expected, UUID sessionId, SessionDbClient cli
158208
}
159209
}
160210

211+
public static void testGetJobIds(int expected, UUID sessionId, SessionDbClient client) {
212+
try {
213+
client.getJobIds(sessionId);
214+
assertEquals(true, false);
215+
} catch (RestException e) {
216+
assertEquals(expected, e.getResponse().getStatus());
217+
}
218+
}
219+
220+
public static void testGetJobsWithIds(int expected, UUID sessionId, List<UUID> jobIds, SessionDbClient client) {
221+
try {
222+
client.getJobs(sessionId, jobIds);
223+
assertEquals(true, false);
224+
} catch (RestException e) {
225+
assertEquals(expected, e.getResponse().getStatus());
226+
}
227+
}
228+
161229
@Test
162230
public void put() throws RestException {
163231

0 commit comments

Comments
 (0)