Development: Fix slow database query for Iris pipeline (#10930)

Author: tobias-lippert

src/main/java/de/tum/cit/aet/artemis/atlas/api/CompetencyApi.java

@@ -1,9 +1,13 @@
 package de.tum.cit.aet.artemis.atlas.api;
 
+import java.util.Set;
+
 import org.springframework.context.annotation.Conditional;
 import org.springframework.stereotype.Controller;
 
 import de.tum.cit.aet.artemis.atlas.config.AtlasEnabled;
+import de.tum.cit.aet.artemis.atlas.domain.competency.Competency;
+import de.tum.cit.aet.artemis.atlas.repository.CompetencyRepository;
 import de.tum.cit.aet.artemis.atlas.service.competency.CompetencyService;
 import de.tum.cit.aet.artemis.lecture.domain.Lecture;
 
@@ -13,11 +17,18 @@ public class CompetencyApi extends AbstractAtlasApi {
 
     private final CompetencyService competencyService;
 
-    public CompetencyApi(CompetencyService competencyService) {
+    private final CompetencyRepository competencyRepository;
+
+    public CompetencyApi(CompetencyService competencyService, CompetencyRepository competencyRepository) {
         this.competencyService = competencyService;
+        this.competencyRepository = competencyRepository;
     }
 
     public void addCompetencyLinksToExerciseUnits(Lecture lecture) {
         competencyService.addCompetencyLinksToExerciseUnits(lecture);
     }
+
+    public Set<Competency> findAllByCourseId(long courseId) {
+        return competencyRepository.findAllByCourseId(courseId);
+    }
 }

src/main/java/de/tum/cit/aet/artemis/atlas/api/PrerequisitesApi.java

@@ -1,5 +1,7 @@
 package de.tum.cit.aet.artemis.atlas.api;
 
+import java.util.Set;
+
 import org.springframework.context.annotation.Conditional;
 import org.springframework.stereotype.Controller;
 
@@ -25,4 +27,8 @@ public long countByCourse(Course course) {
     public void deleteAll(Iterable<Prerequisite> prerequisites) {
         prerequisiteRepository.deleteAll(prerequisites);
     }
+
+    public Set<Prerequisite> findAllByCourseId(long courseId) {
+        return prerequisiteRepository.findAllByCourseId(courseId);
+    }
 }

src/main/java/de/tum/cit/aet/artemis/atlas/repository/CompetencyRepository.java

@@ -55,6 +55,13 @@ public interface CompetencyRepository extends ArtemisJpaRepository<Competency, L
             """)
     Optional<Competency> findByIdWithLectureUnitsAndExercises(@Param("competencyId") long competencyId);
 
+    @Query("""
+            SELECT c
+            FROM Competency c
+            WHERE c.course.id = :courseId
+            """)
+    Set<Competency> findAllByCourseId(@Param("courseId") long courseId);
+
     default Competency findByIdWithLectureUnitsAndExercisesElseThrow(long competencyId) {
         return getValueElseThrow(findByIdWithLectureUnitsAndExercises(competencyId), competencyId);
     }

src/main/java/de/tum/cit/aet/artemis/atlas/repository/PrerequisiteRepository.java

@@ -54,6 +54,13 @@ public interface PrerequisiteRepository extends ArtemisJpaRepository<Prerequisit
             """)
     Optional<Prerequisite> findByIdWithLectureUnits(@Param("competencyId") long competencyId);
 
+    @Query("""
+            SELECT p
+            FROM Prerequisite p
+            WHERE p.course.id = :courseId
+            """)
+    Set<Prerequisite> findAllByCourseId(@Param("courseId") long courseId);
+
     default Prerequisite findByIdWithLectureUnitsAndExercisesElseThrow(long competencyId) {
         return getValueElseThrow(findByIdWithLectureUnitsAndExercises(competencyId), competencyId);
     }

src/main/java/de/tum/cit/aet/artemis/core/repository/CourseRepository.java

@@ -143,9 +143,6 @@ SELECT COUNT(c) > 0
     @EntityGraph(type = LOAD, attributePaths = { "exercises", "lectures", "lectures.lectureUnits", "lectures.attachments", "competencies", "prerequisites" })
     Optional<Course> findWithEagerExercisesAndLecturesAndLectureUnitsAndCompetenciesById(long courseId);
 
-    @EntityGraph(type = LOAD, attributePaths = { "exercises", "lectures", "lectures.lectureUnits", "lectures.attachments", "competencies", "prerequisites", "exams" })
-    Optional<Course> findWithEagerExercisesAndLecturesAndAttachmentsAndLectureUnitsAndCompetenciesAndExamsById(long courseId);
-
     @Query("""
             SELECT course
             FROM Course course

src/main/java/de/tum/cit/aet/artemis/exam/api/ExamApi.java

@@ -1,11 +1,16 @@
 package de.tum.cit.aet.artemis.exam.api;
 
+import java.time.ZonedDateTime;
+import java.util.Set;
+
 import org.springframework.context.annotation.Conditional;
 import org.springframework.stereotype.Controller;
 
 import de.tum.cit.aet.artemis.exam.config.ExamEnabled;
+import de.tum.cit.aet.artemis.exam.domain.Exam;
 import de.tum.cit.aet.artemis.exam.domain.StudentExam;
 import de.tum.cit.aet.artemis.exam.dto.StudentExamWithGradeDTO;
+import de.tum.cit.aet.artemis.exam.repository.ExamRepository;
 import de.tum.cit.aet.artemis.exam.service.ExamService;
 import de.tum.cit.aet.artemis.exercise.domain.participation.StudentParticipation;
 
@@ -15,8 +20,11 @@ public class ExamApi extends AbstractExamApi {
 
     private final ExamService examService;
 
-    public ExamApi(ExamService examService) {
+    private final ExamRepository examRepository;
+
+    public ExamApi(ExamService examService, ExamRepository examRepository) {
         this.examService = examService;
+        this.examRepository = examRepository;
     }
 
     public StudentExamWithGradeDTO getStudentExamGradeForDataExport(StudentExam studentExam) {
@@ -26,4 +34,8 @@ public StudentExamWithGradeDTO getStudentExamGradeForDataExport(StudentExam stud
     public boolean shouldStudentSeeResult(StudentExam studentExam, StudentParticipation participation) {
         return ExamService.shouldStudentSeeResult(studentExam, participation);
     }
+
+    public Set<Exam> findAllVisibleByCourseId(long courseId, ZonedDateTime now) {
+        return examRepository.findAllVisibleByCourseId(courseId, now);
+    }
 }

src/main/java/de/tum/cit/aet/artemis/exam/repository/ExamRepository.java

@@ -397,6 +397,14 @@ SELECT COUNT(studentExam)
     @Cacheable(cacheNames = "examTitle", key = "#examId", unless = "#result == null")
     String getExamTitle(@Param("examId") long examId);
 
+    @Query("""
+            SELECT e
+            FROM Exam e
+            WHERE e.course.id = :courseId
+                AND e.visibleDate <= :now
+            """)
+    Set<Exam> findAllVisibleByCourseId(@Param("courseId") long courseId, @Param("now") ZonedDateTime now);
+
     /**
      * Get one exam by id with exercise groups.
      *

src/main/java/de/tum/cit/aet/artemis/exercise/repository/ExerciseRepository.java

@@ -74,6 +74,14 @@ public interface ExerciseRepository extends ArtemisJpaRepository<Exercise, Long>
             """)
     Set<Exercise> findAllExercisesByCourseId(@Param("courseId") long courseId);
 
+    @Query("""
+            SELECT e
+            FROM Exercise e
+            WHERE e.course.id = :courseId
+                AND (e.releaseDate <=:now  OR e.releaseDate IS NULL)
+             """)
+    Set<Exercise> findAllReleasedExercisesByCourseId(@Param("courseId") long courseId, @Param("now") ZonedDateTime now);
+
     @Query("""
             SELECT e
             FROM Exercise e

src/main/java/de/tum/cit/aet/artemis/iris/service/pyris/PyrisPipelineService.java

@@ -4,6 +4,7 @@
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.time.ZonedDateTime;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -20,15 +21,22 @@
 import org.springframework.stereotype.Service;
 
 import de.tum.cit.aet.artemis.atlas.api.LearningMetricsApi;
+import de.tum.cit.aet.artemis.atlas.api.PrerequisitesApi;
 import de.tum.cit.aet.artemis.atlas.config.AtlasNotPresentException;
+import de.tum.cit.aet.artemis.atlas.domain.competency.Competency;
 import de.tum.cit.aet.artemis.atlas.domain.competency.CompetencyJol;
+import de.tum.cit.aet.artemis.atlas.domain.competency.Prerequisite;
 import de.tum.cit.aet.artemis.atlas.dto.CompetencyJolDTO;
+import de.tum.cit.aet.artemis.atlas.repository.CompetencyRepository;
 import de.tum.cit.aet.artemis.communication.domain.Post;
 import de.tum.cit.aet.artemis.core.domain.Course;
 import de.tum.cit.aet.artemis.core.repository.CourseRepository;
 import de.tum.cit.aet.artemis.core.repository.UserRepository;
+import de.tum.cit.aet.artemis.exam.domain.Exam;
+import de.tum.cit.aet.artemis.exam.repository.ExamRepository;
 import de.tum.cit.aet.artemis.exercise.domain.Exercise;
 import de.tum.cit.aet.artemis.exercise.domain.participation.StudentParticipation;
+import de.tum.cit.aet.artemis.exercise.repository.ExerciseRepository;
 import de.tum.cit.aet.artemis.exercise.repository.StudentParticipationRepository;
 import de.tum.cit.aet.artemis.iris.domain.session.IrisCourseChatSession;
 import de.tum.cit.aet.artemis.iris.domain.session.IrisProgrammingExerciseChatSession;
@@ -47,6 +55,8 @@
 import de.tum.cit.aet.artemis.iris.service.pyris.dto.data.PyrisUserDTO;
 import de.tum.cit.aet.artemis.iris.service.pyris.dto.status.PyrisStageDTO;
 import de.tum.cit.aet.artemis.iris.service.websocket.IrisChatWebsocketService;
+import de.tum.cit.aet.artemis.lecture.domain.Lecture;
+import de.tum.cit.aet.artemis.lecture.repository.LectureRepository;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingExercise;
 import de.tum.cit.aet.artemis.programming.domain.ProgrammingSubmission;
 
@@ -76,12 +86,25 @@ public class PyrisPipelineService {
 
     private final UserRepository userRepository;
 
+    // we cannot use the apis here in most cases as this leads to circular dependencies, so we use the repositories instead and ignore this class in the architecture tests
+    private final Optional<LectureRepository> lectureRepository;
+
+    private final Optional<CompetencyRepository> competencyRepository;
+
+    private final Optional<PrerequisitesApi> prerequisitesApi;
+
+    private final Optional<ExamRepository> examRepository;
+
+    private final ExerciseRepository exerciseRepository;
+
     @Value("${server.url}")
     private String artemisBaseUrl;
 
     public PyrisPipelineService(PyrisConnectorService pyrisConnectorService, PyrisJobService pyrisJobService, PyrisDTOService pyrisDTOService,
             IrisChatWebsocketService irisChatWebsocketService, CourseRepository courseRepository, Optional<LearningMetricsApi> learningMetricsApi,
-            StudentParticipationRepository studentParticipationRepository, UserRepository userRepository) {
+            StudentParticipationRepository studentParticipationRepository, UserRepository userRepository, Optional<LectureRepository> lectureRepository,
+            Optional<CompetencyRepository> competencyRepository, Optional<PrerequisitesApi> prerequisitesApi, Optional<ExamRepository> examRepository,
+            ExerciseRepository exerciseRepository) {
         this.pyrisConnectorService = pyrisConnectorService;
         this.pyrisJobService = pyrisJobService;
         this.pyrisDTOService = pyrisDTOService;
@@ -90,6 +113,11 @@ public PyrisPipelineService(PyrisConnectorService pyrisConnectorService, PyrisJo
         this.learningMetricsApi = learningMetricsApi;
         this.studentParticipationRepository = studentParticipationRepository;
         this.userRepository = userRepository;
+        this.lectureRepository = lectureRepository;
+        this.competencyRepository = competencyRepository;
+        this.prerequisitesApi = prerequisitesApi;
+        this.examRepository = examRepository;
+        this.exerciseRepository = exerciseRepository;
     }
 
     /**
@@ -301,7 +329,31 @@ public void executeCourseChatPipeline(String variant, String customInstructions,
      * @param studentId the id of the student
      */
     private Course loadCourseWithParticipationOfStudent(long courseId, long studentId) {
-        Course course = courseRepository.findWithEagerExercisesAndLecturesAndAttachmentsAndLectureUnitsAndCompetenciesAndExamsById(courseId).orElseThrow();
+        ZonedDateTime now = ZonedDateTime.now();
+        Course course = courseRepository.findById(courseId).orElseThrow();
+        Set<Exercise> releasedExercises = exerciseRepository.findAllReleasedExercisesByCourseId(courseId, now);
+        Set<Lecture> visibleLectures = new HashSet<>();
+        if (lectureRepository.isPresent()) {
+            visibleLectures = lectureRepository.orElseThrow().findAllVisibleByCourseIdWithEagerLectureUnits(courseId, now);
+        }
+        Set<Competency> competencies = new HashSet<>();
+        if (competencyRepository.isPresent()) {
+            competencies = competencyRepository.orElseThrow().findAllByCourseId(courseId);
+        }
+        Set<Prerequisite> prerequisites = new HashSet<>();
+        if (prerequisitesApi.isPresent()) {
+            prerequisites = prerequisitesApi.orElseThrow().findAllByCourseId(courseId);
+        }
+        Set<Exam> visibleExams = new HashSet<>();
+        if (examRepository.isPresent()) {
+            visibleExams = examRepository.orElseThrow().findAllVisibleByCourseId(courseId, now);
+        }
+        course.setExercises(releasedExercises);
+        course.setLectures(visibleLectures);
+        course.setCompetencies(competencies);
+        course.setPrerequisites(prerequisites);
+        course.setExams(visibleExams);
+
         List<StudentParticipation> participations = studentParticipationRepository.findByStudentIdAndIndividualExercisesWithEagerSubmissionsResultIgnoreTestRuns(studentId,
                 course.getExercises());
 

src/main/java/de/tum/cit/aet/artemis/lecture/api/LectureApi.java

@@ -10,6 +10,7 @@
 import de.tum.cit.aet.artemis.core.domain.Course;
 import de.tum.cit.aet.artemis.core.domain.User;
 import de.tum.cit.aet.artemis.lecture.domain.Lecture;
+import de.tum.cit.aet.artemis.lecture.repository.LectureRepository;
 import de.tum.cit.aet.artemis.lecture.service.LectureImportService;
 import de.tum.cit.aet.artemis.lecture.service.LectureService;
 
@@ -24,9 +25,12 @@ public class LectureApi extends AbstractLectureApi {
 
     private final LectureImportService lectureImportService;
 
-    public LectureApi(LectureService lectureService, LectureImportService lectureImportService) {
+    private final LectureRepository lectureRepository;
+
+    public LectureApi(LectureService lectureService, LectureImportService lectureImportService, LectureRepository lectureRepository) {
         this.lectureService = lectureService;
         this.lectureImportService = lectureImportService;
+        this.lectureRepository = lectureRepository;
     }
 
     public Set<Lecture> filterVisibleLecturesWithActiveAttachments(Course course, Set<Lecture> lecturesWithAttachments, User user) {

src/main/java/de/tum/cit/aet/artemis/lecture/repository/LectureRepository.java

@@ -35,6 +35,15 @@ public interface LectureRepository extends ArtemisJpaRepository<Lecture, Long> {
             """)
     Set<Lecture> findAllByCourseId(@Param("courseId") Long courseId);
 
+    @Query("""
+            SELECT lecture
+            FROM Lecture lecture
+            LEFT JOIN FETCH lecture.lectureUnits
+            WHERE lecture.course.id = :courseId
+                AND (lecture.visibleDate IS NULL OR lecture.visibleDate <= :now)
+            """)
+    Set<Lecture> findAllVisibleByCourseIdWithEagerLectureUnits(@Param("courseId") long courseId, @Param("now") ZonedDateTime now);
+
     @Query("""
             SELECT lecture
             FROM Lecture lecture

src/test/java/de/tum/cit/aet/artemis/atlas/architecture/AtlasApiArchitectureTest.java

@@ -3,6 +3,7 @@
 import java.util.Set;
 
 import de.tum.cit.aet.artemis.atlas.config.AtlasNotPresentException;
+import de.tum.cit.aet.artemis.atlas.repository.CompetencyRepository;
 import de.tum.cit.aet.artemis.shared.architecture.module.AbstractModuleAccessArchitectureTest;
 
 class AtlasApiArchitectureTest extends AbstractModuleAccessArchitectureTest {
@@ -14,6 +15,6 @@ public String getModulePackage() {
 
     @Override
     protected Set<Class<?>> getIgnoredClasses() {
-        return Set.of(AtlasNotPresentException.class);
+        return Set.of(AtlasNotPresentException.class, CompetencyRepository.class);
     }
 }

src/test/java/de/tum/cit/aet/artemis/exam/architecture/ExamApiArchitectureTest.java

@@ -3,6 +3,7 @@
 import java.util.Set;
 
 import de.tum.cit.aet.artemis.exam.config.ExamApiNotPresentException;
+import de.tum.cit.aet.artemis.exam.repository.ExamRepository;
 import de.tum.cit.aet.artemis.shared.architecture.module.AbstractModuleAccessArchitectureTest;
 
 class ExamApiArchitectureTest extends AbstractModuleAccessArchitectureTest {
@@ -14,6 +15,6 @@ public String getModulePackage() {
 
     @Override
     protected Set<Class<?>> getIgnoredClasses() {
-        return Set.of(ExamApiNotPresentException.class);
+        return Set.of(ExamApiNotPresentException.class, ExamRepository.class);
     }
 }

src/test/java/de/tum/cit/aet/artemis/lecture/architecture/LectureApiArchitectureTest.java

@@ -3,6 +3,7 @@
 import java.util.Set;
 
 import de.tum.cit.aet.artemis.lecture.config.LectureApiNotPresentException;
+import de.tum.cit.aet.artemis.lecture.repository.LectureRepository;
 import de.tum.cit.aet.artemis.lecture.service.SlideUnhideScheduleService;
 import de.tum.cit.aet.artemis.shared.architecture.module.AbstractModuleAccessArchitectureTest;
 
@@ -15,6 +16,6 @@ public String getModulePackage() {
 
     @Override
     protected Set<Class<?>> getIgnoredClasses() {
-        return Set.of(LectureApiNotPresentException.class, SlideUnhideScheduleService.class);
+        return Set.of(LectureApiNotPresentException.class, SlideUnhideScheduleService.class, LectureRepository.class);
     }
 }