Merge branch 'release/2.11.0'

Author: vmalloc

CHANGES.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Version 2.11.0
+
+* Added search option to user view
+* Added support for discarding sessions. Admins can now mark sessions for deletion after a certain period of time (30 days by default). This is useful for temporary sessions that are not interesting for keeping forever.
+* Sessions can have a TTL, automatically marking their discard date on keepalives and session ends
+* Support reporting of remote and local SCM branches
+* Removed the "mark archived" and "mark investigated" features. Investigation will be added in a future release as a part of a stricter workflow, and archiving can now be replaced with discarding sessions
+
 ## Version 2.10.0
 
 * Added pagination navigation to session and test error views

_lib/celery.py

@@ -13,8 +13,9 @@ def celery():
 
 @celery.command()
 @click.argument('name')
+@click.option('--defer', default=False, is_flag=True)
 @requires_env('app')
-def task(name):
+def task(name, defer):
 
     from flask_app import tasks
     from flask_app.app import create_app
@@ -24,5 +25,9 @@ def task(name):
     if task is None:
         click.echo('Could not find task named {}'.format(task))
         raise click.Abort()
-    with create_app().app_context():
-        task()
+
+    with create_app().app_context(), logbook.StderrHandler(level='DEBUG'):
+        if defer:
+            task.delay()
+        else:
+            task()

_lib/frontend_tmux.yml

@@ -7,7 +7,7 @@ windows:
       - python manage.py testserver --no-tmux --port 8800 --no-livereload
   - window_name: celery worker
     panes:
-      - .env/bin/celery -A flask_app.tasks worker --loglevel=info -B
+      - .env/bin/celery -A flask_app.tasks worker --loglevel=info -B --max-tasks-per-child=500
       - .env/bin/celery -A flask_app.tasks shell
   - window_name: redis
     panes:

_lib/slash_running.py

@@ -34,7 +34,7 @@ def _get_initial_session_metadata(self):
             return returned
 
         def _get_extra_session_start_kwargs(self):
-            returned = {}
+            returned = super()._get_extra_session_start_kwargs()
             if use_subjects:
                 returned['subjects'] = [
                     {

deps/develop.txt

@@ -2,7 +2,7 @@ pytest
 pytest-cov
 pytest-selenium
 Flask-Loopback
-backslash>=2.28.0
+backslash>=2.31.1
 tmuxp
 livereload
 ipython

docker/docker-compose.yml

@@ -26,7 +26,7 @@ services:
 
   worker:
     image: getslash/backslash
-    command: dockerize -timeout 3600s -wait tcp://rabbitmq:5672 .env/bin/celery -A flask_app.tasks worker -B --loglevel=info
+    command: dockerize -timeout 3600s -wait tcp://rabbitmq:5672 .env/bin/celery -A flask_app.tasks worker -B --loglevel=info --max-tasks-per-child=500
     logging:
       driver: syslog
     environment:

flask_app/activity.py

@@ -4,85 +4,9 @@
 
 from flask import g
 
-from sqlalchemy.sql import select, union_all, literal_column, text
-from sqlalchemy import and_
-
 from flask_security import current_user
 
 
-
-ACTION_ARCHIVED, ACTION_UNARCHIVED, ACTION_INVESTIGATED, ACTION_UNINVESTIGATED, MAX_ACTIVITY, ACTION_COMMENTED = range(6)
-
-_ACTION_STRINGS = {
-    ACTION_COMMENTED: 'commented',
-    ACTION_ARCHIVED: 'archived',
-    ACTION_UNARCHIVED: 'unarchived',
-    ACTION_INVESTIGATED: 'investigated',
-    ACTION_UNINVESTIGATED: 'uninvestigated',
-}
-
-def register_user_activity(action, **kw):
-    from . import models
-    assert action < MAX_ACTIVITY
-    models.db.session.add(
-        models.Activity(
-            action=action,
-            user_id=current_user.id,
-            **kw))
-
-def get_action_string(action):
-    return _ACTION_STRINGS.get(action)
-
-
-def get_activity_query(user_id=None, session_id=None, test_id=None):
-    # pylint: disable=no-member
-    from .models import Activity, Comment, User
-
-    _filter = functools.partial(_apply_filters, user_id=user_id, session_id=session_id, test_id=test_id)
-
-    comments = select([
-        literal_column("('comment:' || comment.id)").label('id'),
-        literal_column(str(ACTION_COMMENTED)).label('action'),
-        Comment.user_id.label('user_id'),
-        Comment.session_id.label('session_id'),
-        Comment.test_id.label('test_id'),
-        Comment.timestamp.label('timestamp'),
-        Comment.comment.label('text'),
-        (User.first_name + ' ' + User.last_name).label('user_name'),
-        User.email.label('user_email'),
-    ]).select_from(Comment.__table__.join(User, User.id == Comment.user_id))
-
-    comments = _filter(Comment, comments)
-
-    activity = select([
-        literal_column("('activity:' || activity.id)").label('id'),
-        Activity.action.label('action'),
-        Activity.user_id.label('user_id'),
-        Activity.session_id.label('session_id'),
-        Activity.test_id.label('test_id'),
-        Activity.timestamp.label('timestamp'),
-        literal_column("NULL").label('text'),
-        (User.first_name + ' ' + User.last_name).label('user_name'),
-        User.email.label('user_email'),
-    ]).select_from(Activity.__table__.join(User, User.id == Activity.user_id))
-
-    activity = _filter(Activity, activity)
-
-    u = union_all(comments, activity).alias('u')
-
-    return select([u]).order_by(u.c.timestamp)
-
-
-def _apply_filters(model, query, **filters):
-    whereclause = []
-
-    for key, value in filters.items():
-        if value is None:
-            continue
-        whereclause.append(getattr(model, key) == value)
-    return query.where(and_(*whereclause))
-
-
 def updates_last_active(func):
     from . import models
 
@@ -96,7 +20,7 @@ def new_func(*args, **kwargs):
             u = None
 
         if u is not None:
-            u.last_activity = flux.current_timeline.time()
+            u.last_activity = flux.current_timeline.time() # pylint: disable=no-member
             models.db.session.add(u)
         return func(*args, **kwargs)
 

flask_app/blueprints/api/main.py

@@ -1,6 +1,5 @@
 from contextlib import contextmanager
 import logbook
-import multiprocessing
 
 import requests
 from flask import abort, current_app
@@ -11,8 +10,6 @@
 from sqlalchemy.exc import DataError
 
 from .blueprint import API
-from ... import activity
-from ... import models
 from ... import metrics
 from ...models import db, Session, Test, Comment, User, Role, Warning, Entity, TestVariation, TestMetadata
 from ...utils import get_current_time, statuses
@@ -100,7 +97,7 @@ def append_upcoming_tests(tests: list, session_id: int):
     except DataError:
         raise
 
-@API(version=2)
+@API(version=3)
 def report_test_start(
         session_id: int,
         name: str,
@@ -111,6 +108,8 @@ def report_test_start(
         file_hash: (str, NoneType)=None,
         scm_revision: (str, NoneType)=None,
         scm_dirty: bool=False,
+        scm_local_branch: (str, NoneType)=None,
+        scm_remote_branch: (str, NoneType)=None,
         is_interactive: bool=False,
         variation: (dict, NoneType)=None,
         metadata: (dict, NoneType)=None,
@@ -135,6 +134,8 @@ def report_test_start(
     returned.status = statuses.RUNNING
     returned.scm_dirty = scm_dirty
     returned.scm_revision = scm_revision
+    returned.scm_local_branch = scm_local_branch
+    returned.scm_remote_branch = scm_remote_branch
     returned.scm = scm
     returned.is_interactive = is_interactive
     returned.file_hash = file_hash
@@ -243,27 +244,6 @@ def report_test_interrupted(id: int):
     _update_running_test_status(id, statuses.INTERRUPTED)
     db.session.commit()
 
-@API(require_real_login=True)
-@requires_role('moderator')
-def toggle_archived(session_id: int):
-    returned = _toggle_session_attribute(session_id, 'archived', activity.ACTION_ARCHIVED, activity.ACTION_UNARCHIVED)
-    db.session.commit()
-    return returned
-
-@API(require_real_login=True)
-def toggle_investigated(session_id: int):
-    _toggle_session_attribute(session_id, 'investigated', activity.ACTION_INVESTIGATED, activity.ACTION_UNINVESTIGATED)
-    db.session.commit()
-
-def _toggle_session_attribute(session_id, attr, on_action, off_action):
-    session = Session.query.get_or_404(session_id)
-    new_value = not getattr(session, attr)
-    setattr(session, attr, new_value)
-    db.session.add(session)
-    activity.register_user_activity(on_action if new_value else off_action, session_id=session_id)
-    return new_value
-
-
 
 def _update_running_test_status(test_id, status, ignore_conflict=False, additional_updates=None):
     logbook.debug('marking test {} as {}', test_id, status)

flask_app/blueprints/api/sessions.py

@@ -1,23 +1,27 @@
-import requests
-
 from flask import g, request
 from flask_simple_api import error_abort
+import flux
+import requests
 
 from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.exc import IntegrityError
 from ...auth import get_or_create_user
 
+from ...search import get_orm_query_from_search_string
 from ...models import Session, Test, db, SessionMetadata
 from ...utils import get_current_time, statuses
+from ...utils.api_utils import requires_role
 from ...utils.subjects import get_or_create_subject_instance
 from ...utils.users import has_role
 from ... import metrics
 from .blueprint import API
 
 NoneType = type(None)
 
+_DEFAULT_DELETE_GRACE_PERIOD_SECONDS = 60 * 60 * 24 * 30
 
-@API(version=2)
+
+@API(version=3)
 def report_session_start(logical_id: str=None,
                          parent_logical_id: (NoneType, str)=None,
                          is_parent_session: bool=False,
@@ -29,6 +33,7 @@ def report_session_start(logical_id: str=None,
                          keepalive_interval: (NoneType, int)=None,
                          subjects: (list, NoneType)=None,
                          infrastructure: (str, NoneType)=None,
+                         ttl_seconds: (int, NoneType)=None,
                          ):
     if hostname is None:
         hostname = request.remote_addr
@@ -44,6 +49,9 @@ def report_session_start(logical_id: str=None,
         user_id = g.token_user.id
         real_user_id = None
 
+    if keepalive_interval is None and ttl_seconds is not None:
+        error_abort("Cannot specify session TTL when keepalive isn't used")
+
     returned = Session(
         hostname=hostname,
         parent_logical_id=parent_logical_id,
@@ -56,10 +64,11 @@ def report_session_start(logical_id: str=None,
         status=statuses.RUNNING,
         logical_id=logical_id,
         keepalive_interval=keepalive_interval,
-        next_keepalive=None if keepalive_interval is None else get_current_time() +
-        keepalive_interval,
+        ttl_seconds=ttl_seconds,
     )
 
+    returned.update_keepalive()
+
     returned.mark_started()
 
     if subjects:
@@ -117,6 +126,8 @@ def report_session_end(id: int, duration: (int, NoneType)=None, has_fatal_errors
         session.status = statuses.SUCCESS
     session.has_fatal_errors = has_fatal_errors
     session.in_pdb = False
+    if session.ttl_seconds is not None:
+        session.delete_at = flux.current_timeline.time() + session.ttl_seconds
     db.session.add(session)
     db.session.commit()
 
@@ -143,8 +154,7 @@ def send_keepalive(session_id: int):
     if s.end_time is not None:
         return
     timestamp = get_current_time() + s.keepalive_interval
-    s.next_keepalive = timestamp
-    s.extend_timespan_to(timestamp)
+    s.update_keepalive()
     for test in Test.query.filter(Test.session_id==session_id,
                                   Test.end_time == None,
                                   Test.start_time != None):
@@ -161,5 +171,33 @@ def report_session_interrupted(id: int):
     s.status = statuses.INTERRUPTED
     if s.parent:
         s.parent.status = statuses.INTERRUPTED
-    db.session.add(s)
+    db.session.commit()
+
+
+@API(require_real_login=True)
+@requires_role('admin')
+def discard_session(session_id: int, grace_period_seconds: int=_DEFAULT_DELETE_GRACE_PERIOD_SECONDS):
+    session = Session.query.get_or_404(session_id)
+    session.delete_at = flux.current_timeline.time() + grace_period_seconds # pylint: disable=undefined-variable
+    db.session.commit()
+
+
+@API(require_real_login=True)
+@requires_role('admin')
+def discard_sessions_search(search_string: str, grace_period_seconds: int=_DEFAULT_DELETE_GRACE_PERIOD_SECONDS):
+    if not search_string:
+        error_abort('Invadlid search string')
+    delete_at = flux.current_timeline.time() + grace_period_seconds
+    search_query = get_orm_query_from_search_string('session', search_string).filter(Session.delete_at == None)
+    Session.query.filter(Session.id.in_(db.session.query(search_query.subquery().c.id))).update({
+        'delete_at': delete_at
+    }, synchronize_session=False)
+    db.session.commit()
+
+
+@API(require_real_login=True)
+@requires_role('admin')
+def preserve_session(session_id: int):
+    session = Session.query.get_or_404(session_id)
+    session.delete_at = None
     db.session.commit()

flask_app/blueprints/filter_configs.py

@@ -2,7 +2,8 @@
 
 from .. import models
 from ..utils import statuses
-from ..utils.filtering import ConstFilter, ToggleFilter, FilterConfig, in_, notin_
+from ..utils.filtering import ConstFilter, FilterConfig, in_, notin_
+
 
 _STATUS_FILTERS = {
         'unsuccessful': (notin_, (statuses.SUCCESS, statuses.SKIPPED, statuses.RUNNING)),
@@ -12,12 +13,7 @@
 
 
 SESSION_FILTERS = FilterConfig({
-    'investigated': ConstFilter(models.Session.investigated, {
-        'not investigated': False,
-        'investigated': True,
-    }),
     'status': ConstFilter(models.Session.status, _STATUS_FILTERS),
-    'archived': ToggleFilter(models.Session.archived, default=False),
 })
 
 TEST_FILTERS = FilterConfig({