Tasks API Endpoint (#6230)

* Add API endpoint for background task overview

* Cleanup other pending heartbeat tasks

* Adds API endpoint for queued tasks

* Adds API endpoint for scheduled tasks

* Add API endpoint for failed tasks

* Update API version info

* Add table for displaying pending tasks

* Add failed tasks table

* Use accordion

* Annotate extra data to scheduled tasks serializer

* Extend API functionality

* Update tasks.py

- Allow skipping of static file step in "invoke update"
- Allows for quicker updates in dev mode

* Display task result error for failed tasks

* Allow delete of failed tasks

* Remove old debug message

* Adds ability to delete pending tasks

* Update table columns

* Fix unused imports

* Prevent multiple heartbeat functions from being added to the queue at startup

* Add unit tests for API

.gitignore

@@ -26,6 +26,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
+*.DS_Store
 
 # Django stuff:
 *.log

InvenTree/InvenTree/api_version.py

@@ -1,11 +1,14 @@
 """InvenTree API version information."""
 
 # InvenTree API version
-INVENTREE_API_VERSION = 161
+INVENTREE_API_VERSION = 162
 """Increment this API version number whenever there is a significant change to the API that any clients need to know about."""
 
 INVENTREE_API_TEXT = """
 
+v162 -> 2024-01-14 : https://github.com/inventree/InvenTree/pull/6230
+    - Adds API endpoints to provide information on background tasks
+
 v161 -> 2024-01-13 : https://github.com/inventree/InvenTree/pull/6222
     - Adds API endpoint for system error information
 

InvenTree/InvenTree/apps.py

@@ -138,12 +138,22 @@ class InvenTreeConfig(AppConfig):
             Schedule.objects.bulk_update(tasks_to_update, ['schedule_type', 'minutes'])
             logger.info('Updated %s existing scheduled tasks', len(tasks_to_update))
 
-        # Put at least one task onto the background worker stack,
+        self.add_heartbeat()
-        # which will be processed as soon as the worker comes online
-        InvenTree.tasks.offload_task(InvenTree.tasks.heartbeat, force_async=True)
 
         logger.info('Started %s scheduled background tasks...', len(tasks))
 
+    def add_heartbeat(self):
+        """Ensure there is at least one background task in the queue."""
+        import django_q.models
+
+        try:
+            if django_q.models.OrmQ.objects.count() == 0:
+                InvenTree.tasks.offload_task(
+                    InvenTree.tasks.heartbeat, force_async=True
+                )
+        except Exception:
+            pass
+
     def collect_tasks(self):
         """Collect all background tasks."""
         for app_name, app in apps.app_configs.items():

InvenTree/InvenTree/tasks.py

@@ -347,7 +347,7 @@ def heartbeat():
     (There is probably a less "hacky" way of achieving this)?
     """
     try:
-        from django_q.models import Success
+        from django_q.models import OrmQ, Success
     except AppRegistryNotReady:  # pragma: no cover
         logger.info('Could not perform heartbeat task - App registry not ready')
         return
@@ -362,6 +362,11 @@ def heartbeat():
 
     heartbeats.delete()
 
+    # Clear out any other pending heartbeat tasks
+    for task in OrmQ.objects.all():
+        if task.func() == 'InvenTree.tasks.heartbeat':
+            task.delete()
+
 
 @scheduled_task(ScheduledTask.DAILY)
 def delete_successful_tasks():

InvenTree/common/api.py

@@ -8,6 +8,7 @@ from django.urls import include, path, re_path
 from django.utils.decorators import method_decorator
 from django.views.decorators.csrf import csrf_exempt
 
+import django_q.models
 from django_q.tasks import async_task
 from djmoney.contrib.exchange.models import ExchangeBackend, Rate
 from error_report.models import Error
@@ -509,6 +510,71 @@ class ErrorMessageDetail(RetrieveUpdateDestroyAPI):
     permission_classes = [permissions.IsAuthenticated, IsAdminUser]
 
 
+class BackgroundTaskOverview(APIView):
+    """Provides an overview of the background task queue status."""
+
+    permission_classes = [permissions.IsAuthenticated, IsAdminUser]
+
+    def get(self, request, format=None):
+        """Return information about the current status of the background task queue."""
+        import django_q.models as q_models
+
+        import InvenTree.status
+
+        serializer = common.serializers.TaskOverviewSerializer({
+            'is_running': InvenTree.status.is_worker_running(),
+            'pending_tasks': q_models.OrmQ.objects.count(),
+            'scheduled_tasks': q_models.Schedule.objects.count(),
+            'failed_tasks': q_models.Failure.objects.count(),
+        })
+
+        return Response(serializer.data)
+
+
+class PendingTaskList(BulkDeleteMixin, ListAPI):
+    """Provides a read-only list of currently pending tasks."""
+
+    permission_classes = [permissions.IsAuthenticated, IsAdminUser]
+
+    queryset = django_q.models.OrmQ.objects.all()
+    serializer_class = common.serializers.PendingTaskSerializer
+
+
+class ScheduledTaskList(ListAPI):
+    """Provides a read-only list of currently scheduled tasks."""
+
+    permission_classes = [permissions.IsAuthenticated, IsAdminUser]
+
+    queryset = django_q.models.Schedule.objects.all()
+    serializer_class = common.serializers.ScheduledTaskSerializer
+
+    filter_backends = SEARCH_ORDER_FILTER
+
+    ordering_fields = ['pk', 'func', 'last_run', 'next_run']
+
+    search_fields = ['func']
+
+    def get_queryset(self):
+        """Return annotated queryset."""
+        queryset = super().get_queryset()
+        return common.serializers.ScheduledTaskSerializer.annotate_queryset(queryset)
+
+
+class FailedTaskList(BulkDeleteMixin, ListAPI):
+    """Provides a read-only list of currently failed tasks."""
+
+    permission_classes = [permissions.IsAuthenticated, IsAdminUser]
+
+    queryset = django_q.models.Failure.objects.all()
+    serializer_class = common.serializers.FailedTaskSerializer
+
+    filter_backends = SEARCH_ORDER_FILTER
+
+    ordering_fields = ['pk', 'func', 'started', 'stopped']
+
+    search_fields = ['func']
+
+
 class FlagList(ListAPI):
     """List view for feature flags."""
 
@@ -590,6 +656,24 @@ common_api_urls = [
     re_path(
         r'^notes-image-upload/', NotesImageList.as_view(), name='api-notes-image-list'
     ),
+    # Background task information
+    re_path(
+        r'^background-task/',
+        include([
+            re_path(
+                r'^pending/', PendingTaskList.as_view(), name='api-pending-task-list'
+            ),
+            re_path(
+                r'^scheduled/',
+                ScheduledTaskList.as_view(),
+                name='api-scheduled-task-list',
+            ),
+            re_path(r'^failed/', FailedTaskList.as_view(), name='api-failed-task-list'),
+            re_path(
+                r'^.*$', BackgroundTaskOverview.as_view(), name='api-task-overview'
+            ),
+        ]),
+    ),
     # Project codes
     re_path(
         r'^project-code/',

InvenTree/common/serializers.py

@@ -1,7 +1,10 @@
 """JSON serializers for common components."""
 
+from django.db.models import OuterRef, Subquery
 from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
 
+import django_q.models
 from error_report.models import Error
 from flags.state import flag_state
 from rest_framework import serializers
@@ -316,3 +319,120 @@ class ErrorMessageSerializer(InvenTreeModelSerializer):
         fields = ['when', 'info', 'data', 'path', 'pk']
 
         read_only_fields = ['when', 'info', 'data', 'path', 'pk']
+
+
+class TaskOverviewSerializer(serializers.Serializer):
+    """Serializer for background task overview."""
+
+    is_running = serializers.BooleanField(
+        label=_('Is Running'),
+        help_text='Boolean value to indicate if the background worker process is running.',
+        read_only=True,
+    )
+
+    pending_tasks = serializers.IntegerField(
+        label=_('Pending Tasks'),
+        help_text='Number of active background tasks',
+        read_only=True,
+    )
+
+    scheduled_tasks = serializers.IntegerField(
+        label=_('Scheduled Tasks'),
+        help_text='Number of scheduled background tasks',
+        read_only=True,
+    )
+
+    failed_tasks = serializers.IntegerField(
+        label=_('Failed Tasks'),
+        help_text='Number of failed background tasks',
+        read_only=True,
+    )
+
+
+class PendingTaskSerializer(InvenTreeModelSerializer):
+    """Serializer for an individual pending task object."""
+
+    class Meta:
+        """Metaclass options for the serializer."""
+
+        model = django_q.models.OrmQ
+        fields = ['pk', 'key', 'lock', 'task_id', 'name', 'func', 'args', 'kwargs']
+
+    task_id = serializers.CharField(label=_('Task ID'), help_text=_('Unique task ID'))
+
+    lock = serializers.DateTimeField(label=_('Lock'), help_text=_('Lock time'))
+
+    name = serializers.CharField(label=_('Name'), help_text=_('Task name'))
+
+    func = serializers.CharField(label=_('Function'), help_text=_('Function name'))
+
+    args = serializers.CharField(label=_('Arguments'), help_text=_('Task arguments'))
+
+    kwargs = serializers.CharField(
+        label=_('Keyword Arguments'), help_text=_('Task keyword arguments')
+    )
+
+
+class ScheduledTaskSerializer(InvenTreeModelSerializer):
+    """Serializer for an individual scheduled task object."""
+
+    class Meta:
+        """Metaclass options for the serializer."""
+
+        model = django_q.models.Schedule
+        fields = [
+            'pk',
+            'name',
+            'func',
+            'args',
+            'kwargs',
+            'schedule_type',
+            'repeats',
+            'last_run',
+            'next_run',
+            'success',
+            'task',
+        ]
+
+    last_run = serializers.DateTimeField()
+    success = serializers.BooleanField()
+
+    @staticmethod
+    def annotate_queryset(queryset):
+        """Add custom annotations to the queryset.
+
+        - last_run: The last time the task was run
+        - success: The outcome status of the last run
+        """
+        task = django_q.models.Task.objects.filter(id=OuterRef('task'))
+
+        queryset = queryset.annotate(
+            last_run=Subquery(task.values('started')[:1]),
+            success=Subquery(task.values('success')[:1]),
+        )
+
+        return queryset
+
+
+class FailedTaskSerializer(InvenTreeModelSerializer):
+    """Serializer for an individual failed task object."""
+
+    class Meta:
+        """Metaclass options for the serializer."""
+
+        model = django_q.models.Failure
+        fields = [
+            'pk',
+            'name',
+            'func',
+            'args',
+            'kwargs',
+            'started',
+            'stopped',
+            'attempt_count',
+            'result',
+        ]
+
+    pk = serializers.CharField(source='id', read_only=True)
+
+    result = serializers.CharField()

InvenTree/common/tests.py

@@ -658,6 +658,50 @@ class PluginSettingsApiTest(PluginMixin, InvenTreeAPITestCase):
         ...
 
 
+class TaskListApiTests(InvenTreeAPITestCase):
+    """Unit tests for the background task API endpoints."""
+
+    def test_pending_tasks(self):
+        """Test that the pending tasks endpoint is available."""
+        # Schedule some tasks
+        from django_q.models import OrmQ
+
+        from InvenTree.tasks import offload_task
+
+        n = OrmQ.objects.count()
+
+        for i in range(3):
+            offload_task(f'fake_module.test_{i}', force_async=True)
+
+        self.assertEqual(OrmQ.objects.count(), 3)
+
+        url = reverse('api-pending-task-list')
+        response = self.get(url, expected_code=200)
+
+        self.assertEqual(len(response.data), n + 3)
+
+        for task in response.data:
+            self.assertTrue(task['func'].startswith('fake_module.test_'))
+
+    def test_scheduled_tasks(self):
+        """Test that the scheduled tasks endpoint is available."""
+        from django_q.models import Schedule
+
+        for i in range(5):
+            Schedule.objects.create(
+                name='time.sleep', func='time.sleep', args=f'{i + 1}'
+            )
+
+        n = Schedule.objects.count()
+        self.assertGreater(n, 0)
+
+        url = reverse('api-scheduled-task-list')
+        response = self.get(url, expected_code=200)
+
+        for task in response.data:
+            self.assertTrue(task['name'] == 'time.sleep')
+
+
 class WebhookMessageTests(TestCase):
     """Tests for webhooks."""
 

InvenTree/company/api.py

@@ -54,9 +54,7 @@ class CompanyList(ListCreateAPI):
     def get_queryset(self):
         """Return annotated queryset for the company list endpoint."""
         queryset = super().get_queryset()
-        queryset = CompanySerializer.annotate_queryset(queryset)
+        return CompanySerializer.annotate_queryset(queryset)
-
-        return queryset
 
     filter_backends = SEARCH_ORDER_FILTER
 

src/frontend/src/components/tables/InvenTreeTable.tsx

@@ -388,7 +388,9 @@ export function InvenTreeTable<T = any>({
       },
       onConfirm: () => {
         // Delete the selected records
-        let selection = tableState.selectedRecords.map((record) => record.pk);
+        let selection = tableState.selectedRecords.map(
+          (record) => record.pk ?? record.id
+        );
 
         api
           .delete(url, {
@@ -409,6 +411,12 @@ export function InvenTreeTable<T = any>({
           })
           .catch((_error) => {
             console.warn(`Bulk delete operation failed at ${url}`);
+
+            showNotification({
+              title: t`Error`,
+              message: t`Failed to delete records`,
+              color: 'red'
+            });
           });
       }
     });

src/frontend/src/components/tables/settings/ErrorTable.tsx

@@ -80,7 +80,6 @@ export default function ErrorReportTable() {
           enableSelection: true,
           rowActions: rowActions,
           onRowClick: (row) => {
-            console.log(row);
             setError(row.data);
             open();
           }

src/frontend/src/components/tables/settings/FailedTasksTable.tsx

@@ -0,0 +1,79 @@
+import { t } from '@lingui/macro';
+import { Drawer, Text } from '@mantine/core';
+import { useDisclosure } from '@mantine/hooks';
+import { useMemo, useState } from 'react';
+
+import { ApiPaths } from '../../../enums/ApiEndpoints';
+import { useTable } from '../../../hooks/UseTable';
+import { apiUrl } from '../../../states/ApiState';
+import { StylishText } from '../../items/StylishText';
+import { TableColumn } from '../Column';
+import { InvenTreeTable } from '../InvenTreeTable';
+
+export default function FailedTasksTable() {
+  const table = useTable('tasks-failed');
+
+  const [error, setError] = useState<string>('');
+
+  const [opened, { open, close }] = useDisclosure(false);
+
+  const columns: TableColumn[] = useMemo(() => {
+    return [
+      {
+        accessor: 'func',
+        title: t`Task`,
+        sortable: true,
+        switchable: false
+      },
+      {
+        accessor: 'pk',
+        title: t`Task ID`
+      },
+      {
+        accessor: 'started',
+        title: t`Started`,
+        sortable: true,
+        switchable: false
+      },
+      {
+        accessor: 'stopped',
+        title: t`Stopped`,
+        sortable: true,
+        switchable: false
+      },
+      {
+        accessor: 'attempt_count',
+        title: t`Attempts`
+      }
+    ];
+  }, []);
+
+  return (
+    <>
+      <Drawer
+        opened={opened}
+        size="xl"
+        position="right"
+        title={<StylishText>{t`Error Details`}</StylishText>}
+        onClose={close}
+      >
+        {error.split('\n').map((line: string) => {
+          return <Text size="sm">{line}</Text>;
+        })}
+      </Drawer>
+      <InvenTreeTable
+        url={apiUrl(ApiPaths.task_failed_list)}
+        tableState={table}
+        columns={columns}
+        props={{
+          enableBulkDelete: true,
+          enableSelection: true,
+          onRowClick: (row: any) => {
+            setError(row.result);
+            open();
+          }
+        }}
+      />
+    </>
+  );
+}

src/frontend/src/components/tables/settings/PendingTasksTable.tsx

@@ -0,0 +1,56 @@
+import { t } from '@lingui/macro';
+import { useMemo } from 'react';
+
+import { ApiPaths } from '../../../enums/ApiEndpoints';
+import { useTable } from '../../../hooks/UseTable';
+import { apiUrl } from '../../../states/ApiState';
+import { TableColumn } from '../Column';
+import { InvenTreeTable } from '../InvenTreeTable';
+
+export default function PendingTasksTable() {
+  const table = useTable('tasks-pending');
+
+  const columns: TableColumn[] = useMemo(() => {
+    return [
+      {
+        accessor: 'func',
+        title: t`Task`,
+        switchable: false
+      },
+      {
+        accessor: 'task_id',
+        title: t`Task ID`
+      },
+      {
+        accessor: 'name',
+        title: t`Name`
+      },
+      {
+        accessor: 'lock',
+        title: t`Created`,
+        sortable: true,
+        switchable: false
+      },
+      {
+        accessor: 'args',
+        title: t`Arguments`
+      },
+      {
+        accessor: 'kwargs',
+        title: t`Keywords`
+      }
+    ];
+  }, []);
+
+  return (
+    <InvenTreeTable
+      url={apiUrl(ApiPaths.task_pending_list)}
+      tableState={table}
+      columns={columns}
+      props={{
+        enableBulkDelete: true,
+        enableSelection: true
+      }}
+    />
+  );
+}

src/frontend/src/components/tables/settings/ScheduledTasksTable.tsx

@@ -0,0 +1,62 @@
+import { t } from '@lingui/macro';
+import { Group, Text } from '@mantine/core';
+import { IconCircleCheck, IconCircleX } from '@tabler/icons-react';
+import { useMemo } from 'react';
+
+import { ApiPaths } from '../../../enums/ApiEndpoints';
+import { useTable } from '../../../hooks/UseTable';
+import { apiUrl } from '../../../states/ApiState';
+import { TableColumn } from '../Column';
+import { InvenTreeTable } from '../InvenTreeTable';
+
+export default function ScheduledTasksTable() {
+  const table = useTable('tasks-scheduled');
+
+  const columns: TableColumn[] = useMemo(() => {
+    return [
+      {
+        accessor: 'func',
+        title: t`Task`,
+        sortable: true,
+        switchable: false
+      },
+      {
+        accessor: 'last_run',
+        title: t`Last Run`,
+        sortable: true,
+        switchable: false,
+        render: (record: any) => {
+          if (!record.last_run) {
+            return '-';
+          }
+
+          return (
+            <Group position="apart">
+              <Text>{record.last_run}</Text>
+              {record.success ? (
+                <IconCircleCheck color="green" />
+              ) : (
+                <IconCircleX color="red" />
+              )}
+            </Group>
+          );
+        }
+      },
+      {
+        accessor: 'next_run',
+        title: t`Next Run`,
+        sortable: true,
+        switchable: false
+      }
+    ];
+  }, []);
+
+  return (
+    <InvenTreeTable
+      url={apiUrl(ApiPaths.task_scheduled_list)}
+      tableState={table}
+      columns={columns}
+      props={{}}
+    />
+  );
+}

src/frontend/src/enums/ApiEndpoints.tsx

@@ -24,6 +24,11 @@ export enum ApiPaths {
   group_list = 'api-group-list',
   owner_list = 'api-owner-list',
 
+  task_overview = 'api-task-overview',
+  task_pending_list = 'api-task-pending-list',
+  task_scheduled_list = 'api-task-scheduled-list',
+  task_failed_list = 'api-task-failed-list',
+
   settings_global_list = 'api-settings-global-list',
   settings_user_list = 'api-settings-user-list',
   notifications_list = 'api-notifications-list',

src/frontend/src/pages/Index/Settings/AdminCenter/Index.tsx

@@ -1,6 +1,7 @@
 import { Trans, t } from '@lingui/macro';
 import { Divider, Paper, SimpleGrid, Stack, Text, Title } from '@mantine/core';
 import {
+  IconCpu,
   IconExclamationCircle,
   IconList,
   IconListDetails,
@@ -20,6 +21,10 @@ const UserManagementPanel = Loadable(
   lazy(() => import('./UserManagementPanel'))
 );
 
+const TaskManagementPanel = Loadable(
+  lazy(() => import('./TaskManagementPanel'))
+);
+
 const PluginManagementPanel = Loadable(
   lazy(() => import('./PluginManagementPanel'))
 );
@@ -52,6 +57,12 @@ export default function AdminCenter() {
         icon: <IconUsersGroup />,
         content: <UserManagementPanel />
       },
+      {
+        name: 'background',
+        label: t`Background Tasks`,
+        icon: <IconCpu />,
+        content: <TaskManagementPanel />
+      },
       {
         name: 'errors',
         label: t`Error Reports`,
@@ -126,7 +137,7 @@ export default function AdminCenter() {
       <PanelGroup
         pageKey="admin-center"
         panels={adminCenterPanels}
-        collapsible={false}
+        collapsible={true}
       />
     </Stack>
   );

src/frontend/src/pages/Index/Settings/AdminCenter/TaskManagementPanel.tsx

@@ -0,0 +1,51 @@
+import { t } from '@lingui/macro';
+import { Accordion } from '@mantine/core';
+import { lazy } from 'react';
+
+import { StylishText } from '../../../../components/items/StylishText';
+import { Loadable } from '../../../../functions/loading';
+
+const PendingTasksTable = Loadable(
+  lazy(() => import('../../../../components/tables/settings/PendingTasksTable'))
+);
+
+const ScheduledTasksTable = Loadable(
+  lazy(
+    () => import('../../../../components/tables/settings/ScheduledTasksTable')
+  )
+);
+
+const FailedTasksTable = Loadable(
+  lazy(() => import('../../../../components/tables/settings/FailedTasksTable'))
+);
+
+export default function TaskManagementPanel() {
+  return (
+    <Accordion defaultValue="pending">
+      <Accordion.Item value="pending">
+        <Accordion.Control>
+          <StylishText size="lg">{t`Pending Tasks`}</StylishText>
+        </Accordion.Control>
+        <Accordion.Panel>
+          <PendingTasksTable />
+        </Accordion.Panel>
+      </Accordion.Item>
+      <Accordion.Item value="scheduled">
+        <Accordion.Control>
+          <StylishText size="lg">{t`Scheduled Tasks`}</StylishText>
+        </Accordion.Control>
+        <Accordion.Panel>
+          <ScheduledTasksTable />
+        </Accordion.Panel>
+      </Accordion.Item>
+      <Accordion.Item value="failed">
+        <Accordion.Control>
+          <StylishText size="lg">{t`Failed Tasks`}</StylishText>
+        </Accordion.Control>
+        <Accordion.Panel>
+          <FailedTasksTable />
+        </Accordion.Panel>
+      </Accordion.Item>
+    </Accordion>
+  );
+}

src/frontend/src/states/ApiState.tsx

@@ -99,6 +99,14 @@ export function apiEndpoint(path: ApiPaths): string {
       return 'currency/exchange/';
     case ApiPaths.currency_refresh:
       return 'currency/refresh/';
+    case ApiPaths.task_overview:
+      return 'background-task/';
+    case ApiPaths.task_pending_list:
+      return 'background-task/pending/';
+    case ApiPaths.task_scheduled_list:
+      return 'background-task/scheduled/';
+    case ApiPaths.task_failed_list:
+      return 'background-task/failed/';
     case ApiPaths.api_search:
       return 'search/';
     case ApiPaths.settings_global_list:

tasks.py

@@ -376,14 +376,21 @@ def migrate(c):
 
 
 @task(
-    post=[static, clean_settings, translate_stats],
+    post=[clean_settings, translate_stats],
     help={
         'skip_backup': 'Skip database backup step (advanced users)',
         'frontend': 'Force frontend compilation/download step (ignores INVENTREE_DOCKER)',
         'no_frontend': 'Skip frontend compilation/download step',
+        'skip_static': 'Skip static file collection step',
     },
 )
-def update(c, skip_backup=False, frontend: bool = False, no_frontend: bool = False):
+def update(
+    c,
+    skip_backup: bool = False,
+    frontend: bool = False,
+    no_frontend: bool = False,
+    skip_static: bool = False,
+):
     """Update InvenTree installation.
 
     This command should be invoked after source code has been updated,
@@ -394,8 +401,8 @@ def update(c, skip_backup=False, frontend: bool = False, no_frontend: bool = Fal
     - install
     - backup (optional)
     - migrate
-    - frontend_compile or frontend_download
+    - frontend_compile or frontend_download (optional)
-    - static
+    - static (optional)
     - clean_settings
     - translate_stats
     """
@@ -421,6 +428,9 @@ def update(c, skip_backup=False, frontend: bool = False, no_frontend: bool = Fal
     else:
         frontend_download(c)
 
+    if not skip_static:
+        static(c)
+
 
 # Data tasks
 @task(