From 05e294fc81f354be9796608408a7ffc5f652fc16 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 29 Dec 2023 00:51:57 -0800 Subject: [PATCH 01/33] Fix URL validation of empty string --- src/documents/serialisers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index 62e7d75b4..b6be62d9b 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -494,7 +494,11 @@ class CustomFieldInstanceSerializer(serializers.ModelSerializer): """ data = super().validate(data) field: CustomField = data["field"] - if field.data_type == CustomField.FieldDataType.URL: + if ( + field.data_type == CustomField.FieldDataType.URL + and data["value"] is not None + and len(data["value"]) > 0 + ): URLValidator()(data["value"]) return data From cf869b13568191183d12cf5fecaf4356302ffba9 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 29 Dec 2023 09:19:45 -0800 Subject: [PATCH 02/33] Fix: type casting of db values for shared by me filter (#5155) --- src/documents/filters.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/documents/filters.py b/src/documents/filters.py index 0f49c7c27..c63484ee2 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -1,7 +1,9 @@ from django.contrib.contenttypes.models import ContentType +from django.db.models import CharField from django.db.models import Count from django.db.models import OuterRef from django.db.models import Q +from django.db.models.functions import Cast from django_filters.rest_framework import BooleanFilter from django_filters.rest_framework import Filter from django_filters.rest_framework import FilterSet @@ -119,7 +121,7 @@ class SharedByUser(Filter): num_shared_users=Count( UserObjectPermission.objects.filter( content_type=ctype, - object_pk=OuterRef("pk"), + object_pk=Cast(OuterRef("pk"), CharField()), ).values("user_id"), ), ) @@ -127,7 +129,7 @@ class SharedByUser(Filter): num_shared_groups=Count( GroupObjectPermission.objects.filter( content_type=ctype, - object_pk=OuterRef("pk"), + object_pk=Cast(OuterRef("pk"), CharField()), ).values("group_id"), ), ) From da058b915b0606ca935da0eca79559b3946c6fe7 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 29 Dec 2023 14:45:29 -0800 Subject: [PATCH 03/33] Enhancement: improve validation of custom field values (#5166) * Support all URI schemes * Reworks custom field value validation to check and return a 400 error code in more cases and support more URL looking items, not just some basic schemes * Fixes a spelling error in the message --------- Co-authored-by: Trenton H <797416+stumpylog@users.noreply.github.com> --- src/documents/serialisers.py | 34 +++-- src/documents/tests/test_api_custom_fields.py | 134 ++++++++++++++++-- src/documents/validators.py | 29 ++++ 3 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 src/documents/validators.py diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index b6be62d9b..c07b00f78 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -2,6 +2,7 @@ import datetime import math import re import zoneinfo +from decimal import Decimal import magic from celery import states @@ -9,7 +10,9 @@ from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType -from django.core.validators import URLValidator +from django.core.validators import DecimalValidator +from django.core.validators import MaxLengthValidator +from django.core.validators import integer_validator from django.utils.crypto import get_random_string from django.utils.text import slugify from django.utils.translation import gettext as _ @@ -41,6 +44,7 @@ from documents.models import UiSettings from documents.parsers import is_mime_type_supported from documents.permissions import get_groups_with_only_permission from documents.permissions import set_permissions_for_object +from documents.validators import uri_validator # https://www.django-rest-framework.org/api-guide/serializers/#example @@ -489,17 +493,29 @@ class CustomFieldInstanceSerializer(serializers.ModelSerializer): def validate(self, data): """ - For some reason, URLField validation is not run against the value - automatically. Force it to run against the value + Probably because we're kind of doing it odd, validation from the model + doesn't run against the field "value", so we have to re-create it here. + + Don't like it, but it is better than returning an HTTP 500 when the database + hates the value """ data = super().validate(data) field: CustomField = data["field"] - if ( - field.data_type == CustomField.FieldDataType.URL - and data["value"] is not None - and len(data["value"]) > 0 - ): - URLValidator()(data["value"]) + if "value" in data and data["value"] is not None: + if ( + field.data_type == CustomField.FieldDataType.URL + and len(data["value"]) > 0 + ): + uri_validator(data["value"]) + elif field.data_type == CustomField.FieldDataType.INT: + integer_validator(data["value"]) + elif field.data_type == CustomField.FieldDataType.MONETARY: + DecimalValidator(max_digits=12, decimal_places=2)( + Decimal(str(data["value"])), + ) + elif field.data_type == CustomField.FieldDataType.STRING: + MaxLengthValidator(limit_value=128)(data["value"]) + return data def reflect_doclinks( diff --git a/src/documents/tests/test_api_custom_fields.py b/src/documents/tests/test_api_custom_fields.py index 15abcd053..690e92690 100644 --- a/src/documents/tests/test_api_custom_fields.py +++ b/src/documents/tests/test_api_custom_fields.py @@ -333,19 +333,17 @@ class TestCustomField(DirectoriesMixin, APITestCase): }, format="json", ) - from pprint import pprint - pprint(resp.json()) self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(CustomFieldInstance.objects.count(), 0) self.assertEqual(len(doc.custom_fields.all()), 0) - def test_custom_field_value_validation(self): + def test_custom_field_value_url_validation(self): """ GIVEN: - Document & custom field exist WHEN: - - API request to set a field value + - API request to set a field value to something which is or is not a link THEN: - HTTP 400 is returned - No field instance is created or attached to the document @@ -360,31 +358,62 @@ class TestCustomField(DirectoriesMixin, APITestCase): name="Test Custom Field URL", data_type=CustomField.FieldDataType.URL, ) - custom_field_int = CustomField.objects.create( - name="Test Custom Field INT", - data_type=CustomField.FieldDataType.INT, - ) + for value in ["not a url", "file:"]: + with self.subTest(f"Test value {value}"): + resp = self.client.patch( + f"/api/documents/{doc.id}/", + data={ + "custom_fields": [ + { + "field": custom_field_url.id, + "value": value, + }, + ], + }, + format="json", + ) + + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(CustomFieldInstance.objects.count(), 0) + self.assertEqual(len(doc.custom_fields.all()), 0) resp = self.client.patch( f"/api/documents/{doc.id}/", data={ "custom_fields": [ { "field": custom_field_url.id, - "value": "not a url", + "value": "tel:+1-816-555-1212", }, ], }, format="json", ) - self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual(CustomFieldInstance.objects.count(), 0) - self.assertEqual(len(doc.custom_fields.all()), 0) + self.assertEqual(resp.status_code, status.HTTP_200_OK) - self.assertRaises( - Exception, - self.client.patch, + def test_custom_field_value_integer_validation(self): + """ + GIVEN: + - Document & custom field exist + WHEN: + - API request to set a field value to something not an integer + THEN: + - HTTP 400 is returned + - No field instance is created or attached to the document + """ + doc = Document.objects.create( + title="WOW", + content="the content", + checksum="123", + mime_type="application/pdf", + ) + custom_field_int = CustomField.objects.create( + name="Test Custom Field INT", + data_type=CustomField.FieldDataType.INT, + ) + + resp = self.client.patch( f"/api/documents/{doc.id}/", data={ "custom_fields": [ @@ -397,6 +426,81 @@ class TestCustomField(DirectoriesMixin, APITestCase): format="json", ) + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(CustomFieldInstance.objects.count(), 0) + self.assertEqual(len(doc.custom_fields.all()), 0) + + def test_custom_field_value_monetary_validation(self): + """ + GIVEN: + - Document & custom field exist + WHEN: + - API request to set a field value to something not a valid monetary decimal + THEN: + - HTTP 400 is returned + - No field instance is created or attached to the document + """ + doc = Document.objects.create( + title="WOW", + content="the content", + checksum="123", + mime_type="application/pdf", + ) + custom_field_money = CustomField.objects.create( + name="Test Custom Field MONETARY", + data_type=CustomField.FieldDataType.MONETARY, + ) + + resp = self.client.patch( + f"/api/documents/{doc.id}/", + data={ + "custom_fields": [ + { + "field": custom_field_money.id, + # Too many places past decimal + "value": 12.123, + }, + ], + }, + format="json", + ) + + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(CustomFieldInstance.objects.count(), 0) + self.assertEqual(len(doc.custom_fields.all()), 0) + + def test_custom_field_value_short_text_validation(self): + """ + GIVEN: + - Document & custom field exist + WHEN: + - API request to set a field value to a too long string + THEN: + - HTTP 400 is returned + - No field instance is created or attached to the document + """ + doc = Document.objects.create( + title="WOW", + content="the content", + checksum="123", + mime_type="application/pdf", + ) + custom_field_string = CustomField.objects.create( + name="Test Custom Field STRING", + data_type=CustomField.FieldDataType.STRING, + ) + + resp = self.client.patch( + f"/api/documents/{doc.id}/", + data={ + "custom_fields": [ + {"field": custom_field_string.id, "value": "a" * 129}, + ], + }, + format="json", + ) + + self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(CustomFieldInstance.objects.count(), 0) self.assertEqual(len(doc.custom_fields.all()), 0) diff --git a/src/documents/validators.py b/src/documents/validators.py new file mode 100644 index 000000000..0ebf15697 --- /dev/null +++ b/src/documents/validators.py @@ -0,0 +1,29 @@ +from urllib.parse import urlparse + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + + +def uri_validator(value) -> None: + """ + Raises a ValidationError if the given value does not parse as an + URI looking thing, which we're defining as a scheme and either network + location or path value + """ + try: + parts = urlparse(value) + if not parts.scheme: + raise ValidationError( + _(f"Unable to parse URI {value}, missing scheme"), + params={"value": value}, + ) + elif not parts.netloc and not parts.path: + raise ValidationError( + _(f"Unable to parse URI {value}, missing net location or path"), + params={"value": value}, + ) + except Exception as e: + raise ValidationError( + _(f"Unable to parse URI {value}"), + params={"value": value}, + ) from e From 061f33fb0516da84df8aca4575a2a17b2335a4d8 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 29 Dec 2023 15:42:56 -0800 Subject: [PATCH 04/33] Feature: Allow setting backend configuration settings via the UI (#5126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Saving some start on this * At least partially working for the tesseract parser * Problems with migration testing need to figure out * Work around that error * Fixes max m_pixels * Moving the settings to main paperless application * Starting some consumer options * More fixes and work * Fixes these last tests * Fix max_length on OcrSettings.mode field * Fix all fields on Common & Ocr settings serializers * Umbrellla config view * Revert "Umbrellla config view" This reverts commit fbaf9f4be30f89afeb509099180158a3406416a5. * Updates to use a single configuration object for all settings * Squashed commit of the following: commit 8a0a49dd5766094f60462fbfbe62e9921fbd2373 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 23:02:47 2023 -0800 Fix formatting commit 66b2d90c507b8afd9507813ff555e46198ea33b9 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 22:36:35 2023 -0800 Refactor frontend data models commit 5723bd8dd823ee855625e250df39393e26709d48 Author: Adam Bogdał Date: Wed Dec 20 01:17:43 2023 +0100 Fix: speed up admin panel for installs with a large number of documents (#5052) commit 9b08ce176199bf9011a6634bb88f616846150d2b Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 15:18:51 2023 -0800 Update PULL_REQUEST_TEMPLATE.md commit a6248bec2d793b7690feed95fcaf5eb34a75bfb6 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 15:02:05 2023 -0800 Chore: Update Angular to v17 (#4980) commit b1f6f52486d5ba5c04af99b41315eb6428fd1fa8 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 13:53:56 2023 -0800 Fix: Dont allow null custom_fields property via API (#5063) commit 638d9970fd468d8c02c91d19bd28f8b0796bdcb1 Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 13:43:50 2023 -0800 Enhancement: symmetric document links (#4907) commit 5e8de4c1da6eb4eb8f738b20962595c7536b30ec Author: shamoon <4887959+shamoon@users.noreply.github.com> Date: Tue Dec 19 12:45:04 2023 -0800 Enhancement: shared icon & shared by me filter (#4859) commit 088bad90306025d3f6b139cbd0ad264a1cbecfe5 Author: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Tue Dec 19 12:04:03 2023 -0800 Bulk updates all the backend libraries (#5061) * Saving some work on frontend config * Very basic but dynamically-generated config form * Saving work on slightly less ugly frontend config * JSON validation for user_args field * Fully dynamic config form * Adds in some additional validators for a nicer error message * Cleaning up the testing and coverage more * Reverts unintentional change * Adds documentation about the settings and the precedence * Couple more commenting and style fixes --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- docs/configuration.md | 5 + src-ui/src/app/app-routing.module.ts | 12 + src-ui/src/app/app.module.ts | 4 + .../admin/config/config.component.html | 54 ++++ .../admin/config/config.component.scss | 0 .../admin/config/config.component.spec.ts | 103 ++++++++ .../admin/config/config.component.ts | 163 ++++++++++++ .../app-frame/app-frame.component.html | 9 + .../common/input/number/number.component.html | 4 +- .../common/input/switch/switch.component.html | 27 ++ .../common/input/switch/switch.component.scss | 0 .../input/switch/switch.component.spec.ts | 39 +++ .../common/input/switch/switch.component.ts | 21 ++ .../common/input/text/text.component.html | 4 +- src-ui/src/app/data/paperless-config.ts | 183 ++++++++++++++ .../src/app/services/config.service.spec.ts | 42 ++++ src-ui/src/app/services/config.service.ts | 27 ++ src/documents/classifier.py | 4 +- src/documents/consumer.py | 2 +- .../management/commands/document_consumer.py | 2 +- .../management/commands/document_exporter.py | 5 + .../management/commands/loaddata_stdin.py | 2 +- src/documents/parsers.py | 11 +- src/documents/tests/test_consumer.py | 16 +- .../tests/test_management_exporter.py | 16 +- src/paperless/config.py | 88 +++++++ src/paperless/migrations/0001_initial.py | 180 ++++++++++++++ src/paperless/migrations/__init__.py | 0 src/paperless/models.py | 173 +++++++++++++ src/paperless/serialisers.py | 8 + src/paperless/settings.py | 70 +++--- src/paperless/urls.py | 2 + src/paperless/views.py | 13 +- src/paperless_mail/mail.py | 14 +- src/paperless_mail/parsers.py | 6 + src/paperless_tesseract/parsers.py | 90 ++++--- src/paperless_tesseract/tests/test_parser.py | 32 +-- .../tests/test_parser_custom_settings.py | 232 ++++++++++++++++++ src/paperless_text/parsers.py | 6 + src/paperless_tika/parsers.py | 19 +- src/setup.cfg | 1 + 41 files changed, 1570 insertions(+), 119 deletions(-) create mode 100644 src-ui/src/app/components/admin/config/config.component.html create mode 100644 src-ui/src/app/components/admin/config/config.component.scss create mode 100644 src-ui/src/app/components/admin/config/config.component.spec.ts create mode 100644 src-ui/src/app/components/admin/config/config.component.ts create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.html create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.scss create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.spec.ts create mode 100644 src-ui/src/app/components/common/input/switch/switch.component.ts create mode 100644 src-ui/src/app/data/paperless-config.ts create mode 100644 src-ui/src/app/services/config.service.spec.ts create mode 100644 src-ui/src/app/services/config.service.ts create mode 100644 src/paperless/config.py create mode 100644 src/paperless/migrations/0001_initial.py create mode 100644 src/paperless/migrations/__init__.py create mode 100644 src/paperless/models.py create mode 100644 src/paperless_tesseract/tests/test_parser_custom_settings.py diff --git a/docs/configuration.md b/docs/configuration.md index 212508806..a7cf67bf9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -3,6 +3,11 @@ Paperless provides a wide range of customizations. Depending on how you run paperless, these settings have to be defined in different places. +Certain configuration options may be set via the UI. This currently includes +common [OCR](#ocr) related settings. If set, these will take preference over the +settings via environment variables. If not set, the environment setting or applicable +default will be utilized instead. + - If you run paperless on docker, `paperless.conf` is not used. Rather, configure paperless by copying necessary options to `docker-compose.env`. diff --git a/src-ui/src/app/app-routing.module.ts b/src-ui/src/app/app-routing.module.ts index b3952634c..89ed06e39 100644 --- a/src-ui/src/app/app-routing.module.ts +++ b/src-ui/src/app/app-routing.module.ts @@ -25,6 +25,7 @@ import { ConsumptionTemplatesComponent } from './components/manage/consumption-t import { MailComponent } from './components/manage/mail/mail.component' import { UsersAndGroupsComponent } from './components/admin/users-groups/users-groups.component' import { CustomFieldsComponent } from './components/manage/custom-fields/custom-fields.component' +import { ConfigComponent } from './components/admin/config/config.component' export const routes: Routes = [ { path: '', redirectTo: 'dashboard', pathMatch: 'full' }, @@ -179,6 +180,17 @@ export const routes: Routes = [ }, }, }, + { + path: 'config', + component: ConfigComponent, + canActivate: [PermissionsGuard], + data: { + requiredPermission: { + action: PermissionAction.View, + type: PermissionType.Admin, + }, + }, + }, { path: 'tasks', component: TasksComponent, diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts index c3b98549a..6d8d58944 100644 --- a/src-ui/src/app/app.module.ts +++ b/src-ui/src/app/app.module.ts @@ -108,6 +108,8 @@ import { ProfileEditDialogComponent } from './components/common/profile-edit-dia import { PdfViewerComponent } from './components/common/pdf-viewer/pdf-viewer.component' import { DocumentLinkComponent } from './components/common/input/document-link/document-link.component' import { PreviewPopupComponent } from './components/common/preview-popup/preview-popup.component' +import { ConfigComponent } from './components/admin/config/config.component' +import { SwitchComponent } from './components/common/input/switch/switch.component' import localeAf from '@angular/common/locales/af' import localeAr from '@angular/common/locales/ar' @@ -263,6 +265,8 @@ function initializeApp(settings: SettingsService) { PdfViewerComponent, DocumentLinkComponent, PreviewPopupComponent, + ConfigComponent, + SwitchComponent, ], imports: [ BrowserModule, diff --git a/src-ui/src/app/components/admin/config/config.component.html b/src-ui/src/app/components/admin/config/config.component.html new file mode 100644 index 000000000..48cac6bfa --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.html @@ -0,0 +1,54 @@ + + +
+ + +
+ +
diff --git a/src-ui/src/app/components/admin/config/config.component.scss b/src-ui/src/app/components/admin/config/config.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/src-ui/src/app/components/admin/config/config.component.spec.ts b/src-ui/src/app/components/admin/config/config.component.spec.ts new file mode 100644 index 000000000..5d70881b6 --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.spec.ts @@ -0,0 +1,103 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing' + +import { ConfigComponent } from './config.component' +import { ConfigService } from 'src/app/services/config.service' +import { ToastService } from 'src/app/services/toast.service' +import { of, throwError } from 'rxjs' +import { OutputTypeConfig } from 'src/app/data/paperless-config' +import { HttpClientTestingModule } from '@angular/common/http/testing' +import { BrowserModule } from '@angular/platform-browser' +import { NgbModule } from '@ng-bootstrap/ng-bootstrap' +import { NgSelectModule } from '@ng-select/ng-select' +import { TextComponent } from '../../common/input/text/text.component' +import { NumberComponent } from '../../common/input/number/number.component' +import { SwitchComponent } from '../../common/input/switch/switch.component' +import { FormsModule, ReactiveFormsModule } from '@angular/forms' +import { PageHeaderComponent } from '../../common/page-header/page-header.component' +import { SelectComponent } from '../../common/input/select/select.component' + +describe('ConfigComponent', () => { + let component: ConfigComponent + let fixture: ComponentFixture + let configService: ConfigService + let toastService: ToastService + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ + ConfigComponent, + TextComponent, + SelectComponent, + NumberComponent, + SwitchComponent, + PageHeaderComponent, + ], + imports: [ + HttpClientTestingModule, + BrowserModule, + NgbModule, + NgSelectModule, + FormsModule, + ReactiveFormsModule, + ], + }).compileComponents() + + configService = TestBed.inject(ConfigService) + toastService = TestBed.inject(ToastService) + fixture = TestBed.createComponent(ConfigComponent) + component = fixture.componentInstance + fixture.detectChanges() + }) + + it('should load config on init, show error if necessary', () => { + const getSpy = jest.spyOn(configService, 'getConfig') + const errorSpy = jest.spyOn(toastService, 'showError') + getSpy.mockReturnValueOnce( + throwError(() => new Error('Error getting config')) + ) + component.ngOnInit() + expect(getSpy).toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalled() + getSpy.mockReturnValueOnce( + of({ output_type: OutputTypeConfig.PDF_A } as any) + ) + component.ngOnInit() + expect(component.initialConfig).toEqual({ + output_type: OutputTypeConfig.PDF_A, + }) + }) + + it('should save config, show error if necessary', () => { + const saveSpy = jest.spyOn(configService, 'saveConfig') + const errorSpy = jest.spyOn(toastService, 'showError') + saveSpy.mockReturnValueOnce( + throwError(() => new Error('Error saving config')) + ) + component.saveConfig() + expect(saveSpy).toHaveBeenCalled() + expect(errorSpy).toHaveBeenCalled() + saveSpy.mockReturnValueOnce( + of({ output_type: OutputTypeConfig.PDF_A } as any) + ) + component.saveConfig() + expect(component.initialConfig).toEqual({ + output_type: OutputTypeConfig.PDF_A, + }) + }) + + it('should support discard changes', () => { + component.initialConfig = { output_type: OutputTypeConfig.PDF_A2 } as any + component.configForm.patchValue({ output_type: OutputTypeConfig.PDF_A }) + component.discardChanges() + expect(component.configForm.get('output_type').value).toEqual( + OutputTypeConfig.PDF_A2 + ) + }) + + it('should support JSON validation for e.g. user_args', () => { + component.configForm.patchValue({ user_args: '{ foo bar }' }) + expect(component.errors).toEqual({ user_args: 'Invalid JSON' }) + component.configForm.patchValue({ user_args: '{ "foo": "bar" }' }) + expect(component.errors).toEqual({ user_args: null }) + }) +}) diff --git a/src-ui/src/app/components/admin/config/config.component.ts b/src-ui/src/app/components/admin/config/config.component.ts new file mode 100644 index 000000000..66d7b537f --- /dev/null +++ b/src-ui/src/app/components/admin/config/config.component.ts @@ -0,0 +1,163 @@ +import { Component, OnDestroy, OnInit } from '@angular/core' +import { AbstractControl, FormControl, FormGroup } from '@angular/forms' +import { + BehaviorSubject, + Observable, + Subject, + Subscription, + first, + takeUntil, +} from 'rxjs' +import { + PaperlessConfigOptions, + ConfigCategory, + ConfigOption, + ConfigOptionType, + PaperlessConfig, +} from 'src/app/data/paperless-config' +import { ConfigService } from 'src/app/services/config.service' +import { ToastService } from 'src/app/services/toast.service' +import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component' +import { DirtyComponent, dirtyCheck } from '@ngneat/dirty-check-forms' + +@Component({ + selector: 'pngx-config', + templateUrl: './config.component.html', + styleUrl: './config.component.scss', +}) +export class ConfigComponent + extends ComponentWithPermissions + implements OnInit, OnDestroy, DirtyComponent +{ + public readonly ConfigOptionType = ConfigOptionType + + // generated dynamically + public configForm = new FormGroup({}) + + public errors = {} + + get optionCategories(): string[] { + return Object.values(ConfigCategory) + } + + getCategoryOptions(category: string): ConfigOption[] { + return PaperlessConfigOptions.filter((o) => o.category === category) + } + + public loading: boolean = false + + initialConfig: PaperlessConfig + store: BehaviorSubject + storeSub: Subscription + isDirty$: Observable + + private unsubscribeNotifier: Subject = new Subject() + + constructor( + private configService: ConfigService, + private toastService: ToastService + ) { + super() + this.configForm.addControl('id', new FormControl()) + PaperlessConfigOptions.forEach((option) => { + this.configForm.addControl(option.key, new FormControl()) + }) + } + + ngOnInit(): void { + this.loading = true + this.configService + .getConfig() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe({ + next: (config) => { + this.loading = false + this.initialize(config) + }, + error: (e) => { + this.loading = false + this.toastService.showError($localize`Error retrieving config`, e) + }, + }) + + // validate JSON inputs + PaperlessConfigOptions.filter( + (o) => o.type === ConfigOptionType.JSON + ).forEach((option) => { + this.configForm + .get(option.key) + .addValidators((control: AbstractControl) => { + if (!control.value || control.value.toString().length === 0) + return null + try { + JSON.parse(control.value) + } catch (e) { + return [ + { + user_args: e, + }, + ] + } + return null + }) + this.configForm.get(option.key).statusChanges.subscribe((status) => { + this.errors[option.key] = + status === 'INVALID' ? $localize`Invalid JSON` : null + }) + this.configForm.get(option.key).updateValueAndValidity() + }) + } + + ngOnDestroy(): void { + this.unsubscribeNotifier.next(true) + this.unsubscribeNotifier.complete() + } + + private initialize(config: PaperlessConfig) { + if (!this.store) { + this.store = new BehaviorSubject(config) + + this.store + .asObservable() + .pipe(takeUntil(this.unsubscribeNotifier)) + .subscribe((state) => { + this.configForm.patchValue(state, { emitEvent: false }) + }) + + this.isDirty$ = dirtyCheck(this.configForm, this.store.asObservable()) + } + this.configForm.patchValue(config) + + this.initialConfig = config + } + + getDocsUrl(key: string) { + return `https://docs.paperless-ngx.com/configuration/#${key}` + } + + public saveConfig() { + this.loading = true + this.configService + .saveConfig(this.configForm.value as PaperlessConfig) + .pipe(takeUntil(this.unsubscribeNotifier), first()) + .subscribe({ + next: (config) => { + this.loading = false + this.initialize(config) + this.store.next(config) + this.toastService.showInfo($localize`Configuration updated`) + }, + error: (e) => { + this.loading = false + this.toastService.showError( + $localize`An error occurred updating configuration`, + e + ) + }, + }) + } + + public discardChanges() { + this.configForm.reset(this.initialConfig) + } +} diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 2ab3fe0ae..234099d60 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -271,6 +271,15 @@  Settings + - @for (template of templates; track template) { + @for (workflow of workflows; track workflow.id) {
  • -
    -
    {{template.order}}
    -
    {{getSourceList(template)}}
    +
    +
    {{workflow.order}}
    +
    @if(workflow.enabled) { Enabled } @else { Disabled }
    +
    {{getTypesList(workflow)}}
    - -
  • } - @if (templates.length === 0) { -
  • No templates defined.
  • + @if (workflows.length === 0) { +
  • No workflows defined.
  • } diff --git a/src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.scss b/src-ui/src/app/components/manage/workflows/workflows.component.scss similarity index 100% rename from src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.scss rename to src-ui/src/app/components/manage/workflows/workflows.component.scss diff --git a/src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.spec.ts b/src-ui/src/app/components/manage/workflows/workflows.component.spec.ts similarity index 68% rename from src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.spec.ts rename to src-ui/src/app/components/manage/workflows/workflows.component.spec.ts index 2cb365576..4382d56f5 100644 --- a/src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.spec.ts +++ b/src-ui/src/app/components/manage/workflows/workflows.component.spec.ts @@ -9,55 +9,76 @@ import { NgbModalModule, } from '@ng-bootstrap/ng-bootstrap' import { of, throwError } from 'rxjs' -import { - DocumentSource, - ConsumptionTemplate, -} from 'src/app/data/consumption-template' +import { Workflow } from 'src/app/data/workflow' import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive' -import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service' +import { WorkflowService } from 'src/app/services/rest/workflow.service' import { ToastService } from 'src/app/services/toast.service' import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' import { PageHeaderComponent } from '../../common/page-header/page-header.component' -import { ConsumptionTemplatesComponent } from './consumption-templates.component' -import { ConsumptionTemplateEditDialogComponent } from '../../common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component' +import { WorkflowsComponent } from './workflows.component' +import { WorkflowEditDialogComponent } from '../../common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component' import { PermissionsService } from 'src/app/services/permissions.service' +import { + DocumentSource, + WorkflowTriggerType, +} from 'src/app/data/workflow-trigger' +import { WorkflowActionType } from 'src/app/data/workflow-action' -const templates: ConsumptionTemplate[] = [ +const workflows: Workflow[] = [ { - id: 0, - name: 'Template 1', - order: 0, - sources: [ - DocumentSource.ConsumeFolder, - DocumentSource.ApiUpload, - DocumentSource.MailFetch, + name: 'Workflow 1', + id: 1, + order: 1, + enabled: true, + triggers: [ + { + id: 1, + type: WorkflowTriggerType.Consumption, + sources: [DocumentSource.ConsumeFolder], + filter_filename: '*', + }, + ], + actions: [ + { + id: 1, + type: WorkflowActionType.Assignment, + assign_title: 'foo', + }, ], - filter_filename: 'foo', - filter_path: 'bar', - assign_tags: [1, 2, 3], }, { - id: 1, - name: 'Template 2', - order: 1, - sources: [DocumentSource.MailFetch], - filter_filename: null, - filter_path: 'foo/bar', - assign_owner: 1, + name: 'Workflow 2', + id: 2, + order: 2, + enabled: true, + triggers: [ + { + id: 2, + type: WorkflowTriggerType.DocumentAdded, + filter_filename: 'foo', + }, + ], + actions: [ + { + id: 2, + type: WorkflowActionType.Assignment, + assign_title: 'bar', + }, + ], }, ] -describe('ConsumptionTemplatesComponent', () => { - let component: ConsumptionTemplatesComponent - let fixture: ComponentFixture - let consumptionTemplateService: ConsumptionTemplateService +describe('WorkflowsComponent', () => { + let component: WorkflowsComponent + let fixture: ComponentFixture + let workflowService: WorkflowService let modalService: NgbModal let toastService: ToastService beforeEach(() => { TestBed.configureTestingModule({ declarations: [ - ConsumptionTemplatesComponent, + WorkflowsComponent, IfPermissionsDirective, PageHeaderComponent, ConfirmDialogComponent, @@ -81,18 +102,18 @@ describe('ConsumptionTemplatesComponent', () => { ], }) - consumptionTemplateService = TestBed.inject(ConsumptionTemplateService) - jest.spyOn(consumptionTemplateService, 'listAll').mockReturnValue( + workflowService = TestBed.inject(WorkflowService) + jest.spyOn(workflowService, 'listAll').mockReturnValue( of({ - count: templates.length, - all: templates.map((o) => o.id), - results: templates, + count: workflows.length, + all: workflows.map((o) => o.id), + results: workflows, }) ) modalService = TestBed.inject(NgbModal) toastService = TestBed.inject(ToastService) - fixture = TestBed.createComponent(ConsumptionTemplatesComponent) + fixture = TestBed.createComponent(WorkflowsComponent) component = fixture.componentInstance fixture.detectChanges() }) @@ -108,8 +129,7 @@ describe('ConsumptionTemplatesComponent', () => { createButton.triggerEventHandler('click') expect(modal).not.toBeUndefined() - const editDialog = - modal.componentInstance as ConsumptionTemplateEditDialogComponent + const editDialog = modal.componentInstance as WorkflowEditDialogComponent // fail first editDialog.failed.emit({ error: 'error creating item' }) @@ -117,7 +137,7 @@ describe('ConsumptionTemplatesComponent', () => { expect(reloadSpy).not.toHaveBeenCalled() // succeed - editDialog.succeeded.emit(templates[0]) + editDialog.succeeded.emit(workflows[0]) expect(toastInfoSpy).toHaveBeenCalled() expect(reloadSpy).toHaveBeenCalled() }) @@ -133,9 +153,8 @@ describe('ConsumptionTemplatesComponent', () => { editButton.triggerEventHandler('click') expect(modal).not.toBeUndefined() - const editDialog = - modal.componentInstance as ConsumptionTemplateEditDialogComponent - expect(editDialog.object).toEqual(templates[0]) + const editDialog = modal.componentInstance as WorkflowEditDialogComponent + expect(editDialog.object).toEqual(workflows[0]) // fail first editDialog.failed.emit({ error: 'error editing item' }) @@ -143,7 +162,7 @@ describe('ConsumptionTemplatesComponent', () => { expect(reloadSpy).not.toHaveBeenCalled() // succeed - editDialog.succeeded.emit(templates[0]) + editDialog.succeeded.emit(workflows[0]) expect(toastInfoSpy).toHaveBeenCalled() expect(reloadSpy).toHaveBeenCalled() }) @@ -152,7 +171,7 @@ describe('ConsumptionTemplatesComponent', () => { let modal: NgbModalRef modalService.activeInstances.subscribe((m) => (modal = m[m.length - 1])) const toastErrorSpy = jest.spyOn(toastService, 'showError') - const deleteSpy = jest.spyOn(consumptionTemplateService, 'delete') + const deleteSpy = jest.spyOn(workflowService, 'delete') const reloadSpy = jest.spyOn(component, 'reload') const deleteButton = fixture.debugElement.queryAll(By.css('button'))[3] diff --git a/src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.ts b/src-ui/src/app/components/manage/workflows/workflows.component.ts similarity index 52% rename from src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.ts rename to src-ui/src/app/components/manage/workflows/workflows.component.ts index 301699abd..293473888 100644 --- a/src-ui/src/app/components/manage/consumption-templates/consumption-templates.component.ts +++ b/src-ui/src/app/components/manage/workflows/workflows.component.ts @@ -1,33 +1,33 @@ import { Component, OnInit } from '@angular/core' -import { ConsumptionTemplateService } from 'src/app/services/rest/consumption-template.service' +import { WorkflowService } from 'src/app/services/rest/workflow.service' import { ComponentWithPermissions } from '../../with-permissions/with-permissions.component' import { Subject, takeUntil } from 'rxjs' -import { ConsumptionTemplate } from 'src/app/data/consumption-template' +import { Workflow } from 'src/app/data/workflow' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { ToastService } from 'src/app/services/toast.service' import { PermissionsService } from 'src/app/services/permissions.service' import { - ConsumptionTemplateEditDialogComponent, - DOCUMENT_SOURCE_OPTIONS, -} from '../../common/edit-dialog/consumption-template-edit-dialog/consumption-template-edit-dialog.component' + WorkflowEditDialogComponent, + WORKFLOW_TYPE_OPTIONS, +} from '../../common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component' import { ConfirmDialogComponent } from '../../common/confirm-dialog/confirm-dialog.component' import { EditDialogMode } from '../../common/edit-dialog/edit-dialog.component' @Component({ - selector: 'pngx-consumption-templates', - templateUrl: './consumption-templates.component.html', - styleUrls: ['./consumption-templates.component.scss'], + selector: 'pngx-workflows', + templateUrl: './workflows.component.html', + styleUrls: ['./workflows.component.scss'], }) -export class ConsumptionTemplatesComponent +export class WorkflowsComponent extends ComponentWithPermissions implements OnInit { - public templates: ConsumptionTemplate[] = [] + public workflows: Workflow[] = [] private unsubscribeNotifier: Subject = new Subject() constructor( - private consumptionTemplateService: ConsumptionTemplateService, + private workflowService: WorkflowService, public permissionsService: PermissionsService, private modalService: NgbModal, private toastService: ToastService @@ -40,68 +40,74 @@ export class ConsumptionTemplatesComponent } reload() { - this.consumptionTemplateService + this.workflowService .listAll() .pipe(takeUntil(this.unsubscribeNotifier)) .subscribe((r) => { - this.templates = r.results + this.workflows = r.results }) } - getSourceList(template: ConsumptionTemplate): string { - return template.sources - .map((id) => DOCUMENT_SOURCE_OPTIONS.find((s) => s.id === id).name) + getTypesList(template: Workflow): string { + return template.triggers + .map( + (trigger) => + WORKFLOW_TYPE_OPTIONS.find((t) => t.id === trigger.type).name + ) .join(', ') } - editTemplate(rule: ConsumptionTemplate) { - const modal = this.modalService.open( - ConsumptionTemplateEditDialogComponent, - { - backdrop: 'static', - size: 'xl', - } - ) - modal.componentInstance.dialogMode = rule + editWorkflow(workflow: Workflow) { + const modal = this.modalService.open(WorkflowEditDialogComponent, { + backdrop: 'static', + size: 'xl', + }) + modal.componentInstance.dialogMode = workflow ? EditDialogMode.EDIT : EditDialogMode.CREATE - modal.componentInstance.object = rule + if (workflow) { + // quick "deep" clone so original doesnt get modified + const clone = Object.assign({}, workflow) + clone.actions = [...workflow.actions] + clone.triggers = [...workflow.triggers] + modal.componentInstance.object = clone + } modal.componentInstance.succeeded .pipe(takeUntil(this.unsubscribeNotifier)) - .subscribe((newTemplate) => { + .subscribe((newWorkflow) => { this.toastService.showInfo( - $localize`Saved template "${newTemplate.name}".` + $localize`Saved workflow "${newWorkflow.name}".` ) - this.consumptionTemplateService.clearCache() + this.workflowService.clearCache() this.reload() }) modal.componentInstance.failed .pipe(takeUntil(this.unsubscribeNotifier)) .subscribe((e) => { - this.toastService.showError($localize`Error saving template.`, e) + this.toastService.showError($localize`Error saving workflow.`, e) }) } - deleteTemplate(rule: ConsumptionTemplate) { + deleteWorkflow(workflow: Workflow) { const modal = this.modalService.open(ConfirmDialogComponent, { backdrop: 'static', }) - modal.componentInstance.title = $localize`Confirm delete template` - modal.componentInstance.messageBold = $localize`This operation will permanently delete this template.` + modal.componentInstance.title = $localize`Confirm delete workflow` + modal.componentInstance.messageBold = $localize`This operation will permanently delete this workflow.` modal.componentInstance.message = $localize`This operation cannot be undone.` modal.componentInstance.btnClass = 'btn-danger' modal.componentInstance.btnCaption = $localize`Proceed` modal.componentInstance.confirmClicked.subscribe(() => { modal.componentInstance.buttonsEnabled = false - this.consumptionTemplateService.delete(rule).subscribe({ + this.workflowService.delete(workflow).subscribe({ next: () => { modal.close() - this.toastService.showInfo($localize`Deleted template`) - this.consumptionTemplateService.clearCache() + this.toastService.showInfo($localize`Deleted workflow`) + this.workflowService.clearCache() this.reload() }, error: (e) => { - this.toastService.showError($localize`Error deleting template.`, e) + this.toastService.showError($localize`Error deleting workflow.`, e) }, }) }) diff --git a/src-ui/src/app/data/consumption-template.ts b/src-ui/src/app/data/workflow-action.ts similarity index 64% rename from src-ui/src/app/data/consumption-template.ts rename to src-ui/src/app/data/workflow-action.ts index cc85712c8..a0da5f03a 100644 --- a/src-ui/src/app/data/consumption-template.ts +++ b/src-ui/src/app/data/workflow-action.ts @@ -1,23 +1,10 @@ import { ObjectWithId } from './object-with-id' -export enum DocumentSource { - ConsumeFolder = 1, - ApiUpload = 2, - MailFetch = 3, +export enum WorkflowActionType { + Assignment = 1, } - -export interface ConsumptionTemplate extends ObjectWithId { - name: string - - order: number - - sources: DocumentSource[] - - filter_filename: string - - filter_path?: string - - filter_mailrule?: number // MailRule.id +export interface WorkflowAction extends ObjectWithId { + type: WorkflowActionType assign_title?: string diff --git a/src-ui/src/app/data/workflow-trigger.ts b/src-ui/src/app/data/workflow-trigger.ts new file mode 100644 index 000000000..3e3bf8cf8 --- /dev/null +++ b/src-ui/src/app/data/workflow-trigger.ts @@ -0,0 +1,37 @@ +import { ObjectWithId } from './object-with-id' + +export enum DocumentSource { + ConsumeFolder = 1, + ApiUpload = 2, + MailFetch = 3, +} + +export enum WorkflowTriggerType { + Consumption = 1, + DocumentAdded = 2, + DocumentUpdated = 3, +} + +export interface WorkflowTrigger extends ObjectWithId { + type: WorkflowTriggerType + + sources?: DocumentSource[] + + filter_filename?: string + + filter_path?: string + + filter_mailrule?: number // MailRule.id + + match?: string + + matching_algorithm?: number + + is_insensitive?: boolean + + filter_has_tags?: number[] // Tag.id[] + + filter_has_correspondent?: number // Correspondent.id + + filter_has_document_type?: number // DocumentType.id +} diff --git a/src-ui/src/app/data/workflow.ts b/src-ui/src/app/data/workflow.ts new file mode 100644 index 000000000..740507a62 --- /dev/null +++ b/src-ui/src/app/data/workflow.ts @@ -0,0 +1,15 @@ +import { ObjectWithId } from './object-with-id' +import { WorkflowAction } from './workflow-action' +import { WorkflowTrigger } from './workflow-trigger' + +export interface Workflow extends ObjectWithId { + name: string + + order: number + + enabled: boolean + + triggers: WorkflowTrigger[] + + actions: WorkflowAction[] +} diff --git a/src-ui/src/app/services/permissions.service.spec.ts b/src-ui/src/app/services/permissions.service.spec.ts index 968082ae9..66276fbbb 100644 --- a/src-ui/src/app/services/permissions.service.spec.ts +++ b/src-ui/src/app/services/permissions.service.spec.ts @@ -252,10 +252,18 @@ describe('PermissionsService', () => { 'view_sharelink', 'change_sharelink', 'delete_sharelink', - 'add_consumptiontemplate', - 'view_consumptiontemplate', - 'change_consumptiontemplate', - 'delete_consumptiontemplate', + 'add_workflow', + 'view_workflow', + 'change_workflow', + 'delete_workflow', + 'add_workflowtrigger', + 'view_workflowtrigger', + 'change_workflowtrigger', + 'delete_workflowtrigger', + 'add_workflowaction', + 'view_workflowaction', + 'change_workflowaction', + 'delete_workflowaction', 'add_customfield', 'view_customfield', 'change_customfield', diff --git a/src-ui/src/app/services/permissions.service.ts b/src-ui/src/app/services/permissions.service.ts index a4e30d57e..3a1b99377 100644 --- a/src-ui/src/app/services/permissions.service.ts +++ b/src-ui/src/app/services/permissions.service.ts @@ -25,8 +25,10 @@ export enum PermissionType { Group = '%s_group', Admin = '%s_logentry', ShareLink = '%s_sharelink', - ConsumptionTemplate = '%s_consumptiontemplate', CustomField = '%s_customfield', + Workflow = '%s_workflow', + WorkflowTrigger = '%s_workflowtrigger', + WorkflowAction = '%s_workflowaction', } @Injectable({ diff --git a/src-ui/src/app/services/rest/consumption-template.service.spec.ts b/src-ui/src/app/services/rest/consumption-template.service.spec.ts deleted file mode 100644 index 920d0575c..000000000 --- a/src-ui/src/app/services/rest/consumption-template.service.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { HttpTestingController } from '@angular/common/http/testing' -import { TestBed } from '@angular/core/testing' -import { Subscription } from 'rxjs' -import { environment } from 'src/environments/environment' -import { commonAbstractPaperlessServiceTests } from './abstract-paperless-service.spec' -import { ConsumptionTemplateService } from './consumption-template.service' -import { - DocumentSource, - ConsumptionTemplate, -} from 'src/app/data/consumption-template' - -let httpTestingController: HttpTestingController -let service: ConsumptionTemplateService -const endpoint = 'consumption_templates' -const templates: ConsumptionTemplate[] = [ - { - name: 'Template 1', - id: 1, - order: 1, - filter_filename: '*test*', - filter_path: null, - sources: [DocumentSource.ApiUpload], - assign_correspondent: 2, - }, - { - name: 'Template 2', - id: 2, - order: 2, - filter_filename: null, - filter_path: '/test/', - sources: [DocumentSource.ConsumeFolder, DocumentSource.ApiUpload], - assign_document_type: 1, - }, -] - -// run common tests -commonAbstractPaperlessServiceTests( - 'consumption_templates', - ConsumptionTemplateService -) - -describe(`Additional service tests for ConsumptionTemplateService`, () => { - it('should reload', () => { - service.reload() - const req = httpTestingController.expectOne( - `${environment.apiBaseUrl}${endpoint}/?page=1&page_size=100000` - ) - req.flush({ - results: templates, - }) - expect(service.allTemplates).toEqual(templates) - }) - - beforeEach(() => { - // Dont need to setup again - - httpTestingController = TestBed.inject(HttpTestingController) - service = TestBed.inject(ConsumptionTemplateService) - }) - - afterEach(() => { - httpTestingController.verify() - }) -}) diff --git a/src-ui/src/app/services/rest/workflow.service.spec.ts b/src-ui/src/app/services/rest/workflow.service.spec.ts new file mode 100644 index 000000000..cdffda3e1 --- /dev/null +++ b/src-ui/src/app/services/rest/workflow.service.spec.ts @@ -0,0 +1,85 @@ +import { HttpTestingController } from '@angular/common/http/testing' +import { TestBed } from '@angular/core/testing' +import { environment } from 'src/environments/environment' +import { commonAbstractPaperlessServiceTests } from './abstract-paperless-service.spec' +import { WorkflowService } from './workflow.service' +import { Workflow } from 'src/app/data/workflow' +import { + DocumentSource, + WorkflowTriggerType, +} from 'src/app/data/workflow-trigger' +import { WorkflowActionType } from 'src/app/data/workflow-action' + +let httpTestingController: HttpTestingController +let service: WorkflowService +const endpoint = 'workflows' +const workflows: Workflow[] = [ + { + name: 'Workflow 1', + id: 1, + order: 1, + enabled: true, + triggers: [ + { + id: 1, + type: WorkflowTriggerType.Consumption, + sources: [DocumentSource.ConsumeFolder], + filter_filename: '*', + }, + ], + actions: [ + { + id: 1, + type: WorkflowActionType.Assignment, + assign_title: 'foo', + }, + ], + }, + { + name: 'Workflow 2', + id: 2, + order: 2, + enabled: true, + triggers: [ + { + id: 2, + type: WorkflowTriggerType.DocumentAdded, + filter_filename: 'foo', + }, + ], + actions: [ + { + id: 2, + type: WorkflowActionType.Assignment, + assign_title: 'bar', + }, + ], + }, +] + +// run common tests +commonAbstractPaperlessServiceTests(endpoint, WorkflowService) + +describe(`Additional service tests for WorkflowService`, () => { + it('should reload', () => { + service.reload() + const req = httpTestingController.expectOne( + `${environment.apiBaseUrl}${endpoint}/?page=1&page_size=100000` + ) + req.flush({ + results: workflows, + }) + expect(service.allWorkflows).toEqual(workflows) + }) + + beforeEach(() => { + // Dont need to setup again + + httpTestingController = TestBed.inject(HttpTestingController) + service = TestBed.inject(WorkflowService) + }) + + afterEach(() => { + httpTestingController.verify() + }) +}) diff --git a/src-ui/src/app/services/rest/consumption-template.service.ts b/src-ui/src/app/services/rest/workflow.service.ts similarity index 56% rename from src-ui/src/app/services/rest/consumption-template.service.ts rename to src-ui/src/app/services/rest/workflow.service.ts index eb932ebf7..0b489bc67 100644 --- a/src-ui/src/app/services/rest/consumption-template.service.ts +++ b/src-ui/src/app/services/rest/workflow.service.ts @@ -1,42 +1,42 @@ import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { tap } from 'rxjs' -import { ConsumptionTemplate } from 'src/app/data/consumption-template' +import { Workflow } from 'src/app/data/workflow' import { AbstractPaperlessService } from './abstract-paperless-service' @Injectable({ providedIn: 'root', }) -export class ConsumptionTemplateService extends AbstractPaperlessService { +export class WorkflowService extends AbstractPaperlessService { loading: boolean constructor(http: HttpClient) { - super(http, 'consumption_templates') + super(http, 'workflows') } public reload() { this.loading = true this.listAll().subscribe((r) => { - this.templates = r.results + this.workflows = r.results this.loading = false }) } - private templates: ConsumptionTemplate[] = [] + private workflows: Workflow[] = [] - public get allTemplates(): ConsumptionTemplate[] { - return this.templates + public get allWorkflows(): Workflow[] { + return this.workflows } - create(o: ConsumptionTemplate) { + create(o: Workflow) { return super.create(o).pipe(tap(() => this.reload())) } - update(o: ConsumptionTemplate) { + update(o: Workflow) { return super.update(o).pipe(tap(() => this.reload())) } - delete(o: ConsumptionTemplate) { + delete(o: Workflow) { return super.delete(o).pipe(tap(() => this.reload())) } } diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss index e128b27fa..c8e8e8d5c 100644 --- a/src-ui/src/styles.scss +++ b/src-ui/src/styles.scss @@ -647,8 +647,6 @@ code { } .accordion { - --bs-accordion-btn-padding-x: 0.75rem; - --bs-accordion-btn-padding-y: 0.375rem; --bs-accordion-btn-bg: var(--bs-light); --bs-accordion-btn-color: var(--bs-primary); --bs-accordion-color: var(--bs-body-color); diff --git a/src/documents/apps.py b/src/documents/apps.py index d681b9a87..7ed006d06 100644 --- a/src/documents/apps.py +++ b/src/documents/apps.py @@ -9,8 +9,11 @@ class DocumentsConfig(AppConfig): def ready(self): from documents.signals import document_consumption_finished + from documents.signals import document_updated from documents.signals.handlers import add_inbox_tags from documents.signals.handlers import add_to_index + from documents.signals.handlers import run_workflow_added + from documents.signals.handlers import run_workflow_updated from documents.signals.handlers import set_correspondent from documents.signals.handlers import set_document_type from documents.signals.handlers import set_log_entry @@ -24,5 +27,7 @@ class DocumentsConfig(AppConfig): document_consumption_finished.connect(set_storage_path) document_consumption_finished.connect(set_log_entry) document_consumption_finished.connect(add_to_index) + document_consumption_finished.connect(run_workflow_added) + document_updated.connect(run_workflow_updated) AppConfig.ready(self) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 5d6fe7f65..11faeea43 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -26,8 +26,7 @@ from documents.data_models import DocumentMetadataOverrides from documents.file_handling import create_source_path_directory from documents.file_handling import generate_unique_filename from documents.loggers import LoggingMixin -from documents.matching import document_matches_template -from documents.models import ConsumptionTemplate +from documents.matching import document_matches_workflow from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -36,6 +35,8 @@ from documents.models import DocumentType from documents.models import FileInfo from documents.models import StoragePath from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowTrigger from documents.parsers import DocumentParser from documents.parsers import ParseError from documents.parsers import get_parser_class_for_mime_type @@ -602,66 +603,71 @@ class Consumer(LoggingMixin): return document - def get_template_overrides( + def get_workflow_overrides( self, input_doc: ConsumableDocument, ) -> DocumentMetadataOverrides: """ - Match consumption templates to a document based on source and - file name filters, path filters or mail rule filter if specified + Get overrides from matching workflows """ overrides = DocumentMetadataOverrides() - for template in ConsumptionTemplate.objects.all().order_by("order"): + for workflow in Workflow.objects.filter(enabled=True).order_by("order"): template_overrides = DocumentMetadataOverrides() - if document_matches_template(input_doc, template): - if template.assign_title is not None: - template_overrides.title = template.assign_title - if template.assign_tags is not None: - template_overrides.tag_ids = [ - tag.pk for tag in template.assign_tags.all() - ] - if template.assign_correspondent is not None: - template_overrides.correspondent_id = ( - template.assign_correspondent.pk + if document_matches_workflow( + input_doc, + workflow, + WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + ): + for action in workflow.actions.all(): + self.log.info( + f"Applying overrides in {action} from {workflow}", ) - if template.assign_document_type is not None: - template_overrides.document_type_id = ( - template.assign_document_type.pk - ) - if template.assign_storage_path is not None: - template_overrides.storage_path_id = template.assign_storage_path.pk - if template.assign_owner is not None: - template_overrides.owner_id = template.assign_owner.pk - if template.assign_view_users is not None: - template_overrides.view_users = [ - user.pk for user in template.assign_view_users.all() - ] - if template.assign_view_groups is not None: - template_overrides.view_groups = [ - group.pk for group in template.assign_view_groups.all() - ] - if template.assign_change_users is not None: - template_overrides.change_users = [ - user.pk for user in template.assign_change_users.all() - ] - if template.assign_change_groups is not None: - template_overrides.change_groups = [ - group.pk for group in template.assign_change_groups.all() - ] - if template.assign_custom_fields is not None: - template_overrides.custom_field_ids = [ - field.pk for field in template.assign_custom_fields.all() - ] + if action.assign_title is not None: + template_overrides.title = action.assign_title + if action.assign_tags is not None: + template_overrides.tag_ids = [ + tag.pk for tag in action.assign_tags.all() + ] + if action.assign_correspondent is not None: + template_overrides.correspondent_id = ( + action.assign_correspondent.pk + ) + if action.assign_document_type is not None: + template_overrides.document_type_id = ( + action.assign_document_type.pk + ) + if action.assign_storage_path is not None: + template_overrides.storage_path_id = ( + action.assign_storage_path.pk + ) + if action.assign_owner is not None: + template_overrides.owner_id = action.assign_owner.pk + if action.assign_view_users is not None: + template_overrides.view_users = [ + user.pk for user in action.assign_view_users.all() + ] + if action.assign_view_groups is not None: + template_overrides.view_groups = [ + group.pk for group in action.assign_view_groups.all() + ] + if action.assign_change_users is not None: + template_overrides.change_users = [ + user.pk for user in action.assign_change_users.all() + ] + if action.assign_change_groups is not None: + template_overrides.change_groups = [ + group.pk for group in action.assign_change_groups.all() + ] + if action.assign_custom_fields is not None: + template_overrides.custom_field_ids = [ + field.pk for field in action.assign_custom_fields.all() + ] - overrides.update(template_overrides) + overrides.update(template_overrides) return overrides def _parse_title_placeholders(self, title: str) -> str: - """ - Consumption template title placeholders can only include items that are - assigned as part of this template (since auto-matching hasnt happened yet) - """ local_added = timezone.localtime(timezone.now()) correspondent_name = ( @@ -680,20 +686,14 @@ class Consumer(LoggingMixin): else None ) - return title.format( - correspondent=correspondent_name, - document_type=doc_type_name, - added=local_added.isoformat(), - added_year=local_added.strftime("%Y"), - added_year_short=local_added.strftime("%y"), - added_month=local_added.strftime("%m"), - added_month_name=local_added.strftime("%B"), - added_month_name_short=local_added.strftime("%b"), - added_day=local_added.strftime("%d"), - owner_username=owner_username, - original_filename=Path(self.filename).stem, - added_time=local_added.strftime("%H:%M"), - ).strip() + return parse_doc_title_w_placeholders( + title, + correspondent_name, + doc_type_name, + owner_username, + local_added, + self.filename, + ) def _store( self, @@ -846,3 +846,47 @@ class Consumer(LoggingMixin): self.log.warning("Script stderr:") for line in stderr_str: self.log.warning(line) + + +def parse_doc_title_w_placeholders( + title: str, + correspondent_name: str, + doc_type_name: str, + owner_username: str, + local_added: datetime.datetime, + original_filename: str, + created: Optional[datetime.datetime] = None, +) -> str: + """ + Available title placeholders for Workflows depend on what has already been assigned, + e.g. for pre-consumption triggers created will not have been parsed yet, but it will + for added / updated triggers + """ + formatting = { + "correspondent": correspondent_name, + "document_type": doc_type_name, + "added": local_added.isoformat(), + "added_year": local_added.strftime("%Y"), + "added_year_short": local_added.strftime("%y"), + "added_month": local_added.strftime("%m"), + "added_month_name": local_added.strftime("%B"), + "added_month_name_short": local_added.strftime("%b"), + "added_day": local_added.strftime("%d"), + "added_time": local_added.strftime("%H:%M"), + "owner_username": owner_username, + "original_filename": Path(original_filename).stem, + } + if created is not None: + formatting.update( + { + "created": created.isoformat(), + "created_year": created.strftime("%Y"), + "created_year_short": created.strftime("%y"), + "created_month": created.strftime("%m"), + "created_month_name": created.strftime("%B"), + "created_month_name_short": created.strftime("%b"), + "created_day": created.strftime("%d"), + "created_time": created.strftime("%H:%M"), + }, + ) + return title.format(**formatting).strip() diff --git a/src/documents/data_models.py b/src/documents/data_models.py index 0d506cd6a..6bf3f4f96 100644 --- a/src/documents/data_models.py +++ b/src/documents/data_models.py @@ -33,21 +33,20 @@ class DocumentMetadataOverrides: def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides": """ Merges two DocumentMetadataOverrides objects such that object B's overrides - are only applied if the property is empty in object A or merged if multiple - are accepted. + are applied to object A or merged if multiple are accepted. The update is an in-place modification of self """ # only if empty - if self.title is None: + if other.title is not None: self.title = other.title - if self.correspondent_id is None: + if other.correspondent_id is not None: self.correspondent_id = other.correspondent_id - if self.document_type_id is None: + if other.document_type_id is not None: self.document_type_id = other.document_type_id - if self.storage_path_id is None: + if other.storage_path_id is not None: self.storage_path_id = other.storage_path_id - if self.owner_id is None: + if other.owner_id is not None: self.owner_id = other.owner_id # merge diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index bd5e322e3..b08b0b208 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -23,7 +23,6 @@ from guardian.models import UserObjectPermission from documents.file_handling import delete_empty_directories from documents.file_handling import generate_filename -from documents.models import ConsumptionTemplate from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -35,6 +34,9 @@ from documents.models import SavedViewFilterRule from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME @@ -285,7 +287,15 @@ class Command(BaseCommand): ) manifest += json.loads( - serializers.serialize("json", ConsumptionTemplate.objects.all()), + serializers.serialize("json", WorkflowTrigger.objects.all()), + ) + + manifest += json.loads( + serializers.serialize("json", WorkflowAction.objects.all()), + ) + + manifest += json.loads( + serializers.serialize("json", Workflow.objects.all()), ) manifest += json.loads( diff --git a/src/documents/matching.py b/src/documents/matching.py index 9c6e11ca7..ec28f80ca 100644 --- a/src/documents/matching.py +++ b/src/documents/matching.py @@ -1,27 +1,35 @@ import logging import re from fnmatch import fnmatch +from typing import Union from documents.classifier import DocumentClassifier from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource -from documents.models import ConsumptionTemplate from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType from documents.models import MatchingModel from documents.models import StoragePath from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowTrigger from documents.permissions import get_objects_for_user_owner_aware logger = logging.getLogger("paperless.matching") -def log_reason(matching_model: MatchingModel, document: Document, reason: str): +def log_reason( + matching_model: Union[MatchingModel, WorkflowTrigger], + document: Document, + reason: str, +): class_name = type(matching_model).__name__ + name = ( + matching_model.name if hasattr(matching_model, "name") else str(matching_model) + ) logger.debug( - f"{class_name} {matching_model.name} matched on document " - f"{document} because {reason}", + f"{class_name} {name} matched on document {document} because {reason}", ) @@ -237,65 +245,182 @@ def _split_match(matching_model): ] -def document_matches_template( +def consumable_document_matches_workflow( document: ConsumableDocument, - template: ConsumptionTemplate, -) -> bool: + trigger: WorkflowTrigger, +) -> tuple[bool, str]: """ - Returns True if the incoming document matches all filters and - settings from the template, False otherwise + Returns True if the ConsumableDocument matches all filters from the workflow trigger, + False otherwise. Includes a reason if doesn't match """ - def log_match_failure(reason: str): - logger.info(f"Document did not match template {template.name}") - logger.debug(reason) + trigger_matched = True + reason = "" - # Document source vs template source - if document.source not in [int(x) for x in list(template.sources)]: - log_match_failure( + # Document source vs trigger source + if document.source not in [int(x) for x in list(trigger.sources)]: + reason = ( f"Document source {document.source.name} not in" - f" {[DocumentSource(int(x)).name for x in template.sources]}", + f" {[DocumentSource(int(x)).name for x in trigger.sources]}", ) - return False + trigger_matched = False - # Document mail rule vs template mail rule + # Document mail rule vs trigger mail rule if ( document.mailrule_id is not None - and template.filter_mailrule is not None - and document.mailrule_id != template.filter_mailrule.pk + and trigger.filter_mailrule is not None + and document.mailrule_id != trigger.filter_mailrule.pk ): - log_match_failure( + reason = ( f"Document mail rule {document.mailrule_id}" - f" != {template.filter_mailrule.pk}", + f" != {trigger.filter_mailrule.pk}", ) - return False + trigger_matched = False - # Document filename vs template filename + # Document filename vs trigger filename if ( - template.filter_filename is not None - and len(template.filter_filename) > 0 + trigger.filter_filename is not None + and len(trigger.filter_filename) > 0 and not fnmatch( document.original_file.name.lower(), - template.filter_filename.lower(), + trigger.filter_filename.lower(), ) ): - log_match_failure( + reason = ( f"Document filename {document.original_file.name} does not match" - f" {template.filter_filename.lower()}", + f" {trigger.filter_filename.lower()}", ) - return False + trigger_matched = False - # Document path vs template path + # Document path vs trigger path if ( - template.filter_path is not None - and len(template.filter_path) > 0 - and not document.original_file.match(template.filter_path) + trigger.filter_path is not None + and len(trigger.filter_path) > 0 + and not document.original_file.match(trigger.filter_path) ): - log_match_failure( + reason = ( f"Document path {document.original_file}" - f" does not match {template.filter_path}", + f" does not match {trigger.filter_path}", ) - return False + trigger_matched = False - logger.info(f"Document matched template {template.name}") - return True + return (trigger_matched, reason) + + +def existing_document_matches_workflow( + document: Document, + trigger: WorkflowTrigger, +) -> tuple[bool, str]: + """ + Returns True if the Document matches all filters from the workflow trigger, + False otherwise. Includes a reason if doesn't match + """ + + trigger_matched = True + reason = "" + + if trigger.matching_algorithm > MatchingModel.MATCH_NONE and not matches( + trigger, + document, + ): + reason = ( + f"Document content matching settings for algorithm '{trigger.matching_algorithm}' did not match", + ) + trigger_matched = False + + # Document tags vs trigger has_tags + if ( + trigger.filter_has_tags.all().count() > 0 + and document.tags.filter( + id__in=trigger.filter_has_tags.all().values_list("id"), + ).count() + == 0 + ): + reason = ( + f"Document tags {document.tags.all()} do not include" + f" {trigger.filter_has_tags.all()}", + ) + trigger_matched = False + + # Document correpondent vs trigger has_correspondent + if ( + trigger.filter_has_correspondent is not None + and document.correspondent != trigger.filter_has_correspondent + ): + reason = ( + f"Document correspondent {document.correspondent} does not match {trigger.filter_has_correspondent}", + ) + trigger_matched = False + + # Document document_type vs trigger has_document_type + if ( + trigger.filter_has_document_type is not None + and document.document_type != trigger.filter_has_document_type + ): + reason = ( + f"Document doc type {document.document_type} does not match {trigger.filter_has_document_type}", + ) + trigger_matched = False + + # Document original_filename vs trigger filename + if ( + trigger.filter_filename is not None + and len(trigger.filter_filename) > 0 + and document.original_filename is not None + and not fnmatch( + document.original_filename.lower(), + trigger.filter_filename.lower(), + ) + ): + reason = ( + f"Document filename {document.original_filename} does not match" + f" {trigger.filter_filename.lower()}", + ) + trigger_matched = False + + return (trigger_matched, reason) + + +def document_matches_workflow( + document: Union[ConsumableDocument, Document], + workflow: Workflow, + trigger_type: WorkflowTrigger.WorkflowTriggerType, +) -> bool: + """ + Returns True if the ConsumableDocument or Document matches all filters and + settings from the workflow trigger, False otherwise + """ + + trigger_matched = True + if workflow.triggers.filter(type=trigger_type).count() == 0: + trigger_matched = False + logger.info(f"Document did not match {workflow}") + logger.debug(f"No matching triggers with type {trigger_type} found") + else: + for trigger in workflow.triggers.filter(type=trigger_type): + if trigger_type == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION: + trigger_matched, reason = consumable_document_matches_workflow( + document, + trigger, + ) + elif ( + trigger_type == WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED + or trigger_type == WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED + ): + trigger_matched, reason = existing_document_matches_workflow( + document, + trigger, + ) + else: + # New trigger types need to be explicitly checked above + raise Exception(f"Trigger type {trigger_type} not yet supported") + + if trigger_matched: + logger.info(f"Document matched {trigger} from {workflow}") + # matched, bail early + return True + else: + logger.info(f"Document did not match {workflow}") + logger.debug(reason) + + return trigger_matched diff --git a/src/documents/migrations/1044_workflow_workflowaction_workflowtrigger_and_more.py b/src/documents/migrations/1044_workflow_workflowaction_workflowtrigger_and_more.py new file mode 100644 index 000000000..521de61b8 --- /dev/null +++ b/src/documents/migrations/1044_workflow_workflowaction_workflowtrigger_and_more.py @@ -0,0 +1,513 @@ +# Generated by Django 4.2.7 on 2023-12-23 22:51 + +import django.db.models.deletion +import multiselectfield.db.fields +from django.conf import settings +from django.contrib.auth.management import create_permissions +from django.contrib.auth.models import Group +from django.contrib.auth.models import Permission +from django.contrib.auth.models import User +from django.db import migrations +from django.db import models +from django.db import transaction +from django.db.models import Q + +from documents.models import Correspondent +from documents.models import CustomField +from documents.models import DocumentType +from documents.models import StoragePath +from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger +from paperless_mail.models import MailRule + + +def add_workflow_permissions(apps, schema_editor): + # create permissions without waiting for post_migrate signal + for app_config in apps.get_app_configs(): + app_config.models_module = True + create_permissions(app_config, apps=apps, verbosity=0) + app_config.models_module = None + + add_permission = Permission.objects.get(codename="add_document") + workflow_permissions = Permission.objects.filter( + codename__contains="workflow", + ) + + for user in User.objects.filter(Q(user_permissions=add_permission)).distinct(): + user.user_permissions.add(*workflow_permissions) + + for group in Group.objects.filter(Q(permissions=add_permission)).distinct(): + group.permissions.add(*workflow_permissions) + + +def remove_workflow_permissions(apps, schema_editor): + workflow_permissions = Permission.objects.filter( + codename__contains="workflow", + ) + + for user in User.objects.all(): + user.user_permissions.remove(*workflow_permissions) + + for group in Group.objects.all(): + group.permissions.remove(*workflow_permissions) + + +def migrate_consumption_templates(apps, schema_editor): + """ + Migrate consumption templates to workflows. At this point ConsumptionTemplate still exists + but objects are not returned as their true model so we have to manually do that + """ + model_name = "ConsumptionTemplate" + app_name = "documents" + + ConsumptionTemplate = apps.get_model(app_label=app_name, model_name=model_name) + + with transaction.atomic(): + for template in ConsumptionTemplate.objects.all(): + trigger = WorkflowTrigger( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=template.sources, + filter_path=template.filter_path, + filter_filename=template.filter_filename, + ) + if template.filter_mailrule is not None: + trigger.filter_mailrule = MailRule.objects.get( + id=template.filter_mailrule.id, + ) + trigger.save() + + action = WorkflowAction.objects.create( + assign_title=template.assign_title, + ) + if template.assign_document_type is not None: + action.assign_document_type = DocumentType.objects.get( + id=template.assign_document_type.id, + ) + if template.assign_correspondent is not None: + action.assign_correspondent = Correspondent.objects.get( + id=template.assign_correspondent.id, + ) + if template.assign_storage_path is not None: + action.assign_storage_path = StoragePath.objects.get( + id=template.assign_storage_path.id, + ) + if template.assign_owner is not None: + action.assign_owner = User.objects.get(id=template.assign_owner.id) + if template.assign_tags is not None: + action.assign_tags.set( + Tag.objects.filter( + id__in=[t.id for t in template.assign_tags.all()], + ).all(), + ) + if template.assign_view_users is not None: + action.assign_view_users.set( + User.objects.filter( + id__in=[u.id for u in template.assign_view_users.all()], + ).all(), + ) + if template.assign_view_groups is not None: + action.assign_view_groups.set( + Group.objects.filter( + id__in=[g.id for g in template.assign_view_groups.all()], + ).all(), + ) + if template.assign_change_users is not None: + action.assign_change_users.set( + User.objects.filter( + id__in=[u.id for u in template.assign_change_users.all()], + ).all(), + ) + if template.assign_change_groups is not None: + action.assign_change_groups.set( + Group.objects.filter( + id__in=[g.id for g in template.assign_change_groups.all()], + ).all(), + ) + if template.assign_custom_fields is not None: + action.assign_custom_fields.set( + CustomField.objects.filter( + id__in=[cf.id for cf in template.assign_custom_fields.all()], + ).all(), + ) + action.save() + + workflow = Workflow.objects.create( + name=template.name, + order=template.order, + ) + workflow.triggers.set([trigger]) + workflow.actions.set([action]) + workflow.save() + + +def unmigrate_consumption_templates(apps, schema_editor): + model_name = "ConsumptionTemplate" + app_name = "documents" + + ConsumptionTemplate = apps.get_model(app_label=app_name, model_name=model_name) + + for workflow in Workflow.objects.all(): + template = ConsumptionTemplate.objects.create( + name=workflow.name, + order=workflow.order, + sources=workflow.triggers.first().sources, + filter_path=workflow.triggers.first().filter_path, + filter_filename=workflow.triggers.first().filter_filename, + filter_mailrule=workflow.triggers.first().filter_mailrule, + assign_title=workflow.actions.first().assign_title, + assign_document_type=workflow.actions.first().assign_document_type, + assign_correspondent=workflow.actions.first().assign_correspondent, + assign_storage_path=workflow.actions.first().assign_storage_path, + assign_owner=workflow.actions.first().assign_owner, + ) + template.assign_tags.set(workflow.actions.first().assign_tags.all()) + template.assign_view_users.set(workflow.actions.first().assign_view_users.all()) + template.assign_view_groups.set( + workflow.actions.first().assign_view_groups.all(), + ) + template.assign_change_users.set( + workflow.actions.first().assign_change_users.all(), + ) + template.assign_change_groups.set( + workflow.actions.first().assign_change_groups.all(), + ) + template.assign_custom_fields.set( + workflow.actions.first().assign_custom_fields.all(), + ) + template.save() + + +def delete_consumption_template_content_type(apps, schema_editor): + with transaction.atomic(): + apps.get_model("contenttypes", "ContentType").objects.filter( + app_label="documents", + model="consumptiontemplate", + ).delete() + + +def undelete_consumption_template_content_type(apps, schema_editor): + apps.get_model("contenttypes", "ContentType").objects.create( + app_label="documents", + model="consumptiontemplate", + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("paperless_mail", "0023_remove_mailrule_filter_attachment_filename_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("auth", "0012_alter_user_first_name_max_length"), + ("documents", "1043_alter_savedviewfilterrule_rule_type"), + ] + + operations = [ + migrations.CreateModel( + name="Workflow", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField(max_length=256, unique=True, verbose_name="name"), + ), + ("order", models.IntegerField(default=0, verbose_name="order")), + ( + "enabled", + models.BooleanField(default=True, verbose_name="enabled"), + ), + ], + ), + migrations.CreateModel( + name="WorkflowAction", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "type", + models.PositiveIntegerField( + choices=[(1, "Assignment")], + default=1, + verbose_name="Workflow Action Type", + ), + ), + ( + "assign_title", + models.CharField( + blank=True, + help_text="Assign a document title, can include some placeholders, see documentation.", + max_length=256, + null=True, + verbose_name="assign title", + ), + ), + ( + "assign_change_groups", + models.ManyToManyField( + blank=True, + related_name="+", + to="auth.group", + verbose_name="grant change permissions to these groups", + ), + ), + ( + "assign_change_users", + models.ManyToManyField( + blank=True, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="grant change permissions to these users", + ), + ), + ( + "assign_correspondent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.correspondent", + verbose_name="assign this correspondent", + ), + ), + ( + "assign_custom_fields", + models.ManyToManyField( + blank=True, + related_name="+", + to="documents.customfield", + verbose_name="assign these custom fields", + ), + ), + ( + "assign_document_type", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.documenttype", + verbose_name="assign this document type", + ), + ), + ( + "assign_owner", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="assign this owner", + ), + ), + ( + "assign_storage_path", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.storagepath", + verbose_name="assign this storage path", + ), + ), + ( + "assign_tags", + models.ManyToManyField( + blank=True, + to="documents.tag", + verbose_name="assign this tag", + ), + ), + ( + "assign_view_groups", + models.ManyToManyField( + blank=True, + related_name="+", + to="auth.group", + verbose_name="grant view permissions to these groups", + ), + ), + ( + "assign_view_users", + models.ManyToManyField( + blank=True, + related_name="+", + to=settings.AUTH_USER_MODEL, + verbose_name="grant view permissions to these users", + ), + ), + ], + options={ + "verbose_name": "workflow action", + "verbose_name_plural": "workflow actions", + }, + ), + migrations.CreateModel( + name="WorkflowTrigger", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "type", + models.PositiveIntegerField( + choices=[ + (1, "Consumption Started"), + (2, "Document Added"), + (3, "Document Updated"), + ], + default=1, + verbose_name="Workflow Trigger Type", + ), + ), + ( + "sources", + multiselectfield.db.fields.MultiSelectField( + choices=[ + (1, "Consume Folder"), + (2, "Api Upload"), + (3, "Mail Fetch"), + ], + default="1,2,3", + max_length=5, + ), + ), + ( + "filter_path", + models.CharField( + blank=True, + help_text="Only consume documents with a path that matches this if specified. Wildcards specified as * are allowed. Case insensitive.", + max_length=256, + null=True, + verbose_name="filter path", + ), + ), + ( + "filter_filename", + models.CharField( + blank=True, + help_text="Only consume documents which entirely match this filename if specified. Wildcards such as *.pdf or *invoice* are allowed. Case insensitive.", + max_length=256, + null=True, + verbose_name="filter filename", + ), + ), + ( + "filter_mailrule", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="paperless_mail.mailrule", + verbose_name="filter documents from this mail rule", + ), + ), + ( + "matching_algorithm", + models.PositiveIntegerField( + choices=[ + (0, "None"), + (1, "Any word"), + (2, "All words"), + (3, "Exact match"), + (4, "Regular expression"), + (5, "Fuzzy word"), + ], + default=0, + verbose_name="matching algorithm", + ), + ), + ( + "match", + models.CharField(blank=True, max_length=256, verbose_name="match"), + ), + ( + "is_insensitive", + models.BooleanField(default=True, verbose_name="is insensitive"), + ), + ( + "filter_has_tags", + models.ManyToManyField( + blank=True, + to="documents.tag", + verbose_name="has these tag(s)", + ), + ), + ( + "filter_has_document_type", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.documenttype", + verbose_name="has this document type", + ), + ), + ( + "filter_has_correspondent", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="documents.correspondent", + verbose_name="has this correspondent", + ), + ), + ], + options={ + "verbose_name": "workflow trigger", + "verbose_name_plural": "workflow triggers", + }, + ), + migrations.RunPython( + add_workflow_permissions, + remove_workflow_permissions, + ), + migrations.AddField( + model_name="workflow", + name="actions", + field=models.ManyToManyField( + related_name="workflows", + to="documents.workflowaction", + verbose_name="actions", + ), + ), + migrations.AddField( + model_name="workflow", + name="triggers", + field=models.ManyToManyField( + related_name="workflows", + to="documents.workflowtrigger", + verbose_name="triggers", + ), + ), + migrations.RunPython( + migrate_consumption_templates, + unmigrate_consumption_templates, + ), + migrations.DeleteModel("ConsumptionTemplate"), + migrations.RunPython( + delete_consumption_template_content_type, + undelete_consumption_template_content_type, + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index d95bf46e1..b943fa2b5 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -888,15 +888,31 @@ if settings.AUDIT_LOG_ENABLED: auditlog.register(CustomFieldInstance) -class ConsumptionTemplate(models.Model): +class WorkflowTrigger(models.Model): + class WorkflowTriggerMatching(models.IntegerChoices): + # No auto matching + NONE = MatchingModel.MATCH_NONE, _("None") + ANY = MatchingModel.MATCH_ANY, _("Any word") + ALL = MatchingModel.MATCH_ALL, _("All words") + LITERAL = MatchingModel.MATCH_LITERAL, _("Exact match") + REGEX = MatchingModel.MATCH_REGEX, _("Regular expression") + FUZZY = MatchingModel.MATCH_FUZZY, _("Fuzzy word") + + class WorkflowTriggerType(models.IntegerChoices): + CONSUMPTION = 1, _("Consumption Started") + DOCUMENT_ADDED = 2, _("Document Added") + DOCUMENT_UPDATED = 3, _("Document Updated") + class DocumentSourceChoices(models.IntegerChoices): CONSUME_FOLDER = DocumentSource.ConsumeFolder.value, _("Consume Folder") API_UPLOAD = DocumentSource.ApiUpload.value, _("Api Upload") MAIL_FETCH = DocumentSource.MailFetch.value, _("Mail Fetch") - name = models.CharField(_("name"), max_length=256, unique=True) - - order = models.IntegerField(_("order"), default=0) + type = models.PositiveIntegerField( + _("Workflow Trigger Type"), + choices=WorkflowTriggerType.choices, + default=WorkflowTriggerType.CONSUMPTION, + ) sources = MultiSelectField( max_length=5, @@ -936,6 +952,56 @@ class ConsumptionTemplate(models.Model): verbose_name=_("filter documents from this mail rule"), ) + match = models.CharField(_("match"), max_length=256, blank=True) + + matching_algorithm = models.PositiveIntegerField( + _("matching algorithm"), + choices=WorkflowTriggerMatching.choices, + default=WorkflowTriggerMatching.NONE, + ) + + is_insensitive = models.BooleanField(_("is insensitive"), default=True) + + filter_has_tags = models.ManyToManyField( + Tag, + blank=True, + verbose_name=_("has these tag(s)"), + ) + + filter_has_document_type = models.ForeignKey( + DocumentType, + null=True, + blank=True, + on_delete=models.SET_NULL, + verbose_name=_("has this document type"), + ) + + filter_has_correspondent = models.ForeignKey( + Correspondent, + null=True, + blank=True, + on_delete=models.SET_NULL, + verbose_name=_("has this correspondent"), + ) + + class Meta: + verbose_name = _("workflow trigger") + verbose_name_plural = _("workflow triggers") + + def __str__(self): + return f"WorkflowTrigger {self.pk}" + + +class WorkflowAction(models.Model): + class WorkflowActionType(models.IntegerChoices): + ASSIGNMENT = 1, _("Assignment") + + type = models.PositiveIntegerField( + _("Workflow Action Type"), + choices=WorkflowActionType.choices, + default=WorkflowActionType.ASSIGNMENT, + ) + assign_title = models.CharField( _("assign title"), max_length=256, @@ -1022,8 +1088,33 @@ class ConsumptionTemplate(models.Model): ) class Meta: - verbose_name = _("consumption template") - verbose_name_plural = _("consumption templates") + verbose_name = _("workflow action") + verbose_name_plural = _("workflow actions") def __str__(self): - return f"{self.name}" + return f"WorkflowAction {self.pk}" + + +class Workflow(models.Model): + name = models.CharField(_("name"), max_length=256, unique=True) + + order = models.IntegerField(_("order"), default=0) + + triggers = models.ManyToManyField( + WorkflowTrigger, + related_name="workflows", + blank=False, + verbose_name=_("triggers"), + ) + + actions = models.ManyToManyField( + WorkflowAction, + related_name="workflows", + blank=False, + verbose_name=_("actions"), + ) + + enabled = models.BooleanField(_("enabled"), default=True) + + def __str__(self): + return f"Workflow: {self.name}" diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index c65d4d2ff..b1dd9aee9 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -27,7 +27,6 @@ from rest_framework.fields import SerializerMethodField from documents import bulk_edit from documents.data_models import DocumentSource -from documents.models import ConsumptionTemplate from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -41,6 +40,9 @@ from documents.models import ShareLink from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger from documents.parsers import is_mime_type_supported from documents.permissions import get_groups_with_only_permission from documents.permissions import set_permissions_for_object @@ -1278,43 +1280,38 @@ class BulkEditObjectPermissionsSerializer(serializers.Serializer, SetPermissions return attrs -class ConsumptionTemplateSerializer(serializers.ModelSerializer): - order = serializers.IntegerField(required=False) +class WorkflowTriggerSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False, allow_null=True) sources = fields.MultipleChoiceField( - choices=ConsumptionTemplate.DocumentSourceChoices.choices, - allow_empty=False, + choices=WorkflowTrigger.DocumentSourceChoices.choices, + allow_empty=True, default={ DocumentSource.ConsumeFolder, DocumentSource.ApiUpload, DocumentSource.MailFetch, }, ) - assign_correspondent = CorrespondentField(allow_null=True, required=False) - assign_tags = TagsField(many=True, allow_null=True, required=False) - assign_document_type = DocumentTypeField(allow_null=True, required=False) - assign_storage_path = StoragePathField(allow_null=True, required=False) + + type = serializers.ChoiceField( + choices=WorkflowTrigger.WorkflowTriggerType.choices, + label="Trigger Type", + ) class Meta: - model = ConsumptionTemplate + model = WorkflowTrigger fields = [ "id", - "name", - "order", "sources", + "type", "filter_path", "filter_filename", "filter_mailrule", - "assign_title", - "assign_tags", - "assign_correspondent", - "assign_document_type", - "assign_storage_path", - "assign_owner", - "assign_view_users", - "assign_view_groups", - "assign_change_users", - "assign_change_groups", - "assign_custom_fields", + "matching_algorithm", + "match", + "is_insensitive", + "filter_has_tags", + "filter_has_correspondent", + "filter_has_document_type", ] def validate(self, attrs): @@ -1322,12 +1319,6 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer): attrs["sources"] = {DocumentSource.MailFetch.value} # Empty strings treated as None to avoid unexpected behavior - if ( - "assign_title" in attrs - and attrs["assign_title"] is not None - and len(attrs["assign_title"]) == 0 - ): - attrs["assign_title"] = None if ( "filter_filename" in attrs and attrs["filter_filename"] is not None @@ -1342,7 +1333,8 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer): attrs["filter_path"] = None if ( - "filter_mailrule" not in attrs + attrs["type"] == WorkflowTrigger.WorkflowTriggerType.CONSUMPTION + and "filter_mailrule" not in attrs and ("filter_filename" not in attrs or attrs["filter_filename"] is None) and ("filter_path" not in attrs or attrs["filter_path"] is None) ): @@ -1351,3 +1343,144 @@ class ConsumptionTemplateSerializer(serializers.ModelSerializer): ) return attrs + + +class WorkflowActionSerializer(serializers.ModelSerializer): + id = serializers.IntegerField(required=False, allow_null=True) + assign_correspondent = CorrespondentField(allow_null=True, required=False) + assign_tags = TagsField(many=True, allow_null=True, required=False) + assign_document_type = DocumentTypeField(allow_null=True, required=False) + assign_storage_path = StoragePathField(allow_null=True, required=False) + + class Meta: + model = WorkflowAction + fields = [ + "id", + "type", + "assign_title", + "assign_tags", + "assign_correspondent", + "assign_document_type", + "assign_storage_path", + "assign_owner", + "assign_view_users", + "assign_view_groups", + "assign_change_users", + "assign_change_groups", + "assign_custom_fields", + ] + + def validate(self, attrs): + # Empty strings treated as None to avoid unexpected behavior + if ( + "assign_title" in attrs + and attrs["assign_title"] is not None + and len(attrs["assign_title"]) == 0 + ): + attrs["assign_title"] = None + + return attrs + + +class WorkflowSerializer(serializers.ModelSerializer): + order = serializers.IntegerField(required=False) + + triggers = WorkflowTriggerSerializer(many=True) + actions = WorkflowActionSerializer(many=True) + + class Meta: + model = Workflow + fields = [ + "id", + "name", + "order", + "enabled", + "triggers", + "actions", + ] + + def update_triggers_and_actions(self, instance: Workflow, triggers, actions): + set_triggers = [] + set_actions = [] + + if triggers is not None: + for trigger in triggers: + filter_has_tags = trigger.pop("filter_has_tags", None) + trigger_instance, _ = WorkflowTrigger.objects.update_or_create( + id=trigger["id"] if "id" in trigger else None, + defaults=trigger, + ) + if filter_has_tags is not None: + trigger_instance.filter_has_tags.set(filter_has_tags) + set_triggers.append(trigger_instance) + + if actions is not None: + for action in actions: + assign_tags = action.pop("assign_tags", None) + assign_view_users = action.pop("assign_view_users", None) + assign_view_groups = action.pop("assign_view_groups", None) + assign_change_users = action.pop("assign_change_users", None) + assign_change_groups = action.pop("assign_change_groups", None) + assign_custom_fields = action.pop("assign_custom_fields", None) + action_instance, _ = WorkflowAction.objects.update_or_create( + id=action["id"] if "id" in action else None, + defaults=action, + ) + if assign_tags is not None: + action_instance.assign_tags.set(assign_tags) + if assign_view_users is not None: + action_instance.assign_view_users.set(assign_view_users) + if assign_view_groups is not None: + action_instance.assign_view_groups.set(assign_view_groups) + if assign_change_users is not None: + action_instance.assign_change_users.set(assign_change_users) + if assign_change_groups is not None: + action_instance.assign_change_groups.set(assign_change_groups) + if assign_custom_fields is not None: + action_instance.assign_custom_fields.set(assign_custom_fields) + set_actions.append(action_instance) + + instance.triggers.set(set_triggers) + instance.actions.set(set_actions) + instance.save() + + def prune_triggers_and_actions(self): + """ + ManyToMany fields dont support e.g. on_delete so we need to discard unattached + triggers and actionas manually + """ + for trigger in WorkflowTrigger.objects.all(): + if trigger.workflows.all().count() == 0: + trigger.delete() + + for action in WorkflowAction.objects.all(): + if action.workflows.all().count() == 0: + action.delete() + + def create(self, validated_data) -> Workflow: + if "triggers" in validated_data: + triggers = validated_data.pop("triggers") + + if "actions" in validated_data: + actions = validated_data.pop("actions") + + instance = super().create(validated_data) + + self.update_triggers_and_actions(instance, triggers, actions) + + return instance + + def update(self, instance: Workflow, validated_data) -> Workflow: + if "triggers" in validated_data: + triggers = validated_data.pop("triggers") + + if "actions" in validated_data: + actions = validated_data.pop("actions") + + instance = super().update(instance, validated_data) + + self.update_triggers_and_actions(instance, triggers, actions) + + self.prune_triggers_and_actions() + + return instance diff --git a/src/documents/signals/__init__.py b/src/documents/signals/__init__.py index 393630008..fbb55d9fe 100644 --- a/src/documents/signals/__init__.py +++ b/src/documents/signals/__init__.py @@ -3,3 +3,4 @@ from django.dispatch import Signal document_consumption_started = Signal() document_consumption_finished = Signal() document_consumer_declaration = Signal() +document_updated = Signal() diff --git a/src/documents/signals/handlers.py b/src/documents/signals/handlers.py index 117e3c38d..d536a3967 100644 --- a/src/documents/signals/handlers.py +++ b/src/documents/signals/handlers.py @@ -24,14 +24,19 @@ from filelock import FileLock from documents import matching from documents.classifier import DocumentClassifier +from documents.consumer import parse_doc_title_w_placeholders from documents.file_handling import create_source_path_directory from documents.file_handling import delete_empty_directories from documents.file_handling import generate_unique_filename +from documents.models import CustomFieldInstance from documents.models import Document from documents.models import MatchingModel from documents.models import PaperlessTask from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowTrigger from documents.permissions import get_objects_for_user_owner_aware +from documents.permissions import set_permissions_for_object logger = logging.getLogger("paperless.handlers") @@ -514,6 +519,105 @@ def add_to_index(sender, document, **kwargs): index.add_or_update_document(document) +def run_workflow_added(sender, document: Document, logging_group=None, **kwargs): + run_workflow( + WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + document, + logging_group, + ) + + +def run_workflow_updated(sender, document: Document, logging_group=None, **kwargs): + run_workflow( + WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED, + document, + logging_group, + ) + + +def run_workflow( + trigger_type: WorkflowTrigger.WorkflowTriggerType, + document: Document, + logging_group=None, +): + for workflow in Workflow.objects.filter( + enabled=True, + triggers__type=trigger_type, + ).order_by("order"): + if matching.document_matches_workflow( + document, + workflow, + trigger_type, + ): + for action in workflow.actions.all(): + logger.info( + f"Applying {action} from {workflow}", + extra={"group": logging_group}, + ) + if action.assign_tags.all().count() > 0: + document.tags.add(*action.assign_tags.all()) + + if action.assign_correspondent is not None: + document.correspondent = action.assign_correspondent + + if action.assign_document_type is not None: + document.document_type = action.assign_document_type + + if action.assign_storage_path is not None: + document.storage_path = action.assign_storage_path + + if action.assign_owner is not None: + document.owner = action.assign_owner + + if action.assign_title is not None: + document.title = parse_doc_title_w_placeholders( + action.assign_title, + document.correspondent.name + if document.correspondent is not None + else "", + document.document_type.name + if document.document_type is not None + else "", + document.owner.username if document.owner is not None else "", + document.added, + document.original_filename, + document.created, + ) + + if ( + action.assign_view_users is not None + or action.assign_view_groups is not None + or action.assign_change_users is not None + or action.assign_change_groups is not None + ): + permissions = { + "view": { + "users": action.assign_view_users.all().values_list("id") + or [], + "groups": action.assign_view_groups.all().values_list("id") + or [], + }, + "change": { + "users": action.assign_change_users.all().values_list("id") + or [], + "groups": action.assign_change_groups.all().values_list( + "id", + ) + or [], + }, + } + set_permissions_for_object(permissions=permissions, object=document) + + if action.assign_custom_fields is not None: + for field in action.assign_custom_fields.all(): + CustomFieldInstance.objects.create( + field=field, + document=document, + ) # adds to document + + document.save() + + @before_task_publish.connect def before_task_publish_handler(sender=None, headers=None, body=None, **kwargs): """ diff --git a/src/documents/tasks.py b/src/documents/tasks.py index d0728a719..19e40db5b 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -36,6 +36,7 @@ from documents.models import Tag from documents.parsers import DocumentParser from documents.parsers import get_parser_class_for_mime_type from documents.sanity_checker import SanityCheckFailedException +from documents.signals import document_updated if settings.AUDIT_LOG_ENABLED: import json @@ -157,7 +158,7 @@ def consume_file( overrides.asn = reader.asn logger.info(f"Found ASN in barcode: {overrides.asn}") - template_overrides = Consumer().get_template_overrides( + template_overrides = Consumer().get_workflow_overrides( input_doc=input_doc, ) @@ -215,6 +216,11 @@ def bulk_update_documents(document_ids): ix = index.open_index() for doc in documents: + document_updated.send( + sender=None, + document=doc, + logging_group=uuid.uuid4(), + ) post_save.send(Document, instance=doc, created=False) with AsyncWriter(ix) as writer: diff --git a/src/documents/tests/test_api_consumption_templates.py b/src/documents/tests/test_api_consumption_templates.py deleted file mode 100644 index e32294050..000000000 --- a/src/documents/tests/test_api_consumption_templates.py +++ /dev/null @@ -1,236 +0,0 @@ -import json - -from django.contrib.auth.models import Group -from django.contrib.auth.models import User -from rest_framework import status -from rest_framework.test import APITestCase - -from documents.data_models import DocumentSource -from documents.models import ConsumptionTemplate -from documents.models import Correspondent -from documents.models import CustomField -from documents.models import DocumentType -from documents.models import StoragePath -from documents.models import Tag -from documents.tests.utils import DirectoriesMixin -from paperless_mail.models import MailAccount -from paperless_mail.models import MailRule - - -class TestApiConsumptionTemplates(DirectoriesMixin, APITestCase): - ENDPOINT = "/api/consumption_templates/" - - def setUp(self) -> None: - super().setUp() - - user = User.objects.create_superuser(username="temp_admin") - self.client.force_authenticate(user=user) - self.user2 = User.objects.create(username="user2") - self.user3 = User.objects.create(username="user3") - self.group1 = Group.objects.create(name="group1") - - self.c = Correspondent.objects.create(name="Correspondent Name") - self.c2 = Correspondent.objects.create(name="Correspondent Name 2") - self.dt = DocumentType.objects.create(name="DocType Name") - self.t1 = Tag.objects.create(name="t1") - self.t2 = Tag.objects.create(name="t2") - self.t3 = Tag.objects.create(name="t3") - self.sp = StoragePath.objects.create(path="/test/") - self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string") - self.cf2 = CustomField.objects.create( - name="Custom Field 2", - data_type="integer", - ) - - self.ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{int(DocumentSource.ApiUpload)},{int(DocumentSource.ConsumeFolder)},{int(DocumentSource.MailFetch)}", - filter_filename="*simple*", - filter_path="*/samples/*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - self.ct.assign_tags.add(self.t1) - self.ct.assign_tags.add(self.t2) - self.ct.assign_tags.add(self.t3) - self.ct.assign_view_users.add(self.user3.pk) - self.ct.assign_view_groups.add(self.group1.pk) - self.ct.assign_change_users.add(self.user3.pk) - self.ct.assign_change_groups.add(self.group1.pk) - self.ct.assign_custom_fields.add(self.cf1.pk) - self.ct.assign_custom_fields.add(self.cf2.pk) - self.ct.save() - - def test_api_get_consumption_template(self): - """ - GIVEN: - - API request to get all consumption template - WHEN: - - API is called - THEN: - - Existing consumption templates are returned - """ - response = self.client.get(self.ENDPOINT, format="json") - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["count"], 1) - - resp_consumption_template = response.data["results"][0] - self.assertEqual(resp_consumption_template["id"], self.ct.id) - self.assertEqual( - resp_consumption_template["assign_correspondent"], - self.ct.assign_correspondent.pk, - ) - - def test_api_create_consumption_template(self): - """ - GIVEN: - - API request to create a consumption template - WHEN: - - API is called - THEN: - - Correct HTTP response - - New template is created - """ - response = self.client.post( - self.ENDPOINT, - json.dumps( - { - "name": "Template 2", - "order": 1, - "sources": [DocumentSource.ApiUpload], - "filter_filename": "*test*", - }, - ), - content_type="application/json", - ) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConsumptionTemplate.objects.count(), 2) - - def test_api_create_invalid_consumption_template(self): - """ - GIVEN: - - API request to create a consumption template - - Neither file name nor path filter are specified - WHEN: - - API is called - THEN: - - Correct HTTP 400 response - - No template is created - """ - response = self.client.post( - self.ENDPOINT, - json.dumps( - { - "name": "Template 2", - "order": 1, - "sources": [DocumentSource.ApiUpload], - }, - ), - content_type="application/json", - ) - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual(ConsumptionTemplate.objects.count(), 1) - - def test_api_create_consumption_template_empty_fields(self): - """ - GIVEN: - - API request to create a consumption template - - Path or filename filter or assign title are empty string - WHEN: - - API is called - THEN: - - Template is created but filter or title assignment is not set if "" - """ - response = self.client.post( - self.ENDPOINT, - json.dumps( - { - "name": "Template 2", - "order": 1, - "sources": [DocumentSource.ApiUpload], - "filter_filename": "*test*", - "filter_path": "", - "assign_title": "", - }, - ), - content_type="application/json", - ) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - ct = ConsumptionTemplate.objects.get(name="Template 2") - self.assertEqual(ct.filter_filename, "*test*") - self.assertIsNone(ct.filter_path) - self.assertIsNone(ct.assign_title) - - response = self.client.post( - self.ENDPOINT, - json.dumps( - { - "name": "Template 3", - "order": 1, - "sources": [DocumentSource.ApiUpload], - "filter_filename": "", - "filter_path": "*/test/*", - }, - ), - content_type="application/json", - ) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - ct2 = ConsumptionTemplate.objects.get(name="Template 3") - self.assertEqual(ct2.filter_path, "*/test/*") - self.assertIsNone(ct2.filter_filename) - - def test_api_create_consumption_template_with_mailrule(self): - """ - GIVEN: - - API request to create a consumption template with a mail rule but no MailFetch source - WHEN: - - API is called - THEN: - - New template is created with MailFetch as source - """ - account1 = MailAccount.objects.create( - name="Email1", - username="username1", - password="password1", - imap_server="server.example.com", - imap_port=443, - imap_security=MailAccount.ImapSecurity.SSL, - character_set="UTF-8", - ) - rule1 = MailRule.objects.create( - name="Rule1", - account=account1, - folder="INBOX", - filter_from="from@example.com", - filter_to="someone@somewhere.com", - filter_subject="subject", - filter_body="body", - filter_attachment_filename_include="file.pdf", - maximum_age=30, - action=MailRule.MailAction.MARK_READ, - assign_title_from=MailRule.TitleSource.FROM_SUBJECT, - assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING, - order=0, - attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY, - ) - response = self.client.post( - self.ENDPOINT, - json.dumps( - { - "name": "Template 2", - "order": 1, - "sources": [DocumentSource.ApiUpload], - "filter_mailrule": rule1.pk, - }, - ), - content_type="application/json", - ) - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(ConsumptionTemplate.objects.count(), 2) - ct = ConsumptionTemplate.objects.get(name="Template 2") - self.assertEqual(ct.sources, [int(DocumentSource.MailFetch).__str__()]) diff --git a/src/documents/tests/test_api_workflows.py b/src/documents/tests/test_api_workflows.py new file mode 100644 index 000000000..d7a7ad6ff --- /dev/null +++ b/src/documents/tests/test_api_workflows.py @@ -0,0 +1,435 @@ +import json + +from django.contrib.auth.models import Group +from django.contrib.auth.models import User +from rest_framework import status +from rest_framework.test import APITestCase + +from documents.data_models import DocumentSource +from documents.models import Correspondent +from documents.models import CustomField +from documents.models import DocumentType +from documents.models import StoragePath +from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger +from documents.tests.utils import DirectoriesMixin +from paperless_mail.models import MailAccount +from paperless_mail.models import MailRule + + +class TestApiWorkflows(DirectoriesMixin, APITestCase): + ENDPOINT = "/api/workflows/" + ENDPOINT_TRIGGERS = "/api/workflow_triggers/" + ENDPOINT_ACTIONS = "/api/workflow_actions/" + + def setUp(self) -> None: + super().setUp() + + user = User.objects.create_superuser(username="temp_admin") + self.client.force_authenticate(user=user) + self.user2 = User.objects.create(username="user2") + self.user3 = User.objects.create(username="user3") + self.group1 = Group.objects.create(name="group1") + + self.c = Correspondent.objects.create(name="Correspondent Name") + self.c2 = Correspondent.objects.create(name="Correspondent Name 2") + self.dt = DocumentType.objects.create(name="DocType Name") + self.dt2 = DocumentType.objects.create(name="DocType Name 2") + self.t1 = Tag.objects.create(name="t1") + self.t2 = Tag.objects.create(name="t2") + self.t3 = Tag.objects.create(name="t3") + self.sp = StoragePath.objects.create(name="Storage Path 1", path="/test/") + self.sp2 = StoragePath.objects.create(name="Storage Path 2", path="/test2/") + self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string") + self.cf2 = CustomField.objects.create( + name="Custom Field 2", + data_type="integer", + ) + + self.trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{int(DocumentSource.ApiUpload)},{int(DocumentSource.ConsumeFolder)},{int(DocumentSource.MailFetch)}", + filter_filename="*simple*", + filter_path="*/samples/*", + ) + self.action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + self.action.assign_tags.add(self.t1) + self.action.assign_tags.add(self.t2) + self.action.assign_tags.add(self.t3) + self.action.assign_view_users.add(self.user3.pk) + self.action.assign_view_groups.add(self.group1.pk) + self.action.assign_change_users.add(self.user3.pk) + self.action.assign_change_groups.add(self.group1.pk) + self.action.assign_custom_fields.add(self.cf1.pk) + self.action.assign_custom_fields.add(self.cf2.pk) + self.action.save() + + self.workflow = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + self.workflow.triggers.add(self.trigger) + self.workflow.actions.add(self.action) + self.workflow.save() + + def test_api_get_workflow(self): + """ + GIVEN: + - API request to get all workflows + WHEN: + - API is called + THEN: + - Existing workflows are returned + """ + response = self.client.get(self.ENDPOINT, format="json") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + + resp_workflow = response.data["results"][0] + self.assertEqual(resp_workflow["id"], self.workflow.id) + self.assertEqual( + resp_workflow["actions"][0]["assign_correspondent"], + self.action.assign_correspondent.pk, + ) + + def test_api_create_workflow(self): + """ + GIVEN: + - API request to create a workflow, trigger and action separately + WHEN: + - API is called + THEN: + - Correct HTTP response + - New workflow, trigger and action are created + """ + trigger_response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "sources": [DocumentSource.ApiUpload], + "filter_filename": "*", + }, + ), + content_type="application/json", + ) + self.assertEqual(trigger_response.status_code, status.HTTP_201_CREATED) + + action_response = self.client.post( + self.ENDPOINT_ACTIONS, + json.dumps( + { + "assign_title": "Action Title", + }, + ), + content_type="application/json", + ) + self.assertEqual(action_response.status_code, status.HTTP_201_CREATED) + + response = self.client.post( + self.ENDPOINT, + json.dumps( + { + "name": "Workflow 2", + "order": 1, + "triggers": [ + { + "id": trigger_response.data["id"], + "sources": [DocumentSource.ApiUpload], + "type": trigger_response.data["type"], + "filter_filename": trigger_response.data["filter_filename"], + }, + ], + "actions": [ + { + "id": action_response.data["id"], + "assign_title": action_response.data["assign_title"], + }, + ], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(Workflow.objects.count(), 2) + + def test_api_create_workflow_nested(self): + """ + GIVEN: + - API request to create a workflow with nested trigger and action + WHEN: + - API is called + THEN: + - Correct HTTP response + - New workflow, trigger and action are created + """ + + response = self.client.post( + self.ENDPOINT, + json.dumps( + { + "name": "Workflow 2", + "order": 1, + "triggers": [ + { + "sources": [DocumentSource.ApiUpload], + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "filter_filename": "*", + "filter_path": "*/samples/*", + "filter_has_tags": [self.t1.id], + "filter_has_document_type": self.dt.id, + "filter_has_correspondent": self.c.id, + }, + ], + "actions": [ + { + "assign_title": "Action Title", + "assign_tags": [self.t2.id], + "assign_document_type": self.dt2.id, + "assign_correspondent": self.c2.id, + "assign_storage_path": self.sp2.id, + "assign_owner": self.user2.id, + "assign_view_users": [self.user2.id], + "assign_view_groups": [self.group1.id], + "assign_change_users": [self.user2.id], + "assign_change_groups": [self.group1.id], + "assign_custom_fields": [self.cf2.id], + }, + ], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(Workflow.objects.count(), 2) + + def test_api_create_invalid_workflow_trigger(self): + """ + GIVEN: + - API request to create a workflow trigger + - Neither type or file name nor path filter are specified + WHEN: + - API is called + THEN: + - Correct HTTP 400 response + - No objects are created + """ + response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "sources": [DocumentSource.ApiUpload], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "sources": [DocumentSource.ApiUpload], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + self.assertEqual(WorkflowTrigger.objects.count(), 1) + + def test_api_create_workflow_trigger_action_empty_fields(self): + """ + GIVEN: + - API request to create a workflow trigger and action + - Path or filename filter or assign title are empty string + WHEN: + - API is called + THEN: + - Template is created but filter or title assignment is not set if "" + """ + response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "sources": [DocumentSource.ApiUpload], + "filter_filename": "*test*", + "filter_path": "", + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + trigger = WorkflowTrigger.objects.get(id=response.data["id"]) + self.assertEqual(trigger.filter_filename, "*test*") + self.assertIsNone(trigger.filter_path) + + response = self.client.post( + self.ENDPOINT_ACTIONS, + json.dumps( + { + "assign_title": "", + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + action = WorkflowAction.objects.get(id=response.data["id"]) + self.assertIsNone(action.assign_title) + + response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "sources": [DocumentSource.ApiUpload], + "filter_filename": "", + "filter_path": "*/test/*", + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + trigger2 = WorkflowTrigger.objects.get(id=response.data["id"]) + self.assertEqual(trigger2.filter_path, "*/test/*") + self.assertIsNone(trigger2.filter_filename) + + def test_api_create_workflow_trigger_with_mailrule(self): + """ + GIVEN: + - API request to create a workflow trigger with a mail rule but no MailFetch source + WHEN: + - API is called + THEN: + - New trigger is created with MailFetch as source + """ + account1 = MailAccount.objects.create( + name="Email1", + username="username1", + password="password1", + imap_server="server.example.com", + imap_port=443, + imap_security=MailAccount.ImapSecurity.SSL, + character_set="UTF-8", + ) + rule1 = MailRule.objects.create( + name="Rule1", + account=account1, + folder="INBOX", + filter_from="from@example.com", + filter_to="someone@somewhere.com", + filter_subject="subject", + filter_body="body", + filter_attachment_filename_include="file.pdf", + maximum_age=30, + action=MailRule.MailAction.MARK_READ, + assign_title_from=MailRule.TitleSource.FROM_SUBJECT, + assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING, + order=0, + attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY, + ) + response = self.client.post( + self.ENDPOINT_TRIGGERS, + json.dumps( + { + "type": WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + "sources": [DocumentSource.ApiUpload], + "filter_mailrule": rule1.pk, + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(WorkflowTrigger.objects.count(), 2) + trigger = WorkflowTrigger.objects.get(id=response.data["id"]) + self.assertEqual(trigger.sources, [int(DocumentSource.MailFetch).__str__()]) + + def test_api_update_workflow_nested_triggers_actions(self): + """ + GIVEN: + - Existing workflow with trigger and action + WHEN: + - API request to update an existing workflow with nested triggers actions + THEN: + - Triggers and actions are updated + """ + + response = self.client.patch( + f"{self.ENDPOINT}{self.workflow.id}/", + json.dumps( + { + "name": "Workflow Updated", + "order": 1, + "triggers": [ + { + "type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + "filter_has_tags": [self.t1.id], + "filter_has_correspondent": self.c.id, + "filter_has_document_type": self.dt.id, + }, + ], + "actions": [ + { + "assign_title": "Action New Title", + }, + ], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + workflow = Workflow.objects.get(id=response.data["id"]) + self.assertEqual(workflow.name, "Workflow Updated") + self.assertEqual(workflow.triggers.first().filter_has_tags.first(), self.t1) + self.assertEqual(workflow.actions.first().assign_title, "Action New Title") + + def test_api_auto_remove_orphaned_triggers_actions(self): + """ + GIVEN: + - Existing trigger and action + WHEN: + - API request is made which creates new trigger / actions + THEN: + - "Orphaned" triggers and actions are removed + """ + + response = self.client.patch( + f"{self.ENDPOINT}{self.workflow.id}/", + json.dumps( + { + "name": "Workflow Updated", + "order": 1, + "triggers": [ + { + "type": WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + "filter_has_tags": [self.t1.id], + "filter_has_correspondent": self.c.id, + "filter_has_document_type": self.dt.id, + }, + ], + "actions": [ + { + "assign_title": "Action New Title", + }, + ], + }, + ), + content_type="application/json", + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + workflow = Workflow.objects.get(id=response.data["id"]) + self.assertEqual(WorkflowTrigger.objects.all().count(), 1) + self.assertNotEqual(workflow.triggers.first().id, self.trigger.id) + self.assertEqual(WorkflowAction.objects.all().count(), 1) + self.assertNotEqual(workflow.actions.first().id, self.action.id) diff --git a/src/documents/tests/test_consumption_templates.py b/src/documents/tests/test_consumption_templates.py deleted file mode 100644 index 6f671bfc4..000000000 --- a/src/documents/tests/test_consumption_templates.py +++ /dev/null @@ -1,539 +0,0 @@ -from pathlib import Path -from unittest import TestCase -from unittest import mock - -import pytest -from django.contrib.auth.models import Group -from django.contrib.auth.models import User - -from documents import tasks -from documents.data_models import ConsumableDocument -from documents.data_models import DocumentSource -from documents.models import ConsumptionTemplate -from documents.models import Correspondent -from documents.models import CustomField -from documents.models import DocumentType -from documents.models import StoragePath -from documents.models import Tag -from documents.tests.utils import DirectoriesMixin -from documents.tests.utils import FileSystemAssertsMixin -from paperless_mail.models import MailAccount -from paperless_mail.models import MailRule - - -@pytest.mark.django_db -class TestConsumptionTemplates(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - SAMPLE_DIR = Path(__file__).parent / "samples" - - def setUp(self) -> None: - self.c = Correspondent.objects.create(name="Correspondent Name") - self.c2 = Correspondent.objects.create(name="Correspondent Name 2") - self.dt = DocumentType.objects.create(name="DocType Name") - self.t1 = Tag.objects.create(name="t1") - self.t2 = Tag.objects.create(name="t2") - self.t3 = Tag.objects.create(name="t3") - self.sp = StoragePath.objects.create(path="/test/") - self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string") - self.cf2 = CustomField.objects.create( - name="Custom Field 2", - data_type="integer", - ) - - self.user2 = User.objects.create(username="user2") - self.user3 = User.objects.create(username="user3") - self.group1 = Group.objects.create(name="group1") - - account1 = MailAccount.objects.create( - name="Email1", - username="username1", - password="password1", - imap_server="server.example.com", - imap_port=443, - imap_security=MailAccount.ImapSecurity.SSL, - character_set="UTF-8", - ) - self.rule1 = MailRule.objects.create( - name="Rule1", - account=account1, - folder="INBOX", - filter_from="from@example.com", - filter_to="someone@somewhere.com", - filter_subject="subject", - filter_body="body", - filter_attachment_filename_include="file.pdf", - maximum_age=30, - action=MailRule.MailAction.MARK_READ, - assign_title_from=MailRule.TitleSource.NONE, - assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING, - order=0, - attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY, - assign_owner_from_rule=False, - ) - - return super().setUp() - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_match(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that matches is consumed - THEN: - - Template overrides are applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_filename="*simple*", - filter_path="*/samples/*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - ct.assign_tags.add(self.t1) - ct.assign_tags.add(self.t2) - ct.assign_tags.add(self.t3) - ct.assign_view_users.add(self.user3.pk) - ct.assign_view_groups.add(self.group1.pk) - ct.assign_change_users.add(self.user3.pk) - ct.assign_change_groups.add(self.group1.pk) - ct.assign_custom_fields.add(self.cf1.pk) - ct.assign_custom_fields.add(self.cf2.pk) - ct.save() - - self.assertEqual(ct.__str__(), "Template 1") - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertEqual(overrides["override_correspondent_id"], self.c.pk) - self.assertEqual(overrides["override_document_type_id"], self.dt.pk) - self.assertEqual( - overrides["override_tag_ids"], - [self.t1.pk, self.t2.pk, self.t3.pk], - ) - self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) - self.assertEqual(overrides["override_owner_id"], self.user2.pk) - self.assertEqual(overrides["override_view_users"], [self.user3.pk]) - self.assertEqual(overrides["override_view_groups"], [self.group1.pk]) - self.assertEqual(overrides["override_change_users"], [self.user3.pk]) - self.assertEqual(overrides["override_change_groups"], [self.group1.pk]) - self.assertEqual( - overrides["override_title"], - "Doc from {correspondent}", - ) - self.assertEqual( - overrides["override_custom_field_ids"], - [self.cf1.pk, self.cf2.pk], - ) - - info = cm.output[0] - expected_str = f"Document matched template {ct}" - self.assertIn(expected_str, info) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_match_mailrule(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that matches is consumed via mail rule - THEN: - - Template overrides are applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_mailrule=self.rule1, - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - ct.assign_tags.add(self.t1) - ct.assign_tags.add(self.t2) - ct.assign_tags.add(self.t3) - ct.assign_view_users.add(self.user3.pk) - ct.assign_view_groups.add(self.group1.pk) - ct.assign_change_users.add(self.user3.pk) - ct.assign_change_groups.add(self.group1.pk) - ct.save() - - self.assertEqual(ct.__str__(), "Template 1") - - test_file = self.SAMPLE_DIR / "simple.pdf" - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - mailrule_id=self.rule1.pk, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertEqual(overrides["override_correspondent_id"], self.c.pk) - self.assertEqual(overrides["override_document_type_id"], self.dt.pk) - self.assertEqual( - overrides["override_tag_ids"], - [self.t1.pk, self.t2.pk, self.t3.pk], - ) - self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) - self.assertEqual(overrides["override_owner_id"], self.user2.pk) - self.assertEqual(overrides["override_view_users"], [self.user3.pk]) - self.assertEqual(overrides["override_view_groups"], [self.group1.pk]) - self.assertEqual(overrides["override_change_users"], [self.user3.pk]) - self.assertEqual(overrides["override_change_groups"], [self.group1.pk]) - self.assertEqual( - overrides["override_title"], - "Doc from {correspondent}", - ) - - info = cm.output[0] - expected_str = f"Document matched template {ct}" - self.assertIn(expected_str, info) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_match_multiple(self, m): - """ - GIVEN: - - Multiple existing consumption template - WHEN: - - File that matches is consumed - THEN: - - Template overrides are applied with subsequent templates only overwriting empty values - or merging if multiple - """ - ct1 = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_path="*/samples/*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - ) - ct1.assign_tags.add(self.t1) - ct1.assign_tags.add(self.t2) - ct1.assign_view_users.add(self.user2) - ct1.save() - ct2 = ConsumptionTemplate.objects.create( - name="Template 2", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_filename="*simple*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c2, - assign_storage_path=self.sp, - ) - ct2.assign_tags.add(self.t3) - ct1.assign_view_users.add(self.user3) - ct2.save() - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - # template 1 - self.assertEqual(overrides["override_correspondent_id"], self.c.pk) - self.assertEqual(overrides["override_document_type_id"], self.dt.pk) - # template 2 - self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) - # template 1 & 2 - self.assertEqual( - overrides["override_tag_ids"], - [self.t1.pk, self.t2.pk, self.t3.pk], - ) - self.assertEqual( - overrides["override_view_users"], - [self.user2.pk, self.user3.pk], - ) - - expected_str = f"Document matched template {ct1}" - self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document matched template {ct2}" - self.assertIn(expected_str, cm.output[1]) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_no_match_filename(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that does not match on filename is consumed - THEN: - - Template overrides are not applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_filename="*foobar*", - filter_path=None, - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertIsNone(overrides["override_correspondent_id"]) - self.assertIsNone(overrides["override_document_type_id"]) - self.assertIsNone(overrides["override_tag_ids"]) - self.assertIsNone(overrides["override_storage_path_id"]) - self.assertIsNone(overrides["override_owner_id"]) - self.assertIsNone(overrides["override_view_users"]) - self.assertIsNone(overrides["override_view_groups"]) - self.assertIsNone(overrides["override_change_users"]) - self.assertIsNone(overrides["override_change_groups"]) - self.assertIsNone(overrides["override_title"]) - - expected_str = f"Document did not match template {ct}" - self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document filename {test_file.name} does not match" - self.assertIn(expected_str, cm.output[1]) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_no_match_path(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that does not match on path is consumed - THEN: - - Template overrides are not applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_path="*foo/bar*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertIsNone(overrides["override_correspondent_id"]) - self.assertIsNone(overrides["override_document_type_id"]) - self.assertIsNone(overrides["override_tag_ids"]) - self.assertIsNone(overrides["override_storage_path_id"]) - self.assertIsNone(overrides["override_owner_id"]) - self.assertIsNone(overrides["override_view_users"]) - self.assertIsNone(overrides["override_view_groups"]) - self.assertIsNone(overrides["override_change_users"]) - self.assertIsNone(overrides["override_change_groups"]) - self.assertIsNone(overrides["override_title"]) - - expected_str = f"Document did not match template {ct}" - self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document path {test_file} does not match" - self.assertIn(expected_str, cm.output[1]) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_no_match_mail_rule(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that does not match on source is consumed - THEN: - - Template overrides are not applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_mailrule=self.rule1, - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - mailrule_id=99, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertIsNone(overrides["override_correspondent_id"]) - self.assertIsNone(overrides["override_document_type_id"]) - self.assertIsNone(overrides["override_tag_ids"]) - self.assertIsNone(overrides["override_storage_path_id"]) - self.assertIsNone(overrides["override_owner_id"]) - self.assertIsNone(overrides["override_view_users"]) - self.assertIsNone(overrides["override_view_groups"]) - self.assertIsNone(overrides["override_change_users"]) - self.assertIsNone(overrides["override_change_groups"]) - self.assertIsNone(overrides["override_title"]) - - expected_str = f"Document did not match template {ct}" - self.assertIn(expected_str, cm.output[0]) - expected_str = "Document mail rule 99 !=" - self.assertIn(expected_str, cm.output[1]) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_no_match_source(self, m): - """ - GIVEN: - - Existing consumption template - WHEN: - - File that does not match on source is consumed - THEN: - - Template overrides are not applied - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_path="*", - assign_title="Doc from {correspondent}", - assign_correspondent=self.c, - assign_document_type=self.dt, - assign_storage_path=self.sp, - assign_owner=self.user2, - ) - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ApiUpload, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertIsNone(overrides["override_correspondent_id"]) - self.assertIsNone(overrides["override_document_type_id"]) - self.assertIsNone(overrides["override_tag_ids"]) - self.assertIsNone(overrides["override_storage_path_id"]) - self.assertIsNone(overrides["override_owner_id"]) - self.assertIsNone(overrides["override_view_users"]) - self.assertIsNone(overrides["override_view_groups"]) - self.assertIsNone(overrides["override_change_users"]) - self.assertIsNone(overrides["override_change_groups"]) - self.assertIsNone(overrides["override_title"]) - - expected_str = f"Document did not match template {ct}" - self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document source {DocumentSource.ApiUpload.name} not in ['{DocumentSource.ConsumeFolder.name}', '{DocumentSource.MailFetch.name}']" - self.assertIn(expected_str, cm.output[1]) - - @mock.patch("documents.consumer.Consumer.try_consume_file") - def test_consumption_template_repeat_custom_fields(self, m): - """ - GIVEN: - - Existing consumption templates which assign the same custom field - WHEN: - - File that matches is consumed - THEN: - - Custom field is added the first time successfully - """ - ct = ConsumptionTemplate.objects.create( - name="Template 1", - order=0, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_filename="*simple*", - ) - ct.assign_custom_fields.add(self.cf1.pk) - ct.save() - - ct2 = ConsumptionTemplate.objects.create( - name="Template 2", - order=1, - sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", - filter_filename="*simple*", - ) - ct2.assign_custom_fields.add(self.cf1.pk) - ct2.save() - - test_file = self.SAMPLE_DIR / "simple.pdf" - - with mock.patch("documents.tasks.async_to_sync"): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - m.assert_called_once() - _, overrides = m.call_args - self.assertEqual( - overrides["override_custom_field_ids"], - [self.cf1.pk], - ) - - expected_str = f"Document matched template {ct}" - self.assertIn(expected_str, cm.output[0]) - expected_str = f"Document matched template {ct2}" - self.assertIn(expected_str, cm.output[1]) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 898dfbc53..a51bd4662 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -21,7 +21,6 @@ from guardian.models import UserObjectPermission from guardian.shortcuts import assign_perm from documents.management.commands import document_exporter -from documents.models import ConsumptionTemplate from documents.models import Correspondent from documents.models import CustomField from documents.models import CustomFieldInstance @@ -31,6 +30,9 @@ from documents.models import Note from documents.models import StoragePath from documents.models import Tag from documents.models import User +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger from documents.sanity_checker import check_sanity from documents.settings import EXPORTER_FILE_NAME from documents.tests.utils import DirectoriesMixin @@ -109,7 +111,16 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.d4.storage_path = self.sp1 self.d4.save() - self.ct1 = ConsumptionTemplate.objects.create(name="CT 1", filter_path="*") + self.trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=[1], + filter_filename="*", + ) + self.action = WorkflowAction.objects.create(assign_title="new title") + self.workflow = Workflow.objects.create(name="Workflow 1", order="0") + self.workflow.triggers.add(self.trigger) + self.workflow.actions.add(self.action) + self.workflow.save() super().setUp() @@ -168,7 +179,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): manifest = self._do_export(use_filename_format=use_filename_format) - self.assertEqual(len(manifest), 178) + self.assertEqual(len(manifest), 190) # dont include consumer or AnonymousUser users self.assertEqual( @@ -262,7 +273,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(Document.objects.get(id=self.d4.id).title, "wow_dec") self.assertEqual(GroupObjectPermission.objects.count(), 1) self.assertEqual(UserObjectPermission.objects.count(), 1) - self.assertEqual(Permission.objects.count(), 128) + self.assertEqual(Permission.objects.count(), 136) messages = check_sanity() # everything is alright after the test self.assertEqual(len(messages), 0) @@ -694,15 +705,15 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): os.path.join(self.dirs.media_dir, "documents"), ) - self.assertEqual(ContentType.objects.count(), 32) - self.assertEqual(Permission.objects.count(), 128) + self.assertEqual(ContentType.objects.count(), 34) + self.assertEqual(Permission.objects.count(), 136) manifest = self._do_export() with paperless_environment(): self.assertEqual( len(list(filter(lambda e: e["model"] == "auth.permission", manifest))), - 128, + 136, ) # add 1 more to db to show objects are not re-created by import Permission.objects.create( @@ -710,7 +721,7 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): codename="test_perm", content_type_id=1, ) - self.assertEqual(Permission.objects.count(), 129) + self.assertEqual(Permission.objects.count(), 137) # will cause an import error self.user.delete() @@ -719,5 +730,5 @@ class TestExportImport(DirectoriesMixin, FileSystemAssertsMixin, TestCase): with self.assertRaises(IntegrityError): call_command("document_importer", "--no-progress-bar", self.target) - self.assertEqual(ContentType.objects.count(), 32) - self.assertEqual(Permission.objects.count(), 129) + self.assertEqual(ContentType.objects.count(), 34) + self.assertEqual(Permission.objects.count(), 137) diff --git a/src/documents/tests/test_migration_consumption_templates.py b/src/documents/tests/test_migration_consumption_templates.py index 3374530a2..917007116 100644 --- a/src/documents/tests/test_migration_consumption_templates.py +++ b/src/documents/tests/test_migration_consumption_templates.py @@ -33,11 +33,18 @@ class TestReverseMigrateConsumptionTemplate(TestMigrations): self.Permission = apps.get_model("auth", "Permission") self.user = User.objects.create(username="user1") self.group = Group.objects.create(name="group1") - permission = self.Permission.objects.get(codename="add_consumptiontemplate") - self.user.user_permissions.add(permission.id) - self.group.permissions.add(permission.id) + permission = self.Permission.objects.filter( + codename="add_consumptiontemplate", + ).first() + if permission is not None: + self.user.user_permissions.add(permission.id) + self.group.permissions.add(permission.id) def test_remove_consumptiontemplate_permissions(self): - permission = self.Permission.objects.get(codename="add_consumptiontemplate") - self.assertFalse(self.user.has_perm(f"documents.{permission.codename}")) - self.assertFalse(permission in self.group.permissions.all()) + permission = self.Permission.objects.filter( + codename="add_consumptiontemplate", + ).first() + # can be None ? now that CTs removed + if permission is not None: + self.assertFalse(self.user.has_perm(f"documents.{permission.codename}")) + self.assertFalse(permission in self.group.permissions.all()) diff --git a/src/documents/tests/test_migration_workflows.py b/src/documents/tests/test_migration_workflows.py new file mode 100644 index 000000000..742757783 --- /dev/null +++ b/src/documents/tests/test_migration_workflows.py @@ -0,0 +1,131 @@ +from documents.data_models import DocumentSource +from documents.tests.utils import TestMigrations + + +class TestMigrateWorkflow(TestMigrations): + migrate_from = "1043_alter_savedviewfilterrule_rule_type" + migrate_to = "1044_workflow_workflowaction_workflowtrigger_and_more" + dependencies = ( + ("paperless_mail", "0023_remove_mailrule_filter_attachment_filename_and_more"), + ) + + def setUpBeforeMigration(self, apps): + User = apps.get_model("auth", "User") + Group = apps.get_model("auth", "Group") + self.Permission = apps.get_model("auth", "Permission") + self.user = User.objects.create(username="user1") + self.group = Group.objects.create(name="group1") + permission = self.Permission.objects.get(codename="add_document") + self.user.user_permissions.add(permission.id) + self.group.permissions.add(permission.id) + + # create a CT to migrate + c = apps.get_model("documents", "Correspondent").objects.create( + name="Correspondent Name", + ) + dt = apps.get_model("documents", "DocumentType").objects.create( + name="DocType Name", + ) + t1 = apps.get_model("documents", "Tag").objects.create(name="t1") + sp = apps.get_model("documents", "StoragePath").objects.create(path="/test/") + cf1 = apps.get_model("documents", "CustomField").objects.create( + name="Custom Field 1", + data_type="string", + ) + ma = apps.get_model("paperless_mail", "MailAccount").objects.create( + name="MailAccount 1", + ) + mr = apps.get_model("paperless_mail", "MailRule").objects.create( + name="MailRule 1", + order=0, + account=ma, + ) + + user2 = User.objects.create(username="user2") + user3 = User.objects.create(username="user3") + group2 = Group.objects.create(name="group2") + + ConsumptionTemplate = apps.get_model("documents", "ConsumptionTemplate") + + ct = ConsumptionTemplate.objects.create( + name="Template 1", + order=0, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_filename="*simple*", + filter_path="*/samples/*", + filter_mailrule=mr, + assign_title="Doc from {correspondent}", + assign_correspondent=c, + assign_document_type=dt, + assign_storage_path=sp, + assign_owner=user2, + ) + + ct.assign_tags.add(t1) + ct.assign_view_users.add(user3) + ct.assign_view_groups.add(group2) + ct.assign_change_users.add(user3) + ct.assign_change_groups.add(group2) + ct.assign_custom_fields.add(cf1) + ct.save() + + def test_users_with_add_documents_get_add_and_workflow_templates_get_migrated(self): + permission = self.Permission.objects.get(codename="add_workflow") + self.assertTrue(permission in self.user.user_permissions.all()) + self.assertTrue(permission in self.group.permissions.all()) + + Workflow = self.apps.get_model("documents", "Workflow") + self.assertEqual(Workflow.objects.all().count(), 1) + + +class TestReverseMigrateWorkflow(TestMigrations): + migrate_from = "1044_workflow_workflowaction_workflowtrigger_and_more" + migrate_to = "1043_alter_savedviewfilterrule_rule_type" + + def setUpBeforeMigration(self, apps): + User = apps.get_model("auth", "User") + Group = apps.get_model("auth", "Group") + self.Permission = apps.get_model("auth", "Permission") + self.user = User.objects.create(username="user1") + self.group = Group.objects.create(name="group1") + permission = self.Permission.objects.filter( + codename="add_workflow", + ).first() + if permission is not None: + self.user.user_permissions.add(permission.id) + self.group.permissions.add(permission.id) + + Workflow = apps.get_model("documents", "Workflow") + WorkflowTrigger = apps.get_model("documents", "WorkflowTrigger") + WorkflowAction = apps.get_model("documents", "WorkflowAction") + + trigger = WorkflowTrigger.objects.create( + type=0, + sources=[DocumentSource.ConsumeFolder], + filter_path="*/path/*", + filter_filename="*file*", + ) + + action = WorkflowAction.objects.create( + assign_title="assign title", + ) + workflow = Workflow.objects.create( + name="workflow 1", + order=0, + ) + workflow.triggers.set([trigger]) + workflow.actions.set([action]) + workflow.save() + + def test_remove_workflow_permissions_and_migrate_workflows_to_consumption_templates( + self, + ): + permission = self.Permission.objects.filter( + codename="add_workflow", + ).first() + if permission is not None: + self.assertFalse(permission in self.user.user_permissions.all()) + self.assertFalse(permission in self.group.permissions.all()) + + ConsumptionTemplate = self.apps.get_model("documents", "ConsumptionTemplate") + self.assertEqual(ConsumptionTemplate.objects.all().count(), 1) diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py new file mode 100644 index 000000000..2e516e24c --- /dev/null +++ b/src/documents/tests/test_workflows.py @@ -0,0 +1,1017 @@ +from datetime import timedelta +from pathlib import Path +from unittest import mock + +from django.contrib.auth.models import Group +from django.contrib.auth.models import User +from django.utils import timezone +from rest_framework.test import APITestCase + +from documents import tasks +from documents.data_models import ConsumableDocument +from documents.data_models import DocumentSource +from documents.matching import document_matches_workflow +from documents.models import Correspondent +from documents.models import CustomField +from documents.models import Document +from documents.models import DocumentType +from documents.models import MatchingModel +from documents.models import StoragePath +from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger +from documents.signals import document_consumption_finished +from documents.tests.utils import DirectoriesMixin +from documents.tests.utils import FileSystemAssertsMixin +from paperless_mail.models import MailAccount +from paperless_mail.models import MailRule + + +class TestWorkflows(DirectoriesMixin, FileSystemAssertsMixin, APITestCase): + SAMPLE_DIR = Path(__file__).parent / "samples" + + def setUp(self) -> None: + self.c = Correspondent.objects.create(name="Correspondent Name") + self.c2 = Correspondent.objects.create(name="Correspondent Name 2") + self.dt = DocumentType.objects.create(name="DocType Name") + self.t1 = Tag.objects.create(name="t1") + self.t2 = Tag.objects.create(name="t2") + self.t3 = Tag.objects.create(name="t3") + self.sp = StoragePath.objects.create(path="/test/") + self.cf1 = CustomField.objects.create(name="Custom Field 1", data_type="string") + self.cf2 = CustomField.objects.create( + name="Custom Field 2", + data_type="integer", + ) + + self.user2 = User.objects.create(username="user2") + self.user3 = User.objects.create(username="user3") + self.group1 = Group.objects.create(name="group1") + + account1 = MailAccount.objects.create( + name="Email1", + username="username1", + password="password1", + imap_server="server.example.com", + imap_port=443, + imap_security=MailAccount.ImapSecurity.SSL, + character_set="UTF-8", + ) + self.rule1 = MailRule.objects.create( + name="Rule1", + account=account1, + folder="INBOX", + filter_from="from@example.com", + filter_to="someone@somewhere.com", + filter_subject="subject", + filter_body="body", + filter_attachment_filename_include="file.pdf", + maximum_age=30, + action=MailRule.MailAction.MARK_READ, + assign_title_from=MailRule.TitleSource.NONE, + assign_correspondent_from=MailRule.CorrespondentSource.FROM_NOTHING, + order=0, + attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY, + assign_owner_from_rule=False, + ) + + return super().setUp() + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_match(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that matches is consumed + THEN: + - Template overrides are applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_filename="*simple*", + filter_path="*/samples/*", + ) + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.assign_tags.add(self.t1) + action.assign_tags.add(self.t2) + action.assign_tags.add(self.t3) + action.assign_view_users.add(self.user3.pk) + action.assign_view_groups.add(self.group1.pk) + action.assign_change_users.add(self.user3.pk) + action.assign_change_groups.add(self.group1.pk) + action.assign_custom_fields.add(self.cf1.pk) + action.assign_custom_fields.add(self.cf2.pk) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + self.assertEqual(w.__str__(), "Workflow: Workflow 1") + self.assertEqual(trigger.__str__(), "WorkflowTrigger 1") + self.assertEqual(action.__str__(), "WorkflowAction 1") + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertEqual(overrides["override_correspondent_id"], self.c.pk) + self.assertEqual(overrides["override_document_type_id"], self.dt.pk) + self.assertEqual( + overrides["override_tag_ids"], + [self.t1.pk, self.t2.pk, self.t3.pk], + ) + self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) + self.assertEqual(overrides["override_owner_id"], self.user2.pk) + self.assertEqual(overrides["override_view_users"], [self.user3.pk]) + self.assertEqual(overrides["override_view_groups"], [self.group1.pk]) + self.assertEqual(overrides["override_change_users"], [self.user3.pk]) + self.assertEqual(overrides["override_change_groups"], [self.group1.pk]) + self.assertEqual( + overrides["override_title"], + "Doc from {correspondent}", + ) + self.assertEqual( + overrides["override_custom_field_ids"], + [self.cf1.pk, self.cf2.pk], + ) + + info = cm.output[0] + expected_str = f"Document matched {trigger} from {w}" + self.assertIn(expected_str, info) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_match_mailrule(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that matches is consumed via mail rule + THEN: + - Template overrides are applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_mailrule=self.rule1, + ) + + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.assign_tags.add(self.t1) + action.assign_tags.add(self.t2) + action.assign_tags.add(self.t3) + action.assign_view_users.add(self.user3.pk) + action.assign_view_groups.add(self.group1.pk) + action.assign_change_users.add(self.user3.pk) + action.assign_change_groups.add(self.group1.pk) + action.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + mailrule_id=self.rule1.pk, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertEqual(overrides["override_correspondent_id"], self.c.pk) + self.assertEqual(overrides["override_document_type_id"], self.dt.pk) + self.assertEqual( + overrides["override_tag_ids"], + [self.t1.pk, self.t2.pk, self.t3.pk], + ) + self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) + self.assertEqual(overrides["override_owner_id"], self.user2.pk) + self.assertEqual(overrides["override_view_users"], [self.user3.pk]) + self.assertEqual(overrides["override_view_groups"], [self.group1.pk]) + self.assertEqual(overrides["override_change_users"], [self.user3.pk]) + self.assertEqual(overrides["override_change_groups"], [self.group1.pk]) + self.assertEqual( + overrides["override_title"], + "Doc from {correspondent}", + ) + + info = cm.output[0] + expected_str = f"Document matched {trigger} from {w}" + self.assertIn(expected_str, info) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_match_multiple(self, m): + """ + GIVEN: + - Multiple existing workflow + WHEN: + - File that matches is consumed + THEN: + - Template overrides are applied with subsequent templates overwriting previous values + or merging if multiple + """ + trigger1 = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_path="*/samples/*", + ) + action1 = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + ) + action1.assign_tags.add(self.t1) + action1.assign_tags.add(self.t2) + action1.assign_view_users.add(self.user2) + action1.save() + + w1 = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w1.triggers.add(trigger1) + w1.actions.add(action1) + w1.save() + + trigger2 = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_filename="*simple*", + ) + action2 = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c2, + assign_storage_path=self.sp, + ) + action2.assign_tags.add(self.t3) + action2.assign_view_users.add(self.user3) + action2.save() + + w2 = Workflow.objects.create( + name="Workflow 2", + order=0, + ) + w2.triggers.add(trigger2) + w2.actions.add(action2) + w2.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + # template 1 + self.assertEqual(overrides["override_document_type_id"], self.dt.pk) + # template 2 + self.assertEqual(overrides["override_correspondent_id"], self.c2.pk) + self.assertEqual(overrides["override_storage_path_id"], self.sp.pk) + # template 1 & 2 + self.assertEqual( + overrides["override_tag_ids"], + [self.t1.pk, self.t2.pk, self.t3.pk], + ) + self.assertEqual( + overrides["override_view_users"], + [self.user2.pk, self.user3.pk], + ) + + expected_str = f"Document matched {trigger1} from {w1}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document matched {trigger2} from {w2}" + self.assertIn(expected_str, cm.output[1]) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_no_match_filename(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that does not match on filename is consumed + THEN: + - Template overrides are not applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_filename="*foobar*", + filter_path=None, + ) + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertIsNone(overrides["override_correspondent_id"]) + self.assertIsNone(overrides["override_document_type_id"]) + self.assertIsNone(overrides["override_tag_ids"]) + self.assertIsNone(overrides["override_storage_path_id"]) + self.assertIsNone(overrides["override_owner_id"]) + self.assertIsNone(overrides["override_view_users"]) + self.assertIsNone(overrides["override_view_groups"]) + self.assertIsNone(overrides["override_change_users"]) + self.assertIsNone(overrides["override_change_groups"]) + self.assertIsNone(overrides["override_title"]) + + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document filename {test_file.name} does not match" + self.assertIn(expected_str, cm.output[1]) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_no_match_path(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that does not match on path is consumed + THEN: + - Template overrides are not applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_path="*foo/bar*", + ) + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertIsNone(overrides["override_correspondent_id"]) + self.assertIsNone(overrides["override_document_type_id"]) + self.assertIsNone(overrides["override_tag_ids"]) + self.assertIsNone(overrides["override_storage_path_id"]) + self.assertIsNone(overrides["override_owner_id"]) + self.assertIsNone(overrides["override_view_users"]) + self.assertIsNone(overrides["override_view_groups"]) + self.assertIsNone(overrides["override_change_users"]) + self.assertIsNone(overrides["override_change_groups"]) + self.assertIsNone(overrides["override_title"]) + + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document path {test_file} does not match" + self.assertIn(expected_str, cm.output[1]) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_no_match_mail_rule(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that does not match on source is consumed + THEN: + - Template overrides are not applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_mailrule=self.rule1, + ) + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + mailrule_id=99, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertIsNone(overrides["override_correspondent_id"]) + self.assertIsNone(overrides["override_document_type_id"]) + self.assertIsNone(overrides["override_tag_ids"]) + self.assertIsNone(overrides["override_storage_path_id"]) + self.assertIsNone(overrides["override_owner_id"]) + self.assertIsNone(overrides["override_view_users"]) + self.assertIsNone(overrides["override_view_groups"]) + self.assertIsNone(overrides["override_change_users"]) + self.assertIsNone(overrides["override_change_groups"]) + self.assertIsNone(overrides["override_title"]) + + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = "Document mail rule 99 !=" + self.assertIn(expected_str, cm.output[1]) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_no_match_source(self, m): + """ + GIVEN: + - Existing workflow + WHEN: + - File that does not match on source is consumed + THEN: + - Template overrides are not applied + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_path="*", + ) + action = WorkflowAction.objects.create( + assign_title="Doc from {correspondent}", + assign_correspondent=self.c, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ApiUpload, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertIsNone(overrides["override_correspondent_id"]) + self.assertIsNone(overrides["override_document_type_id"]) + self.assertIsNone(overrides["override_tag_ids"]) + self.assertIsNone(overrides["override_storage_path_id"]) + self.assertIsNone(overrides["override_owner_id"]) + self.assertIsNone(overrides["override_view_users"]) + self.assertIsNone(overrides["override_view_groups"]) + self.assertIsNone(overrides["override_change_users"]) + self.assertIsNone(overrides["override_change_groups"]) + self.assertIsNone(overrides["override_title"]) + + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document source {DocumentSource.ApiUpload.name} not in ['{DocumentSource.ConsumeFolder.name}', '{DocumentSource.MailFetch.name}']" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_no_match_trigger_type(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + doc.save() + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_matches_workflow( + doc, + w, + WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"No matching triggers with type {WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED} found" + self.assertIn(expected_str, cm.output[1]) + + @mock.patch("documents.consumer.Consumer.try_consume_file") + def test_workflow_repeat_custom_fields(self, m): + """ + GIVEN: + - Existing workflows which assign the same custom field + WHEN: + - File that matches is consumed + THEN: + - Custom field is added the first time successfully + """ + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, + sources=f"{DocumentSource.ApiUpload},{DocumentSource.ConsumeFolder},{DocumentSource.MailFetch}", + filter_filename="*simple*", + ) + action1 = WorkflowAction.objects.create() + action1.assign_custom_fields.add(self.cf1.pk) + action1.save() + + action2 = WorkflowAction.objects.create() + action2.assign_custom_fields.add(self.cf1.pk) + action2.save() + + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action1, action2) + w.save() + + test_file = self.SAMPLE_DIR / "simple.pdf" + + with mock.patch("documents.tasks.async_to_sync"): + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + m.assert_called_once() + _, overrides = m.call_args + self.assertEqual( + overrides["override_custom_field_ids"], + [self.cf1.pk], + ) + + expected_str = f"Document matched {trigger} from {w}" + self.assertIn(expected_str, cm.output[0]) + + def test_document_added_workflow(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_filename="*sample*", + ) + action = WorkflowAction.objects.create( + assign_title="Doc created in {created_year}", + assign_correspondent=self.c2, + assign_document_type=self.dt, + assign_storage_path=self.sp, + assign_owner=self.user2, + ) + action.assign_tags.add(self.t1) + action.assign_tags.add(self.t2) + action.assign_tags.add(self.t3) + action.assign_view_users.add(self.user3.pk) + action.assign_view_groups.add(self.group1.pk) + action.assign_change_users.add(self.user3.pk) + action.assign_change_groups.add(self.group1.pk) + action.assign_custom_fields.add(self.cf1.pk) + action.assign_custom_fields.add(self.cf2.pk) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + now = timezone.localtime(timezone.now()) + created = now - timedelta(weeks=520) + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + added=now, + created=created, + ) + + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + + self.assertEqual(doc.correspondent, self.c2) + self.assertEqual(doc.title, f"Doc created in {created.year}") + + def test_document_added_no_match_filename(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_filename="*foobar*", + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + doc.tags.set([self.t3]) + doc.save() + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document filename {doc.original_filename} does not match" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_match_content_matching(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + matching_algorithm=MatchingModel.MATCH_LITERAL, + match="foo", + is_insensitive=True, + ) + action = WorkflowAction.objects.create( + assign_title="Doc content matching worked", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + content="Hello world foo bar", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"WorkflowTrigger {trigger} matched on document" + expected_str2 = 'because it contains this string: "foo"' + self.assertIn(expected_str, cm.output[0]) + self.assertIn(expected_str2, cm.output[0]) + expected_str = f"Document matched {trigger} from {w}" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_no_match_content_matching(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + matching_algorithm=MatchingModel.MATCH_LITERAL, + match="foo", + is_insensitive=True, + ) + action = WorkflowAction.objects.create( + assign_title="Doc content matching worked", + assign_owner=self.user2, + ) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + content="Hello world bar", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document content matching settings for algorithm '{trigger.matching_algorithm}' did not match" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_no_match_tags(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + ) + trigger.filter_has_tags.set([self.t1, self.t2]) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + doc.tags.set([self.t3]) + doc.save() + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document tags {doc.tags.all()} do not include {trigger.filter_has_tags.all()}" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_no_match_doctype(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_has_document_type=self.dt, + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document doc type {doc.document_type} does not match {trigger.filter_has_document_type}" + self.assertIn(expected_str, cm.output[1]) + + def test_document_added_no_match_correspondent(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_has_correspondent=self.c, + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + assign_owner=self.user2, + ) + action.save() + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c2, + original_filename="sample.pdf", + ) + + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + expected_str = f"Document did not match {w}" + self.assertIn(expected_str, cm.output[0]) + expected_str = f"Document correspondent {doc.correspondent} does not match {trigger.filter_has_correspondent}" + self.assertIn(expected_str, cm.output[1]) + + def test_document_updated_workflow(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_UPDATED, + filter_has_document_type=self.dt, + ) + action = WorkflowAction.objects.create() + action.assign_custom_fields.add(self.cf1) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + + superuser = User.objects.create_superuser("superuser") + self.client.force_authenticate(user=superuser) + + self.client.patch( + f"/api/documents/{doc.id}/", + {"document_type": self.dt.id}, + format="json", + ) + + self.assertEqual(doc.custom_fields.all().count(), 1) + + def test_workflow_enabled_disabled(self): + trigger = WorkflowTrigger.objects.create( + type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED, + filter_filename="*sample*", + ) + action = WorkflowAction.objects.create( + assign_title="Title assign correspondent", + assign_correspondent=self.c2, + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + enabled=False, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + action2 = WorkflowAction.objects.create( + assign_title="Title assign owner", + assign_owner=self.user2, + ) + w2 = Workflow.objects.create( + name="Workflow 2", + order=0, + enabled=True, + ) + w2.triggers.add(trigger) + w2.actions.add(action2) + w2.save() + + doc = Document.objects.create( + title="sample test", + correspondent=self.c, + original_filename="sample.pdf", + ) + + document_consumption_finished.send( + sender=self.__class__, + document=doc, + ) + + self.assertEqual(doc.correspondent, self.c) + self.assertEqual(doc.title, "Title assign owner") + self.assertEqual(doc.owner, self.user2) + + def test_new_trigger_type_raises_exception(self): + trigger = WorkflowTrigger.objects.create( + type=4, + ) + action = WorkflowAction.objects.create( + assign_title="Doc assign owner", + ) + w = Workflow.objects.create( + name="Workflow 1", + order=0, + ) + w.triggers.add(trigger) + w.actions.add(action) + w.save() + + doc = Document.objects.create( + title="test", + ) + self.assertRaises(Exception, document_matches_workflow, doc, w, 4) diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index fe7dbb059..0b6d8fcad 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -265,6 +265,7 @@ class TestMigrations(TransactionTestCase): return apps.get_containing_app_config(type(self).__module__).name migrate_from = None + dependencies = None migrate_to = None auto_migrate = True @@ -277,6 +278,8 @@ class TestMigrations(TransactionTestCase): type(self).__name__, ) self.migrate_from = [(self.app, self.migrate_from)] + if self.dependencies is not None: + self.migrate_from.extend(self.dependencies) self.migrate_to = [(self.app, self.migrate_to)] executor = MigrationExecutor(connection) old_apps = executor.loader.project_state(self.migrate_from).apps diff --git a/src/documents/views.py b/src/documents/views.py index e8c6db0de..84633cc03 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -76,7 +76,6 @@ from documents.matching import match_correspondents from documents.matching import match_document_types from documents.matching import match_storage_paths from documents.matching import match_tags -from documents.models import ConsumptionTemplate from documents.models import Correspondent from documents.models import CustomField from documents.models import Document @@ -87,6 +86,9 @@ from documents.models import SavedView from documents.models import ShareLink from documents.models import StoragePath from documents.models import Tag +from documents.models import Workflow +from documents.models import WorkflowAction +from documents.models import WorkflowTrigger from documents.parsers import get_parser_class_for_mime_type from documents.parsers import parse_date_generator from documents.permissions import PaperlessAdminPermissions @@ -98,7 +100,6 @@ from documents.serialisers import AcknowledgeTasksViewSerializer from documents.serialisers import BulkDownloadSerializer from documents.serialisers import BulkEditObjectPermissionsSerializer from documents.serialisers import BulkEditSerializer -from documents.serialisers import ConsumptionTemplateSerializer from documents.serialisers import CorrespondentSerializer from documents.serialisers import CustomFieldSerializer from documents.serialisers import DocumentListSerializer @@ -112,6 +113,10 @@ from documents.serialisers import TagSerializer from documents.serialisers import TagSerializerVersion1 from documents.serialisers import TasksViewSerializer from documents.serialisers import UiSettingsViewSerializer +from documents.serialisers import WorkflowActionSerializer +from documents.serialisers import WorkflowSerializer +from documents.serialisers import WorkflowTriggerSerializer +from documents.signals import document_updated from documents.tasks import consume_file from paperless import version from paperless.db import GnuPG @@ -320,6 +325,12 @@ class DocumentViewSet( from documents import index index.add_or_update_document(self.get_object()) + + document_updated.send( + sender=self.__class__, + document=self.get_object(), + ) + return response def destroy(self, request, *args, **kwargs): @@ -1373,25 +1384,50 @@ class BulkEditObjectPermissionsView(GenericAPIView, PassUserMixin): ) -class ConsumptionTemplateViewSet(ModelViewSet): +class WorkflowTriggerViewSet(ModelViewSet): permission_classes = (IsAuthenticated, PaperlessObjectPermissions) - serializer_class = ConsumptionTemplateSerializer + serializer_class = WorkflowTriggerSerializer pagination_class = StandardPagination - model = ConsumptionTemplate + model = WorkflowTrigger + + queryset = WorkflowTrigger.objects.all() + + +class WorkflowActionViewSet(ModelViewSet): + permission_classes = (IsAuthenticated, PaperlessObjectPermissions) + + serializer_class = WorkflowActionSerializer + pagination_class = StandardPagination + + model = WorkflowAction + + queryset = WorkflowAction.objects.all().prefetch_related( + "assign_tags", + "assign_view_users", + "assign_view_groups", + "assign_change_users", + "assign_change_groups", + "assign_custom_fields", + ) + + +class WorkflowViewSet(ModelViewSet): + permission_classes = (IsAuthenticated, PaperlessObjectPermissions) + + serializer_class = WorkflowSerializer + pagination_class = StandardPagination + + model = Workflow queryset = ( - ConsumptionTemplate.objects.prefetch_related( - "assign_tags", - "assign_view_users", - "assign_view_groups", - "assign_change_users", - "assign_change_groups", - "assign_custom_fields", - ) - .all() + Workflow.objects.all() .order_by("order") + .prefetch_related( + "triggers", + "actions", + ) ) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index a317ddfd0..3cb19d63c 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2023-12-09 10:53-0800\n" +"POT-Creation-Date: 2024-01-01 07:54-0800\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -25,27 +25,27 @@ msgstr "" msgid "owner" msgstr "" -#: documents/models.py:53 +#: documents/models.py:53 documents/models.py:894 msgid "None" msgstr "" -#: documents/models.py:54 +#: documents/models.py:54 documents/models.py:895 msgid "Any word" msgstr "" -#: documents/models.py:55 +#: documents/models.py:55 documents/models.py:896 msgid "All words" msgstr "" -#: documents/models.py:56 +#: documents/models.py:56 documents/models.py:897 msgid "Exact match" msgstr "" -#: documents/models.py:57 +#: documents/models.py:57 documents/models.py:898 msgid "Regular expression" msgstr "" -#: documents/models.py:58 +#: documents/models.py:58 documents/models.py:899 msgid "Fuzzy word" msgstr "" @@ -53,20 +53,20 @@ msgstr "" msgid "Automatic" msgstr "" -#: documents/models.py:62 documents/models.py:402 documents/models.py:897 +#: documents/models.py:62 documents/models.py:402 documents/models.py:1099 #: paperless_mail/models.py:18 paperless_mail/models.py:93 msgid "name" msgstr "" -#: documents/models.py:64 +#: documents/models.py:64 documents/models.py:955 msgid "match" msgstr "" -#: documents/models.py:67 +#: documents/models.py:67 documents/models.py:958 msgid "matching algorithm" msgstr "" -#: documents/models.py:72 +#: documents/models.py:72 documents/models.py:963 msgid "is insensitive" msgstr "" @@ -615,118 +615,174 @@ msgstr "" msgid "custom field instances" msgstr "" -#: documents/models.py:893 +#: documents/models.py:902 +msgid "Consumption Started" +msgstr "" + +#: documents/models.py:903 +msgid "Document Added" +msgstr "" + +#: documents/models.py:904 +msgid "Document Updated" +msgstr "" + +#: documents/models.py:907 msgid "Consume Folder" msgstr "" -#: documents/models.py:894 +#: documents/models.py:908 msgid "Api Upload" msgstr "" -#: documents/models.py:895 +#: documents/models.py:909 msgid "Mail Fetch" msgstr "" -#: documents/models.py:899 paperless_mail/models.py:95 -msgid "order" +#: documents/models.py:912 +msgid "Workflow Trigger Type" msgstr "" -#: documents/models.py:908 +#: documents/models.py:924 msgid "filter path" msgstr "" -#: documents/models.py:913 +#: documents/models.py:929 msgid "" "Only consume documents with a path that matches this if specified. Wildcards " "specified as * are allowed. Case insensitive." msgstr "" -#: documents/models.py:920 +#: documents/models.py:936 msgid "filter filename" msgstr "" -#: documents/models.py:925 paperless_mail/models.py:148 +#: documents/models.py:941 paperless_mail/models.py:148 msgid "" "Only consume documents which entirely match this filename if specified. " "Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." msgstr "" -#: documents/models.py:936 +#: documents/models.py:952 msgid "filter documents from this mail rule" msgstr "" -#: documents/models.py:940 +#: documents/models.py:968 +msgid "has these tag(s)" +msgstr "" + +#: documents/models.py:976 +msgid "has this document type" +msgstr "" + +#: documents/models.py:984 +msgid "has this correspondent" +msgstr "" + +#: documents/models.py:988 +msgid "workflow trigger" +msgstr "" + +#: documents/models.py:989 +msgid "workflow triggers" +msgstr "" + +#: documents/models.py:997 +msgid "Assignment" +msgstr "" + +#: documents/models.py:1000 +msgid "Workflow Action Type" +msgstr "" + +#: documents/models.py:1006 msgid "assign title" msgstr "" -#: documents/models.py:945 +#: documents/models.py:1011 msgid "" "Assign a document title, can include some placeholders, see documentation." msgstr "" -#: documents/models.py:953 paperless_mail/models.py:216 +#: documents/models.py:1019 paperless_mail/models.py:216 msgid "assign this tag" msgstr "" -#: documents/models.py:961 paperless_mail/models.py:224 +#: documents/models.py:1027 paperless_mail/models.py:224 msgid "assign this document type" msgstr "" -#: documents/models.py:969 paperless_mail/models.py:238 +#: documents/models.py:1035 paperless_mail/models.py:238 msgid "assign this correspondent" msgstr "" -#: documents/models.py:977 +#: documents/models.py:1043 msgid "assign this storage path" msgstr "" -#: documents/models.py:986 +#: documents/models.py:1052 msgid "assign this owner" msgstr "" -#: documents/models.py:993 +#: documents/models.py:1059 msgid "grant view permissions to these users" msgstr "" -#: documents/models.py:1000 +#: documents/models.py:1066 msgid "grant view permissions to these groups" msgstr "" -#: documents/models.py:1007 +#: documents/models.py:1073 msgid "grant change permissions to these users" msgstr "" -#: documents/models.py:1014 +#: documents/models.py:1080 msgid "grant change permissions to these groups" msgstr "" -#: documents/models.py:1021 +#: documents/models.py:1087 msgid "assign these custom fields" msgstr "" -#: documents/models.py:1025 -msgid "consumption template" +#: documents/models.py:1091 +msgid "workflow action" msgstr "" -#: documents/models.py:1026 -msgid "consumption templates" +#: documents/models.py:1092 +msgid "workflow actions" msgstr "" -#: documents/serialisers.py:105 +#: documents/models.py:1101 paperless_mail/models.py:95 +msgid "order" +msgstr "" + +#: documents/models.py:1107 +msgid "triggers" +msgstr "" + +#: documents/models.py:1114 +msgid "actions" +msgstr "" + +#: documents/models.py:1117 +msgid "enabled" +msgstr "" + +#: documents/serialisers.py:111 #, python-format msgid "Invalid regular expression: %(error)s" msgstr "" -#: documents/serialisers.py:399 +#: documents/serialisers.py:405 msgid "Invalid color." msgstr "" -#: documents/serialisers.py:865 +#: documents/serialisers.py:988 #, python-format msgid "File type %(type)s not supported" msgstr "" -#: documents/serialisers.py:962 +#: documents/serialisers.py:1085 msgid "Invalid variable detected." msgstr "" @@ -869,135 +925,286 @@ msgstr "" msgid "Send me instructions!" msgstr "" +#: documents/validators.py:17 +#, python-brace-format +msgid "Unable to parse URI {value}, missing scheme" +msgstr "" + +#: documents/validators.py:22 +#, python-brace-format +msgid "Unable to parse URI {value}, missing net location or path" +msgstr "" + +#: documents/validators.py:27 +#, python-brace-format +msgid "Unable to parse URI {value}" +msgstr "" + #: paperless/apps.py:10 msgid "Paperless" msgstr "" -#: paperless/settings.py:586 -msgid "English (US)" +#: paperless/models.py:25 +msgid "pdf" msgstr "" -#: paperless/settings.py:587 -msgid "Arabic" +#: paperless/models.py:26 +msgid "pdfa" msgstr "" -#: paperless/settings.py:588 -msgid "Afrikaans" +#: paperless/models.py:27 +msgid "pdfa-1" msgstr "" -#: paperless/settings.py:589 -msgid "Belarusian" +#: paperless/models.py:28 +msgid "pdfa-2" msgstr "" -#: paperless/settings.py:590 -msgid "Bulgarian" +#: paperless/models.py:29 +msgid "pdfa-3" msgstr "" -#: paperless/settings.py:591 -msgid "Catalan" +#: paperless/models.py:38 +msgid "skip" msgstr "" -#: paperless/settings.py:592 -msgid "Czech" +#: paperless/models.py:39 +msgid "redo" msgstr "" -#: paperless/settings.py:593 -msgid "Danish" +#: paperless/models.py:40 +msgid "force" msgstr "" -#: paperless/settings.py:594 -msgid "German" +#: paperless/models.py:41 +msgid "skip_noarchive" msgstr "" -#: paperless/settings.py:595 -msgid "Greek" +#: paperless/models.py:49 +msgid "never" msgstr "" -#: paperless/settings.py:596 -msgid "English (GB)" +#: paperless/models.py:50 +msgid "with_text" msgstr "" -#: paperless/settings.py:597 -msgid "Spanish" +#: paperless/models.py:51 +msgid "always" msgstr "" -#: paperless/settings.py:598 -msgid "Finnish" +#: paperless/models.py:59 +msgid "clean" msgstr "" -#: paperless/settings.py:599 -msgid "French" +#: paperless/models.py:60 +msgid "clean-final" msgstr "" -#: paperless/settings.py:600 -msgid "Hungarian" +#: paperless/models.py:61 +msgid "none" +msgstr "" + +#: paperless/models.py:69 +msgid "LeaveColorUnchanged" +msgstr "" + +#: paperless/models.py:70 +msgid "RGB" +msgstr "" + +#: paperless/models.py:71 +msgid "UseDeviceIndependentColor" +msgstr "" + +#: paperless/models.py:72 +msgid "Gray" +msgstr "" + +#: paperless/models.py:73 +msgid "CMYK" +msgstr "" + +#: paperless/models.py:82 +msgid "Sets the output PDF type" +msgstr "" + +#: paperless/models.py:94 +msgid "Do OCR from page 1 to this value" +msgstr "" + +#: paperless/models.py:100 +msgid "Do OCR using these languages" +msgstr "" + +#: paperless/models.py:107 +msgid "Sets the OCR mode" +msgstr "" + +#: paperless/models.py:115 +msgid "Controls the generation of an archive file" +msgstr "" + +#: paperless/models.py:123 +msgid "Sets image DPI fallback value" +msgstr "" + +#: paperless/models.py:130 +msgid "Controls the unpaper cleaning" +msgstr "" + +#: paperless/models.py:137 +msgid "Enables deskew" +msgstr "" + +#: paperless/models.py:140 +msgid "Enables page rotation" +msgstr "" + +#: paperless/models.py:145 +msgid "Sets the threshold for rotation of pages" +msgstr "" + +#: paperless/models.py:151 +msgid "Sets the maximum image size for decompression" +msgstr "" + +#: paperless/models.py:157 +msgid "Sets the Ghostscript color conversion strategy" +msgstr "" + +#: paperless/models.py:165 +msgid "Adds additional user arguments for OCRMyPDF" +msgstr "" + +#: paperless/models.py:170 +msgid "paperless application settings" msgstr "" #: paperless/settings.py:601 -msgid "Italian" +msgid "English (US)" msgstr "" #: paperless/settings.py:602 -msgid "Luxembourgish" +msgid "Arabic" msgstr "" #: paperless/settings.py:603 -msgid "Norwegian" +msgid "Afrikaans" msgstr "" #: paperless/settings.py:604 -msgid "Dutch" +msgid "Belarusian" msgstr "" #: paperless/settings.py:605 -msgid "Polish" +msgid "Bulgarian" msgstr "" #: paperless/settings.py:606 -msgid "Portuguese (Brazil)" +msgid "Catalan" msgstr "" #: paperless/settings.py:607 -msgid "Portuguese" +msgid "Czech" msgstr "" #: paperless/settings.py:608 -msgid "Romanian" +msgid "Danish" msgstr "" #: paperless/settings.py:609 -msgid "Russian" +msgid "German" msgstr "" #: paperless/settings.py:610 -msgid "Slovak" +msgid "Greek" msgstr "" #: paperless/settings.py:611 -msgid "Slovenian" +msgid "English (GB)" msgstr "" #: paperless/settings.py:612 -msgid "Serbian" +msgid "Spanish" msgstr "" #: paperless/settings.py:613 -msgid "Swedish" +msgid "Finnish" msgstr "" #: paperless/settings.py:614 -msgid "Turkish" +msgid "French" msgstr "" #: paperless/settings.py:615 -msgid "Ukrainian" +msgid "Hungarian" msgstr "" #: paperless/settings.py:616 +msgid "Italian" +msgstr "" + +#: paperless/settings.py:617 +msgid "Luxembourgish" +msgstr "" + +#: paperless/settings.py:618 +msgid "Norwegian" +msgstr "" + +#: paperless/settings.py:619 +msgid "Dutch" +msgstr "" + +#: paperless/settings.py:620 +msgid "Polish" +msgstr "" + +#: paperless/settings.py:621 +msgid "Portuguese (Brazil)" +msgstr "" + +#: paperless/settings.py:622 +msgid "Portuguese" +msgstr "" + +#: paperless/settings.py:623 +msgid "Romanian" +msgstr "" + +#: paperless/settings.py:624 +msgid "Russian" +msgstr "" + +#: paperless/settings.py:625 +msgid "Slovak" +msgstr "" + +#: paperless/settings.py:626 +msgid "Slovenian" +msgstr "" + +#: paperless/settings.py:627 +msgid "Serbian" +msgstr "" + +#: paperless/settings.py:628 +msgid "Swedish" +msgstr "" + +#: paperless/settings.py:629 +msgid "Turkish" +msgstr "" + +#: paperless/settings.py:630 +msgid "Ukrainian" +msgstr "" + +#: paperless/settings.py:631 msgid "Chinese Simplified" msgstr "" -#: paperless/urls.py:194 +#: paperless/urls.py:205 msgid "Paperless-ngx administration" msgstr "" diff --git a/src/paperless/urls.py b/src/paperless/urls.py index 1c0dafd65..25190e0d8 100644 --- a/src/paperless/urls.py +++ b/src/paperless/urls.py @@ -15,7 +15,6 @@ from documents.views import AcknowledgeTasksView from documents.views import BulkDownloadView from documents.views import BulkEditObjectPermissionsView from documents.views import BulkEditView -from documents.views import ConsumptionTemplateViewSet from documents.views import CorrespondentViewSet from documents.views import CustomFieldViewSet from documents.views import DocumentTypeViewSet @@ -34,6 +33,9 @@ from documents.views import TagViewSet from documents.views import TasksViewSet from documents.views import UiSettingsView from documents.views import UnifiedSearchViewSet +from documents.views import WorkflowActionViewSet +from documents.views import WorkflowTriggerViewSet +from documents.views import WorkflowViewSet from paperless.consumers import StatusConsumer from paperless.views import ApplicationConfigurationViewSet from paperless.views import FaviconView @@ -59,7 +61,9 @@ api_router.register(r"groups", GroupViewSet, basename="groups") api_router.register(r"mail_accounts", MailAccountViewSet) api_router.register(r"mail_rules", MailRuleViewSet) api_router.register(r"share_links", ShareLinkViewSet) -api_router.register(r"consumption_templates", ConsumptionTemplateViewSet) +api_router.register(r"workflow_triggers", WorkflowTriggerViewSet) +api_router.register(r"workflow_actions", WorkflowActionViewSet) +api_router.register(r"workflows", WorkflowViewSet) api_router.register(r"custom_fields", CustomFieldViewSet) api_router.register(r"config", ApplicationConfigurationViewSet) From bbf64b7e93da7cb99183caef942d7da22b353299 Mon Sep 17 00:00:00 2001 From: Bevan Kay Date: Wed, 3 Jan 2024 19:31:56 +1100 Subject: [PATCH 19/33] Enhancement: add `storage_path` parameter to post_document API (#5217) * Feature: add `storage_path` parameter to post_document API * Complete coverage for validate_storage_path --------- Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com> --- docs/api.md | 1 + src/documents/serialisers.py | 14 +++++++ src/documents/tests/test_api_documents.py | 51 ++++++++++++++++++++++- src/documents/views.py | 2 + 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 39a06f37f..97ccf4c3a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -274,6 +274,7 @@ The endpoint supports the following optional form fields: - `correspondent`: Specify the ID of a correspondent that the consumer should use for the document. - `document_type`: Similar to correspondent. +- `storage_path`: Similar to correspondent. - `tags`: Similar to correspondent. Specify this multiple times to have multiple tags added to the document. - `archive_serial_number`: An optional archive serial number to set. diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py index b1dd9aee9..41d3139b3 100644 --- a/src/documents/serialisers.py +++ b/src/documents/serialisers.py @@ -966,6 +966,14 @@ class PostDocumentSerializer(serializers.Serializer): required=False, ) + storage_path = serializers.PrimaryKeyRelatedField( + queryset=StoragePath.objects.all(), + label="Storage path", + allow_null=True, + write_only=True, + required=False, + ) + tags = serializers.PrimaryKeyRelatedField( many=True, queryset=Tag.objects.all(), @@ -1005,6 +1013,12 @@ class PostDocumentSerializer(serializers.Serializer): else: return None + def validate_storage_path(self, storage_path): + if storage_path: + return storage_path.id + else: + return None + def validate_tags(self, tags): if tags: return [tag.id for tag in tags] diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index 8415b9a71..f19711a96 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -819,7 +819,13 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): ) as f: response = self.client.post( "/api/documents/post_document/", - {"document": f, "title": "", "correspondent": "", "document_type": ""}, + { + "document": f, + "title": "", + "correspondent": "", + "document_type": "", + "storage_path": "", + }, ) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -833,6 +839,7 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.assertIsNone(overrides.title) self.assertIsNone(overrides.correspondent_id) self.assertIsNone(overrides.document_type_id) + self.assertIsNone(overrides.storage_path_id) self.assertIsNone(overrides.tag_ids) def test_upload_invalid_form(self): @@ -975,6 +982,48 @@ class TestDocumentApi(DirectoriesMixin, DocumentConsumeDelayMixin, APITestCase): self.consume_file_mock.assert_not_called() + def test_upload_with_storage_path(self): + self.consume_file_mock.return_value = celery.result.AsyncResult( + id=str(uuid.uuid4()), + ) + + sp = StoragePath.objects.create(name="invoices") + with open( + os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), + "rb", + ) as f: + response = self.client.post( + "/api/documents/post_document/", + {"document": f, "storage_path": sp.id}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.consume_file_mock.assert_called_once() + + _, overrides = self.get_last_consume_delay_call_args() + + self.assertEqual(overrides.storage_path_id, sp.id) + self.assertIsNone(overrides.correspondent_id) + self.assertIsNone(overrides.title) + self.assertIsNone(overrides.tag_ids) + + def test_upload_with_invalid_storage_path(self): + self.consume_file_mock.return_value = celery.result.AsyncResult( + id=str(uuid.uuid4()), + ) + + with open( + os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), + "rb", + ) as f: + response = self.client.post( + "/api/documents/post_document/", + {"document": f, "storage_path": 34578}, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + self.consume_file_mock.assert_not_called() + def test_upload_with_tags(self): self.consume_file_mock.return_value = celery.result.AsyncResult( id=str(uuid.uuid4()), diff --git a/src/documents/views.py b/src/documents/views.py index 84633cc03..83f2fc321 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -850,6 +850,7 @@ class PostDocumentView(GenericAPIView): doc_name, doc_data = serializer.validated_data.get("document") correspondent_id = serializer.validated_data.get("correspondent") document_type_id = serializer.validated_data.get("document_type") + storage_path_id = serializer.validated_data.get("storage_path") tag_ids = serializer.validated_data.get("tags") title = serializer.validated_data.get("title") created = serializer.validated_data.get("created") @@ -876,6 +877,7 @@ class PostDocumentView(GenericAPIView): title=title, correspondent_id=correspondent_id, document_type_id=document_type_id, + storage_path_id=storage_path_id, tag_ids=tag_ids, created=created, asn=archive_serial_number, From ba2f51bed1d008a6506665386b3458f4cfe82b5b Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 3 Jan 2024 00:45:22 -0800 Subject: [PATCH 20/33] Fix: consistent workflow action name display --- src-ui/messages.xlf | 50 +++++++++---------- .../workflow-edit-dialog.component.html | 2 +- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 763576677..7e45a4550 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -444,7 +444,7 @@ src/app/components/document-detail/document-detail.component.html - 294 + 339 @@ -503,7 +503,7 @@ src/app/components/document-detail/document-detail.component.html - 286 + 331 src/app/components/document-list/save-view-config-dialog/save-view-config-dialog.component.html @@ -608,7 +608,7 @@ src/app/components/document-detail/document-detail.component.html - 303 + 348 src/app/components/document-list/document-list.component.html @@ -917,7 +917,7 @@ src/app/components/document-detail/document-detail.component.html - 252 + 297 src/app/components/document-list/bulk-editor/bulk-editor.component.html @@ -2965,10 +2965,6 @@ src/app/components/common/edit-dialog/mail-rule-edit-dialog/mail-rule-edit-dialog.component.html 28 - - src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html - 84 - Action is only performed when documents are consumed from the mail. Mails without attachments remain entirely untouched. @@ -4615,14 +4611,14 @@ Content src/app/components/document-detail/document-detail.component.html - 161 + 206 Metadata src/app/components/document-detail/document-detail.component.html - 170 + 215 src/app/components/document-detail/metadata-collapse/metadata-collapse.component.ts @@ -4633,116 +4629,116 @@ Date modified src/app/components/document-detail/document-detail.component.html - 177 + 222 Date added src/app/components/document-detail/document-detail.component.html - 181 + 226 Media filename src/app/components/document-detail/document-detail.component.html - 185 + 230 Original filename src/app/components/document-detail/document-detail.component.html - 189 + 234 Original MD5 checksum src/app/components/document-detail/document-detail.component.html - 193 + 238 Original file size src/app/components/document-detail/document-detail.component.html - 197 + 242 Original mime type src/app/components/document-detail/document-detail.component.html - 201 + 246 Archive MD5 checksum src/app/components/document-detail/document-detail.component.html - 206 + 251 Archive file size src/app/components/document-detail/document-detail.component.html - 212 + 257 Original document metadata src/app/components/document-detail/document-detail.component.html - 221 + 266 Archived document metadata src/app/components/document-detail/document-detail.component.html - 224 + 269 Preview src/app/components/document-detail/document-detail.component.html - 231 + 276 Notes src/app/components/document-detail/document-detail.component.html - 241,244 + 286,289 Enter Password src/app/components/document-detail/document-detail.component.html - 275 + 320 src/app/components/document-detail/document-detail.component.html - 332 + 377 Save & next src/app/components/document-detail/document-detail.component.html - 288 + 333 Save & close src/app/components/document-detail/document-detail.component.html - 291 + 336 diff --git a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html index 4663a7f77..81a5eca25 100644 --- a/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html +++ b/src-ui/src/app/components/common/edit-dialog/workflow-edit-dialog/workflow-edit-dialog.component.html @@ -81,7 +81,7 @@ @for (action of object?.actions; track action; let i = $index){
    -