diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c0d31bf..5e6c5cec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,9 @@ jobs: strategy: matrix: - python-version: ["3.7", "3.8"] - django-version: ["2.1.1", "3.1.4"] - database-engine: ["postgres", "mysql"] + python-version: [ "3.7", "3.8" ] + django-version: [ "2.1.1", "3.1.4" ] + database-engine: [ "postgres", "mysql", "mssql"] services: postgres: @@ -37,6 +37,14 @@ jobs: ports: - 3306:3306 + + mssqldb: + image: mcr.microsoft.com/mssql/server:2017-latest + env: + ACCEPT_EULA: y + SA_PASSWORD: Test + + steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/binder/exceptions.py b/binder/exceptions.py index d5aec299..75e4e572 100644 --- a/binder/exceptions.py +++ b/binder/exceptions.py @@ -235,3 +235,15 @@ def __add__(self, other): else: errors[model] = other.errors[model] return BinderValidationError(errors) + + +class BinderSkipSave(BinderException): + """Used to abort the database transaction when validation was successfull. + Validation is possible when saving (post, put, multi-put) or deleting models.""" + + http_code = 200 + code = 'SkipSave' + + def __init__(self): + super().__init__() + self.fields['message'] = 'No validation errors were encountered.' diff --git a/binder/models.py b/binder/models.py index ab1e05e1..1cb7c430 100644 --- a/binder/models.py +++ b/binder/models.py @@ -468,8 +468,12 @@ class Meta: abstract = True ordering = ['pk'] - def save(self, *args, **kwargs): - self.full_clean() # Never allow saving invalid models! + def save(self, *args, only_validate=False, **kwargs): + # A validation model might not require all validation checks as it is not a full model + # _validation_model can be used to skip validation checks that are meant for complete models that are actually being saved + self._validation_model = only_validate # Set the model as a validation model when we only want to validate the model + + self.full_clean() # Never allow saving invalid models! return super().save(*args, **kwargs) diff --git a/binder/plugins/models/__init__.py b/binder/plugins/models/__init__.py new file mode 100644 index 00000000..7ac605e7 --- /dev/null +++ b/binder/plugins/models/__init__.py @@ -0,0 +1 @@ +from .html_field import HtmlField # noqa: F401 diff --git a/binder/plugins/models/html_field.py b/binder/plugins/models/html_field.py new file mode 100644 index 00000000..d6cd5c65 --- /dev/null +++ b/binder/plugins/models/html_field.py @@ -0,0 +1,157 @@ +from typing import List + +from django.db.models import TextField +from html.parser import HTMLParser +from django.core.exceptions import ValidationError +from django.utils.translation import gettext as _ + +ALLOWED_LINK_PREFIXES = [ + 'http://', + 'https://', + 'mailto:' +] + + +def link_rel_validator(tag, attribute_name, attribute_value) -> List[ValidationError]: + validation_errors = [] + + rels = attribute_value.split(' ') + + if 'noopener' not in rels: + + validation_errors.append(ValidationError( + _('Link needs rel="noopener"'), + code='invalid_attribute', + params={ + 'tag': tag, + }, + )) + + if 'noreferrer' not in rels: + validation_errors.append(ValidationError( + _('Link needs rel="noreferer"'), + code='invalid_attribute', + params={ + 'tag': tag, + }, + )) + + + return validation_errors + + +def link_validator(tag, attribute_name, attribute_value) -> List[ValidationError]: + validation_errors = [] + if not any(map(lambda prefix: attribute_value.startswith(prefix), ALLOWED_LINK_PREFIXES)): + validation_errors.append(ValidationError( + _('Link is not valid'), + code='invalid_attribute', + params={ + 'tag': tag, + }, + )) + return validation_errors + + +class HtmlValidator(HTMLParser): + allowed_tags = [ + # General setup + 'p', 'br', + # Headers + 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'h7', + + # text decoration + 'b', 'strong', 'i', 'em', 'u', + # Lists + 'ol', 'ul', 'li', + + # Special + 'a', + ] + + allowed_attributes = { + 'a': ['href', 'rel', 'target'] + } + + required_attributes = { + 'a': ['rel'], + } + + special_validators = { + ('a', 'href'): link_validator, + ('a', 'rel'): link_rel_validator, + } + + error_messages = { + 'invalid_tag': _('Tag %(tag)s is not allowed'), + 'missing_attribute': _('Attribute %(attribute)s is required for tag %(tag)s'), + 'invalid_attribute': _('Attribute %(attribute)s not allowed for tag %(tag)s'), + } + + def validate(self, value: str) -> List[ValidationError]: + """ + Validates html, and gives a list of validation errors + """ + + self.errors = [] + + self.feed(value) + + return self.errors + + def handle_starttag(self, tag: str, attrs: list) -> None: + tag_errors = [] + if tag not in self.allowed_tags: + tag_errors.append(ValidationError( + self.error_messages['invalid_tag'], + code='invalid_tag', + params={ + 'tag': tag + }, + )) + + set_attributes = set(map(lambda attr: attr[0], attrs)) + required_attributes = set(self.required_attributes.get(tag, [])) + missing_attributes = required_attributes - set_attributes + for missing_attribute in missing_attributes: + tag_errors.append( + ValidationError( + self.error_messages['missing_attribute'], + code='missing_attribute', + params={ + 'tag': tag, + 'attribute': missing_attribute + }, + ) + ) + + allowed_attributes_for_tag = self.allowed_attributes.get(tag, []) + + for (attribute_name, attribute_content) in attrs: + if attribute_name not in allowed_attributes_for_tag: + tag_errors.append(ValidationError( + self.error_messages['invalid_attribute'], + code='invalid_attribute', + params={ + 'tag': tag, + 'attribute': attribute_name + }, + )) + if (tag, attribute_name) in self.special_validators: + tag_errors += self.special_validators[(tag, attribute_name)](tag, attribute_name, attribute_content) + + self.errors += tag_errors + + +class HtmlField(TextField): + """ + Determine a safe way to save "secure" user provided HTML input, and prevent XSS injections + """ + + def validate(self, value: str, _): + # Validate all html tags + validator = HtmlValidator() + errors = validator.validate(value) + + if errors: + raise ValidationError(errors) diff --git a/binder/plugins/views/csvexport.py b/binder/plugins/views/csvexport.py index aa413470..1741cbf6 100644 --- a/binder/plugins/views/csvexport.py +++ b/binder/plugins/views/csvexport.py @@ -103,7 +103,7 @@ def __init__(self, request: HttpRequest): # self.writer = self.pandas.ExcelWriter(self.response) self.work_book = self.openpyxl.Workbook() - self.sheet = self.work_book.create_sheet() + self.sheet = self.work_book._sheets[0] # The row number we are currently writing to self._row_number = 0 @@ -285,12 +285,23 @@ def get_datum(data, key, prefix=''): else: # Assume that we have a mapping now fk_ids = data[head_key] - if type(fk_ids) != list: + + if fk_ids is None: + # This case happens if we have a nullable foreign key that is null. Treat this as a many + # to one relation with no values. + fk_ids = [] + elif type(fk_ids) != list: fk_ids = [fk_ids] # if head_key not in key_mapping: prefix_key = parent_data['with_mapping'][new_prefix[1:]] - datums = [str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix)) for fk_id in fk_ids] + datums = [] + for fk_id in fk_ids: + try: + datums.append(str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix))) + except KeyError: + pass + # datums = [str(get_datum(key_mapping[prefix_key][fk_id], subkey, new_prefix)) for fk_id in fk_ids] return self.csv_settings.multi_value_delimiter.join( datums ) diff --git a/binder/views.py b/binder/views.py index 429cbd1c..04922aa4 100644 --- a/binder/views.py +++ b/binder/views.py @@ -17,7 +17,7 @@ from django.core.exceptions import ObjectDoesNotExist, FieldError, ValidationError, FieldDoesNotExist from django.core.files.base import File, ContentFile from django.http import HttpResponse, StreamingHttpResponse, HttpResponseForbidden -from django.http.request import RawPostDataException +from django.http.request import RawPostDataException, QueryDict from django.http.multipartparser import MultiPartParser from django.db import models, connections from django.db.models import Q, F @@ -28,7 +28,11 @@ from django.db.models.fields.reverse_related import ForeignObjectRel -from .exceptions import BinderException, BinderFieldTypeError, BinderFileSizeExceeded, BinderForbidden, BinderImageError, BinderImageSizeExceeded, BinderInvalidField, BinderIsDeleted, BinderIsNotDeleted, BinderMethodNotAllowed, BinderNotAuthenticated, BinderNotFound, BinderReadOnlyFieldError, BinderRequestError, BinderValidationError, BinderFileTypeIncorrect, BinderInvalidURI +from .exceptions import ( + BinderException, BinderFieldTypeError, BinderFileSizeExceeded, BinderForbidden, BinderImageError, BinderImageSizeExceeded, + BinderInvalidField, BinderIsDeleted, BinderIsNotDeleted, BinderMethodNotAllowed, BinderNotAuthenticated, BinderNotFound, + BinderReadOnlyFieldError, BinderRequestError, BinderValidationError, BinderFileTypeIncorrect, BinderInvalidURI, BinderSkipSave +) from . import history from .orderable_agg import OrderableArrayAgg, GroupConcat, StringAgg from .models import FieldFilter, BinderModel, ContextAnnotation, OptionalAnnotation, BinderFileField, BinderImageField @@ -280,7 +284,6 @@ def prefix_q_expression(value, prefix, antiprefix=None, model=None): children.append((prefix + '__' + child[0], child[1])) return Q(*children, _connector=value.connector, _negated=value.negated) - class ModelView(View): # Model this is a view for. Use None for views not tied to a particular model. model = None @@ -330,6 +333,9 @@ class ModelView(View): # NOTE: custom _store__foo() methods will still be called for unupdatable fields. unupdatable_fields = [] + # Allow validation without saving. + allow_standalone_validation = False + # Fields to use for ?search=foo. Empty tuple for disabled search. # NOTE: only string fields and 'id' are supported. # id is hardcoded to be treated as an integer. @@ -436,6 +442,10 @@ def dispatch(self, request, *args, **kwargs): response = None try: + # only allow standalone validation if you know what you are doing + if 'validate' in request.GET and request.GET['validate'] == 'true' and not self.allow_standalone_validation: + raise BinderRequestError('Standalone validation not enabled. You must enable this feature explicitly.') + #### START TRANSACTION with ExitStack() as stack, history.atomic(source='http', user=request.user, uuid=request.request_id): transaction_dbs = ['default'] @@ -1428,6 +1438,18 @@ def binder_validation_error(self, obj, validation_error, pk=None): }) + + def _abort_when_standalone_validation(self, request): + """Raise a `BinderSkipSave` exception when this is a validation request.""" + if 'validate' in request.GET and request.GET['validate'] == 'true': + if self.allow_standalone_validation: + params = QueryDict(request.body) + raise BinderSkipSave + else: + raise BinderRequestError('Standalone validation not enabled. You must enable this feature explicitly.') + + + # Deserialize JSON to Django Model objects. # obj: Model object to update (for PUT), newly created object (for POST) # values: Python dict of {field name: value} (parsed JSON) @@ -1437,6 +1459,9 @@ def _store(self, obj, values, request, ignore_unknown_fields=False, pk=None): ignored_fields = [] validation_errors = [] + # When only validating and not saving we attach a parameter so that we can skip or add validation checks + only_validate = request.GET.get('validate') == 'true' or request.GET.get('validate') == 'True' + if obj.pk is None: self._require_model_perm('add', request, obj.pk) else: @@ -1474,8 +1499,8 @@ def store_m2m_field(obj, field, value, request): raise sum(validation_errors, None) try: - obj.save() - assert(obj.pk is not None) # At this point, the object must have been created. + obj.save(only_validate=only_validate) + assert(obj.pk is not None) # At this point, the object must have been created. except ValidationError as ve: validation_errors.append(self.binder_validation_error(obj, ve, pk=pk)) @@ -1520,6 +1545,9 @@ def store_m2m_field(obj, field, value, request): def _store_m2m_field(self, obj, field, value, request): validation_errors = [] + # When only validating and not saving we attach a parameter so that we can skip or add validation checks + only_validate = request.GET.get('validate') == 'true' or request.GET.get('validate') == 'True' + # Can't use isinstance() because apparantly ManyToManyDescriptor is a subclass of # ReverseManyToOneDescriptor. Yes, really. if getattr(obj._meta.model, field).__class__ == models.fields.related.ReverseManyToOneDescriptor: @@ -1562,11 +1590,11 @@ def _store_m2m_field(self, obj, field, value, request): for addobj in obj_field.model.objects.filter(id__in=new_ids - old_ids): setattr(addobj, obj_field.field.name, obj) try: - addobj.save() + addobj.save(only_validate=only_validate) except ValidationError as ve: validation_errors.append(self.binder_validation_error(addobj, ve)) else: - addobj.save() + addobj.save(only_validate=only_validate) elif getattr(obj._meta.model, field).__class__ == models.fields.related.ReverseOneToOneDescriptor: #### XXX FIXME XXX ugly quick fix for reverse relation + multiput issue if any(v for v in value if v is not None and v < 0): @@ -1582,7 +1610,7 @@ def _store_m2m_field(self, obj, field, value, request): remote_obj = field_descriptor.related.remote_field.model.objects.get(pk=value[0]) setattr(remote_obj, field_descriptor.related.remote_field.name, obj) try: - remote_obj.save() + remote_obj.save(only_validate=only_validate) remote_obj.refresh_from_db() except ValidationError as ve: validation_errors.append(self.binder_validation_error(remote_obj, ve)) @@ -2279,7 +2307,7 @@ def _multi_put_deletions(self, deletions, new_id_map, request): def multi_put(self, request): - logger.info('ACTIVATING THE MULTI-PUT!!!1!') + logger.info('ACTIVATING THE MULTI-PUT!!!!!') # Hack to communicate to _store() that we're not interested in # the new data (for perf reasons). @@ -2287,13 +2315,15 @@ def multi_put(self, request): data, deletions = self._multi_put_parse_request(request) objects = self._multi_put_collect_objects(data) - objects, overrides = self._multi_put_override_superclass(objects) + objects, overrides = self._multi_put_override_superclass(objects) # model inheritance objects = self._multi_put_convert_backref_to_forwardref(objects) dependencies = self._multi_put_calculate_dependencies(objects) ordered_objects = self._multi_put_order_dependencies(dependencies) - new_id_map = self._multi_put_save_objects(ordered_objects, objects, request) - self._multi_put_id_map_add_overrides(new_id_map, overrides) - new_id_map = self._multi_put_deletions(deletions, new_id_map, request) + new_id_map = self._multi_put_save_objects(ordered_objects, objects, request) # may raise validation errors + self._multi_put_id_map_add_overrides(new_id_map, overrides) # model inheritance + new_id_map = self._multi_put_deletions(deletions, new_id_map, request) # may raise validation errors + + self._abort_when_standalone_validation(request) output = defaultdict(list) for (model, oid), nid in new_id_map.items(): @@ -2387,6 +2417,8 @@ def put(self, request, pk=None): data = self._store(obj, values, request) + self._abort_when_standalone_validation(request) + new = dict(data) new.pop('_meta', None) @@ -2417,6 +2449,8 @@ def post(self, request, pk=None): data = self._store(self.model(), values, request) + self._abort_when_standalone_validation(request) + new = dict(data) new.pop('_meta', None) @@ -2454,6 +2488,9 @@ def delete(self, request, pk=None, undelete=False, skip_body_check=False): raise BinderNotFound() self.delete_obj(obj, undelete, request) + + self._abort_when_standalone_validation(request) + logger.info('{}DELETEd {} #{}'.format('UN' if undelete else '', self._model_name(), pk)) return HttpResponse(status=204) # No content @@ -2466,6 +2503,10 @@ def delete_obj(self, obj, undelete, request): def soft_delete(self, obj, undelete, request): + + # When only validating and not saving we attach a parameter so that we can skip or add validation checks + only_validate = request.GET.get('validate') == 'true' or request.GET.get('validate') == 'True' + # Not only for soft delets, actually handles all deletions try: if obj.deleted and not undelete: @@ -2497,7 +2538,7 @@ def soft_delete(self, obj, undelete, request): obj.deleted = not undelete try: - obj.save() + obj.save(only_validate=only_validate) except ValidationError as ve: raise self.binder_validation_error(obj, ve) diff --git a/docs/api.md b/docs/api.md index b5dddb3b..eb6ba843 100644 --- a/docs/api.md +++ b/docs/api.md @@ -67,7 +67,7 @@ Ordering is a simple matter of enumerating the fields in the `order_by` query pa The default sort order is ascending. If you want to sort in descending order, simply prefix the attribute name with a minus sign. This honors the scoping, so `api/animal?order_by=-name,id` will sort by `name` in descending order and by `id` in ascending order. -### Saving a model +### Saving or updating a model Creating a new model is possible with `POST api/animal/`, and updating a model with `PUT api/animal/`. Both requests accept a JSON body, like this: @@ -161,6 +161,17 @@ If this request succeeds, you'll get back a mapping of the fake ids and the real It is also possible to update existing models with multi PUT. If you use a "real" id instead of a fake one, the model will be updated instead of created. + +#### Standalone Validation (without saving models) + +Sometimes you want to validate the model that you are going to save without actually saving it. This is useful, for example, when you want to inform the user of validation errors on the frontend, without having to implement the validation logic again. You may check for validation errors by sending a `POST`, `PUT` or `PATCH` request with an additional query parameter `validate`. + +Currently this is implemented by raising an `BinderValidateOnly` exception, which makes sure that the atomic database transaction is aborted. Ideally, you would only want to call the validation logic on the models, so only calling validation for fields and validation for model (`clean()`). But for now, we do it this way, at the cost of a performance penalty. + +It is important to realize that in this way, the normal `save()` function is called on a model, so it is possible that possible side effects are triggered, when these are implemented directly in `save()`, as opposed to in a signal method, which would be preferable. In other words, we cannot guarantee that the request will be idempotent. Therefore, the validation only feature is disabled by default and must be enabled by setting `allow_standalone_validation=True` on the view. + +When a model is being validated and not actually being saved the `_validation_model property` of the binder model is set to True. This allows whitelisting of certain validation checks such as with certain relations that are not included with the validation model. + ### Uploading files To upload a file, you have to add it to the `file_fields` of the `ModelView`: diff --git a/docs/plugins/html_field.md b/docs/plugins/html_field.md new file mode 100644 index 00000000..3c830213 --- /dev/null +++ b/docs/plugins/html_field.md @@ -0,0 +1,5 @@ +# HTML Field + +The HTML field provides a django model field optimized for user posted HTML code. Its aim is to provide a safe +way to implement a CMS system, where the end user can create pages, but cannot do XSS injections. + diff --git a/tests/plugins/test_csvexport.py b/tests/plugins/test_csvexport.py index fbe86bfb..8c37a87b 100644 --- a/tests/plugins/test_csvexport.py +++ b/tests/plugins/test_csvexport.py @@ -7,7 +7,7 @@ from django.core.files import File from django.contrib.auth.models import User -from ..testapp.models import Picture, Animal, Caretaker +from ..testapp.models import Picture, Animal, PictureBook, Caretaker import csv import openpyxl @@ -31,8 +31,10 @@ def setUp(self): self.pictures = [] + picture_book = PictureBook.objects.create(name='Holiday 2012') + for i in range(3): - picture = Picture(animal=animal) + picture = Picture(animal=animal, picture_book=picture_book) file = CsvExportTest.temp_imagefile(50, 50, 'jpeg') picture.file.save('picture.jpg', File(file), save=False) picture.original_file.save('picture_copy.jpg', File(file), save=False) @@ -48,6 +50,7 @@ def setUp(self): def test_csv_download(self): response = self.client.get('/picture/download_csv/') + print(response.content) self.assertEqual(200, response.status_code) response_data = csv.reader(io.StringIO(response.content.decode("utf-8"))) @@ -146,3 +149,24 @@ def test_context_aware_download_xlsx(self): [self.pictures[1].id, str(self.pictures[1].animal_id), (self.pictures[1].id ** 2)]) self.assertEqual(list(_values[3]), [self.pictures[2].id, str(self.pictures[2].animal_id), (self.pictures[2].id ** 2)]) + + def test_none_foreign_key(self): + """ + If we have a relation that we have to include which is nullable, and we have a foreign key, we do not want the + csv export to crash. I.e. we also want for a picture to export the picture book name, even though not all pioctures + belong to a picture book + :return: + """ + self.pictures[0].picture_book = None + self.pictures[0].save() + + response = self.client.get('/picture/download/') + self.assertEqual(200, response.status_code) + response_data = csv.reader(io.StringIO(response.content.decode("utf-8"))) + + data = list(response_data) + content = data[1:] + + picture_books = [c[-1] for c in content] + + self.assertEqual(['', 'Holiday 2012', 'Holiday 2012'], picture_books) diff --git a/tests/test_html_field.py b/tests/test_html_field.py new file mode 100644 index 00000000..fa10ab7d --- /dev/null +++ b/tests/test_html_field.py @@ -0,0 +1,142 @@ +from django.contrib.auth.models import User +from django.test import TestCase, Client + +import json +from .testapp.models import Zoo, WebPage + + +class HtmlFieldTestCase(TestCase): + + def setUp(self): + super().setUp() + u = User(username='testuser', is_active=True, is_superuser=True) + u.set_password('test') + u.save() + self.client = Client() + r = self.client.login(username='testuser', password='test') + self.assertTrue(r) + + self.zoo = Zoo(name='Apenheul') + self.zoo.save() + + self.webpage = WebPage.objects.create(zoo=self.zoo, content='') + + + + def test_save_normal_text_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', data=json.dumps({'content': 'Artis'})) + self.assertEqual(response.status_code, 200) + + def test_simple_html_is_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': '

Artis

Artis is a zoo in amsterdam'})) + self.assertEqual(response.status_code, 200) + + def test_wrong_attribute_not_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': 'test'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('invalid_attribute', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + def test_simple_link_is_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', data=json.dumps( + {'content': 'Visit artis website'})) + + self.assertEqual(response.status_code, 200) + + + + def test_javascript_link_is_not_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({ + 'content': 'Visit artis website'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + self.assertEqual('ValidationError', parsed_response['code']) + + self.assertEqual('invalid_attribute', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + + + def test_script_is_not_ok(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': ''})) + + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('invalid_tag', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + def test_script_is_not_ok_nested(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': ''})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('invalid_tag', parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + + def test_can_handle_reallife_data(self): + """ + This is the worst case that we could produce on the WYIWYG edittor + """ + content = '

normal text


HEADing 1


HEADING 2


HEADING 3


bold


italic


underlined


Link


  1. ol1
  2. ol2


subscripttgege

g

"' + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': content})) + + self.assertEqual(response.status_code, 200) + + def test_multiple_errors(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({ + 'content': 'Visit artis website'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + self.assertEqual('ValidationError', parsed_response['code']) + + + self.assertEqual('invalid_tag', + parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + self.assertEqual('invalid_tag', + parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][1]['code']) + + + def test_link_no_rel_errors(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': 'bla'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('missing_attribute', + parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + def test_link_noopener_required(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': 'bla'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('invalid_attribute', + parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) + + def test_link_noreferrer_required(self): + response = self.client.put(f'/web_page/{self.webpage.id}/', + data=json.dumps({'content': 'bla'})) + self.assertEqual(response.status_code, 400) + + parsed_response = json.loads(response.content) + + self.assertEqual('ValidationError', parsed_response['code']) + self.assertEqual('invalid_attribute', + parsed_response['errors']['web_page'][f'{self.webpage.id}']['content'][0]['code']) diff --git a/tests/test_model_validation.py b/tests/test_model_validation.py new file mode 100644 index 00000000..b05dba5a --- /dev/null +++ b/tests/test_model_validation.py @@ -0,0 +1,271 @@ +from re import I +from tests.testapp.models import contact_person +from tests.testapp.models.contact_person import ContactPerson +from django.test import TestCase, Client + +import json +from binder.json import jsonloads +from django.contrib.auth.models import User +from .testapp.models import Animal, Caretaker, ContactPerson + + +class TestModelValidation(TestCase): + """ + Test the validate-only functionality. + + We check that the validation is executed as normal, but that the models + are not created when the validate query paramter is set to true. + + We check validation for: + - post + - put + - multi-put + - delete + """ + + + def setUp(self): + super().setUp() + u = User(username='testuser', is_active=True, is_superuser=True) + u.set_password('test') + u.save() + self.client = Client() + r = self.client.login(username='testuser', password='test') + self.assertTrue(r) + + # some update payload + self.model_data_with_error = { + 'name': 'very_special_forbidden_contact_person_name', # see `contact_person.py` + } + self.model_data_with_non_validation_error = { + 'name': 'very_special_validation_contact_person_name', # see `contact_person.py` + } + self.model_data = { + 'name': 'Scooooooby', + } + + + ### helpers ### + + + def assert_validation_error(self, response, person_id=None): + if person_id is None: + person_id = 'null' # for post + + self.assertEqual(response.status_code, 400) + + returned_data = jsonloads(response.content) + + # check that there were validation errors + self.assertEqual(returned_data.get('code'), 'ValidationError') + + # check that the validation error is present + validation_error = returned_data.get('errors').get('contact_person').get(str(person_id)).get('__all__')[0] + self.assertEqual(validation_error.get('code'), 'invalid') + self.assertEqual(validation_error.get('message'), 'Very special validation check that we need in `tests.M2MStoreErrorsTest`.') + + + def assert_multi_put_validation_error(self, response): + self.assertEqual(response.status_code, 400) + + returned_data = jsonloads(response.content) + + # check that there were validation errors + self.assertEqual(returned_data.get('code'), 'ValidationError') + + # check that all (two) the validation errors are present + for error in returned_data.get('errors').get('contact_person').values(): + validation_error = error.get('__all__')[0] + self.assertEqual(validation_error.get('code'), 'invalid') + self.assertEqual(validation_error.get('message'), 'Very special validation check that we need in `tests.M2MStoreErrorsTest`.') + + + ### tests ### + + + def assert_no_validation_error(self, response): + self.assertEqual(response.status_code, 200) + + # check that the validation was successful + returned_data = jsonloads(response.content) + self.assertEqual(returned_data.get('code'), 'SkipSave') + self.assertEqual(returned_data.get('message'), 'No validation errors were encountered.') + + + def test_validate_on_post(self): + self.assertEqual(0, ContactPerson.objects.count()) + + # trigger a validation error + response = self.client.post('/contact_person/?validate=true', data=json.dumps(self.model_data_with_error), content_type='application/json') + self.assert_validation_error(response) + self.assertEqual(0, ContactPerson.objects.count()) + + # now without validation errors + response = self.client.post('/contact_person/?validate=true', data=json.dumps(self.model_data), content_type='application/json') + self.assert_no_validation_error(response) + self.assertEqual(0, ContactPerson.objects.count()) + + # now for real + response = self.client.post('/contact_person/', data=json.dumps(self.model_data), content_type='application/json') + self.assertEqual(response.status_code, 200) + self.assertEqual('Scooooooby', ContactPerson.objects.first().name) + + + def test_validate_on_put(self): + person_id = ContactPerson.objects.create(name='Scooby Doo').id + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + # trigger a validation error + response = self.client.put(f'/contact_person/{person_id}/?validate=true', data=json.dumps(self.model_data_with_error), content_type='application/json') + self.assert_validation_error(response, person_id) + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + # now without validation errors + response = self.client.put(f'/contact_person/{person_id}/?validate=true', data=json.dumps(self.model_data), content_type='application/json') + self.assert_no_validation_error(response) + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + # now for real + response = self.client.put(f'/contact_person/{person_id}/', data=json.dumps(self.model_data), content_type='application/json') + self.assertEqual(response.status_code, 200) + self.assertEqual('Scooooooby', ContactPerson.objects.first().name) + + def test_validate_model_validation_whitelisting(self): + person_id = ContactPerson.objects.create(name='Scooby Doo').id + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + # the normal request should give a validation error + response = self.client.put(f'/contact_person/{person_id}/', data=json.dumps(self.model_data_with_non_validation_error), content_type='application/json') + self.assert_validation_error(response, person_id) + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + # when just validating we want to ignore this validation error, so with validation it should not throw an error + response = self.client.put(f'/contact_person/{person_id}/?validate=true', data=json.dumps(self.model_data), content_type='application/json') + self.assert_no_validation_error(response) + self.assertEqual('Scooby Doo', ContactPerson.objects.first().name) + + + + def test_validate_on_multiput(self): + person_1_id = ContactPerson.objects.create(name='Scooby Doo 1').id + person_2_id = ContactPerson.objects.create(name='Scooby Doo 2').id + + multi_put_data = {'data': [ + { + 'id': person_1_id, + 'name': 'New Scooby', + }, + { + 'id': person_2_id, + 'name': 'New Doo' + } + ]} + + multi_put_data_with_error = {'data': [ + { + 'id': person_1_id, + 'name': 'very_special_forbidden_contact_person_name', + }, + { + 'id': person_2_id, + 'name': 'very_special_forbidden_contact_person_name' + } + ]} + + multi_put_data_with_validation_whitelist = {'data': [ + { + 'id': person_1_id, + 'name': 'very_special_validation_contact_person_name', + }, + { + 'id': person_2_id, + 'name': 'very_special_validation_contact_person_other_name' + } + ]} + + # trigger a validation error + response = self.client.put(f'/contact_person/?validate=true', data=json.dumps(multi_put_data_with_error), content_type='application/json') + self.assert_multi_put_validation_error(response) + self.assertEqual('Scooby Doo 1', ContactPerson.objects.get(id=person_1_id).name) + self.assertEqual('Scooby Doo 2', ContactPerson.objects.get(id=person_2_id).name) + + + # now without validation error + response = self.client.put(f'/contact_person/?validate=true', data=json.dumps(multi_put_data), content_type='application/json') + self.assert_no_validation_error(response) + self.assertEqual('Scooby Doo 1', ContactPerson.objects.get(id=person_1_id).name) + self.assertEqual('Scooby Doo 2', ContactPerson.objects.get(id=person_2_id).name) + + # multi put validation whitelist test + response = self.client.put(f'/contact_person/?validate=true', data=json.dumps(multi_put_data_with_validation_whitelist), content_type='application/json') + self.assert_no_validation_error(response) + self.assertEqual('Scooby Doo 1', ContactPerson.objects.get(id=person_1_id).name) + self.assertEqual('Scooby Doo 2', ContactPerson.objects.get(id=person_2_id).name) + + # multi put non validation whitelist test error + response = self.client.put(f'/contact_person/', + data=json.dumps(multi_put_data_with_validation_whitelist), + content_type='application/json') + self.assert_multi_put_validation_error(response) + self.assertEqual('Scooby Doo 1', ContactPerson.objects.get(id=person_1_id).name) + self.assertEqual('Scooby Doo 2', ContactPerson.objects.get(id=person_2_id).name) + + # now for real + response = self.client.put(f'/contact_person/', data=json.dumps(multi_put_data), content_type='application/json') + self.assertEqual(response.status_code, 200) + self.assertEqual('New Scooby', ContactPerson.objects.get(id=person_1_id).name) + self.assertEqual('New Doo', ContactPerson.objects.get(id=person_2_id).name) + + + def test_validate_on_delete(self): + '''Check if deletion is cancelled when we only attempt to validate + the delete operation. This test only covers validation of the + on_delete=PROTECT constraint of a fk.''' + + def is_deleted(obj): + '''Whether the obj was soft-deleted, so when the 'deleted' + attribute is not present, or when it is True.''' + + try: + obj.refresh_from_db() + except obj.DoesNotExist: + return True # hard-deleted + return animal.__dict__.get('deleted') or False + + + # animal has a fk to caretaker with on_delete=PROTECT + caretaker = Caretaker.objects.create(name='Connie Care') + animal = Animal.objects.create(name='Pony', caretaker=caretaker) + + + ### with validation error + + response = self.client.delete(f'/caretaker/{caretaker.id}/?validate=true') + # assert validation error + # and check that it was about the PROTECTED constraint + self.assertEqual(response.status_code, 400) + returned_data = jsonloads(response.content) + self.assertEqual(returned_data.get('code'), 'ValidationError') + self.assertEqual(returned_data.get('errors').get('caretaker').get(str(caretaker.id)).get('id')[0].get('code'), 'protected') + + self.assertFalse(is_deleted(caretaker)) + + + ### without validation error + + # now we delete the animal to make sure that deletion is possible + # note that soft-deleting will of course not remove the validation error + animal.delete() + + # now no validation error should be trown + response = self.client.delete(f'/caretaker/{caretaker.id}/?validate=true') + print(response.content) + self.assert_no_validation_error(response) + + self.assertFalse(is_deleted(caretaker)) + + + ### now for real + + response = self.client.delete(f'/caretaker/{caretaker.id}/') + self.assertTrue(is_deleted(caretaker)) diff --git a/tests/testapp/models/__init__.py b/tests/testapp/models/__init__.py index ed41bbe5..bb6cf47f 100644 --- a/tests/testapp/models/__init__.py +++ b/tests/testapp/models/__init__.py @@ -9,11 +9,13 @@ from .gate import Gate from .nickname import Nickname, NullableNickname from .lion import Lion -from .picture import Picture +from .picture import Picture, PictureBook from .zoo import Zoo from .zoo_employee import ZooEmployee from .city import City, CityState, PermanentCity from .country import Country +from .web_page import WebPage + # This is Postgres-specific if os.environ.get('BINDER_TEST_MYSQL', '0') != '1': from .timetable import TimeTable diff --git a/tests/testapp/models/animal.py b/tests/testapp/models/animal.py index 85d844e0..49d48987 100644 --- a/tests/testapp/models/animal.py +++ b/tests/testapp/models/animal.py @@ -14,7 +14,7 @@ class Animal(LoadedValuesMixin, BinderModel): name = models.TextField(max_length=64) zoo = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='animals', blank=True, null=True) zoo_of_birth = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='+', blank=True, null=True) # might've been born outside captivity - caretaker = models.ForeignKey('Caretaker', on_delete=models.PROTECT, related_name='animals', blank=True, null=True) + caretaker = models.ForeignKey('Caretaker', on_delete=models.PROTECT, related_name='animals', blank=True, null=True) # we use the fact that this one is PROTECT in `test_model_validation.py` deleted = models.BooleanField(default=False) # Softdelete def __str__(self): diff --git a/tests/testapp/models/contact_person.py b/tests/testapp/models/contact_person.py index 60db9897..d810c745 100644 --- a/tests/testapp/models/contact_person.py +++ b/tests/testapp/models/contact_person.py @@ -16,3 +16,10 @@ def clean(self): code='invalid', message='Very special validation check that we need in `tests.M2MStoreErrorsTest`.' ) + + # Should only give an error when model is not a validation model + if (self.name == 'very_special_validation_contact_person_name' or self.name == 'very_special_validation_contact_person_other_name') and not self._validation_model: + raise ValidationError( + code='invalid', + message='Very special validation check that we need in `tests.M2MStoreErrorsTest`.' + ) diff --git a/tests/testapp/models/picture.py b/tests/testapp/models/picture.py index ffbe59bb..862bfb3e 100644 --- a/tests/testapp/models/picture.py +++ b/tests/testapp/models/picture.py @@ -13,6 +13,18 @@ def delete_files(sender, instance=None, **kwargs): except Exception: pass + +class PictureBook(BinderModel): + """ + Sometimes customers like to commemorate their visit to the zoo. Of course there are always some shitty pictures that + we do not want in a picture album + + """ + + name = models.TextField() + + + # At the website of the zoo there are some pictures of animals. This model links the picture to an animal. # # A picture has two files, the original uploaded file, and the modified file. This model is used for testing the @@ -21,6 +33,7 @@ class Picture(BinderModel): animal = models.ForeignKey('Animal', on_delete=models.CASCADE, related_name='picture') file = models.ImageField(upload_to='floor-plans') original_file = models.ImageField(upload_to='floor-plans') + picture_book = models.ForeignKey('PictureBook', on_delete=models.CASCADE, null=True, blank=True) def __str__(self): return 'picture %d: (Picture for animal %s)' % (self.pk or 0, self.animal.name) diff --git a/tests/testapp/models/web_page.py b/tests/testapp/models/web_page.py new file mode 100644 index 00000000..811cdeb6 --- /dev/null +++ b/tests/testapp/models/web_page.py @@ -0,0 +1,13 @@ + +from binder.models import BinderModel +from django.db import models + +from binder.plugins.models import HtmlField + + +class WebPage(BinderModel): + """ + Every zoo has a webpage containing some details about the zoo + """ + zoo = models.OneToOneField('Zoo', related_name='web_page', on_delete=models.CASCADE) + content = HtmlField() diff --git a/tests/testapp/views/__init__.py b/tests/testapp/views/__init__.py index 7dd1f4f0..90ed137d 100644 --- a/tests/testapp/views/__init__.py +++ b/tests/testapp/views/__init__.py @@ -15,7 +15,8 @@ from .gate import GateView from .lion import LionView from .nickname import NicknameView -from .picture import PictureView +from .picture import PictureView, PictureBookView from .user import UserView from .zoo import ZooView from .zoo_employee import ZooEmployeeView +from .web_page import WebPageView diff --git a/tests/testapp/views/caretaker.py b/tests/testapp/views/caretaker.py index 85698354..04b9769b 100644 --- a/tests/testapp/views/caretaker.py +++ b/tests/testapp/views/caretaker.py @@ -9,6 +9,10 @@ class CaretakerView(CsvExportView, ModelView): unupdatable_fields = ['first_seen'] model = Caretaker + + # see `test_model_validation.py` + allow_standalone_validation = True + csv_settings = CsvExportView.CsvExportSettings( withs=[], column_map=[ @@ -18,3 +22,4 @@ class CaretakerView(CsvExportView, ModelView): ], extra_params={'include_annotations': 'scary'}, ) + diff --git a/tests/testapp/views/contact_person.py b/tests/testapp/views/contact_person.py index c1e90c5b..bc2f25fe 100644 --- a/tests/testapp/views/contact_person.py +++ b/tests/testapp/views/contact_person.py @@ -6,3 +6,6 @@ class ContactPersonView(ModelView): model = ContactPerson m2m_fields = ['zoos'] unwritable_fields = ['created_at', 'updated_at'] + + # see `test_model_validation.py` + allow_standalone_validation = True diff --git a/tests/testapp/views/picture.py b/tests/testapp/views/picture.py index 7dadfe99..0ba35f99 100644 --- a/tests/testapp/views/picture.py +++ b/tests/testapp/views/picture.py @@ -3,16 +3,19 @@ from binder.views import ModelView from binder.plugins.views import ImageView, CsvExportView -from ..models import Picture +from ..models import Picture, PictureBook +class PictureBookView(ModelView): + model = PictureBook class PictureView(ModelView, ImageView, CsvExportView): model = Picture file_fields = ['file', 'original_file'] - csv_settings = CsvExportView.CsvExportSettings(['animal'], [ + csv_settings = CsvExportView.CsvExportSettings(['animal', 'picture_book'], [ ('id', 'picture identifier'), ('animal.id', 'animal identifier'), ('id', 'squared picture identifier', lambda id, row, mapping: id**2), + ('picture_book.name', 'Picturebook name') ]) @list_route(name='download_csv', methods=['GET']) diff --git a/tests/testapp/views/web_page.py b/tests/testapp/views/web_page.py new file mode 100644 index 00000000..d7613031 --- /dev/null +++ b/tests/testapp/views/web_page.py @@ -0,0 +1,7 @@ +from binder.views import ModelView + +from ..models import WebPage + +# From the api docs +class WebPageView(ModelView): + model = WebPage