Snapshot: alpha-2026-9
This commit is contained in:
parent
9acf5a97e2
commit
d00b5c7961
241 changed files with 85546 additions and 2409 deletions
|
|
@ -1,6 +1,17 @@
|
|||
import io
|
||||
import os
|
||||
from datetime import timedelta
|
||||
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import default_storage
|
||||
from django.http import HttpResponse
|
||||
from django.urls import path
|
||||
from django.db.models import Q
|
||||
from django.conf import settings
|
||||
from django.utils.http import http_date
|
||||
from django.utils.timezone import now
|
||||
from drf_yasg.utils import swagger_auto_schema
|
||||
from PIL import Image
|
||||
from rest_framework import status
|
||||
from rest_framework.decorators import api_view, permission_classes, authentication_classes
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
|
|
@ -9,27 +20,129 @@ from rest_framework.response import Response
|
|||
from authentication.signature_auth import SignatureAuthentication
|
||||
from files.models import File
|
||||
|
||||
THUMBNAIL_SIZES = (32, 64, 256)
|
||||
|
||||
|
||||
def _accessible_files(request):
|
||||
# Shared by media_urls and thumbnail_urls: a file is visible if the requester is
|
||||
# friends-or-self with whatever references it (item, profile picture), a member of the group
|
||||
# that owns the item it's attached to, or it's their own staged photo.
|
||||
return File.objects.filter(
|
||||
Q(connected_items__owner__in=request.user.friends_or_self(), connected_items__is_deleted=False) |
|
||||
Q(connected_items__owner_group__in=request.user.member_of_groups.all(), connected_items__is_deleted=False) |
|
||||
Q(profile_picture_users__in=request.user.friends_or_self()) |
|
||||
Q(staged_by_workflows__owner__in=request.user.user.all())
|
||||
).distinct()
|
||||
|
||||
|
||||
def _cache_headers(etag):
|
||||
# Content is hash-addressed and can never change under a given URL, so it's cacheable forever.
|
||||
return {
|
||||
'ETag': etag,
|
||||
'Cache-Control': 'max-age=31536000, private, immutable',
|
||||
'Expires': http_date((now() + timedelta(days=365)).timestamp()),
|
||||
}
|
||||
|
||||
|
||||
@swagger_auto_schema(method='GET', auto_schema=None)
|
||||
@api_view(['GET'])
|
||||
@permission_classes([IsAuthenticated])
|
||||
@authentication_classes([SignatureAuthentication])
|
||||
def media_urls(request, hash_path):
|
||||
# CORS is added automatically by middleware, except via X-Accel-Redirect, where nginx's
|
||||
# /redirect_media/ block must set it instead.
|
||||
#
|
||||
# Looked up by the derived storage path, not the raw hash, to match FileSerializer.name
|
||||
# (used for AuthenticatedImage's `src`) and the existing test suite (MediaUrlTestCase).
|
||||
try:
|
||||
file = File.objects.filter(connected_items__owner__in=request.user.friends_or_self()).distinct().get(
|
||||
file=hash_path)
|
||||
file = _accessible_files(request).get(file=hash_path)
|
||||
|
||||
return HttpResponse(status=status.HTTP_200_OK,
|
||||
content_type=file.mime_type,
|
||||
headers={
|
||||
'X-Accel-Redirect': f'/redirect_media/{hash_path}',
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
}) # TODO Expires and Cache-Control
|
||||
# Must run before this check, else a bare hash + If-None-Match would let anyone probe
|
||||
# file existence for files they can't see.
|
||||
if request.META.get('HTTP_IF_NONE_MATCH') == file.hash:
|
||||
return HttpResponse(status=status.HTTP_304_NOT_MODIFIED)
|
||||
|
||||
cache_headers = _cache_headers(file.hash)
|
||||
|
||||
if settings.SERVE_X_ACCEL_REDIRECT:
|
||||
return HttpResponse(status=status.HTTP_200_OK,
|
||||
content_type=file.mime_type,
|
||||
headers={
|
||||
'X-Accel-Redirect': f'/redirect_media/{hash_path}',
|
||||
**cache_headers,
|
||||
})
|
||||
else:
|
||||
# Reads via FieldFile.open() (not file.file.path) since tests swap in an in-memory storage backend.
|
||||
with file.file.open('rb') as fh:
|
||||
content = fh.read()
|
||||
return HttpResponse(status=status.HTTP_200_OK,
|
||||
content_type=file.mime_type,
|
||||
headers=cache_headers,
|
||||
content=content)
|
||||
|
||||
except File.DoesNotExist:
|
||||
return Response(status=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
|
||||
def _thumbnail_rel_path(file_hash, size):
|
||||
# Mirrors hash_upload()'s sharding under thumbnails/<size>/, reachable via the same nginx alias as originals.
|
||||
return os.path.join('thumbnails', str(size), file_hash[:2], file_hash[2:4], file_hash[4:6],
|
||||
file_hash[6:] + '.jpg')
|
||||
|
||||
|
||||
@swagger_auto_schema(method='GET', auto_schema=None)
|
||||
@api_view(['GET'])
|
||||
@permission_classes([IsAuthenticated])
|
||||
@authentication_classes([SignatureAuthentication])
|
||||
def thumbnail_urls(request, size, hash_path):
|
||||
if size not in THUMBNAIL_SIZES:
|
||||
return Response(status=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
try:
|
||||
file = _accessible_files(request).get(file=hash_path)
|
||||
|
||||
etag = f'{file.hash}_{size}'
|
||||
if request.META.get('HTTP_IF_NONE_MATCH') == etag:
|
||||
return HttpResponse(status=status.HTTP_304_NOT_MODIFIED)
|
||||
|
||||
# Read/write via default_storage, not a hand-rolled path, to work with both real-disk
|
||||
# and in-memory test storage.
|
||||
rel_path = _thumbnail_rel_path(file.hash, size)
|
||||
if not default_storage.exists(rel_path):
|
||||
# Always re-encoded as JPEG regardless of original format - simpler than preserving transparency at this scale.
|
||||
with file.file.open('rb') as fh:
|
||||
image = Image.open(fh)
|
||||
image.thumbnail((size, size))
|
||||
# Flatten through RGBA before dropping to RGB. See docs/implementation.md#rgba-flattening-avoids-revealing-black-under-transparent-pixels.
|
||||
rgba = image.convert('RGBA')
|
||||
flattened = Image.new('RGB', rgba.size, (255, 255, 255))
|
||||
flattened.paste(rgba, mask=rgba.getchannel('A'))
|
||||
buffer = io.BytesIO()
|
||||
flattened.save(buffer, 'JPEG', quality=90)
|
||||
default_storage.save(rel_path, ContentFile(buffer.getvalue()))
|
||||
|
||||
cache_headers = _cache_headers(etag)
|
||||
|
||||
if settings.SERVE_X_ACCEL_REDIRECT:
|
||||
return HttpResponse(status=status.HTTP_200_OK,
|
||||
content_type='image/jpeg',
|
||||
headers={
|
||||
'X-Accel-Redirect': f'/redirect_media/{rel_path}',
|
||||
**cache_headers,
|
||||
})
|
||||
else:
|
||||
with default_storage.open(rel_path, 'rb') as fh:
|
||||
content = fh.read()
|
||||
return HttpResponse(status=status.HTTP_200_OK,
|
||||
content_type='image/jpeg',
|
||||
headers=cache_headers,
|
||||
content=content)
|
||||
|
||||
except File.DoesNotExist:
|
||||
return Response(status=status.HTTP_404_NOT_FOUND)
|
||||
|
||||
|
||||
urlpatterns = [
|
||||
path('<int:size>/<path:hash_path>/', thumbnail_urls),
|
||||
path('<path:hash_path>', media_urls),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import default_storage
|
||||
from django.db import models, IntegrityError
|
||||
from django.db.models import Model
|
||||
|
||||
|
|
@ -40,6 +43,10 @@ class FileManager(models.Manager):
|
|||
else:
|
||||
raise ValueError('data must be a base64 encoded string or file and hash must be provided')
|
||||
if not self.filter(hash=kwargs['hash']).exists():
|
||||
# Clears a stale orphan already at this hash's canonical path before saving. See docs/implementation.md#stale-orphan-cleanup-at-the-canonical-hash-path.
|
||||
expected_path = hash_upload(SimpleNamespace(hash=kwargs['hash']), '')
|
||||
if default_storage.exists(expected_path):
|
||||
default_storage.delete(expected_path)
|
||||
return super().create(**kwargs)
|
||||
else:
|
||||
raise IntegrityError('File with this hash already exists')
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ class FileSerializer(serializers.Serializer):
|
|||
|
||||
def to_representation(self, instance):
|
||||
return {'id': instance.id, 'name': instance.file.url, 'size': instance.file.size,
|
||||
'mime_type': instance.mime_type}
|
||||
'mime_type': instance.mime_type, 'hash': instance.hash}
|
||||
|
||||
def create(self, validated_data):
|
||||
return File.objects.get_or_create(**validated_data)[0]
|
||||
|
|
|
|||
|
|
@ -1,13 +1,21 @@
|
|||
import io
|
||||
import os
|
||||
import zlib
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import DefaultStorage
|
||||
from django.core.files.storage import DefaultStorage, default_storage
|
||||
from django.db import IntegrityError, transaction
|
||||
from django.test import Client
|
||||
from authentication.tests import SignatureAuthClient, ToolshedTestCase, UserTestMixin
|
||||
from django.test import Client, override_settings
|
||||
from authentication.tests import SignatureAuthClient, ToolshedTestCase, UserTestMixin, GroupTestMixin
|
||||
from toolshed.models import InventoryItem
|
||||
from toolshed.tests import InventoryTestMixin
|
||||
from nacl.hash import sha256
|
||||
from nacl.encoding import HexEncoder
|
||||
from PIL import Image
|
||||
import base64
|
||||
|
||||
from files.media_urls import THUMBNAIL_SIZES
|
||||
from files.models import File
|
||||
|
||||
anonymous_client = Client()
|
||||
|
|
@ -105,6 +113,18 @@ class FilesTestCase(FilesTestMixin, ToolshedTestCase):
|
|||
self.assertEqual(File.objects.count(), 3)
|
||||
self.assertEqual(countdir(DefaultStorage(), ''), 3)
|
||||
|
||||
def test_file_upload_reclaims_stale_orphan_at_canonical_path(self):
|
||||
# Regression test for a stale orphan at the canonical hash path. See docs/implementation.md#stale-orphan-cleanup-at-the-canonical-hash-path.
|
||||
expected_path = f"{self.f['hash4'][:2]}/{self.f['hash4'][2:4]}/{self.f['hash4'][4:6]}/{self.f['hash4'][6:]}"
|
||||
default_storage.save(expected_path, ContentFile(self.f['test_content4']))
|
||||
self.assertTrue(default_storage.exists(expected_path))
|
||||
self.assertFalse(File.objects.filter(hash=self.f['hash4']).exists())
|
||||
|
||||
file = File.objects.create(mime_type='text/plain', data=self.f['encoded_content4'])
|
||||
|
||||
self.assertEqual(file.file.name, expected_path)
|
||||
self.assertEqual(file.file.read(), self.f['test_content4'])
|
||||
|
||||
|
||||
class MediaUrlTestCase(FilesTestMixin, UserTestMixin, InventoryTestMixin, ToolshedTestCase):
|
||||
def setUp(self):
|
||||
|
|
@ -120,6 +140,7 @@ class MediaUrlTestCase(FilesTestMixin, UserTestMixin, InventoryTestMixin, Toolsh
|
|||
self.f['item2'].files.add(self.f['test_file1'])
|
||||
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=True)
|
||||
def test_file_url(self):
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash1'][:2]}/{self.f['hash1'][2:4]}/{self.f['hash1'][4:6]}/{self.f['hash1'][6:]}",
|
||||
|
|
@ -165,3 +186,198 @@ class MediaUrlTestCase(FilesTestMixin, UserTestMixin, InventoryTestMixin, Toolsh
|
|||
self.f['ext_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
self.assertTrue('X-Accel-Redirect' not in reply.headers)
|
||||
|
||||
def test_file_url_only_connected_via_deleted_item(self):
|
||||
# test_file2 is only reachable through item1; soft-deleting it doesn't sever the files
|
||||
# M2M row, so this would regress to serving test_file2 as if item1 were still live if
|
||||
# _accessible_files ever stops excluding soft-deleted items again.
|
||||
self.f['item1'].delete()
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash2'][:2]}/{self.f['hash2'][2:4]}/{self.f['hash2'][4:6]}/{self.f['hash2'][6:]}",
|
||||
self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=True)
|
||||
def test_profile_picture_url(self):
|
||||
self.f['local_user1'].profile_picture = self.f['test_file3']
|
||||
self.f['local_user1'].save()
|
||||
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash3'][:2]}/{self.f['hash3'][2:4]}/{self.f['hash3'][4:6]}/{self.f['hash3'][6:]}",
|
||||
self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=True)
|
||||
def test_profile_picture_url_friend(self):
|
||||
self.f['local_user1'].profile_picture = self.f['test_file3']
|
||||
self.f['local_user1'].save()
|
||||
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash3'][:2]}/{self.f['hash3'][2:4]}/{self.f['hash3'][4:6]}/{self.f['hash3'][6:]}",
|
||||
self.f['local_user2'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
|
||||
def test_profile_picture_url_not_friend(self):
|
||||
self.f['local_user1'].profile_picture = self.f['test_file3']
|
||||
self.f['local_user1'].save()
|
||||
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash3'][:2]}/{self.f['hash3'][2:4]}/{self.f['hash3'][4:6]}/{self.f['hash3'][6:]}",
|
||||
self.f['ext_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
|
||||
|
||||
class GroupOwnedMediaUrlTestCase(FilesTestMixin, UserTestMixin, GroupTestMixin, ToolshedTestCase):
|
||||
"""_accessible_files() only checked connected_items__owner (personal items) before, never
|
||||
connected_items__owner_group - a group-owned item's own files were unreachable via /media/ or
|
||||
/thumbnails/ for every member, including ones who could see and edit the item itself."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.prepare_files()
|
||||
self.prepare_users()
|
||||
self.prepare_groups()
|
||||
self.f['group1'].members.add(self.f['local_user2'].public_identity)
|
||||
self.f['group_item'] = InventoryItem.create_for_owner(
|
||||
owner_group=self.f['group1'], owned_quantity=1, name='group-drill')
|
||||
self.f['group_item'].files.add(self.f['test_file1'])
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=True)
|
||||
def test_group_member_can_view_group_item_file(self):
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash1'][:2]}/{self.f['hash1'][2:4]}/{self.f['hash1'][4:6]}/{self.f['hash1'][6:]}",
|
||||
self.f['local_user2'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
|
||||
def test_non_member_cannot_view_group_item_file(self):
|
||||
reply = client.get(
|
||||
f"/media/{self.f['hash1'][:2]}/{self.f['hash1'][2:4]}/{self.f['hash1'][4:6]}/{self.f['hash1'][6:]}",
|
||||
self.f['ext_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
|
||||
|
||||
class ThumbnailUrlTestCase(FilesTestMixin, UserTestMixin, InventoryTestMixin, ToolshedTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.prepare_files()
|
||||
self.prepare_users()
|
||||
self.prepare_categories()
|
||||
self.prepare_tags()
|
||||
self.prepare_properties()
|
||||
self.prepare_inventory()
|
||||
|
||||
# Each test uses a distinct seeded image (own hash/cache path) since InMemoryStorage
|
||||
# isn't reset between test methods, so a shared image risks one test's cached thumbnail leaking into another's assertions.
|
||||
seed = zlib.crc32(self._testMethodName.encode()) % 256
|
||||
buffer = io.BytesIO()
|
||||
Image.new('RGB', (800, 600), (seed, 255 - seed, 128)).save(buffer, 'PNG')
|
||||
image_bytes = buffer.getvalue()
|
||||
self.f['image_hash'] = sha256(image_bytes, encoder=HexEncoder).decode('utf-8')
|
||||
self.f['image_file'] = File.objects.create(
|
||||
mime_type='image/png', data=base64.b64encode(image_bytes).decode('utf-8'))
|
||||
self.f['item1'].files.add(self.f['image_file'])
|
||||
|
||||
def _thumb_url(self, size, image_hash=None):
|
||||
h = image_hash or self.f['image_hash']
|
||||
return f"/media/{size}/{h[:2]}/{h[2:4]}/{h[4:6]}/{h[6:]}/"
|
||||
|
||||
def _thumb_rel_path(self, size):
|
||||
h = self.f['image_hash']
|
||||
return os.path.join('thumbnails', str(size), h[:2], h[2:4], h[4:6], h[6:] + '.jpg')
|
||||
|
||||
def test_thumbnail_sizes_available(self):
|
||||
# Fixed size allow-list this suite exercises - update both if media_urls.py's THUMBNAIL_SIZES changes.
|
||||
self.assertEqual(THUMBNAIL_SIZES, (32, 64, 256))
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=False)
|
||||
def test_thumbnail_generates_resized_jpeg(self):
|
||||
self.assertFalse(default_storage.exists(self._thumb_rel_path(64)))
|
||||
|
||||
reply = client.get(self._thumb_url(64), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
self.assertEqual(reply.headers['Content-Type'], 'image/jpeg')
|
||||
|
||||
generated = Image.open(io.BytesIO(reply.content))
|
||||
self.assertEqual(generated.format, 'JPEG')
|
||||
# Aspect-ratio-preserving fit within a 64x64 box, not a crop to exactly 64x64.
|
||||
self.assertLessEqual(max(generated.size), 64)
|
||||
self.assertAlmostEqual(generated.size[0] / generated.size[1], 800 / 600, places=2)
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=False)
|
||||
def test_thumbnail_flattens_transparency_instead_of_going_black(self):
|
||||
# Regression test for an 'LA' source with zeroed transparent-region luminance. See docs/implementation.md#rgba-flattening-avoids-revealing-black-under-transparent-pixels.
|
||||
half_transparent = Image.new('LA', (200, 200))
|
||||
pixels = half_transparent.load()
|
||||
for x in range(200):
|
||||
for y in range(200):
|
||||
if x < 100:
|
||||
pixels[x, y] = (0, 0) # transparent, zeroed-out luminance underneath
|
||||
else:
|
||||
pixels[x, y] = (255, 255) # fully opaque, bright content
|
||||
|
||||
buffer = io.BytesIO()
|
||||
half_transparent.save(buffer, 'PNG')
|
||||
image_bytes = buffer.getvalue()
|
||||
image_hash = sha256(image_bytes, encoder=HexEncoder).decode('utf-8')
|
||||
image_file = File.objects.create(
|
||||
mime_type='image/png', data=base64.b64encode(image_bytes).decode('utf-8'))
|
||||
self.f['item1'].files.add(image_file)
|
||||
|
||||
reply = client.get(self._thumb_url(64, image_hash=image_hash), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
|
||||
generated = Image.open(io.BytesIO(reply.content)).convert('L')
|
||||
# The opaque (right) half must stay bright; a naive RGB conversion would blacken it too.
|
||||
self.assertGreater(generated.getpixel((generated.width - 1, generated.height // 2)), 200)
|
||||
self.assertNotEqual(generated.getextrema(), (0, 0))
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=False)
|
||||
def test_thumbnail_served_from_cache_on_second_request(self):
|
||||
client.get(self._thumb_url(64), self.f['local_user1'])
|
||||
rel_path = self._thumb_rel_path(64)
|
||||
with default_storage.open(rel_path, 'rb') as f:
|
||||
cached_bytes = f.read()
|
||||
|
||||
# Overwrites the cache with a marker so a correct implementation must serve it back, not regenerate.
|
||||
default_storage.delete(rel_path)
|
||||
default_storage.save(rel_path, ContentFile(cached_bytes + b'MARKER'))
|
||||
|
||||
reply = client.get(self._thumb_url(64), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
self.assertTrue(reply.content.endswith(b'MARKER'))
|
||||
|
||||
def test_thumbnail_invalid_size(self):
|
||||
reply = client.get(self._thumb_url(100), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
self.assertFalse(default_storage.exists(self._thumb_rel_path(100)))
|
||||
|
||||
def test_thumbnail_not_found(self):
|
||||
reply = client.get(self._thumb_url(64, image_hash='0' * 64), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
|
||||
def test_thumbnail_anonymous(self):
|
||||
reply = anonymous_client.get(self._thumb_url(64))
|
||||
self.assertEqual(reply.status_code, 403)
|
||||
|
||||
def test_thumbnail_not_friend(self):
|
||||
# local_user1/local_user2 are friends here (see prepare_inventory), so the denied case needs a stranger instead.
|
||||
reply = client.get(self._thumb_url(64), self.f['ext_user1'])
|
||||
self.assertEqual(reply.status_code, 404)
|
||||
self.assertFalse(default_storage.exists(self._thumb_rel_path(64)))
|
||||
|
||||
def test_thumbnail_conditional_get(self):
|
||||
reply = client.get(self._thumb_url(64), self.f['local_user1'])
|
||||
etag = reply.headers['ETag']
|
||||
self.assertEqual(etag, f"{self.f['image_hash']}_64")
|
||||
|
||||
reply = client.get(self._thumb_url(64), self.f['local_user1'], HTTP_IF_NONE_MATCH=etag)
|
||||
self.assertEqual(reply.status_code, 304)
|
||||
|
||||
@override_settings(SERVE_X_ACCEL_REDIRECT=True)
|
||||
def test_thumbnail_x_accel_redirect(self):
|
||||
reply = client.get(self._thumb_url(64), self.f['local_user1'])
|
||||
self.assertEqual(reply.status_code, 200)
|
||||
h = self.f['image_hash']
|
||||
self.assertEqual(reply.headers['X-Accel-Redirect'],
|
||||
f"/redirect_media/thumbnails/64/{h[:2]}/{h[2:4]}/{h[4:6]}/{h[6:]}.jpg")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue