diff --git a/lockbox/common/constants.py b/lockbox/common/constants.py index ed02950..e0647cf 100644 --- a/lockbox/common/constants.py +++ b/lockbox/common/constants.py @@ -1,7 +1,7 @@ import re CONTENT_RANGE_HEADER = "HTTP_CONTENT_RANGE" -CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P\d+)-(?P\d+)/(?P\d+)$") +CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P\d+)-(?P\d+)$") class UPLOAD_STATUS_TYPES: UPLOADING = "uploading" diff --git a/lockbox/lockbox/settings.py b/lockbox/lockbox/settings.py index dd62e96..c91150f 100644 --- a/lockbox/lockbox/settings.py +++ b/lockbox/lockbox/settings.py @@ -2,10 +2,12 @@ from pathlib import Path - -from common.utils import get_config from dotenv import load_dotenv +from lockbox.setup import validate_paths +from common.utils import get_config + + load_dotenv() # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -107,15 +109,26 @@ STATICFILES_DIRS = [ STATIC_ROOT = BASE_DIR / "staticfiles" STATIC_URL = 'static/' STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, "staticfiles": { "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", }, } # Storage -MEDIA_ROOT = BASE_DIR / "media" +MEDIA_ROOT = Path("/home/kitty/src/lockbox/FILES") MEDIA_URL = "files/" +validate_paths(MEDIA_ROOT) + # Default primary key field type DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' AUTH_USER_MODEL = 'user.LockboxUser' + + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', + 'PAGE_SIZE': 50 +} \ No newline at end of file diff --git a/lockbox/lockbox/setup.py b/lockbox/lockbox/setup.py new file mode 100644 index 0000000..0e9fd78 --- /dev/null +++ b/lockbox/lockbox/setup.py @@ -0,0 +1,11 @@ +import os + +# TODO: LOG MEEEEE +# TODO: Figure out file owner in system, permissions, GUID +# Whats the default path if not provided? // docker volume +def validate_paths(media_path): + if not os.path.isdir(media_path): + try: + os.makedirs(media_path) + except Exception as e: + raise e \ No newline at end of file diff --git a/lockbox/storage/forms.py b/lockbox/storage/forms.py deleted file mode 100644 index 9aaddfa..0000000 --- a/lockbox/storage/forms.py +++ /dev/null @@ -1,12 +0,0 @@ -from django import forms - -from storage.models import File - - -class FileForm(forms.ModelForm): - - set_name = forms.BooleanField() - - class Meta: - model = File - exclude = File.readonly_fields diff --git a/lockbox/storage/migrations/0001_initial.py b/lockbox/storage/migrations/0001_initial.py index 5f607b0..7f458eb 100644 --- a/lockbox/storage/migrations/0001_initial.py +++ b/lockbox/storage/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-02-16 08:15 +# Generated by Django 4.2.15 on 2024-09-16 11:24 import common.utils from django.conf import settings @@ -23,7 +23,7 @@ class Migration(migrations.Migration): ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), - ('filename', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')), + ('filename', models.CharField(help_text='Name of the file', max_length=255, verbose_name='name')), ('extension', models.CharField(blank=True, help_text='reported filesystem extension (not mime type)', max_length=128, null=True, verbose_name='extension')), ('file', models.FileField(blank=True, help_text='actual file', null=True, upload_to='', verbose_name='file')), ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('processing', 'processing'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=10, verbose_name='status')), @@ -32,7 +32,7 @@ class Migration(migrations.Migration): ('delete_on_expiration', models.BooleanField(default=False, help_text='will be deleted if expired and expires is true', verbose_name='delete on expiration')), ('size_on_disk', models.PositiveBigIntegerField(blank=True, help_text='total size on disk for this file', null=True, verbose_name='size on disk (bytes)')), ('max_size_chunk_bytes', models.PositiveBigIntegerField(default=common.utils.get_max_size_chunk_bytes, help_text='max size of each individual chunk for this file', verbose_name='maximum size of chunks (bytes)')), - ('owner', models.ForeignKey(blank=True, help_text='owner of this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')), + ('owner', models.ForeignKey(blank=True, help_text='Who owns this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')), ], options={ 'verbose_name': 'file', @@ -45,13 +45,12 @@ class Migration(migrations.Migration): ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), - ('chunk', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path_chunk, verbose_name='file')), - ('chunk_id', models.BigIntegerField(help_text='part of chunk', verbose_name='chunk id')), - ('size', models.BigIntegerField(help_text='size for this chunk', verbose_name='size')), - ('start', models.BigIntegerField(help_text='start for this chunk', verbose_name='start')), - ('end', models.BigIntegerField(help_text='end for this chunk', verbose_name='end')), + ('chunk', models.FileField(help_text='chunk file', upload_to=storage.models.get_upload_path_chunk, verbose_name='chunk file')), + ('chunk_id', models.BigIntegerField(help_text='chunk id', verbose_name='chunk id')), + ('size', models.BigIntegerField(help_text='chunk size', verbose_name='size')), + ('start_bytes', models.BigIntegerField(help_text='part of file start', verbose_name='start bytes')), + ('end_bytes', models.BigIntegerField(help_text='part of file end', verbose_name='end bytes')), ('file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='storage.file')), - ('owner', models.ForeignKey(blank=True, help_text='owner of this file chunk', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='chunks_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')), ], options={ 'verbose_name': 'file chunk', diff --git a/lockbox/storage/models.py b/lockbox/storage/models.py index 802e135..8ea2ace 100644 --- a/lockbox/storage/models.py +++ b/lockbox/storage/models.py @@ -6,19 +6,26 @@ from common.models import LockboxBase from common.utils import get_config, get_max_size_chunk_bytes from django.conf import settings from django.core.files.uploadedfile import UploadedFile -from django.db import models +from django.db import models, transaction from django.utils import timezone from django.utils.translation import gettext_lazy as _ def get_upload_path_chunk(instance, filename): - file_subdir = settings.MEDIA_ROOT / str(instance.file.lid) + # TODO: How do we reconcile storage? + # TODO: Do we autodetect existing files task? + # TODO: Figure out absolute storage :(, custom storage and custom filefield? why is this not a def behaviour? - if not Path.exists(file_subdir): - Path.mkdir(file_subdir) + filename = f"{instance.chunk_id}.chunk" + chunk_dir = settings.MEDIA_ROOT / str(instance.file.lid) + + if not Path.exists(chunk_dir): + Path.mkdir(chunk_dir) + + target_path = Path(chunk_dir) / Path(filename) + print(target_path) + return target_path - filename = f"{FileChunk.last_chunk_id(instance.file)}.chunk" - return Path(str(instance.lid)) / Path(filename) class File(LockboxBase): filename = models.CharField( @@ -149,19 +156,40 @@ class File(LockboxBase): if last_chunk_id: return last_chunk_id.get("chunk_id") return - 1 + + def create_chunk(self, chunk_file, chunk_data): + chunk = FileChunk( + file=self, + chunk=chunk_file, + chunk_id=self.last_chunk_id, + **chunk_data + ) + + chunk.save() + return chunk + + + def save(self, *args, **kwargs): + self.max_size_chunk_bytes = get_max_size_chunk_bytes() + return super().save(*args, **kwargs) + + + def delete(self, *args, **kwargs): - def delete(self, *args, delete_file=True, **kwargs): - # TODO: make this baby an atomic transaction if self.file: storage, path = self.file.storage, self.file.path - super().delete(*args, **kwargs) - if self.file and delete_file: + + if self.file: + # TODO: Figure out if file exists and try to delete it if error, report error. storage.delete(path) - # clean up chunks in case they have not been cleaned up by task. - self.chunks.all().delete() + with transaction.atomic(): + self.chunks.all().delete() + result = super().delete(*args, **kwargs) + return result def handler_bytes(self): + # TODO: This is a naive approach, we almost never want to do this. self.file.close() self.file.open(mode="rb") return UploadedFile(file=self.file, name=self.filename, size=self.offset) @@ -197,28 +225,18 @@ class FileChunk(LockboxBase): help_text=_("chunk size"), ) - start = models.BigIntegerField( + start_bytes = models.BigIntegerField( null=False, blank=False, - verbose_name=("start"), - help_text=_("chunk start"), + verbose_name=("start bytes"), + help_text=_("part of file start"), ) - end = models.BigIntegerField( + end_bytes = models.BigIntegerField( null=False, blank=False, - verbose_name=("end"), - help_text=_("chunk end"), - ) - - owner = models.ForeignKey( - "user.LockboxUser", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="chunks_owned", - verbose_name=_("owner"), - help_text=_("chunk owner"), + verbose_name=("end bytes"), + help_text=_("part of file end"), ) readonly_fields = [ @@ -239,8 +257,17 @@ class FileChunk(LockboxBase): unique_together = ("file", "chunk_id") def save(self, *args, **kwargs): - self.chunk_id = self.file.last_chunk_id() + 1 + self.chunk_id = self.file.last_chunk_id + 1 return super().save(*args, **kwargs) + + def delete(self, *args, **kwargs): + if self.file: + storage, path = self.file.storage, self.file.path + + if self.file: + # TODO: Figure out if file exists and try to delete it if error, report error. + storage.delete(path) + return super().delete(*args, **kwargs) # class FileShare(LockboxBase): # file = models.ForeignKey( diff --git a/lockbox/storage/views_api.py b/lockbox/storage/views_api.py index 9de0d9b..fd5f7f7 100644 --- a/lockbox/storage/views_api.py +++ b/lockbox/storage/views_api.py @@ -1,5 +1,6 @@ from common.constants import ( - UPLOAD_STATUS_TYPES, + CONTENT_RANGE_HEADER, + CONTENT_RANGE_HEADER_PATTERN ) # from common.utils import get_config @@ -7,6 +8,8 @@ from rest_framework import status from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from rest_framework.exceptions import NotFound, ValidationError +from rest_framework.parsers import FileUploadParser # from user.models import LockboxUser from storage.models import File, FileChunk @@ -18,16 +21,60 @@ class FileModelViewSet(ModelViewSet): queryset = File.objects.all() serializer_class = FileSerializer - @action(detail=True, methods=["post"]) - def finalize(self, *args, **kwargs): + @action(detail=True, methods=["GET"]) + def last_chunk_position(self, request, pk=None): file = self.get_object() - file.status = UPLOAD_STATUS_TYPES.PROCESSING - file.save() - return Response(status=status.HTTP_200_OK) + last_chunk_id = file.last_chunk_id + last_postion = 0 + if last_chunk_id != -1: + last_chunk = self.chunks.order_by("-chunk_id").values("end_bytes").first() + if last_chunk: + last_postion = last_chunk_id.get("end_bytes") + return Response({"last_chunk_position": last_postion}, status=status.HTTP_200_OK) + class FileChunkViewSet(ModelViewSet): model = FileChunk queryset = FileChunk.objects.all() serializer_class = FileChunkSerializer + parser_classes = (FileUploadParser,) + def create(self, request, filename="DUMMY", format=None, file_pk=None): + file = File.objects.filter(lid=str(file_pk)).first() + if not file: + raise NotFound(f"File with ID {file_pk} was not found") + chunk_data = self.get_content_range(request) + if not chunk_data: + raise ValidationError( + f"Missing content range headers" + ) + + chunk_file = request.FILES["file"] + if chunk_file.size > file.max_size_chunk_bytes: + raise ValidationError( + f"Chunk size is greater than files max chunk size: {chunk_file.size} > {file.max_size_chunk_bytes}") + + range_size = chunk_data["end_bytes"] - chunk_data["start_bytes"] + if chunk_file.size != range_size: + raise ValidationError( + f"Actual chunk size mismatches content-range header: {chunk_file.size} != {range_size}" + ) + + chunk_data["size"] = chunk_file.size + file.create_chunk(chunk_file=chunk_file, chunk_data=chunk_data) + return Response(status=status.HTTP_201_CREATED) + + def get_content_range(self, request): + content_range = request.META.get(CONTENT_RANGE_HEADER, None) + if not content_range: + return None + + match = CONTENT_RANGE_HEADER_PATTERN.match(content_range) + if not match: + return None + + return { + "start_bytes": int(match.group('start')), + "end_bytes": int(match.group('end')), + } \ No newline at end of file diff --git a/lockbox/storage/views_client.py b/lockbox/storage/views_client.py index 576ae1c..3ae393b 100644 --- a/lockbox/storage/views_client.py +++ b/lockbox/storage/views_client.py @@ -2,14 +2,7 @@ from common.utils import get_config from django.shortcuts import render from django.views import View -from storage.forms import FileForm - - +# Static view class FileUploadView(View): def get(self, request): - context = { - "form": FileForm, - "max_chunk_bytes": get_config("MAX_CHUNK_BYTES"), - "max_file_bytes": get_config("MAX_FILE_BYTES"), - } - return render(request, "storage/upload.html", context=context) + return render(request, "storage/upload.html") diff --git a/lockbox/user/migrations/0001_initial.py b/lockbox/user/migrations/0001_initial.py index fb3d32e..03a347a 100644 --- a/lockbox/user/migrations/0001_initial.py +++ b/lockbox/user/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-02-12 08:39 +# Generated by Django 4.2.15 on 2024-09-16 10:45 import django.contrib.auth.validators from django.db import migrations, models @@ -28,7 +28,7 @@ class Migration(migrations.Migration): ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), - ('alias', models.SlugField(blank=True, help_text='an alias or nickname to remember who this is', max_length=32, null=True, unique=True, verbose_name='name')), + ('alias', models.SlugField(blank=True, help_text='an alias or nickname to remember who this is', max_length=32, null=True, unique=True, verbose_name='alias')), ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')), ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')), ], diff --git a/lockbox/user/models.py b/lockbox/user/models.py index 9216734..da916fb 100644 --- a/lockbox/user/models.py +++ b/lockbox/user/models.py @@ -8,7 +8,7 @@ from user.managers import LockboxUserManager class LockboxUser(AbstractUser, LockboxBase): alias = models.SlugField( - verbose_name=_("name"), + verbose_name=_("alias"), max_length=32, unique=True, null=True,