Added pagination, simplified models, rebuild chunking view

lockbox/common/constants.py

@@ -1,7 +1,7 @@
 import re
 
 CONTENT_RANGE_HEADER = "HTTP_CONTENT_RANGE"
-CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P<start>\d+)-(?P<end>\d+)/(?P<total>\d+)$")
+CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P<start>\d+)-(?P<end>\d+)$")
 
 class UPLOAD_STATUS_TYPES:
     UPLOADING = "uploading"

lockbox/lockbox/settings.py

@@ -2,10 +2,12 @@
 
 
 from pathlib import Path
-
-from common.utils import get_config
 from dotenv import load_dotenv
 
+from lockbox.setup import validate_paths
+from common.utils import get_config
+
+
 load_dotenv()
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -107,15 +109,26 @@ STATICFILES_DIRS = [
 STATIC_ROOT = BASE_DIR / "staticfiles"
 STATIC_URL = 'static/'
 STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+    },
     "staticfiles": {
         "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
     },
 }
 
 # Storage
-MEDIA_ROOT = BASE_DIR / "media"
+MEDIA_ROOT = Path("/home/kitty/src/lockbox/FILES")
 MEDIA_URL = "files/"
 
+validate_paths(MEDIA_ROOT)
+
 # Default primary key field type
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 AUTH_USER_MODEL = 'user.LockboxUser'
+
+
+REST_FRAMEWORK = {
+    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
+    'PAGE_SIZE': 50
+}

lockbox/lockbox/setup.py

@@ -0,0 +1,11 @@
+import os
+
+# TODO: LOG MEEEEE
+# TODO: Figure out file owner in system, permissions, GUID
+# Whats the default path if not provided? // docker volume
+def validate_paths(media_path):
+    if not os.path.isdir(media_path):
+        try:
+            os.makedirs(media_path)
+        except Exception as e:
+            raise e

lockbox/storage/forms.py

@@ -1,12 +0,0 @@
-from django import forms
-
-from storage.models import File
-
-
-class FileForm(forms.ModelForm):
-
-    set_name = forms.BooleanField()
-
-    class Meta:
-        model = File
-        exclude = File.readonly_fields

lockbox/storage/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.10 on 2024-02-16 08:15
+# Generated by Django 4.2.15 on 2024-09-16 11:24
 
 import common.utils
 from django.conf import settings
@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
                 ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
                 ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
                 ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
-                ('filename', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')),
+                ('filename', models.CharField(help_text='Name of the file', max_length=255, verbose_name='name')),
                 ('extension', models.CharField(blank=True, help_text='reported filesystem extension (not mime type)', max_length=128, null=True, verbose_name='extension')),
                 ('file', models.FileField(blank=True, help_text='actual file', null=True, upload_to='', verbose_name='file')),
                 ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('processing', 'processing'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=10, verbose_name='status')),
@@ -32,7 +32,7 @@ class Migration(migrations.Migration):
                 ('delete_on_expiration', models.BooleanField(default=False, help_text='will be deleted if expired and expires is true', verbose_name='delete on expiration')),
                 ('size_on_disk', models.PositiveBigIntegerField(blank=True, help_text='total size on disk for this file', null=True, verbose_name='size on disk (bytes)')),
                 ('max_size_chunk_bytes', models.PositiveBigIntegerField(default=common.utils.get_max_size_chunk_bytes, help_text='max size of each individual chunk for this file', verbose_name='maximum size of chunks (bytes)')),
-                ('owner', models.ForeignKey(blank=True, help_text='owner of this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')),
+                ('owner', models.ForeignKey(blank=True, help_text='Who owns this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')),
             ],
             options={
                 'verbose_name': 'file',
@@ -45,13 +45,12 @@ class Migration(migrations.Migration):
                 ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
                 ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
                 ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
-                ('chunk', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path_chunk, verbose_name='file')),
+                ('chunk', models.FileField(help_text='chunk file', upload_to=storage.models.get_upload_path_chunk, verbose_name='chunk file')),
-                ('chunk_id', models.BigIntegerField(help_text='part of chunk', verbose_name='chunk id')),
+                ('chunk_id', models.BigIntegerField(help_text='chunk id', verbose_name='chunk id')),
-                ('size', models.BigIntegerField(help_text='size for this chunk', verbose_name='size')),
+                ('size', models.BigIntegerField(help_text='chunk size', verbose_name='size')),
-                ('start', models.BigIntegerField(help_text='start for this chunk', verbose_name='start')),
+                ('start_bytes', models.BigIntegerField(help_text='part of file start', verbose_name='start bytes')),
-                ('end', models.BigIntegerField(help_text='end for this chunk', verbose_name='end')),
+                ('end_bytes', models.BigIntegerField(help_text='part of file end', verbose_name='end bytes')),
                 ('file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='storage.file')),
-                ('owner', models.ForeignKey(blank=True, help_text='owner of this file chunk', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='chunks_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')),
             ],
             options={
                 'verbose_name': 'file chunk',

lockbox/storage/models.py

@@ -6,19 +6,26 @@ from common.models import LockboxBase
 from common.utils import get_config, get_max_size_chunk_bytes
 from django.conf import settings
 from django.core.files.uploadedfile import UploadedFile
-from django.db import models
+from django.db import models, transaction
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 
 def get_upload_path_chunk(instance, filename):
-    file_subdir = settings.MEDIA_ROOT / str(instance.file.lid)
+    # TODO: How do we reconcile storage?
+    # TODO: Do we autodetect existing files task?
+    # TODO: Figure out absolute storage :(, custom storage and custom filefield? why is this not a def behaviour?
 
-    if not Path.exists(file_subdir):
+    filename = f"{instance.chunk_id}.chunk"
-        Path.mkdir(file_subdir)
+    chunk_dir = settings.MEDIA_ROOT / str(instance.file.lid)
+
+    if not Path.exists(chunk_dir):
+        Path.mkdir(chunk_dir)
+
+    target_path = Path(chunk_dir) / Path(filename)
+    print(target_path)
+    return target_path
 
-    filename = f"{FileChunk.last_chunk_id(instance.file)}.chunk"
-    return Path(str(instance.lid)) / Path(filename)
 
 class File(LockboxBase):
     filename = models.CharField(
@@ -150,18 +157,39 @@ class File(LockboxBase):
             return last_chunk_id.get("chunk_id")
         return - 1
     
-    def delete(self, *args, delete_file=True, **kwargs):
+    def create_chunk(self, chunk_file, chunk_data):
-        # TODO: make this baby an atomic transaction 
+        chunk = FileChunk(
+            file=self,
+            chunk=chunk_file,
+            chunk_id=self.last_chunk_id,
+            **chunk_data
+        )
+
+        chunk.save()
+        return chunk
+    
+
+    def save(self, *args, **kwargs):
+        self.max_size_chunk_bytes = get_max_size_chunk_bytes()
+        return super().save(*args, **kwargs)
+        
+
+    def delete(self, *args, **kwargs):
+
         if self.file:
             storage, path = self.file.storage, self.file.path
-        super().delete(*args, **kwargs)
+
-        if self.file and delete_file:
+        if self.file:
+            # TODO: Figure out if file exists and try to delete it if error, report error.
             storage.delete(path)
 
-        # clean up chunks in case they have not been cleaned up by task.
+        with transaction.atomic():
-        self.chunks.all().delete()
+            self.chunks.all().delete()
+            result = super().delete(*args, **kwargs)
+        return result
 
     def handler_bytes(self):
+        # TODO: This is a naive approach, we almost never want to do this.
         self.file.close()
         self.file.open(mode="rb")
         return UploadedFile(file=self.file, name=self.filename, size=self.offset)
@@ -197,28 +225,18 @@ class FileChunk(LockboxBase):
         help_text=_("chunk size"),
     )
 
-    start = models.BigIntegerField(
+    start_bytes = models.BigIntegerField(
         null=False,
         blank=False,
-        verbose_name=("start"),
+        verbose_name=("start bytes"),
-        help_text=_("chunk start"),
+        help_text=_("part of file start"),
     )
 
-    end = models.BigIntegerField(
+    end_bytes = models.BigIntegerField(
         null=False,
         blank=False,
-        verbose_name=("end"),
+        verbose_name=("end bytes"),
-        help_text=_("chunk end"),
+        help_text=_("part of file end"),
-    )
-
-    owner = models.ForeignKey(
-        "user.LockboxUser",
-        null=True,
-        blank=True,
-        on_delete=models.SET_NULL,
-        related_name="chunks_owned",
-        verbose_name=_("owner"),
-        help_text=_("chunk owner"),
     )
 
     readonly_fields = [
@@ -239,9 +257,18 @@ class FileChunk(LockboxBase):
         unique_together = ("file", "chunk_id")
 
     def save(self, *args, **kwargs):
-        self.chunk_id = self.file.last_chunk_id() + 1
+        self.chunk_id = self.file.last_chunk_id + 1
         return super().save(*args, **kwargs)
     
+    def delete(self, *args, **kwargs):
+        if self.file:
+            storage, path = self.file.storage, self.file.path
+
+        if self.file:
+            # TODO: Figure out if file exists and try to delete it if error, report error.
+            storage.delete(path)
+        return super().delete(*args, **kwargs)
+
 # class FileShare(LockboxBase):
 #     file = models.ForeignKey(
 #         "storage.File",

lockbox/storage/views_api.py

@@ -1,5 +1,6 @@
 from common.constants import (
-    UPLOAD_STATUS_TYPES,
+    CONTENT_RANGE_HEADER,
+    CONTENT_RANGE_HEADER_PATTERN
 )
 
 # from common.utils import get_config
@@ -7,6 +8,8 @@ from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet
+from rest_framework.exceptions import NotFound, ValidationError
+from rest_framework.parsers import FileUploadParser
 
 # from user.models import LockboxUser
 from storage.models import File, FileChunk
@@ -18,16 +21,60 @@ class FileModelViewSet(ModelViewSet):
     queryset = File.objects.all()
     serializer_class = FileSerializer
 
-    @action(detail=True, methods=["post"])
+    @action(detail=True, methods=["GET"])
-    def finalize(self, *args, **kwargs):
+    def last_chunk_position(self, request, pk=None):
         file = self.get_object()
-        file.status = UPLOAD_STATUS_TYPES.PROCESSING
+        last_chunk_id = file.last_chunk_id
-        file.save()
+        last_postion = 0
-        return Response(status=status.HTTP_200_OK)
+        if  last_chunk_id != -1:
+            last_chunk = self.chunks.order_by("-chunk_id").values("end_bytes").first()
+            if last_chunk:
+                last_postion = last_chunk_id.get("end_bytes")
+        return Response({"last_chunk_position": last_postion}, status=status.HTTP_200_OK)
+
 
 class FileChunkViewSet(ModelViewSet):
     model = FileChunk
     queryset = FileChunk.objects.all()
     serializer_class = FileChunkSerializer
+    parser_classes = (FileUploadParser,)
 
+    def create(self, request, filename="DUMMY", format=None, file_pk=None):
+        file = File.objects.filter(lid=str(file_pk)).first()
+        if not file:
+            raise NotFound(f"File with ID {file_pk} was not found")
 
+        chunk_data = self.get_content_range(request)
+        if not chunk_data:
+            raise ValidationError(
+                f"Missing content range headers"
+            )
+
+        chunk_file = request.FILES["file"]
+        if chunk_file.size > file.max_size_chunk_bytes:
+            raise ValidationError(
+                f"Chunk size is greater than files max chunk size: {chunk_file.size} > {file.max_size_chunk_bytes}")
+        
+        range_size = chunk_data["end_bytes"] - chunk_data["start_bytes"]
+        if chunk_file.size != range_size:
+            raise ValidationError(
+                f"Actual chunk size mismatches content-range header: {chunk_file.size} != {range_size}"
+            )
+
+        chunk_data["size"] = chunk_file.size
+        file.create_chunk(chunk_file=chunk_file, chunk_data=chunk_data)
+        return Response(status=status.HTTP_201_CREATED)
+
+    def get_content_range(self, request):
+        content_range = request.META.get(CONTENT_RANGE_HEADER, None)
+        if not content_range:
+            return None
+        
+        match = CONTENT_RANGE_HEADER_PATTERN.match(content_range)
+        if not match:
+            return None
+        
+        return {
+            "start_bytes": int(match.group('start')),
+            "end_bytes": int(match.group('end')),
+        }

lockbox/storage/views_client.py

@@ -2,14 +2,7 @@ from common.utils import get_config
 from django.shortcuts import render
 from django.views import View
 
-from storage.forms import FileForm
+# Static view
-
-
 class FileUploadView(View):
     def get(self, request):
-        context = {
+        return render(request, "storage/upload.html")
-            "form": FileForm,
-            "max_chunk_bytes": get_config("MAX_CHUNK_BYTES"),
-            "max_file_bytes": get_config("MAX_FILE_BYTES"),
-        }
-        return render(request, "storage/upload.html", context=context)

lockbox/user/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.10 on 2024-02-12 08:39
+# Generated by Django 4.2.15 on 2024-09-16 10:45
 
 import django.contrib.auth.validators
 from django.db import migrations, models
@@ -28,7 +28,7 @@ class Migration(migrations.Migration):
                 ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
                 ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
                 ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
-                ('alias', models.SlugField(blank=True, help_text='an alias or nickname to remember who this is', max_length=32, null=True, unique=True, verbose_name='name')),
+                ('alias', models.SlugField(blank=True, help_text='an alias or nickname to remember who this is', max_length=32, null=True, unique=True, verbose_name='alias')),
                 ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.group', verbose_name='groups')),
                 ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.permission', verbose_name='user permissions')),
             ],

lockbox/user/models.py

@@ -8,7 +8,7 @@ from user.managers import LockboxUserManager
 
 class LockboxUser(AbstractUser, LockboxBase):
     alias = models.SlugField(
-        verbose_name=_("name"),
+        verbose_name=_("alias"),
         max_length=32,
         unique=True,
         null=True,