Swapped to async chunk uploading.

Swapped to api/client model.

.gitignore

@@ -93,3 +93,5 @@ venv.bak/
 .vscode
 
 lockbox/media
+lockbox/staticfiles
+TODO.txt

Makefile

@@ -1,5 +1,5 @@
 lint:
-	@ruff check $(shell git diff --diff-filter=ACM --name-only HEAD | grep '\.py$$' )
+	@ruff check $(shell git diff --diff-filter=ACM --name-only HEAD | grep '\.py$$' ) --config=./pyproject.toml
 
 stampreqs:
 	poetry export --without-hashes --format=requirements.txt > requirements.txt

lockbox/common/constants.py

@@ -1,9 +1,13 @@
+import re
+
+CONTENT_RANGE_HEADER = "HTTP_CONTENT_RANGE"
+CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P<start>\d+)-(?P<end>\d+)/(?P<total>\d+)$")
 
 class UPLOAD_STATUS_TYPES:
     UPLOADING = "uploading"
     COMPLETED = "completed"
     ABANDONED = "abandoned"
-
+    PROCESSING = "processing"
 
 # Config
 
@@ -14,18 +18,22 @@ CONFIG_KEYS = {
         "native_type": int,
         "default": 120,
     },
-
     "ABANDONED_DELTA_MINUTES": {
         "description": "Date created + this delta at which a file is marked as abandoned",
         "verbose_name": "Uncompleted file abandoned max age",
         "native_type": int,
         "default": 20,
     },
-
     "ABANDONED_EXPIRED_SCAN_INTERVAL": {
         "description": "Scan and scrub abandoned or expired uploads",
         "verbose_name": "Scan interval for abandoned/expired files",
         "native_type": int,
         "default": 20,
     },
+    "MAX_UPLOAD_BYTES": {
+        "description": "Max bytes that can be uploaded in one go",
+        "verbose_name": "Max upload size in bytes",
+        "native_type": int,
+        "default": 2000000, # 2 MB
+    },
 }

lockbox/common/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.10 on 2024-02-12 08:39
+# Generated by Django 4.2.10 on 2024-02-13 09:47
 
 from django.db import migrations, models
 import uuid
@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
                 ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
                 ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
                 ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
-                ('key', models.CharField(choices=[('EXPIRATION_DELTA_MINUTES', 'Date created + this delta at which file expires'), ('ABANDONED_DELTA_MINUTES', 'Date created + this delta at which a file is marked as abandoned'), ('ABANDONED_EXPIRED_SCAN_INTERVAL', 'Scan and scrub abandoned or expired uploads')], help_text='internal configuration key name', max_length=50)),
+                ('key', models.CharField(choices=[('EXPIRATION_DELTA_MINUTES', 'Date created + this delta at which file expires'), ('ABANDONED_DELTA_MINUTES', 'Date created + this delta at which a file is marked as abandoned'), ('ABANDONED_EXPIRED_SCAN_INTERVAL', 'Scan and scrub abandoned or expired uploads'), ('MAX_UPLOAD_BYTES', 'Max bytes that can be uploaded in one go')], help_text='internal configuration key name', max_length=50)),
                 ('value', models.CharField(help_text='actual DB config value', max_length=1024)),
             ],
             options={

lockbox/common/models.py

@@ -1,4 +1,3 @@
-from typing import Any, NamedTuple
 from uuid import uuid4
 
 from django.db import models
@@ -54,13 +53,26 @@ class LockboxBase(models.Model):  # pragma: no cover
         return f"{self.__class__.__name__} Object {self.lid}"
 
 
-class Config(NamedTuple):
+class Config:
-    key: str
+    def __init__(
-    value: Any
+            self,
-    native_type: type # change to type
+            key,
-    description: str
+            description,
-    source: str
+            verbose_name,
-    default: Any
+            native_type,
+            default,
+            value=None,
+            source=None,
+    ):
+
+        self.key = key
+        self.description = description
+        self.verbose_name = verbose_name
+        self.native_type = native_type
+        self.default = default
+        self.value = value
+        self.source = source
+
 
 
 class Configuration(LockboxBase):

lockbox/common/utils.py

@@ -19,7 +19,7 @@ def cast_to_native_type(key, value, native_type):
 
 def get_config(key):
     from common.models import Config, Configuration
-    config = Config(CONFIG_KEYS[key])
+    config = Config(**CONFIG_KEYS[key], key=key)
 
     obj = Configuration.objects.filter(key=key).first()
 
@@ -38,3 +38,7 @@ def get_config(key):
     config.value = config.default
     config.source = "default"
     return config
+
+
+def get_max_size_chunk_bytes():
+    return get_config("MAX_UPLOAD_BYTES").value

lockbox/lockbox/settings.py

@@ -7,37 +7,39 @@ from pathlib import Path
 BASE_DIR = Path(__file__).resolve().parent.parent
 
 SECRET_KEY = os.getenv("LOCKBOX_SECRET_KEY")
-DEBUG = os.getenv("LOCKBOX_DEBUG")
+# DEBUG = os.getenv("LOCKBOX_DEBUG")
-
+DEBUG = True
-ALLOWED_HOSTS = []
 
+ALLOWED_HOSTS = ["*"]
 
 # Application definition
 
+ENABLE_BROWSABLE_API = os.getenv("ENABLE_BROWSABLE_API")
+
 INSTALLED_APPS = [
-    'django.contrib.admin',
+    "django.contrib.admin",
-    'django.contrib.auth',
+    "django.contrib.auth",
-    'django.contrib.contenttypes',
+    "django.contrib.contenttypes",
-    'django.contrib.sessions',
+    "django.contrib.sessions",
-    'django.contrib.messages',
+    "django.contrib.messages",
-    'django.contrib.staticfiles',
+    "django.contrib.staticfiles",
-
     # Vendors
-
+    "rest_framework",
     # Apps
-    'common',
+    "common",
-    'user',
+    "user",
-    'storage',
+    "storage",
 ]
 
 MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
+    "django.middleware.security.SecurityMiddleware",
-    'django.contrib.sessions.middleware.SessionMiddleware',
+    "whitenoise.middleware.WhiteNoiseMiddleware",
-    'django.middleware.common.CommonMiddleware',
+    "django.contrib.sessions.middleware.SessionMiddleware",
-    'django.middleware.csrf.CsrfViewMiddleware',
+    "django.middleware.common.CommonMiddleware",
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
-    'django.contrib.messages.middleware.MessageMiddleware',
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
 ROOT_URLCONF = 'lockbox.urls'
@@ -45,7 +47,7 @@ ROOT_URLCONF = 'lockbox.urls'
 TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': [],
+        'DIRS': [BASE_DIR / "templates"],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -98,7 +100,12 @@ USE_TZ = True
 
 
 # Static files (CSS, JavaScript, Images)
+STATICFILES_DIRS = [
+    BASE_DIR / "static",
+]
+STATIC_ROOT = BASE_DIR / "staticfiles"
 STATIC_URL = 'static/'
+STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage"
 
 # Storage
 MEDIA_ROOT = BASE_DIR / "media"

lockbox/lockbox/urls.py

@@ -1,26 +1,15 @@
-"""
-URL configuration for lockbox project.
-
-The `urlpatterns` list routes URLs to views. For more information please see:
-    https://docs.djangoproject.com/en/4.2/topics/http/urls/
-Examples:
-Function views
-    1. Add an import:  from my_app import views
-    2. Add a URL to urlpatterns:  path('', views.home, name='home')
-Class-based views
-    1. Add an import:  from other_app.views import Home
-    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
-Including another URLconf
-    1. Import the include() function: from django.urls import include, path
-    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
-"""
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
-from django.urls import path
+from django.urls import include, path
 
 urlpatterns = [
-    path('admin/', admin.site.urls),
+    path("admin/", admin.site.urls),
+    path("storage/", include("storage.urls")),
 ]
 
+
+if settings.ENABLE_BROWSABLE_API:
+    urlpatterns.extend(path('api-auth/', include('rest_framework.urls')))
+
 urlpatterns.extend(static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT))

lockbox/static/css/main.css

@@ -0,0 +1,3 @@
+p {
+    color: red;
+}

lockbox/static/js/chunked_uploader.js

@@ -0,0 +1,49 @@
+const fileInput = document.getElementById('file-upload');
+fileInput.addEventListener('change', handleFileUpload);
+const csrftoken = getCookie('csrftoken');
+let file_id;
+
+function handleFileUpload(event) {
+  const file = event.target.files[0];
+  let start = 0;
+  let end = 0;
+  let chunk;
+
+  while (start < file.size) {
+    chunk = file.slice(start, start + chunk_size);
+    end = chunk.size - start;
+    console.log("LID: ", file_id);
+    file_id = uploadChunk(chunk, start, end, file.size, file_id);
+    start += chunk_size;
+  }
+}
+
+function uploadChunk(chunk, start, end, total, file_id=null) {
+    const formData = new FormData();
+    const range_header = `bytes ${start}-${end}/${total}`;
+    formData.append('file', chunk);
+
+
+    if (file_id) {
+        formData.append("lid", file_id);
+    }
+
+    let request = new Request(".", {
+        method: 'POST',
+        body: formData,
+        headers: {
+            'X-CSRFToken': csrftoken,
+            'Content-range': range_header
+        }
+    })
+    return _uploadChunk(request);
+}   
+
+async function _uploadChunk(request) {
+    const _response = await fetch(request)
+    .then(async (response)=>response.json())
+    .then((data) =>{
+        return data.lid;
+    })
+    return _response;
+}

lockbox/static/js/utils.js

@@ -0,0 +1,15 @@
+function getCookie(name) {
+    let cookieValue = null;
+    if (document.cookie && document.cookie !== '') {
+        const cookies = document.cookie.split(';');
+        for (let i = 0; i < cookies.length; i++) {
+            const cookie = cookies[i].trim();
+            // Does this cookie string begin with the name we want?
+            if (cookie.substring(0, name.length + 1) === (name + '=')) {
+                cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                break;
+            }
+        }
+    }
+    return cookieValue;
+}

lockbox/storage/migrations/0001_initial.py

@@ -1,5 +1,6 @@
-# Generated by Django 4.2.10 on 2024-02-12 09:59
+# Generated by Django 4.2.10 on 2024-02-16 08:15
 
+import common.utils
 from django.conf import settings
 from django.db import migrations, models
 import django.db.models.deletion
@@ -22,15 +23,15 @@ class Migration(migrations.Migration):
                 ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
                 ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
                 ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
-                ('name', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')),
+                ('filename', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')),
                 ('extension', models.CharField(blank=True, help_text='reported filesystem extension (not mime type)', max_length=128, null=True, verbose_name='extension')),
-                ('file', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path, verbose_name='file')),
+                ('file', models.FileField(blank=True, help_text='actual file', null=True, upload_to='', verbose_name='file')),
-                ('position', models.PositiveBigIntegerField(default=0, help_text='current position of uploaded bytes', verbose_name='position')),
+                ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('processing', 'processing'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=10, verbose_name='status')),
-                ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=9, verbose_name='status')),
                 ('date_completed', models.DateTimeField(blank=True, help_text="datetime at which this file's upload was completed", null=True, verbose_name='completed on')),
                 ('expires', models.BooleanField(default=False, help_text="will be scrubbed on 'date_expires'", verbose_name='expires')),
                 ('delete_on_expiration', models.BooleanField(default=False, help_text='will be deleted if expired and expires is true', verbose_name='delete on expiration')),
                 ('size_on_disk', models.PositiveBigIntegerField(blank=True, help_text='total size on disk for this file', null=True, verbose_name='size on disk (bytes)')),
+                ('max_size_chunk_bytes', models.PositiveBigIntegerField(default=common.utils.get_max_size_chunk_bytes, help_text='max size of each individual chunk for this file', verbose_name='maximum size of chunks (bytes)')),
                 ('owner', models.ForeignKey(blank=True, help_text='owner of this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')),
             ],
             options={
@@ -38,4 +39,24 @@ class Migration(migrations.Migration):
                 'verbose_name_plural': 'files',
             },
         ),
+        migrations.CreateModel(
+            name='FileChunk',
+            fields=[
+                ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
+                ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
+                ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
+                ('chunk', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path_chunk, verbose_name='file')),
+                ('chunk_id', models.BigIntegerField(help_text='part of chunk', verbose_name='chunk id')),
+                ('size', models.BigIntegerField(help_text='size for this chunk', verbose_name='size')),
+                ('start', models.BigIntegerField(help_text='start for this chunk', verbose_name='start')),
+                ('end', models.BigIntegerField(help_text='end for this chunk', verbose_name='end')),
+                ('file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='storage.file')),
+                ('owner', models.ForeignKey(blank=True, help_text='owner of this file chunk', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='chunks_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')),
+            ],
+            options={
+                'verbose_name': 'file chunk',
+                'verbose_name_plural': 'file chunks',
+                'unique_together': {('file', 'chunk_id')},
+            },
+        ),
     ]

lockbox/storage/models.py

@@ -1,9 +1,9 @@
-import hashlib
+from datetime import timedelta
 from pathlib import Path
 
 from common.constants import CONFIG_KEYS, UPLOAD_STATUS_TYPES
 from common.models import LockboxBase
-from common.utils import get_config, normalize_string
+from common.utils import get_config, get_max_size_chunk_bytes, normalize_string
 from django.conf import settings
 from django.core.files.uploadedfile import UploadedFile
 from django.db import models
@@ -11,18 +11,17 @@ from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 
-def get_upload_path(instance, filename):
+def get_upload_path_chunk(instance, filename):
-    filename = normalize_string(filename) + ".part"
+    file_subdir = settings.MEDIA_ROOT / str(instance.file.lid)
-    file_subdir = settings.MEDIA_ROOT / str(instance.lid)
 
     if not Path.exists(file_subdir):
         Path.mkdir(file_subdir)
 
+    filename = f"{FileChunk.last_chunk_id(instance.file)}.chunk"
     return Path(str(instance.lid)) / Path(filename)
 
-
 class File(LockboxBase):
-    name = models.CharField(
+    filename = models.CharField(
         max_length=255,
         null=False,
         blank=False,
@@ -39,29 +38,21 @@ class File(LockboxBase):
     )
 
     file = models.FileField(
-        upload_to=get_upload_path,
+        null=True,
-        null=False,
+        blank=True,
-        blank=False,
         verbose_name=_("file"),
         help_text=_("actual file"),
     )
 
-    position = models.PositiveBigIntegerField(
-        default=0,
-        blank=False,
-        null=False,
-        verbose_name=_("position"),
-        help_text=_("current position of uploaded bytes"),
-    )
-
     UPLOAD_CHOICES = (
         (UPLOAD_STATUS_TYPES.UPLOADING, _(UPLOAD_STATUS_TYPES.UPLOADING)),
         (UPLOAD_STATUS_TYPES.COMPLETED, _(UPLOAD_STATUS_TYPES.COMPLETED)),
+        (UPLOAD_STATUS_TYPES.PROCESSING, _(UPLOAD_STATUS_TYPES.PROCESSING)),
         (UPLOAD_STATUS_TYPES.ABANDONED, _(UPLOAD_STATUS_TYPES.ABANDONED)),
     )
 
     status = models.CharField(
-        max_length=9,
+        max_length=10,
         choices=UPLOAD_CHOICES,
         default=UPLOAD_STATUS_TYPES.UPLOADING,
         blank=False,
@@ -110,38 +101,42 @@ class File(LockboxBase):
         help_text=_("total size on disk for this file"),
     )
 
+    max_size_chunk_bytes = models.PositiveBigIntegerField(
+        null=False,
+        blank=False,
+        default=get_max_size_chunk_bytes,
+        verbose_name=_("maximum size of chunks (bytes)"),
+        help_text=_("max size of each individual chunk for this file"),
+    )
+
     readonly_fields = [
         "extension",
-        "position",
         "status",
         "date_completed",
         "size_on_disk",
+        "file",
+        "max_size_chunk_bytes",
         *LockboxBase.readonly_fields,
     ]
 
     def __str__(self):
-        return self.name
+        return self.filename
 
     class Meta:
         verbose_name = _("file")
         verbose_name_plural = _("files")
 
     @property
-    def md5(self):
+    def checksum(self):
-        if getattr(self, "_md5", None) is None:
+        return 0
-            md5 = hashlib.md5()  # noqa:S324 this is needed due to how chunked files work.
-            for chunk in self.file.chunks():
-                md5.update(chunk)
-            self._md5 = md5.hexdigest()
-        return self._md5
 
     @property
     def date_expires(self):
-        return self.date_created + get_config(CONFIG_KEYS["EXPIRATION_DELTA_MINUTES"])
+        return self.date_created + timedelta(minutes=get_config("EXPIRATION_DELTA_MINUTES").value)
 
     @property
     def abandoned(self):
-        return self.date_created + get_config(CONFIG_KEYS["ABANDONED_DELTA_MINUTES"])
+        return self.date_created + timedelta(minutes=get_config("ABANDONED_DELTA_MINUTES").value)
 
     @property
     def expired(self):
@@ -154,32 +149,114 @@ class File(LockboxBase):
         if self.file and delete_file:
             storage.delete(path)
 
+        # clean up chunks in case they have not been cleaned up by task.
+        self.chunks.all().delete()
+
     def get_file_handler_bytes(self):
         self.file.close()
         self.file.open(mode="rb")
         return UploadedFile(file=self.file, name=self.filename, size=self.offset)
 
-    def append_chunk(self, chunk, chunk_size=None, save=None):
-        # file handler might be open for some bizzare reason
-        self.file.close()
 
-        with Path.open(self.file.path, mode="ab") as fh:
+class FileChunk(LockboxBase):
-            fh.write(
+    file = models.ForeignKey(
-                chunk.read(),
+        "storage.File",
+        null=False,
+        blank=False,
+        on_delete=models.CASCADE,
+        related_name="chunks",
     )
 
-        if chunk_size is not None:
+    chunk = models.FileField(
-            # file is chunked
+        upload_to=get_upload_path_chunk,
-            self.postition += chunk_size
+        null=False,
-        elif hasattr(chunk, "size"):
+        blank=False,
-            self.postition += chunk.size
+        verbose_name=_("file"),
-        else:
+        help_text=_("actual file"),
-            # file is one shot (small file)
+    )
-            self.postition = self.file.size
+
-        self._md5 = None
+    chunk_id = models.BigIntegerField(
-        if save:
+        null=False,
-            self.save()
+        blank=False,
-        self.file.close()
+        verbose_name=_("chunk id"),
+        help_text=_("part of chunk"),
+    )
+
+    size = models.BigIntegerField(
+        null=False,
+        blank=False,
+        verbose_name=("size"),
+        help_text=_("size for this chunk"),
+    )
+
+    start = models.BigIntegerField(
+        null=False,
+        blank=False,
+        verbose_name=("start"),
+        help_text=_("start for this chunk"),
+    )
+
+    end = models.BigIntegerField(
+        null=False,
+        blank=False,
+        verbose_name=("end"),
+        help_text=_("end for this chunk"),
+    )
+
+    owner = models.ForeignKey(
+        "user.LockboxUser",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="chunks_owned",
+        verbose_name=_("owner"),
+        help_text=_("owner of this file chunk"),
+    )
+
+    readonly_fields = [
+        "file",
+        "chunk_id",
+        "start",
+        "end",
+        "size",
+        *LockboxBase.readonly_fields,
+    ]
+
+    def __str__(self):
+        return f"{self.file.filename}.{self.chunk_id}.chunk"
+
+    class Meta:
+        verbose_name = _("file chunk")
+        verbose_name_plural = _("file chunks")
+        unique_together = ("file", "chunk_id")
+
+    def save(self, *args, **kwargs):
+        # nasty hack lol
+        self.chunk_id = int(Path(self.file.name).stem)
+        return super().save(*args, **kwargs)
+
+    def delete(self, *args, delete_file=True, **kwargs):
+        if self.chunk:
+            storage, path = self.chunk.storage, self.chunk.path
+        super().delete(*args, **kwargs)
+        if self.chunk and delete_file:
+            storage.delete(path)
+
+    @staticmethod
+    def last_chunk_id(file_lid):
+        last_chunk = (
+            FileChunk.objects.filter(
+                file__lid=file_lid,
+            )
+            .order_by("-chunk_id")
+            .values("chunk_id")
+            .first()
+            .get("chunk_id")
+        )
+
+        if last_chunk:
+            return last_chunk + 1
+        return 1
 
 
 # class FileShare(LockboxBase):

lockbox/storage/serializers.py

@@ -0,0 +1,28 @@
+from rest_framework import serializers
+
+from storage.models import File, FileChunk
+
+
+class FileSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = File
+        fields = "__all__"
+        read_only_fields = File.readonly_fields
+
+
+class FileChunkSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = FileChunk
+        fields = "__all__"
+        read_only_fields = FileChunk.readonly_fields
+
+    def validate(self, data):
+        data = super().validate(data)
+        file = File.objects.get(lid=data["file"])
+
+        if data["size"] > file.max_size_chunk_bytes:
+            detail = f"'size' param is larger than max chunk size for file, {data["size"]} > {file.max_size_chunk_bytes}"
+            raise serializers.ValidationError(detail)
+        return data

lockbox/storage/storage_backend.py

@@ -1,2 +0,0 @@
-from django.core.files.storage import FileSystemStorage
-

lockbox/storage/urls.py

@@ -0,0 +1,17 @@
+from django.urls import include, path, re_path
+from rest_framework.routers import SimpleRouter
+from rest_framework_nested.routers import NestedSimpleRouter
+
+from storage import views_api, views_client
+
+router = SimpleRouter()
+router.register(r'files', views_api.FileModelViewSet)
+
+chunk_router = NestedSimpleRouter(router, r'files', lookup="file")
+chunk_router.register(r'chunks', views_api.FileChunkViewSet, basename="file-chunks")
+
+urlpatterns = [
+    re_path(r"api/", include(router.urls)),
+    re_path(r"api/", include(chunk_router.urls)),
+    path("client/files/", views_client.FileUploadView.as_view, name="client-fileupload"),
+]

lockbox/storage/views.py

@@ -1,3 +0,0 @@
-# from django.shortcuts import render
-
-# Create your views here.

lockbox/storage/views_api.py

@@ -0,0 +1,32 @@
+from common.constants import (
+    UPLOAD_STATUS_TYPES,
+)
+from common.utils import get_config
+from rest_framework import status
+from rest_framework.decorators import action
+from rest_framework.response import Response
+from rest_framework.viewsets import ModelViewSet
+from user.models import LockboxUser
+
+from storage.models import File, FileChunk
+from storage.serializers import FileChunkSerializer, FileSerializer
+
+
+class FileModelViewSet(ModelViewSet):
+    model = File
+    queryset = File.objects.all()
+    serializer_class = FileSerializer
+
+    @action(detail=True, methods=["post"])
+    def finalize(self, *args, **kwargs):  #noqa: ARG002
+        file = self.get_object()
+        file.status = UPLOAD_STATUS_TYPES.PROCESSING
+        file.save()
+        return Response(status=status.HTTP_200_OK)
+
+class FileChunkViewSet(ModelViewSet):
+    model = FileChunk
+    queryset = FileChunk.objects.all()
+    serializer_class = FileChunkSerializer
+
+

lockbox/storage/views_client.py

@@ -0,0 +1,8 @@
+from django.shortcuts import render
+from django.views import View
+
+
+class FileUploadView(View):
+    def get(self, request):
+        context = {}
+        return render(request, "storage/upload.html", context=context)

lockbox/templates/base.html

@@ -0,0 +1,36 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+
+    {% block css  %}
+    {% load static %}
+    <link rel="stylesheet" href="{% static 'css/main.css' %}">
+    {% endblock %}
+
+    {% block prejs  %}
+    {% endblock %}
+    
+    <title>{% block title %}Lockbox{% endblock %}</title>
+</head>
+
+<body>
+    <div id="sidebar">
+        {% block sidebar %}
+        <ul>
+            <li><a href="/">Files</a></li>
+            <li><a href="/files/upload/">Upload</a></li>
+        </ul>
+        {% endblock %}
+    </div>
+
+    <div id="content">
+        {% block content %}{% endblock %}
+    </div>
+</body>
+
+{% block postjs  %}
+{% load static %}
+<script src="{% static 'js/utils.js' %}"></script>
+{% endblock %}
+
+</html>

lockbox/templates/storage/upload.html

@@ -0,0 +1,24 @@
+{% extends "base.html" %}
+
+{% block title %}Upload a file{% endblock %}
+
+{% block postjs  %}
+{% load static %}
+{{ block.super }}
+
+<script>
+    const chunk_size = 1024 * 1024
+</script>
+<script src="{% static 'js/chunked_uploader.js' %}"></script>
+{% endblock %}
+
+
+{% block content %}
+<p> Upload file </p>
+<form method="post">{% csrf_token %}
+<input type="file" id="file-upload">
+</form>
+{% endblock %}
+
+
+

poetry.lock

@@ -109,6 +109,36 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""}
 argon2 = ["argon2-cffi (>=19.1.0)"]
 bcrypt = ["bcrypt"]
 
+[[package]]
+name = "djangorestframework"
+version = "3.14.0"
+description = "Web APIs for Django, made easy."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "djangorestframework-3.14.0-py3-none-any.whl", hash = "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08"},
+    {file = "djangorestframework-3.14.0.tar.gz", hash = "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8"},
+]
+
+[package.dependencies]
+django = ">=3.0"
+pytz = "*"
+
+[[package]]
+name = "drf-nested-routers"
+version = "0.93.5"
+description = "Nested resources for the Django Rest Framework"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "drf-nested-routers-0.93.5.tar.gz", hash = "sha256:1407565abc7bada37c162c7e11bf214ae71625a17fdec6d9a47a17f4a3627d32"},
+    {file = "drf_nested_routers-0.93.5-py2.py3-none-any.whl", hash = "sha256:9a6813554020134a02e62f8c2934b2047717f7da06f8b801752c521e43735c63"},
+]
+
+[package.dependencies]
+Django = ">=3.2"
+djangorestframework = ">=3.14.0"
+
 [[package]]
 name = "iniconfig"
 version = "2.0.0"
@@ -202,6 +232,17 @@ pytest = ">=7.0.0"
 docs = ["sphinx", "sphinx-rtd-theme"]
 testing = ["Django", "django-configurations (>=2.0)"]
 
+[[package]]
+name = "pytz"
+version = "2024.1"
+description = "World timezone definitions, modern and historical"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"},
+    {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"},
+]
+
 [[package]]
 name = "ruff"
 version = "0.2.1"
@@ -255,7 +296,21 @@ files = [
     {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"},
 ]
 
+[[package]]
+name = "whitenoise"
+version = "6.6.0"
+description = "Radically simplified static file serving for WSGI applications"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "whitenoise-6.6.0-py3-none-any.whl", hash = "sha256:b1f9db9bf67dc183484d760b99f4080185633136a273a03f6436034a41064146"},
+    {file = "whitenoise-6.6.0.tar.gz", hash = "sha256:8998f7370973447fac1e8ef6e8ded2c5209a7b1f67c1012866dbcd09681c3251"},
+]
+
+[package.extras]
+brotli = ["Brotli"]
+
 [metadata]
 lock-version = "2.0"
 python-versions = "3.12"
-content-hash = "1a0f3ef0953e311eaf5daaff9b7980e0211fb0e5195344a28881b731bca3b7eb"
+content-hash = "600fd336e3593b6fc5503b70e73dd2c22a2922a1662240f540ac6399520ae6d2"

pyproject.toml

@@ -9,6 +9,9 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = "3.12"
 django = "~4.2.0"
+whitenoise = "^6.6.0"
+djangorestframework = "^3.14.0"
+drf-nested-routers = "^0.93.5"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
@@ -44,18 +47,18 @@ omit = [
 
     "lockbox/asgi.py",
     "lockbox/wsgi.py",
-
 ]
 
 [tool.ruff]
 exclude = [
-    "*/migrations/[0-9]*",
+    "*/migrations/*",
     ".pyscripts/*",
     "pyenv*",
     ".pyenv*",
     ".git",
     ".venv",
 ]
+force-exclude = true
 
 line-length = 120
 target-version = "py312"
@@ -68,10 +71,14 @@ ignore = [
     "DJ001",
     "DJ012",
     "ERA001",
+    "FIX",
     "N801",
+    "PLR0913",
     "Q000",
     "RUF012",
     "TRY",
+    "T201",
+    "TD",
     "S101",
     "SLF001",
 ]