From 35bc178845e03550c2229b4bc9e4ce221d4136b6 Mon Sep 17 00:00:00 2001 From: Jordi Loyzaga Date: Fri, 16 Feb 2024 02:46:29 -0600 Subject: [PATCH] Swapped to async chunk uploading. Swapped to api/client model. --- .gitignore | 4 +- Makefile | 2 +- lockbox/common/constants.py | 16 +- lockbox/common/migrations/0001_initial.py | 4 +- lockbox/common/models.py | 28 +++- lockbox/common/utils.py | 6 +- lockbox/lockbox/settings.py | 51 +++--- lockbox/lockbox/urls.py | 25 +-- lockbox/static/css/main.css | 3 + lockbox/static/js/chunked_uploader.js | 49 ++++++ lockbox/static/js/utils.js | 15 ++ lockbox/storage/migrations/0001_initial.py | 31 +++- lockbox/storage/models.py | 173 +++++++++++++++------ lockbox/storage/serializers.py | 28 ++++ lockbox/storage/storage_backend.py | 2 - lockbox/storage/urls.py | 17 ++ lockbox/storage/views.py | 3 - lockbox/storage/views_api.py | 32 ++++ lockbox/storage/views_client.py | 8 + lockbox/templates/base.html | 36 +++++ lockbox/templates/storage/upload.html | 24 +++ poetry.lock | 57 ++++++- pyproject.toml | 11 +- 23 files changed, 507 insertions(+), 118 deletions(-) create mode 100644 lockbox/static/css/main.css create mode 100644 lockbox/static/js/chunked_uploader.js create mode 100644 lockbox/static/js/utils.js create mode 100644 lockbox/storage/serializers.py delete mode 100644 lockbox/storage/storage_backend.py create mode 100644 lockbox/storage/urls.py delete mode 100644 lockbox/storage/views.py create mode 100644 lockbox/storage/views_api.py create mode 100644 lockbox/storage/views_client.py create mode 100644 lockbox/templates/base.html create mode 100644 lockbox/templates/storage/upload.html diff --git a/.gitignore b/.gitignore index 64e22ef..48c0084 100644 --- a/.gitignore +++ b/.gitignore @@ -92,4 +92,6 @@ venv.bak/ .vscode -lockbox/media \ No newline at end of file +lockbox/media +lockbox/staticfiles +TODO.txt \ No newline at end of file diff --git a/Makefile b/Makefile index 947146c..5e368ec 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ lint: - @ruff check $(shell git diff --diff-filter=ACM --name-only HEAD | grep '\.py$$' ) + @ruff check $(shell git diff --diff-filter=ACM --name-only HEAD | grep '\.py$$' ) --config=./pyproject.toml stampreqs: poetry export --without-hashes --format=requirements.txt > requirements.txt diff --git a/lockbox/common/constants.py b/lockbox/common/constants.py index 3ec6ef2..58e5dba 100644 --- a/lockbox/common/constants.py +++ b/lockbox/common/constants.py @@ -1,31 +1,39 @@ +import re + +CONTENT_RANGE_HEADER = "HTTP_CONTENT_RANGE" +CONTENT_RANGE_HEADER_PATTERN = re.compile(r"^bytes (?P\d+)-(?P\d+)/(?P\d+)$") class UPLOAD_STATUS_TYPES: UPLOADING = "uploading" COMPLETED = "completed" ABANDONED = "abandoned" - + PROCESSING = "processing" # Config CONFIG_KEYS = { - "EXPIRATION_DELTA_MINUTES": { + "EXPIRATION_DELTA_MINUTES": { "description": "Date created + this delta at which file expires", "verbose_name": "File expiration delta (minutes)", "native_type": int, "default": 120, }, - "ABANDONED_DELTA_MINUTES": { "description": "Date created + this delta at which a file is marked as abandoned", "verbose_name": "Uncompleted file abandoned max age", "native_type": int, "default": 20, }, - "ABANDONED_EXPIRED_SCAN_INTERVAL": { "description": "Scan and scrub abandoned or expired uploads", "verbose_name": "Scan interval for abandoned/expired files", "native_type": int, "default": 20, }, + "MAX_UPLOAD_BYTES": { + "description": "Max bytes that can be uploaded in one go", + "verbose_name": "Max upload size in bytes", + "native_type": int, + "default": 2000000, # 2 MB + }, } diff --git a/lockbox/common/migrations/0001_initial.py b/lockbox/common/migrations/0001_initial.py index 50d4aae..9ed9f24 100644 --- a/lockbox/common/migrations/0001_initial.py +++ b/lockbox/common/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-02-12 08:39 +# Generated by Django 4.2.10 on 2024-02-13 09:47 from django.db import migrations, models import uuid @@ -18,7 +18,7 @@ class Migration(migrations.Migration): ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), - ('key', models.CharField(choices=[('EXPIRATION_DELTA_MINUTES', 'Date created + this delta at which file expires'), ('ABANDONED_DELTA_MINUTES', 'Date created + this delta at which a file is marked as abandoned'), ('ABANDONED_EXPIRED_SCAN_INTERVAL', 'Scan and scrub abandoned or expired uploads')], help_text='internal configuration key name', max_length=50)), + ('key', models.CharField(choices=[('EXPIRATION_DELTA_MINUTES', 'Date created + this delta at which file expires'), ('ABANDONED_DELTA_MINUTES', 'Date created + this delta at which a file is marked as abandoned'), ('ABANDONED_EXPIRED_SCAN_INTERVAL', 'Scan and scrub abandoned or expired uploads'), ('MAX_UPLOAD_BYTES', 'Max bytes that can be uploaded in one go')], help_text='internal configuration key name', max_length=50)), ('value', models.CharField(help_text='actual DB config value', max_length=1024)), ], options={ diff --git a/lockbox/common/models.py b/lockbox/common/models.py index 43606a9..fdff843 100644 --- a/lockbox/common/models.py +++ b/lockbox/common/models.py @@ -1,4 +1,3 @@ -from typing import Any, NamedTuple from uuid import uuid4 from django.db import models @@ -54,13 +53,26 @@ class LockboxBase(models.Model): # pragma: no cover return f"{self.__class__.__name__} Object {self.lid}" -class Config(NamedTuple): - key: str - value: Any - native_type: type # change to type - description: str - source: str - default: Any +class Config: + def __init__( + self, + key, + description, + verbose_name, + native_type, + default, + value=None, + source=None, + ): + + self.key = key + self.description = description + self.verbose_name = verbose_name + self.native_type = native_type + self.default = default + self.value = value + self.source = source + class Configuration(LockboxBase): diff --git a/lockbox/common/utils.py b/lockbox/common/utils.py index dd775af..f3c346a 100644 --- a/lockbox/common/utils.py +++ b/lockbox/common/utils.py @@ -19,7 +19,7 @@ def cast_to_native_type(key, value, native_type): def get_config(key): from common.models import Config, Configuration - config = Config(CONFIG_KEYS[key]) + config = Config(**CONFIG_KEYS[key], key=key) obj = Configuration.objects.filter(key=key).first() @@ -38,3 +38,7 @@ def get_config(key): config.value = config.default config.source = "default" return config + + +def get_max_size_chunk_bytes(): + return get_config("MAX_UPLOAD_BYTES").value diff --git a/lockbox/lockbox/settings.py b/lockbox/lockbox/settings.py index 2a00491..ba3fe04 100644 --- a/lockbox/lockbox/settings.py +++ b/lockbox/lockbox/settings.py @@ -7,37 +7,39 @@ from pathlib import Path BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = os.getenv("LOCKBOX_SECRET_KEY") -DEBUG = os.getenv("LOCKBOX_DEBUG") - -ALLOWED_HOSTS = [] +# DEBUG = os.getenv("LOCKBOX_DEBUG") +DEBUG = True +ALLOWED_HOSTS = ["*"] # Application definition +ENABLE_BROWSABLE_API = os.getenv("ENABLE_BROWSABLE_API") + INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", # Vendors - + "rest_framework", # Apps - 'common', - 'user', - 'storage', + "common", + "user", + "storage", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] ROOT_URLCONF = 'lockbox.urls' @@ -45,7 +47,7 @@ ROOT_URLCONF = 'lockbox.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': [BASE_DIR / "templates"], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -98,7 +100,12 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) +STATICFILES_DIRS = [ + BASE_DIR / "static", +] +STATIC_ROOT = BASE_DIR / "staticfiles" STATIC_URL = 'static/' +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" # Storage MEDIA_ROOT = BASE_DIR / "media" diff --git a/lockbox/lockbox/urls.py b/lockbox/lockbox/urls.py index 04d0042..2780b3e 100644 --- a/lockbox/lockbox/urls.py +++ b/lockbox/lockbox/urls.py @@ -1,26 +1,15 @@ -""" -URL configuration for lockbox project. - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/4.2/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" from django.conf import settings from django.conf.urls.static import static from django.contrib import admin -from django.urls import path +from django.urls import include, path urlpatterns = [ - path('admin/', admin.site.urls), + path("admin/", admin.site.urls), + path("storage/", include("storage.urls")), ] + +if settings.ENABLE_BROWSABLE_API: + urlpatterns.extend(path('api-auth/', include('rest_framework.urls'))) + urlpatterns.extend(static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/lockbox/static/css/main.css b/lockbox/static/css/main.css new file mode 100644 index 0000000..d364947 --- /dev/null +++ b/lockbox/static/css/main.css @@ -0,0 +1,3 @@ +p { + color: red; +} \ No newline at end of file diff --git a/lockbox/static/js/chunked_uploader.js b/lockbox/static/js/chunked_uploader.js new file mode 100644 index 0000000..0b0a898 --- /dev/null +++ b/lockbox/static/js/chunked_uploader.js @@ -0,0 +1,49 @@ +const fileInput = document.getElementById('file-upload'); +fileInput.addEventListener('change', handleFileUpload); +const csrftoken = getCookie('csrftoken'); +let file_id; + +function handleFileUpload(event) { + const file = event.target.files[0]; + let start = 0; + let end = 0; + let chunk; + + while (start < file.size) { + chunk = file.slice(start, start + chunk_size); + end = chunk.size - start; + console.log("LID: ", file_id); + file_id = uploadChunk(chunk, start, end, file.size, file_id); + start += chunk_size; + } +} + +function uploadChunk(chunk, start, end, total, file_id=null) { + const formData = new FormData(); + const range_header = `bytes ${start}-${end}/${total}`; + formData.append('file', chunk); + + + if (file_id) { + formData.append("lid", file_id); + } + + let request = new Request(".", { + method: 'POST', + body: formData, + headers: { + 'X-CSRFToken': csrftoken, + 'Content-range': range_header + } + }) + return _uploadChunk(request); +} + +async function _uploadChunk(request) { + const _response = await fetch(request) + .then(async (response)=>response.json()) + .then((data) =>{ + return data.lid; + }) + return _response; +} \ No newline at end of file diff --git a/lockbox/static/js/utils.js b/lockbox/static/js/utils.js new file mode 100644 index 0000000..b672cc3 --- /dev/null +++ b/lockbox/static/js/utils.js @@ -0,0 +1,15 @@ +function getCookie(name) { + let cookieValue = null; + if (document.cookie && document.cookie !== '') { + const cookies = document.cookie.split(';'); + for (let i = 0; i < cookies.length; i++) { + const cookie = cookies[i].trim(); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) === (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; +} \ No newline at end of file diff --git a/lockbox/storage/migrations/0001_initial.py b/lockbox/storage/migrations/0001_initial.py index e00cf19..5f607b0 100644 --- a/lockbox/storage/migrations/0001_initial.py +++ b/lockbox/storage/migrations/0001_initial.py @@ -1,5 +1,6 @@ -# Generated by Django 4.2.10 on 2024-02-12 09:59 +# Generated by Django 4.2.10 on 2024-02-16 08:15 +import common.utils from django.conf import settings from django.db import migrations, models import django.db.models.deletion @@ -22,15 +23,15 @@ class Migration(migrations.Migration): ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), - ('name', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')), + ('filename', models.CharField(help_text='display name of this file', max_length=255, verbose_name='name')), ('extension', models.CharField(blank=True, help_text='reported filesystem extension (not mime type)', max_length=128, null=True, verbose_name='extension')), - ('file', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path, verbose_name='file')), - ('position', models.PositiveBigIntegerField(default=0, help_text='current position of uploaded bytes', verbose_name='position')), - ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=9, verbose_name='status')), + ('file', models.FileField(blank=True, help_text='actual file', null=True, upload_to='', verbose_name='file')), + ('status', models.CharField(choices=[('uploading', 'uploading'), ('completed', 'completed'), ('processing', 'processing'), ('abandoned', 'abandoned')], default='uploading', help_text='upload status for file', max_length=10, verbose_name='status')), ('date_completed', models.DateTimeField(blank=True, help_text="datetime at which this file's upload was completed", null=True, verbose_name='completed on')), ('expires', models.BooleanField(default=False, help_text="will be scrubbed on 'date_expires'", verbose_name='expires')), ('delete_on_expiration', models.BooleanField(default=False, help_text='will be deleted if expired and expires is true', verbose_name='delete on expiration')), ('size_on_disk', models.PositiveBigIntegerField(blank=True, help_text='total size on disk for this file', null=True, verbose_name='size on disk (bytes)')), + ('max_size_chunk_bytes', models.PositiveBigIntegerField(default=common.utils.get_max_size_chunk_bytes, help_text='max size of each individual chunk for this file', verbose_name='maximum size of chunks (bytes)')), ('owner', models.ForeignKey(blank=True, help_text='owner of this file', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='files_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')), ], options={ @@ -38,4 +39,24 @@ class Migration(migrations.Migration): 'verbose_name_plural': 'files', }, ), + migrations.CreateModel( + name='FileChunk', + fields=[ + ('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')), + ('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')), + ('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')), + ('chunk', models.FileField(help_text='actual file', upload_to=storage.models.get_upload_path_chunk, verbose_name='file')), + ('chunk_id', models.BigIntegerField(help_text='part of chunk', verbose_name='chunk id')), + ('size', models.BigIntegerField(help_text='size for this chunk', verbose_name='size')), + ('start', models.BigIntegerField(help_text='start for this chunk', verbose_name='start')), + ('end', models.BigIntegerField(help_text='end for this chunk', verbose_name='end')), + ('file', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='storage.file')), + ('owner', models.ForeignKey(blank=True, help_text='owner of this file chunk', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='chunks_owned', to=settings.AUTH_USER_MODEL, verbose_name='owner')), + ], + options={ + 'verbose_name': 'file chunk', + 'verbose_name_plural': 'file chunks', + 'unique_together': {('file', 'chunk_id')}, + }, + ), ] diff --git a/lockbox/storage/models.py b/lockbox/storage/models.py index 1a6e70e..508d547 100644 --- a/lockbox/storage/models.py +++ b/lockbox/storage/models.py @@ -1,9 +1,9 @@ -import hashlib +from datetime import timedelta from pathlib import Path from common.constants import CONFIG_KEYS, UPLOAD_STATUS_TYPES from common.models import LockboxBase -from common.utils import get_config, normalize_string +from common.utils import get_config, get_max_size_chunk_bytes, normalize_string from django.conf import settings from django.core.files.uploadedfile import UploadedFile from django.db import models @@ -11,18 +11,17 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ -def get_upload_path(instance, filename): - filename = normalize_string(filename) + ".part" - file_subdir = settings.MEDIA_ROOT / str(instance.lid) +def get_upload_path_chunk(instance, filename): + file_subdir = settings.MEDIA_ROOT / str(instance.file.lid) if not Path.exists(file_subdir): Path.mkdir(file_subdir) + filename = f"{FileChunk.last_chunk_id(instance.file)}.chunk" return Path(str(instance.lid)) / Path(filename) - class File(LockboxBase): - name = models.CharField( + filename = models.CharField( max_length=255, null=False, blank=False, @@ -39,29 +38,21 @@ class File(LockboxBase): ) file = models.FileField( - upload_to=get_upload_path, - null=False, - blank=False, + null=True, + blank=True, verbose_name=_("file"), help_text=_("actual file"), ) - position = models.PositiveBigIntegerField( - default=0, - blank=False, - null=False, - verbose_name=_("position"), - help_text=_("current position of uploaded bytes"), - ) - UPLOAD_CHOICES = ( (UPLOAD_STATUS_TYPES.UPLOADING, _(UPLOAD_STATUS_TYPES.UPLOADING)), (UPLOAD_STATUS_TYPES.COMPLETED, _(UPLOAD_STATUS_TYPES.COMPLETED)), + (UPLOAD_STATUS_TYPES.PROCESSING, _(UPLOAD_STATUS_TYPES.PROCESSING)), (UPLOAD_STATUS_TYPES.ABANDONED, _(UPLOAD_STATUS_TYPES.ABANDONED)), ) status = models.CharField( - max_length=9, + max_length=10, choices=UPLOAD_CHOICES, default=UPLOAD_STATUS_TYPES.UPLOADING, blank=False, @@ -110,38 +101,42 @@ class File(LockboxBase): help_text=_("total size on disk for this file"), ) + max_size_chunk_bytes = models.PositiveBigIntegerField( + null=False, + blank=False, + default=get_max_size_chunk_bytes, + verbose_name=_("maximum size of chunks (bytes)"), + help_text=_("max size of each individual chunk for this file"), + ) + readonly_fields = [ "extension", - "position", "status", "date_completed", "size_on_disk", + "file", + "max_size_chunk_bytes", *LockboxBase.readonly_fields, ] def __str__(self): - return self.name + return self.filename class Meta: verbose_name = _("file") verbose_name_plural = _("files") @property - def md5(self): - if getattr(self, "_md5", None) is None: - md5 = hashlib.md5() # noqa:S324 this is needed due to how chunked files work. - for chunk in self.file.chunks(): - md5.update(chunk) - self._md5 = md5.hexdigest() - return self._md5 + def checksum(self): + return 0 @property def date_expires(self): - return self.date_created + get_config(CONFIG_KEYS["EXPIRATION_DELTA_MINUTES"]) + return self.date_created + timedelta(minutes=get_config("EXPIRATION_DELTA_MINUTES").value) @property def abandoned(self): - return self.date_created + get_config(CONFIG_KEYS["ABANDONED_DELTA_MINUTES"]) + return self.date_created + timedelta(minutes=get_config("ABANDONED_DELTA_MINUTES").value) @property def expired(self): @@ -154,32 +149,114 @@ class File(LockboxBase): if self.file and delete_file: storage.delete(path) + # clean up chunks in case they have not been cleaned up by task. + self.chunks.all().delete() + def get_file_handler_bytes(self): self.file.close() self.file.open(mode="rb") return UploadedFile(file=self.file, name=self.filename, size=self.offset) - def append_chunk(self, chunk, chunk_size=None, save=None): - # file handler might be open for some bizzare reason - self.file.close() - with Path.open(self.file.path, mode="ab") as fh: - fh.write( - chunk.read(), +class FileChunk(LockboxBase): + file = models.ForeignKey( + "storage.File", + null=False, + blank=False, + on_delete=models.CASCADE, + related_name="chunks", + ) + + chunk = models.FileField( + upload_to=get_upload_path_chunk, + null=False, + blank=False, + verbose_name=_("file"), + help_text=_("actual file"), + ) + + chunk_id = models.BigIntegerField( + null=False, + blank=False, + verbose_name=_("chunk id"), + help_text=_("part of chunk"), + ) + + size = models.BigIntegerField( + null=False, + blank=False, + verbose_name=("size"), + help_text=_("size for this chunk"), + ) + + start = models.BigIntegerField( + null=False, + blank=False, + verbose_name=("start"), + help_text=_("start for this chunk"), + ) + + end = models.BigIntegerField( + null=False, + blank=False, + verbose_name=("end"), + help_text=_("end for this chunk"), + ) + + owner = models.ForeignKey( + "user.LockboxUser", + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name="chunks_owned", + verbose_name=_("owner"), + help_text=_("owner of this file chunk"), + ) + + readonly_fields = [ + "file", + "chunk_id", + "start", + "end", + "size", + *LockboxBase.readonly_fields, + ] + + def __str__(self): + return f"{self.file.filename}.{self.chunk_id}.chunk" + + class Meta: + verbose_name = _("file chunk") + verbose_name_plural = _("file chunks") + unique_together = ("file", "chunk_id") + + def save(self, *args, **kwargs): + # nasty hack lol + self.chunk_id = int(Path(self.file.name).stem) + return super().save(*args, **kwargs) + + def delete(self, *args, delete_file=True, **kwargs): + if self.chunk: + storage, path = self.chunk.storage, self.chunk.path + super().delete(*args, **kwargs) + if self.chunk and delete_file: + storage.delete(path) + + @staticmethod + def last_chunk_id(file_lid): + last_chunk = ( + FileChunk.objects.filter( + file__lid=file_lid, ) + .order_by("-chunk_id") + .values("chunk_id") + .first() + .get("chunk_id") + ) - if chunk_size is not None: - # file is chunked - self.postition += chunk_size - elif hasattr(chunk, "size"): - self.postition += chunk.size - else: - # file is one shot (small file) - self.postition = self.file.size - self._md5 = None - if save: - self.save() - self.file.close() + if last_chunk: + return last_chunk + 1 + return 1 # class FileShare(LockboxBase): diff --git a/lockbox/storage/serializers.py b/lockbox/storage/serializers.py new file mode 100644 index 0000000..96601b0 --- /dev/null +++ b/lockbox/storage/serializers.py @@ -0,0 +1,28 @@ +from rest_framework import serializers + +from storage.models import File, FileChunk + + +class FileSerializer(serializers.ModelSerializer): + + class Meta: + model = File + fields = "__all__" + read_only_fields = File.readonly_fields + + +class FileChunkSerializer(serializers.ModelSerializer): + + class Meta: + model = FileChunk + fields = "__all__" + read_only_fields = FileChunk.readonly_fields + + def validate(self, data): + data = super().validate(data) + file = File.objects.get(lid=data["file"]) + + if data["size"] > file.max_size_chunk_bytes: + detail = f"'size' param is larger than max chunk size for file, {data["size"]} > {file.max_size_chunk_bytes}" + raise serializers.ValidationError(detail) + return data diff --git a/lockbox/storage/storage_backend.py b/lockbox/storage/storage_backend.py deleted file mode 100644 index 92e4dd1..0000000 --- a/lockbox/storage/storage_backend.py +++ /dev/null @@ -1,2 +0,0 @@ -from django.core.files.storage import FileSystemStorage - diff --git a/lockbox/storage/urls.py b/lockbox/storage/urls.py new file mode 100644 index 0000000..e89d7d7 --- /dev/null +++ b/lockbox/storage/urls.py @@ -0,0 +1,17 @@ +from django.urls import include, path, re_path +from rest_framework.routers import SimpleRouter +from rest_framework_nested.routers import NestedSimpleRouter + +from storage import views_api, views_client + +router = SimpleRouter() +router.register(r'files', views_api.FileModelViewSet) + +chunk_router = NestedSimpleRouter(router, r'files', lookup="file") +chunk_router.register(r'chunks', views_api.FileChunkViewSet, basename="file-chunks") + +urlpatterns = [ + re_path(r"api/", include(router.urls)), + re_path(r"api/", include(chunk_router.urls)), + path("client/files/", views_client.FileUploadView.as_view, name="client-fileupload"), +] diff --git a/lockbox/storage/views.py b/lockbox/storage/views.py deleted file mode 100644 index fd0e044..0000000 --- a/lockbox/storage/views.py +++ /dev/null @@ -1,3 +0,0 @@ -# from django.shortcuts import render - -# Create your views here. diff --git a/lockbox/storage/views_api.py b/lockbox/storage/views_api.py new file mode 100644 index 0000000..11cf802 --- /dev/null +++ b/lockbox/storage/views_api.py @@ -0,0 +1,32 @@ +from common.constants import ( + UPLOAD_STATUS_TYPES, +) +from common.utils import get_config +from rest_framework import status +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.viewsets import ModelViewSet +from user.models import LockboxUser + +from storage.models import File, FileChunk +from storage.serializers import FileChunkSerializer, FileSerializer + + +class FileModelViewSet(ModelViewSet): + model = File + queryset = File.objects.all() + serializer_class = FileSerializer + + @action(detail=True, methods=["post"]) + def finalize(self, *args, **kwargs): #noqa: ARG002 + file = self.get_object() + file.status = UPLOAD_STATUS_TYPES.PROCESSING + file.save() + return Response(status=status.HTTP_200_OK) + +class FileChunkViewSet(ModelViewSet): + model = FileChunk + queryset = FileChunk.objects.all() + serializer_class = FileChunkSerializer + + diff --git a/lockbox/storage/views_client.py b/lockbox/storage/views_client.py new file mode 100644 index 0000000..6b2ee17 --- /dev/null +++ b/lockbox/storage/views_client.py @@ -0,0 +1,8 @@ +from django.shortcuts import render +from django.views import View + + +class FileUploadView(View): + def get(self, request): + context = {} + return render(request, "storage/upload.html", context=context) diff --git a/lockbox/templates/base.html b/lockbox/templates/base.html new file mode 100644 index 0000000..ce62f9d --- /dev/null +++ b/lockbox/templates/base.html @@ -0,0 +1,36 @@ + + + + + {% block css %} + {% load static %} + + {% endblock %} + + {% block prejs %} + {% endblock %} + + {% block title %}Lockbox{% endblock %} + + + + + +
+ {% block content %}{% endblock %} +
+ + +{% block postjs %} +{% load static %} + +{% endblock %} + + \ No newline at end of file diff --git a/lockbox/templates/storage/upload.html b/lockbox/templates/storage/upload.html new file mode 100644 index 0000000..5a9de54 --- /dev/null +++ b/lockbox/templates/storage/upload.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} + +{% block title %}Upload a file{% endblock %} + +{% block postjs %} +{% load static %} +{{ block.super }} + + + +{% endblock %} + + +{% block content %} +

Upload file

+
{% csrf_token %} + +
+{% endblock %} + + + diff --git a/poetry.lock b/poetry.lock index fa67480..276adee 100644 --- a/poetry.lock +++ b/poetry.lock @@ -109,6 +109,36 @@ tzdata = {version = "*", markers = "sys_platform == \"win32\""} argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +[[package]] +name = "djangorestframework" +version = "3.14.0" +description = "Web APIs for Django, made easy." +optional = false +python-versions = ">=3.6" +files = [ + {file = "djangorestframework-3.14.0-py3-none-any.whl", hash = "sha256:eb63f58c9f218e1a7d064d17a70751f528ed4e1d35547fdade9aaf4cd103fd08"}, + {file = "djangorestframework-3.14.0.tar.gz", hash = "sha256:579a333e6256b09489cbe0a067e66abe55c6595d8926be6b99423786334350c8"}, +] + +[package.dependencies] +django = ">=3.0" +pytz = "*" + +[[package]] +name = "drf-nested-routers" +version = "0.93.5" +description = "Nested resources for the Django Rest Framework" +optional = false +python-versions = ">=3.8" +files = [ + {file = "drf-nested-routers-0.93.5.tar.gz", hash = "sha256:1407565abc7bada37c162c7e11bf214ae71625a17fdec6d9a47a17f4a3627d32"}, + {file = "drf_nested_routers-0.93.5-py2.py3-none-any.whl", hash = "sha256:9a6813554020134a02e62f8c2934b2047717f7da06f8b801752c521e43735c63"}, +] + +[package.dependencies] +Django = ">=3.2" +djangorestframework = ">=3.14.0" + [[package]] name = "iniconfig" version = "2.0.0" @@ -202,6 +232,17 @@ pytest = ">=7.0.0" docs = ["sphinx", "sphinx-rtd-theme"] testing = ["Django", "django-configurations (>=2.0)"] +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + [[package]] name = "ruff" version = "0.2.1" @@ -255,7 +296,21 @@ files = [ {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] +[[package]] +name = "whitenoise" +version = "6.6.0" +description = "Radically simplified static file serving for WSGI applications" +optional = false +python-versions = ">=3.8" +files = [ + {file = "whitenoise-6.6.0-py3-none-any.whl", hash = "sha256:b1f9db9bf67dc183484d760b99f4080185633136a273a03f6436034a41064146"}, + {file = "whitenoise-6.6.0.tar.gz", hash = "sha256:8998f7370973447fac1e8ef6e8ded2c5209a7b1f67c1012866dbcd09681c3251"}, +] + +[package.extras] +brotli = ["Brotli"] + [metadata] lock-version = "2.0" python-versions = "3.12" -content-hash = "1a0f3ef0953e311eaf5daaff9b7980e0211fb0e5195344a28881b731bca3b7eb" +content-hash = "600fd336e3593b6fc5503b70e73dd2c22a2922a1662240f540ac6399520ae6d2" diff --git a/pyproject.toml b/pyproject.toml index 787098f..7a53819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,9 @@ readme = "README.md" [tool.poetry.dependencies] python = "3.12" django = "~4.2.0" +whitenoise = "^6.6.0" +djangorestframework = "^3.14.0" +drf-nested-routers = "^0.93.5" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0" @@ -44,18 +47,18 @@ omit = [ "lockbox/asgi.py", "lockbox/wsgi.py", - ] [tool.ruff] exclude = [ - "*/migrations/[0-9]*", + "*/migrations/*", ".pyscripts/*", "pyenv*", ".pyenv*", ".git", ".venv", ] +force-exclude = true line-length = 120 target-version = "py312" @@ -68,10 +71,14 @@ ignore = [ "DJ001", "DJ012", "ERA001", + "FIX", "N801", + "PLR0913", "Q000", "RUF012", "TRY", + "T201", + "TD", "S101", "SLF001", ]