kitty/lockbox
kitty
/
lockbox
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

whoops
continuous-integration/drone/push Build is failing Details

This commit is contained in:
Jordi Loyzaga 2024-04-22 03:11:33 -06:00
parent 837b756b0d
commit 4055f7e966
26 changed files with 292 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.drone.yml Normal file
View File

@ -0,0 +1,9 @@
kind: pipeline
name: default
steps:
- name: unittest
image: python
commands:
- pip install -r requirements_test.txt
- pytest --cov=. --cov-report term-missing

4
Makefile
View File

@ -4,5 +4,5 @@ lint:
stampreqs:
poetry export --without-hashes --format=requirements.txt > requirements.txt
test:
pytest --cov=. --cov-report term-missing
stampreqsci:
poetry export --without-hashes --with dev --format=requirements.txt > requirements_test.txt

10
lockbox/common/admin.py
View File

@ -1,10 +0,0 @@
from django.contrib import admin
from common.models import Configuration
class LockboxModelAdmin(admin.ModelAdmin):
readonly_fields = Configuration.readonly_fields
admin.site.register(Configuration, LockboxModelAdmin)

52
lockbox/common/constants.py
View File

@ -16,24 +16,70 @@ CONFIG_KEYS = {
"description": "Date created + this delta at which file expires",
"verbose_name": "File expiration delta (minutes)",
"native_type": int,
"sensitive": False,
"default": 120,
},
"ABANDONED_DELTA_MINUTES": {
"description": "Date created + this delta at which a file is marked as abandoned",
"verbose_name": "Uncompleted file abandoned max age",
"native_type": int,
"sensitive": False,
"default": 20,
},
"ABANDONED_EXPIRED_SCAN_INTERVAL": {
"description": "Scan and scrub abandoned or expired uploads",
"verbose_name": "Scan interval for abandoned/expired files",
"native_type": int,
"sensitive": False,
"default": 20,
},
"MAX_UPLOAD_BYTES": {
"description": "Max bytes that can be uploaded in one go",
"MAX_CHUNK_BYTES": {
"description": "Max bytes that can be uploaded in one go, too large of a value might result in a timeout",
"verbose_name": "Max per chunk size in bytes",
"native_type": int,
"sensitive": False,
"default": 1024 * 1024 * 20, # 20 MB
},
"MAX_FILE_BYTES": {
"description": "Max total file size in bytes",
"verbose_name": "Max upload size in bytes",
"native_type": int,
"default": 2000000, # 2 MB
"sensitive": False,
"default": 1024 * 1024 * 200, # 200 MB
},
"ENABLE_BROWSABLE_API": {
"description": "REST Framework browsable API is enabled (Always enabled if DEBUG is true)",
"verbose_name": "Enable browsable API",
"native_type": bool,
"sensitive": False,
"default": False,
},
"DEBUG": {
"description": "Django DEBUG flag is enabled (Do not use in production!)",
"verbose_name": "Django DEBUG flag value",
"native_type": bool,
"sensitive": False,
"default": False,
},
"SECRET_KEY": {
"description": "Django SECRET_KEY used for crypto singning",
"verbose_name": "Django SECRET_KEY",
"native_type": str,
"sensitive": True,
"default": None,
},
"ALLOWED_HOSTS": {
"description": "Where is this app being served from",
"verbose_name": "Allowed hosts",
"native_type": list,
"sensitive": False,
"default": ["*"],
},
"DB_SQLITE_ABSOLUTE_PATH": {
"description": "Path where db.sqlite3 is stored",
"verbose_name": "db.sqlite3 path",
"native_type": str,
"sensitive": False,
"default": ".",
},
}

28
lockbox/common/migrations/0001_initial.py
View File

@ -1,28 +0,0 @@
# Generated by Django 4.2.10 on 2024-02-13 09:47
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Configuration',
fields=[
('lid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='lockbox ID')),
('date_created', models.DateTimeField(blank=True, help_text='date at which this object was created', verbose_name='date created')),
('date_updated', models.DateTimeField(blank=True, help_text='date at which this object was last updated', verbose_name='date updated')),
('key', models.CharField(choices=[('EXPIRATION_DELTA_MINUTES', 'Date created + this delta at which file expires'), ('ABANDONED_DELTA_MINUTES', 'Date created + this delta at which a file is marked as abandoned'), ('ABANDONED_EXPIRED_SCAN_INTERVAL', 'Scan and scrub abandoned or expired uploads'), ('MAX_UPLOAD_BYTES', 'Max bytes that can be uploaded in one go')], help_text='internal configuration key name', max_length=50)),
('value', models.CharField(help_text='actual DB config value', max_length=1024)),
],
options={
'abstract': False,
},
),
]

54
lockbox/common/models.py
View File

@ -4,9 +4,6 @@ from django.db import models
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from common.constants import CONFIG_KEYS
from common.utils import cast_to_native_type
class LockboxBase(models.Model): # pragma: no cover
lid = models.UUIDField(
@ -51,54 +48,3 @@ class LockboxBase(models.Model): # pragma: no cover
def __str__(self):
return f"{self.__class__.__name__} Object {self.lid}"
class Config:
def __init__(
self,
key,
description,
verbose_name,
native_type,
default,
value=None,
source=None,
):
self.key = key
self.description = description
self.verbose_name = verbose_name
self.native_type = native_type
self.default = default
self.value = value
self.source = source
class Configuration(LockboxBase):
CONFIG_KEY_CHOICES = (
(key, value["description"]) for key, value in CONFIG_KEYS.items()
)
key = models.CharField(
choices=CONFIG_KEY_CHOICES,
max_length=50,
null=False,
blank=False,
help_text=_("internal configuration key name"),
)
value = models.CharField(
max_length=1024,
null=False,
blank=False,
help_text=_("actual DB config value"),
)
readonly_fields = LockboxBase.readonly_fields
def save(self, *args, **kwargs):
native_type = CONFIG_KEYS[self.key]["native_type"]
cast_to_native_type(self.key, self.value, native_type)
return super().save(*args, **kwargs)

24
lockbox/common/serializers.py Normal file
View File

@ -0,0 +1,24 @@
from rest_framework import serializers
class ConfigSerializer(serializers.Serializer):
key = serializers.CharField(max_length=50)
description = serializers.CharField(max_length=120)
verbose_name = serializers.CharField(max_length=120)
native_type = serializers.CharField(max_length=10)
sensitive = serializers.BooleanField()
default = serializers.CharField(max_length=120)
value = serializers.CharField(max_length=120)
source = serializers.CharField(max_length=10)
def to_representation(self, instance):
if instance["default"] is not None:
instance["default"] = str(instance["default"])
if instance["sensitive"] and instance["value"] is not None:
instance["value"] = "***************"
elif instance["value"] is not None:
instance["value"] = str(instance["value"])
return super().to_representation(instance)

8
lockbox/common/urls.py Normal file
View File

@ -0,0 +1,8 @@
from django.urls import path
from common import views_api
urlpatterns = [
path("api/configs/", views_api.configs, name="api-config"),
path("api/configs/<str:key>/", views_api.configs, name="api-config"),
]

53
lockbox/common/utils.py
View File

@ -1,13 +1,33 @@
from os import getenv
from typing import Any, NamedTuple
from unicodedata import normalize
from common.constants import CONFIG_KEYS
class Config(NamedTuple):
key: str
description: str
verbose_name: str
native_type: type
sensitive: bool
default: Any
value: Any
source: str
def normalize_string(string, form="NFKC"):
return normalize(form, string)
def cast_to_native_type(key, value, native_type):
if native_type == list:
value = value.split(",")
if native_type == bool:
if value == "false":
return False
return True
try:
return native_type(value)
except ValueError as e:
@ -17,28 +37,21 @@ def cast_to_native_type(key, value, native_type):
raise ValueError(message) from e
def get_config(key):
from common.models import Config, Configuration
config = Config(**CONFIG_KEYS[key], key=key)
def get_config(key, value_only=True): # noqa: FBT002
default = CONFIG_KEYS[key]
from_env = getenv(key)
obj = Configuration.objects.filter(key=key).first()
if from_env:
value = cast_to_native_type(key, from_env, default["native_type"])
source = "env"
else:
value = default["default"]
source = "default"
if obj:
config.value = cast_to_native_type(key, obj.value, config.native_type)
config.source = "db"
return config
value = getenv(key)
if value:
config.value = cast_to_native_type(key, value, config.native_type)
config.source = "env_variable"
return config
config.value = config.default
config.source = "default"
return config
if value_only:
return value
return Config(key=key, value=value, source=source, **default)
def get_max_size_chunk_bytes():
return get_config("MAX_UPLOAD_BYTES").value
return get_config("MAX_CHUNK_BYTES")

3
lockbox/common/views.py
View File

@ -1,3 +0,0 @@
# from django.shortcuts import render
# Create your views here.

26
lockbox/common/views_api.py Normal file
View File

@ -0,0 +1,26 @@
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from common.constants import CONFIG_KEYS
from common.serializers import ConfigSerializer
from common.utils import get_config
def get_all_configs():
return [get_config(key, value_only=False)._asdict() for key in CONFIG_KEYS]
@api_view(["GET"])
def configs(request, key=None):
if key:
if key not in CONFIG_KEYS:
return Response(status=status.HTTP_404_NOT_FOUND)
payload = ConfigSerializer(get_config(key, value_only=False)._asdict()).data
else:
payload = ConfigSerializer(get_all_configs(), many=True).data
return Response(
payload,
status=status.HTTP_200_OK,
)

19
lockbox/lockbox/settings.py
View File

@ -1,21 +1,23 @@
"""Lockbox File Sharing"""
import os
from pathlib import Path
from common.utils import get_config
from dotenv import load_dotenv
load_dotenv()
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
SECRET_KEY = os.getenv("LOCKBOX_SECRET_KEY")
# DEBUG = os.getenv("LOCKBOX_DEBUG")
DEBUG = True
SECRET_KEY = get_config("SECRET_KEY")
DEBUG = get_config("DEBUG")
ALLOWED_HOSTS = ["*"]
ALLOWED_HOSTS = get_config("ALLOWED_HOSTS")
# Application definition
ENABLE_BROWSABLE_API = os.getenv("ENABLE_BROWSABLE_API")
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
@ -64,6 +66,9 @@ WSGI_APPLICATION = 'lockbox.wsgi.application'
# Database
DB_SQLITE_ABSOLUTE_PATH = get_config("DB_SQLITE_ABSOLUTE_PATH")
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',

4
lockbox/lockbox/urls.py
View File

@ -1,3 +1,4 @@
from common.utils import get_config
from django.conf import settings
from django.conf.urls.static import static
from django.contrib import admin
@ -6,10 +7,11 @@ from django.urls import include, path
urlpatterns = [
path("admin/", admin.site.urls),
path("storage/", include("storage.urls")),
path("common/", include("common.urls")),
]
if settings.ENABLE_BROWSABLE_API:
if get_config("ENABLE_BROWSABLE_API"):
urlpatterns.extend(path('api-auth/', include('rest_framework.urls')))
urlpatterns.extend(static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT))

3
lockbox/static/css/main.css
View File

@ -1,3 +0,0 @@
p {
color: red;
}

43
lockbox/static/js/chunked_uploader.js
View File

@ -1,21 +1,36 @@
const fileInput = document.getElementById('file-upload');
fileInput.addEventListener('change', handleFileUpload);
const csrftoken = getCookie('csrftoken');
let file_id;
const upload_ready = false;
fileInput.addEventListener('change', handleFileChange);
function handleFileChange(event) {
const file = event.target.files[0];
const file_size = file.size;
console.log("Max file bytes is : ", max_file_bytes);
console.log("File size is: ", file_size);
if (file_size > max_file_bytes){
console.log("PLACEHOLDER: Size too big man.");
return
}
console.log("PLACEHOLDER: Ready!");
}
function handleFileUpload(event) {
const file = event.target.files[0];
let start = 0;
let end = 0;
let chunk;
const file = event.target.files[0];
let start = 0;
let end = 0;
let chunk;
while (start < file.size) {
chunk = file.slice(start, start + chunk_size);
end = chunk.size - start;
console.log("LID: ", file_id);
file_id = uploadChunk(chunk, start, end, file.size, file_id);
start += chunk_size;
}
while (start < file.size) {
chunk = file.slice(start, start + chunk_size);
end = chunk.size - start;
console.log("LID: ", file_id);
file_id = uploadChunk(chunk, start, end, file.size, file_id);
start += chunk_size;
}
}
function uploadChunk(chunk, start, end, total, file_id=null) {

12
lockbox/storage/forms.py Normal file
View File

@ -0,0 +1,12 @@
from django import forms
from storage.models import File
class FileForm(forms.ModelForm):
set_name = forms.BooleanField()
class Meta:
model = File
exclude = File.readonly_fields

12
lockbox/storage/models.py
View File

@ -1,9 +1,9 @@
from datetime import timedelta
from pathlib import Path
from common.constants import CONFIG_KEYS, UPLOAD_STATUS_TYPES
from common.constants import UPLOAD_STATUS_TYPES
from common.models import LockboxBase
from common.utils import get_config, get_max_size_chunk_bytes, normalize_string
from common.utils import get_config, get_max_size_chunk_bytes
from django.conf import settings
from django.core.files.uploadedfile import UploadedFile
from django.db import models
@ -26,7 +26,7 @@ class File(LockboxBase):
null=False,
blank=False,
verbose_name = _("name"),
help_text=_("display name of this file"),
help_text=_("Name of the file"),
)
extension = models.CharField(
@ -75,7 +75,7 @@ class File(LockboxBase):
on_delete=models.SET_NULL,
related_name="files_owned",
verbose_name=_("owner"),
help_text=_("owner of this file"),
help_text=_("Who owns this file"),
)
expires = models.BooleanField(
@ -132,11 +132,11 @@ class File(LockboxBase):
@property
def date_expires(self):
return self.date_created + timedelta(minutes=get_config("EXPIRATION_DELTA_MINUTES").value)
return self.date_created + timedelta(minutes=get_config("EXPIRATION_DELTA_MINUTES"))
@property
def abandoned(self):
return self.date_created + timedelta(minutes=get_config("ABANDONED_DELTA_MINUTES").value)
return self.date_created + timedelta(minutes=get_config("ABANDONED_DELTA_MINUTES"))
@property
def expired(self):

8
lockbox/storage/urls.py
View File

@ -1,4 +1,4 @@
from django.urls import include, path, re_path
from django.urls import include, path
from rest_framework.routers import SimpleRouter
from rest_framework_nested.routers import NestedSimpleRouter
@ -11,7 +11,7 @@ chunk_router = NestedSimpleRouter(router, r'files', lookup="file")
chunk_router.register(r'chunks', views_api.FileChunkViewSet, basename="file-chunks")
urlpatterns = [
re_path(r"api/", include(router.urls)),
re_path(r"api/", include(chunk_router.urls)),
path("client/files/", views_client.FileUploadView.as_view, name="client-fileupload"),
path("api/", include(router.urls)),
path("api/", include(chunk_router.urls)),
path("upload/", views_client.FileUploadView.as_view(), name="client-fileupload"),
]

2
lockbox/storage/views_api.py
View File

@ -18,7 +18,7 @@ class FileModelViewSet(ModelViewSet):
serializer_class = FileSerializer
@action(detail=True, methods=["post"])
def finalize(self, *args, **kwargs): #noqa: ARG002
def finalize(self, *args, **kwargs):
file = self.get_object()
file.status = UPLOAD_STATUS_TYPES.PROCESSING
file.save()

9
lockbox/storage/views_client.py
View File

@ -1,8 +1,15 @@
from common.utils import get_config
from django.shortcuts import render
from django.views import View
from storage.forms import FileForm
class FileUploadView(View):
def get(self, request):
context = {}
context = {
"form": FileForm,
"max_chunk_bytes": get_config("MAX_CHUNK_BYTES"),
"max_file_bytes": get_config("MAX_FILE_BYTES"),
}
return render(request, "storage/upload.html", context=context)

2
lockbox/templates/base.html
View File

@ -18,7 +18,7 @@
{% block sidebar %}
<ul>
<li><a href="/">Files</a></li>
<li><a href="/files/upload/">Upload</a></li>
<li><a href="{% url 'client-fileupload' %}">Upload</a></li>
</ul>
{% endblock %}
</div>

17
lockbox/templates/storage/upload.html
View File

@ -3,21 +3,34 @@
{% block title %}Upload a file{% endblock %}
{% block postjs %}
{% load static %}
{{ block.super }}
<script>
const chunk_size = 1024 * 1024
const chunk_size = {{ max_chunk_bytes }};
const max_file_bytes = {{ max_file_bytes }};
</script>
<script src="{% static 'js/chunked_uploader.js' %}"></script>
{% endblock %}
{% block content %}
<p> Upload file </p>
<table>
{{form}}
</table>
<p id="file-size"></p>
<form method="post">{% csrf_token %}
<input type="file" id="file-upload">
<input type="file" id="file-upload">
</form>
{% endblock %}

16
poetry.lock generated
View File

@ -232,6 +232,20 @@ pytest = ">=7.0.0"
docs = ["sphinx", "sphinx-rtd-theme"]
testing = ["Django", "django-configurations (>=2.0)"]
[[package]]
name = "python-dotenv"
version = "1.0.1"
description = "Read key-value pairs from a .env file and set them as environment variables"
optional = false
python-versions = ">=3.8"
files = [
{file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
{file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
]
[package.extras]
cli = ["click (>=5.0)"]
[[package]]
name = "pytz"
version = "2024.1"
@ -313,4 +327,4 @@ brotli = ["Brotli"]
[metadata]
lock-version = "2.0"
python-versions = "3.12"
content-hash = "600fd336e3593b6fc5503b70e73dd2c22a2922a1662240f540ac6399520ae6d2"
content-hash = "e338f5cc37553ef6a4799746f6feb537427330934b43caee4aa73c3b74a0fb9e"

11
pyproject.toml
View File

@ -12,6 +12,7 @@ django = "~4.2.0"
whitenoise = "^6.6.0"
djangorestframework = "^3.14.0"
drf-nested-routers = "^0.93.5"
python-dotenv = "^1.0.1"
[tool.poetry.group.dev.dependencies]
pytest = "^8.0.0"
@ -67,8 +68,11 @@ target-version = "py312"
select = ["ALL"]
ignore = [
"ANN",
"ARG001",
"ARG002",
"D",
"DJ001",
"DJ006",
"DJ012",
"ERA001",
"FIX",
@ -76,11 +80,12 @@ ignore = [
"PLR0913",
"Q000",
"RUF012",
"TRY",
"S101",
"SIM102",
"SLF001",
"T201",
"TD",
"S101",
"SLF001",
"TRY",
]
[build-system]

9
requirements.txt Normal file
View File

@ -0,0 +1,9 @@
asgiref==3.7.2 ; python_version == "3.12"
django==4.2.10 ; python_version == "3.12"
djangorestframework==3.14.0 ; python_version == "3.12"
drf-nested-routers==0.93.5 ; python_version == "3.12"
python-dotenv==1.0.1 ; python_version == "3.12"
pytz==2024.1 ; python_version == "3.12"
sqlparse==0.4.4 ; python_version == "3.12"
tzdata==2023.4 ; sys_platform == "win32" and python_version == "3.12"
whitenoise==6.6.0 ; python_version == "3.12"

18
requirements_test.txt Normal file
View File

@ -0,0 +1,18 @@
asgiref==3.7.2 ; python_version == "3.12"
colorama==0.4.6 ; python_version == "3.12" and sys_platform == "win32"
coverage[toml]==7.4.1 ; python_version == "3.12"
django==4.2.10 ; python_version == "3.12"
djangorestframework==3.14.0 ; python_version == "3.12"
drf-nested-routers==0.93.5 ; python_version == "3.12"
iniconfig==2.0.0 ; python_version == "3.12"
packaging==23.2 ; python_version == "3.12"
pluggy==1.4.0 ; python_version == "3.12"
pytest-cov==4.1.0 ; python_version == "3.12"
pytest-django==4.8.0 ; python_version == "3.12"
pytest==8.0.0 ; python_version == "3.12"
python-dotenv==1.0.1 ; python_version == "3.12"
pytz==2024.1 ; python_version == "3.12"
ruff==0.2.1 ; python_version == "3.12"
sqlparse==0.4.4 ; python_version == "3.12"
tzdata==2023.4 ; sys_platform == "win32" and python_version == "3.12"
whitenoise==6.6.0 ; python_version == "3.12"