Feature/s3 integration (#2)

* feat: add s3 storage for media * refactor: update letter decryption test to look for key request properties * fix: update db url to be ipv4 for ssl context match * ci: output backend logs to the console * ci: unset email host creds for local testing --------- Co-authored-by: ramvignesh-b <ramvignesh-b@github.com>
2026-05-04 00:56:34 +00:00 · 2026-04-26 21:40:00 +05:30
parent 1f47b6f4dd
commit 48b6a06571
8 changed files with 156 additions and 23 deletions
@@ -2,7 +2,7 @@
 DB_NAME=piku_test_db
 DB_USER=test
 DB_PASSWORD=password123
-DB_HOST=localhost
+DB_HOST=127.0.0.1
 DB_PORT=5433

 # SSL
@@ -17,8 +17,8 @@ BACKEND_PORT=8001
 # EMAIL
 EMAIL_HOST=127.0.0.1
 EMAIL_PORT=1026
-EMAIL_HOST_USER=test
-EMAIL_HOST_PASSWORD=password123
+EMAIL_HOST_USER=
+EMAIL_HOST_PASSWORD=
 FROM_EMAIL="Test <test@pi-ku.app>"
 EMAIL_API_PORT=8026

@@ -2,23 +2,30 @@
 DB_NAME=piku
 DB_USER=user
 DB_PASSWORD=password123
-DB_HOST=localhost
+DB_HOST=127.0.0.1
 DB_PORT=5432

 # SSL
 SSL_ENABLED=true
+S3_ENABLED=false

 # DJANGO
 DEBUG=True
 SECRET_KEY=django-secret-key
 BACKEND_DOMAIN=127.0.0.1
 BACKEND_PORT=8000
+# S3
+R2_ACCESS_KEY_ID=
+R2_SECRET_ACCESS_KEY=
+R2_REGION_NAME=
+R2_ENDPOINT_URL=
+R2_PUBLIC_URL=

 # EMAIL
 EMAIL_HOST=127.0.0.1
 EMAIL_PORT=1025
-EMAIL_HOST_USER=test
-EMAIL_HOST_PASSWORD=password123
+EMAIL_HOST_USER=
+EMAIL_HOST_PASSWORD=
 FROM_EMAIL="Pi Ku <no-reply@test.com>"

 # FRONTEND
@@ -145,3 +145,7 @@ jobs:
          name: playwright-report
          path: frontend/playwright-report/
          retention-days: 10
+
+      - name: Print Backend Logs on Failure
+        if: failure()
+        run: cat tmp/logs/backend.log || true
@@ -25,8 +25,11 @@ env_file = os.environ.get("PIKU_ENV_FILE", os.path.join(BASE_DIR.parent, ".env")
 if os.path.exists(env_file):
    environ.Env.read_env(env_file, overwrite=False)

-ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[])
-ALLOWED_HOSTS.append(env("FRONTEND_DOMAIN"))
+ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=["127.0.0.1"])
+ALLOWED_HOSTS.append(env("FRONTEND_DOMAIN", default="127.0.0.1"))
+ALLOWED_HOSTS.append(env("BACKEND_DOMAIN", default="127.0.0.1"))
+
+CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[])

 SSL_ENABLED = env.bool("SSL_ENABLED", default=False)
 URI_SCHEME = "https://" if SSL_ENABLED else "http://"
@@ -98,6 +101,7 @@ DATABASES = {
 }

 CORS_ALLOWED_ORIGINS = FRONTEND_URLS
+CSRF_TRUSTED_ORIGINS += FRONTEND_URLS
 CORS_ALLOW_CREDENTIALS = True

 AUTH_USER_MODEL = "users.User"
@@ -172,4 +176,31 @@ USE_TZ = True

 STATIC_URL = "static/"
 MEDIA_URL = "/media/"
+
+if env.bool("S3_ENABLED", default=False):
+    MEDIA_URL = f"{env('R2_PUBLIC_URL')}/media/"
+    # HACK: S3 auto pre-pends the url scheme forcefully and this prevents double https
+    R2_HOST = env("R2_PUBLIC_URL").replace("https://", "")
+    STORAGES = {
+        "default": {
+            "BACKEND": "storages.backends.s3.S3Storage",
+            "OPTIONS": {
+                "access_key": env("R2_ACCESS_KEY_ID"),
+                "secret_key": env("R2_SECRET_ACCESS_KEY"),
+                "bucket_name": env("R2_STORAGE_BUCKET_NAME"),
+                "region_name": env("R2_REGION_NAME"),
+                "endpoint_url": env("R2_ENDPOINT_URL"),
+                "location": "media",
+                "signature_version": "s3v4",
+                "file_overwrite": False,
+                "custom_domain": R2_HOST,
+                "querystring_auth": False,
+            },
+        },
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+        },
+    }
+    DEFAULT_FILE_STORAGE = "storages.backends.s3.S3Storage"
+
 MEDIA_ROOT = BASE_DIR / "media"
@@ -6,11 +6,13 @@ readme = "README.md"
 requires-python = ">=3.14"
 dependencies = [
    "apscheduler>=3.11.2",
+    "boto3>=1.42.96",
    "django>=6.0.4",
    "django-apscheduler>=0.7.0",
    "django-cors-headers>=4.9.0",
    "django-environ>=0.13.0",
    "django-extensions>=4.1",
+    "django-storages>=1.14.6",
    "django-structlog>=10.0.0",
    "djangorestframework>=3.17.1",
    "djangorestframework-simplejwt>=5.5.1",
@@ -23,6 +23,34 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" },
 ]

+[[package]]
+name = "boto3"
+version = "1.42.96"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+    { name = "jmespath" },
+    { name = "s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a6/2d/69fb3acd50bab83fb295c167d33c4b653faeb5fb0f42bfca4d9b69d6fb68/boto3-1.42.96.tar.gz", hash = "sha256:b38a9e4a3fbbee9017252576f1379780d0a5814768676c08df2f539d31fcdd68", size = 113203, upload-time = "2026-04-24T19:47:18.677Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2b/9d/b3f617d011c42eb804d993103b8fa9acdce153e181a3042f58bfe33d7cb4/boto3-1.42.96-py3-none-any.whl", hash = "sha256:2f4566da2c209a98bdbfc874d813ef231c84ad24e4f815e9bc91de5f63351a24", size = 140557, upload-time = "2026-04-24T19:47:15.824Z" },
+]
+
+[[package]]
+name = "botocore"
+version = "1.42.96"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "jmespath" },
+    { name = "python-dateutil" },
+    { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/77/2c333622a1d47cf5bf73cdcab0cb6c92addafbef2ec05f81b9f75687d9e5/botocore-1.42.96.tar.gz", hash = "sha256:75b3b841ffacaa944f645196655a21ca777591dd8911e732bfb6614545af0250", size = 15263344, upload-time = "2026-04-24T19:47:05.283Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/45/56/152c3a859ca1b9d77ed16deac3cf81682013677c68cf5715698781fc81bd/botocore-1.42.96-py3-none-any.whl", hash = "sha256:db2c3e2006628be6fde81a24124a6563c363d6982fb92728837cf174bad9d98a", size = 14945920, upload-time = "2026-04-24T19:47:00.323Z" },
+]
+
 [[package]]
 name = "cffi"
 version = "2.0.0"
@@ -182,6 +210,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/11/33/bf539925b102d68200da5b1d3eacb8aa5d5d9a065972e8b8724d0d53bb0d/django_ipware-7.0.1-py2.py3-none-any.whl", hash = "sha256:db16bbee920f661ae7f678e4270460c85850f03c6761a4eaeb489bdc91f64709", size = 6425, upload-time = "2024-04-19T20:02:47.469Z" },
 ]

+[[package]]
+name = "django-storages"
+version = "1.14.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "django" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ff/d6/2e50e378fff0408d558f36c4acffc090f9a641fd6e084af9e54d45307efa/django_storages-1.14.6.tar.gz", hash = "sha256:7a25ce8f4214f69ac9c7ce87e2603887f7ae99326c316bc8d2d75375e09341c9", size = 87587, upload-time = "2025-04-02T02:34:55.103Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/1f/21/3cedee63417bc5553eed0c204be478071c9ab208e5e259e97287590194f1/django_storages-1.14.6-py3-none-any.whl", hash = "sha256:11b7b6200e1cb5ffcd9962bd3673a39c7d6a6109e8096f0e03d46fab3d3aabd9", size = 33095, upload-time = "2025-04-02T02:34:53.291Z" },
+]
+
 [[package]]
 name = "django-structlog"
 version = "10.0.0"
@@ -289,6 +329,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/43/c8/8aaf447698c4d59aa853fd318eed300b5c9e44459f242ab8ead6c9c09792/gunicorn-25.3.0-py3-none-any.whl", hash = "sha256:cacea387dab08cd6776501621c295a904fe8e3b7aae9a1a3cbb26f4e7ed54660", size = 208403, upload-time = "2026-03-27T00:00:27.386Z" },
 ]

+[[package]]
+name = "jmespath"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "4.0.0"
@@ -355,11 +404,13 @@ version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
    { name = "apscheduler" },
+    { name = "boto3" },
    { name = "django" },
    { name = "django-apscheduler" },
    { name = "django-cors-headers" },
    { name = "django-environ" },
    { name = "django-extensions" },
+    { name = "django-storages" },
    { name = "django-structlog" },
    { name = "djangorestframework" },
    { name = "djangorestframework-simplejwt" },
@@ -377,11 +428,13 @@ dependencies = [
 [package.metadata]
 requires-dist = [
    { name = "apscheduler", specifier = ">=3.11.2" },
+    { name = "boto3", specifier = ">=1.42.96" },
    { name = "django", specifier = ">=6.0.4" },
    { name = "django-apscheduler", specifier = ">=0.7.0" },
    { name = "django-cors-headers", specifier = ">=4.9.0" },
    { name = "django-environ", specifier = ">=0.13.0" },
    { name = "django-extensions", specifier = ">=4.1" },
+    { name = "django-storages", specifier = ">=1.14.6" },
    { name = "django-structlog", specifier = ">=10.0.0" },
    { name = "djangorestframework", specifier = ">=3.17.1" },
    { name = "djangorestframework-simplejwt", specifier = ">=5.5.1" },
@@ -513,6 +566,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/03/36/76704c4f312257d6dbaae3c959add2a622f63fcca9d864659ce6d8d97d3d/ruff-0.15.9-py3-none-win_arm64.whl", hash = "sha256:0694e601c028fd97dc5c6ee244675bc241aeefced7ef80cd9c6935a871078f53", size = 11005870, upload-time = "2026-04-02T18:17:15.773Z" },
 ]

+[[package]]
+name = "s3transfer"
+version = "0.16.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/46/29/af14f4ef3c11a50435308660e2cc68761c9a7742475e0585cd4396b91777/s3transfer-0.16.1.tar.gz", hash = "sha256:8e424355754b9ccb32467bdc568edf55be82692ef2002d934b1311dbb3b9e524", size = 154801, upload-time = "2026-04-22T20:36:06.475Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/03/19/90d7d4ed51932c022d53f1d02d564b62d10e272692a1f9b76425c1ad2a02/s3transfer-0.16.1-py3-none-any.whl", hash = "sha256:61bcd00ccb83b21a0fe7e91a553fff9729d46c83b4e0106e7c314a733891f7c2", size = 86825, upload-time = "2026-04-22T20:36:04.992Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -579,6 +644,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" },
 ]

+[[package]]
+name = "urllib3"
+version = "2.6.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
+]
+
 [[package]]
 name = "werkzeug"
 version = "3.1.8"
@@ -152,9 +152,10 @@ describe("letterLogic image helpers", () => {
        crypto,
      );

-      expect(api.get).toHaveBeenCalledWith("https://remote/photo.png.bin", {
-        responseType: "blob",
-      });
+      expect(api.get).toHaveBeenCalledWith(
+        "https://remote/photo.png.bin",
+        expect.objectContaining({ responseType: "blob" }),
+      );
      expect(CryptoUtils.prototype.decryptImage).toHaveBeenCalledWith(
        expect.any(Blob),
        "wrapped-dek",
@@ -238,9 +239,10 @@ describe("letterLogic image helpers", () => {
        crypto,
      );

-      expect(api.get).toHaveBeenCalledWith("https://remote/photo.png.bin", {
-        responseType: "blob",
-      });
+      expect(api.get).toHaveBeenCalledWith(
+        "https://remote/photo.png.bin",
+        expect.objectContaining({ responseType: "blob" }),
+      );
      expect(
        CryptoUtils.prototype.decryptImageWithSharingKey,
      ).toHaveBeenCalledWith(expect.any(Blob), "raw-sharing-key");
@@ -28,14 +28,18 @@ export async function decryptCanvasImages(
    remoteImages.map((img) => [img.file_name, img.file]),
  );

-  const decryptionPromises = canvasData.objects.map(async (obj, index) => {
+  const imageDecryptionPromises = canvasData.objects.map(async (obj, index) => {
    if (obj.type !== "Image") return;
    const imgObj = obj as FabricImageJSON;
    const remoteUrl = imageMap.get(imgObj.src);
    if (!remoteUrl) return;

    try {
-      const res = await api.get(remoteUrl, { responseType: "blob" });
+      // HACK: For S3 Storage fetch and avoiding CORS error
+      const res = await api.get(remoteUrl, {
+        responseType: "blob",
+        withCredentials: false,
+      });
      const originalSrc = imgObj.src;

      const blobUrl = await cryptoUtils.decryptImage(
@@ -56,7 +60,7 @@ export async function decryptCanvasImages(
    }
  });

-  await Promise.all(decryptionPromises);
+  await Promise.all(imageDecryptionPromises);
  canvasData.objects = canvasData.objects.filter(Boolean);
  return { isDecryptionPartialFailure, error };
 }
@@ -66,14 +70,16 @@ export async function decryptCanvasImagesWithSharingKey(
  remoteImages: { file_name: string; file: string }[],
  sharingKey: string,
  cryptoUtils: CryptoUtils,
-) {
-  if (!canvasData?.objects) return;
-
+): Promise<{ isDecryptionPartialFailure: boolean; error: string }> {
+  if (!canvasData?.objects)
+    return { isDecryptionPartialFailure: false, error: "" };
+  let isDecryptionPartialFailure = false;
+  let error = "";
  const imageMap = new Map(
    remoteImages.map((img) => [img.file_name, img.file]),
  );

-  const decryptionPromises = canvasData.objects.map(async (obj) => {
+  const decryptionPromises = canvasData.objects.map(async (obj, index) => {
    if (obj.type !== "Image") return;

    const imgObj = obj as FabricImageJSON;
@@ -81,17 +87,24 @@ export async function decryptCanvasImagesWithSharingKey(
    if (!remoteUrl) return;

    try {
-      const res = await api.get(remoteUrl, { responseType: "blob" });
+      const res = await api.get(remoteUrl, {
+        responseType: "blob",
+        withCredentials: false,
+      });
      imgObj.src = await cryptoUtils.decryptImageWithSharingKey(
        res.data,
        sharingKey,
      );
    } catch (_error) {
-      // Keep original or handle failure
+      delete canvasData.objects[index];
+      isDecryptionPartialFailure = true;
+      error = _error instanceof Error ? _error.message : "Unknown error";
    }
  });

  await Promise.all(decryptionPromises);
+  canvasData.objects = canvasData.objects.filter(Boolean);
+  return { isDecryptionPartialFailure, error };
 }

 export async function encryptCanvasImages(