upload to storj

.flake8

@@ -1,6 +0,0 @@
-[flake8]
-ignore = E203,W504,W191,W503
-exclude = .git,__pycache__,orm/rbac.py
-max-complexity = 12
-max-line-length = 108
-indent-string = '  '

.lintstagedrc

@@ -1,4 +1,4 @@
 {
-  "*.{js,ts,tsx,json,scss,css,html}": "prettier --write",
+  "*.{js,mjs,ts,tsx,json,scss,css,html}": "prettier --write",
   "package.json": "sort-package-json"
 }

api/upload.mjs

@@ -0,0 +1,88 @@
+import { S3Client } from '@aws-sdk/client-s3'
+import { Upload } from '@aws-sdk/lib-storage'
+import formidable from 'formidable'
+import { Writable } from 'stream'
+import path from 'path'
+import crypto from 'crypto'
+
+const { STORJ_ACCESS_KEY, STORJ_SECRET_KEY, STORJ_END_POINT, STORJ_BUCKET_NAME, CDN_DOMAIN } = process.env
+
+const storjS3Client = new S3Client({
+  endpoint: STORJ_END_POINT,
+  credentials: {
+    accessKeyId: STORJ_ACCESS_KEY,
+    secretAccessKey: STORJ_SECRET_KEY
+  }
+})
+
+const formidablePromise = async (request, options) => {
+  return new Promise((resolve, reject) => {
+    const form = formidable(options)
+
+    form.parse(request, (err, fields, files) => {
+      if (err) {
+        return reject(err)
+      }
+      return resolve({ fields, files })
+    })
+  })
+}
+
+const fileConsumer = (acc) => {
+  return new Writable({
+    write: (chunk, _enc, next) => {
+      acc.push(chunk)
+      next()
+    }
+  })
+}
+
+const formidableConfig = {
+  keepExtensions: true,
+  maxFileSize: 10_000_000,
+  maxFieldsSize: 10_000_000,
+  maxFields: 7,
+  allowEmptyFiles: false,
+  multiples: false
+}
+
+const handleFileUpload = async (request) => {
+  const chunks = []
+  const { files } = await formidablePromise(request, {
+    ...formidableConfig,
+    // consume this, otherwise formidable tries to save the file to disk
+    fileWriteStreamHandler: () => fileConsumer(chunks)
+  })
+
+  const data = Buffer.concat(chunks)
+
+  const { originalFilename, mimetype } = files.file
+
+  const fileExtension = path.extname(originalFilename)
+
+  const filename = crypto.randomUUID() + fileExtension
+
+  const params = {
+    Bucket: STORJ_BUCKET_NAME,
+    Key: filename,
+    Body: data,
+    ContentType: mimetype
+  }
+
+  const upload = new Upload({ params, client: storjS3Client })
+  await upload.done()
+
+  return `http://${CDN_DOMAIN}/${filename}`
+}
+
+const handler = async (request, response) => {
+  try {
+    const location = await handleFileUpload(request)
+    return response.status(200).json(location)
+  } catch (error) {
+    console.error(error)
+    response.status(500).json({ error: error.message })
+  }
+}
+
+export default handler

api/upload.py

@@ -1,83 +0,0 @@
-from flask import Flask, request, jsonify
-from werkzeug.utils import secure_filename
-import boto3
-from botocore.exceptions import ClientError, WaiterError
-import tempfile
-import os
-
-app = Flask(__name__)
-session = boto3.Session()
-storj_resource = session.resource('s3')
-storj_client = boto3.client('s3',
-                            aws_access_key_id=os.environ['STORJ_ACCESS_KEY'],
-                            aws_secret_access_key=os.environ['STORJ_SECRET_KEY'],
-                            endpoint_url=os.environ['STORJ_END_POINT']
-                            )
-
-
-def upload_storj(filecontent, filename, bucket_name):
-    head = None
-    bucket_obj = None
-    try:
-        bucket = storj_resource.Bucket(bucket_name)
-    except ClientError:
-        bucket = None
-
-    try:
-        # In case filename already exists, get current etag to check if the
-        # contents change after upload
-        head = storj_client.head_object(Bucket=bucket_name, Key=filename)
-    except ClientError:
-        etag = ''
-    else:
-        etag = head['ETag'].strip('"')
-
-    try:
-        bucket_obj = bucket.Object(filename)
-    except (ClientError, AttributeError):
-        bucket_obj = None
-
-    try:
-        # Use the upload_fileobj method to safely upload the file
-        storj_client.upload_fileobj(
-            Fileobj=filecontent,
-            Bucket=bucket_name,
-            Key=filename
-        )
-    except (ClientError, AttributeError):
-        pass
-    else:
-        try:
-            bucket_obj.wait_until_exists(IfNoneMatch=etag)
-        except WaiterError:
-            pass
-        else:
-            head = storj_client.head_object(Bucket=bucket_name, Key=filename)
-    return head
-
-
-@app.route('/api/upload', methods=['post'])
-def upload():
-    print(request.files.to_dict())
-    # check if the post request has the file part
-    if 'file' not in request.files:
-        return {'error': 'No file part'}, 400
-    file = request.files.get('file')
-    if file:
-        # save the file
-        filename = secure_filename(file.name or file.filename)
-        # if user does not select file, browser also
-        # submit a empty part without filename
-        if not filename:
-            return {'error': 'No selected file'}, 400
-        else:
-            # Save the file to a temporary location
-            with tempfile.TemporaryDirectory() as temp_dir:
-                temp_path = os.path.join(temp_dir, filename)
-                file.save(temp_path)
-                # Open the file in binary mode
-                with open(temp_path, 'rb') as filecontent:
-                    result = upload_storj(filecontent, filename, 'discoursio')
-    else:
-        return {'error': 'No selected file'}, 400
-    return {'message': 'File uploaded', 'result': jsonify(result)}, 200

package.json

@@ -23,19 +23,18 @@
     "pre-push-old": "npm run typecheck",
     "prepare": "husky install",
     "preview": "astro preview",
-    "server": "node server/server.mjs",
     "start": "astro dev",
     "start:dev": "cross-env PUBLIC_API_URL=http://127.0.0.1:8080 vercel dev",
     "start:local": "cross-env PUBLIC_API_URL=http://127.0.0.1:8080 astro dev",
     "start:production": "cross-env PUBLIC_API_URL=https://v2.discours.io astro dev",
     "start:staging": "cross-env PUBLIC_API_URL=https://testapi.discours.io astro dev",
     "typecheck": "astro check && tsc --noEmit",
-    "typecheck:watch": "tsc --noEmit --watch",
+    "typecheck:watch": "tsc --noEmit --watch"
-    "vercel-build": "astro build"
   },
   "dependencies": {
+    "@aws-sdk/abort-controller": "^3.266.0",
     "@aws-sdk/client-s3": "^3.216.0",
-    "@aws-sdk/lib-storage": "^3.223.0",
+    "@aws-sdk/lib-storage": "^3.266.0",
     "@connorskees/grass": "^0.12.0",
     "@solid-primitives/share": "^2.0.1",
     "astro-seo-meta": "^2.0.0",

requirements.txt

@@ -1,2 +0,0 @@
-Flask==2.2.2
-boto3

src/components/Pages/profile/ProfileSettingsPage.tsx

@@ -8,6 +8,7 @@ import { clsx } from 'clsx'
 import styles from './Settings.module.scss'
 import { useProfileForm } from '../../../context/profile'
 import validateUrl from '../../../utils/validateUrl'
+import { createFileUploader, UploadFile } from '@solid-primitives/upload'
 
 export const ProfileSettingsPage = (props: PageProps) => {
   const [addLinkForm, setAddLinkForm] = createSignal<boolean>(false)
@@ -26,30 +27,28 @@ export const ProfileSettingsPage = (props: PageProps) => {
     event.preventDefault()
     submit(form)
   }
-  let userpicFile: HTMLInputElement
+
-  const handleFileUpload = async (file: File) => {
+  const { selectFiles } = createFileUploader({ multiple: false, accept: 'image/*' })
+
+  const handleFileUpload = async (uploadFile: UploadFile) => {
     const formData = new FormData()
-    formData.append('file', file)
+    formData.append('file', uploadFile.file, uploadFile.name)
-    console.log(formData)
     const response = await fetch('/api/upload', {
       method: 'POST',
-      body: formData,
+      body: formData
-      headers: {
+    })
-        'Content-Type': 'multipart/form-data'
+    return response.json()
+  }
+
+  const handleUserpicUpload = async () => {
+    await selectFiles(async ([uploadFile]) => {
+      try {
+        const fileUrl = await handleFileUpload(uploadFile)
+        updateFormField('userpic', fileUrl)
+      } catch (error) {
+        console.error('[upload avatar] error', error)
       }
     })
-    const json = await response.json()
-    console.debug(json)
-  }
-  const handleUserpicUpload = async (ev) => {
-    // TODO: show progress
-    console.debug('handleUserpicUpload')
-    try {
-      const f = ev.target.files[0]
-      if (f) await handleFileUpload(f)
-    } catch (error) {
-      console.error('[upload] error', error)
-    }
   }
 
   const [hostname, setHostname] = createSignal('new.discours.io')
@@ -73,20 +72,9 @@ export const ProfileSettingsPage = (props: PageProps) => {
                   <h4>{t('Userpic')}</h4>
                   <div class="pretty-form__item">
                     <div class={styles.avatarContainer}>
-                      <img
+                      <button role="button" onClick={() => handleUserpicUpload()}>
-                        class={styles.avatar}
+                        <img class={styles.avatar} src={form.userpic} alt={form.name} />
-                        src={form.userpic}
+                      </button>
-                        alt={form.name}
-                        onClick={() => userpicFile.click()}
-                      />
-                      <input
-                        ref={userpicFile}
-                        type="file"
-                        name="file"
-                        value="file"
-                        hidden
-                        onChange={handleUserpicUpload}
-                      />
                     </div>
                   </div>
                   <h4>{t('Name')}</h4>

vercel.json

@@ -2,8 +2,7 @@
   "routes": [
     {
       "src": "/api/upload",
-      "headers": { "Content-Type": "application/json" },
+      "dest": "api/upload.mjs"
-      "dest": "api/upload.py"
     },
     {
       "src": "/api/newsletter",