diff --git a/.env-dist b/.env-dist index 9849c7ad..650154d4 100644 --- a/.env-dist +++ b/.env-dist @@ -1,4 +1,5 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_STORAGE_BUCKET_NAME= -S3DIRECT_REGION= +AWS_S3_REGION_NAME= +AWS_S3_ENDPOINT_URL= \ No newline at end of file diff --git a/README.md b/README.md index c02f1505..8a4f3078 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,29 @@ django-s3direct =============== -Upload files directly to S3 from Django +Upload files directly to S3 (or compatible service) from Django. ------------------------------------- [![Build Status](https://travis-ci.org/bradleyg/django-s3direct.svg?branch=master)](https://travis-ci.org/bradleyg/django-s3direct) -Add direct uploads to AWS S3 functionality with a progress bar to file input fields. - +Directly upload files to S3 and other compatible services (such as [Digital Ocean's Spaces](https://www.digitalocean.com/docs/spaces/)) with Django. ## Installation -Install with Pip: - +Install with Pip: ```pip install django-s3direct``` -## AWS Setup - -### Access Credentials +## Access setup -You have two options of providing access to AWS resources: +### When setting up access credentials you have two options: -1. Add credentials of an IAM user to your Django settings -2. Use the EC2 instance profile and its attached IAM role - -Whether you are using an IAM user or a role, there needs to be an IAM policy -in effect that grants permission to upload to S3. Remember to swap out __YOUR_BUCKET_NAME__ for your bucket. +### Option 1: +__Generate access credentials and add them directly to your Django settings__ +If you're not using AWS S3 you can skip to [CORS setup](#cors-setup). If using +Amazon S3 you'll also need to create an IAM policy which grants permission to +upload to your bucket for your newly created credentials. Remember to swap out +__YOUR_BUCKET_NAME__ for your actual bucket. ```json { @@ -47,8 +44,13 @@ in effect that grants permission to upload to S3. Remember to swap out __YOUR_BU } ``` -If the instance profile is to be used, the IAM role needs to have a -Trust Relationship configuration applied: +### Option 2: +__Use the EC2 instance profile and its attached IAM role (AWS only)__ +You'll need to ensure the following trust policy is in place in additon to the +policy above. You'll also need to ensure you have the +[botocore](https://github.com/boto/botocore) package installed. You already +have `botocore` installed if `boto3` +is a dependency of your project. ```json { @@ -65,16 +67,17 @@ Trust Relationship configuration applied: } ``` -Note that in order to use the EC2 instance profile, django-s3direct needs -to query the EC2 instance metadata using utility functions from the -[botocore](https://github.com/boto/botocore) package. You already have `botocore` installed if `boto3` -is a dependency of your project. +### CORS setup -### S3 CORS +You'll need to add a CORS policy on your bucket. Note the ETag header is +particularly important as it is used for multipart uploads. For more information +see [here](https://github.com/TTLabs/EvaporateJS/wiki/Configuring-The-AWS-S3-Bucket). +Remember to swap out YOURDOMAIN.COM in the example below with your domain, +including port if developing locally. -Setup a CORS policy on your S3 bucket. Note the ETag header is particularly -important as it is used for multipart uploads by EvaporateJS. For more information -see [here](https://github.com/TTLabs/EvaporateJS/wiki/Configuring-The-AWS-S3-Bucket). Remember to swap out YOURDOMAIN.COM for your domain, including port if developing locally. +If using Digital Ocean Spaces you must upload the CORS config via the API/s3cmd +CLI. See [here](https://www.digitalocean.com/community/questions/why-can-i-use-http-localhost-port-with-cors-in-spaces) +for more details. ```xml @@ -111,49 +114,70 @@ TEMPLATES = [{ # AWS # If these are set to None, the EC2 instance profile and IAM role are used. -# This requires you to add boto3 (or botocore, which is a dependency of boto3) -# to your project dependencies. AWS_ACCESS_KEY_ID = 'your-aws-access-key-id' AWS_SECRET_ACCESS_KEY = 'your-aws-secret-access-key' + +# Bucket name AWS_STORAGE_BUCKET_NAME = 'your-aws-s3-bucket-name' # The region of your bucket, more info: # http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -S3DIRECT_REGION = 'us-east-1' - -# Destinations, with the following keys: -# -# key [required] Where to upload the file to, can be either: -# 1. '/' = Upload to root with the original filename. -# 2. 'some/path' = Upload to some/path with the original filename. -# 3. functionName = Pass a function and create your own path/filename. -# key_args [optional] Arguments to be passed to 'key' if it's a function. -# auth [optional] An ACL function to whether the current Django user can perform this action. -# allowed [optional] List of allowed MIME types. -# acl [optional] Give the object another ACL rather than 'public-read'. -# cache_control [optional] Cache control headers, eg 'max-age=2592000'. -# content_disposition [optional] Useful for sending files as attachments. -# bucket [optional] Specify a different bucket for this particular object. -# server_side_encryption [optional] Encryption headers for buckets that require it. +AWS_S3_REGION_NAME = 'eu-west-1' + +# The endpoint of your bucket, more info: +# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region +AWS_S3_ENDPOINT_URL = 'https://s3-eu-west-1.amazonaws.com' S3DIRECT_DESTINATIONS = { 'example_destination': { - # REQUIRED + # "key" [required] The location to upload file + # 1. String: folder path to upload to + # 2. Function: generate folder path + filename using a function 'key': 'uploads/images', - - # OPTIONAL - 'auth': lambda u: u.is_staff, # Default allow anybody to upload - 'allowed': ['image/jpeg', 'image/png', 'video/mp4'], # Default allow all mime types - 'bucket': 'pdf-bucket', # Default is 'AWS_STORAGE_BUCKET_NAME' - 'acl': 'private', # Defaults to 'public-read' - 'cache_control': 'max-age=2592000', # Default no cache-control - 'content_disposition': lambda x: 'attachment; filename="{}"'.format(x), # Default no content disposition - 'content_length_range': (5000, 20000000), # Default allow any size - 'server_side_encryption': 'AES256', # Default no encryption + + # "auth" [optional] Limit to specfic Django users + # Function: ACL function + 'auth': lambda u: u.is_staff, + + # "allowed" [optional] Limit to specific mime types + # List: list of mime types + 'allowed': ['image/jpeg', 'image/png', 'video/mp4'], + + # "bucket" [optional] Bucket if different from AWS_STORAGE_BUCKET_NAME + # String: bucket name + 'bucket': 'custom-bucket', + + # "endpoint" [optional] Endpoint if different from AWS_S3_ENDPOINT_URL + # String: endpoint URL + 'endpoint': 'custom-endpoint', + + # "region" [optional] Region if different from AWS_S3_REGION_NAME + # String: region name + 'region': 'custom-region', # Default is 'AWS_S3_REGION_NAME' + + # "acl" [optional] Custom ACL for object, default is 'public-read' + # String: ACL + 'acl': 'private', + + # "cache_control" [optional] Custom cache control header + # String: header + 'cache_control': 'max-age=2592000', + + # "content_disposition" [optional] Custom content disposition header + # String: header + 'content_disposition': lambda x: 'attachment; filename="{}"'.format(x), + + # "content_length_range" [optional] Limit file size + # Tuple: (from, to) in bytes + 'content_length_range': (5000, 20000000), + + # "server_side_encryption" [optional] Use serverside encryption + # String: encrytion standard + 'server_side_encryption': 'AES256', }, - 'example_other': { + 'example_destination_two': { 'key': lambda filename, args: args + '/' + filename, - 'key_args': 'uploads/images', # Only if 'key' is a function + 'key_args': 'uploads/images', } } ``` @@ -235,18 +259,20 @@ $ cd django-s3direct $ python setup.py install $ cd example -# Add your AWS keys to your environment +# Add config to your environment export AWS_ACCESS_KEY_ID='…' export AWS_SECRET_ACCESS_KEY='…' export AWS_STORAGE_BUCKET_NAME='…' -export S3DIRECT_REGION='…' # e.g. 'eu-west-1' +export AWS_S3_REGION_NAME='…' +export AWS_S3_ENDPOINT_URL='…' $ python manage.py migrate $ python manage.py createsuperuser $ python manage.py runserver ``` -Visit ```http://localhost:8000/admin``` to view the admin widget and ```http://localhost:8000/form``` to view the custom form widget. +Visit ```http://localhost:8000/admin``` to view the admin widget and +```http://localhost:8000/form``` to view the custom form widget. ## Development ```shell @@ -258,6 +284,9 @@ $ docker build . --build-arg SKIP_TOX=true -t s3direct $ docker run -itv $(pwd):/code -p 8000:8000 s3direct bash $ npm i +# Install locally +$ python setup.py develop + # Add your AWS keys/details to .env file and export $ cp .env-dist .env $ export $(cat .env) diff --git a/cors_example.xml b/cors_example.xml new file mode 100644 index 00000000..4119aa8b --- /dev/null +++ b/cors_example.xml @@ -0,0 +1,13 @@ + + + + http://localhost:8000 + GET + HEAD + PUT + POST + 3000 + ETag + * + + diff --git a/example/example/settings.py b/example/example/settings.py index 7e36dfc1..f38553aa 100644 --- a/example/example/settings.py +++ b/example/example/settings.py @@ -107,9 +107,9 @@ # django-s3direct will attempt to use the EC2 instance profile instead. AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID', '') AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY', '') - -AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME', 'test-bucket') -S3DIRECT_REGION = os.environ.get('S3DIRECT_REGION', 'us-east-1') +AWS_STORAGE_BUCKET_NAME = os.environ.get('AWS_STORAGE_BUCKET_NAME', '') +AWS_S3_ENDPOINT_URL = os.environ.get('AWS_S3_ENDPOINT_URL', '') +AWS_S3_REGION_NAME = os.environ.get('AWS_S3_REGION_NAME', '') def create_filename(filename): diff --git a/package.json b/package.json index 97acf3c2..e968de4e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "django-s3direct", - "version": "1.0.10", + "version": "1.1.0", "description": "Add direct uploads to S3 functionality with a progress bar to file input fields.", "directories": { "example": "example" diff --git a/runtests.py b/runtests.py index 7ee676e9..4247ed7a 100644 --- a/runtests.py +++ b/runtests.py @@ -47,42 +47,51 @@ def is_authenticated(user): } }, ], - AWS_ACCESS_KEY_ID=environ.get('AWS_ACCESS_KEY_ID', '123'), - AWS_SECRET_ACCESS_KEY=environ.get('AWS_SECRET_ACCESS_KEY', '123'), - AWS_STORAGE_BUCKET_NAME=environ.get( - 'AWS_STORAGE_BUCKET_NAME', 'test-bucket'), - S3DIRECT_REGION='us-east-1', + AWS_ACCESS_KEY_ID='123', + AWS_SECRET_ACCESS_KEY='123', + AWS_STORAGE_BUCKET_NAME='test-bucket', + AWS_S3_REGION_NAME='us-east-1', + AWS_S3_ENDPOINT_URL='https://s3.amazonaws.com', S3DIRECT_DESTINATIONS={ - 'misc': { - 'key': lambda original_filename: 'images/unique.jpg' + 'generic': { + 'key': '/' }, - 'files': { - 'key': '/', - 'auth': lambda u: u.is_staff + 'missing-key': { + 'key': None }, - 'protected': { + 'login-required': { 'key': '/', 'auth': lambda u: u.is_staff }, - 'not_protected': { - 'key': '/', + 'login-not-required': { + 'key': '/' }, - 'imgs': { - 'key': 'uploads/imgs', + 'only-images': { + 'key': '/', 'allowed': ['image/jpeg', 'image/png'] }, - 'thumbs': { - 'key': 'uploads/thumbs', - 'allowed': ['image/jpeg'], + 'limited-size': { + 'key': '/', 'content_length_range': (1000, 50000) }, - 'vids': { - 'key': 'uploads/vids', - 'auth': is_authenticated, - 'allowed': ['video/mp4'] + 'folder-upload' : { + 'key': 'uploads/folder' + }, + 'accidental-leading-slash': { + 'key': '/uploads/folder' + }, + 'accidental-trailing-slash': { + 'key': 'uploads/folder/' + }, + 'function-object-key': { + 'key': lambda original_filename: 'images/unique.jpg' + }, + 'function-object-key-args': { + 'key': lambda original_filename, args: args + '/' + 'filename.jpg', + 'key_args': 'uploads/folder' }, - 'cached': { - 'key': 'uploads/vids', + 'policy-conditions': { + 'key': '/', 'auth': is_authenticated, 'allowed': '*', 'acl': 'authenticated-read', @@ -91,27 +100,19 @@ def is_authenticated(user): 'content_disposition': 'attachment', 'server_side_encryption': 'AES256' }, - 'accidental-leading-slash': { - 'key': '/directory/leading' - }, - 'accidental-trailing-slash': { - 'key': 'directory/trailing/' - }, - 'region-cn': { - 'key': 'uploads/vids', - 'region': 'cn-north-1' - }, - 'region-eu': { - 'key': 'uploads/vids', - 'region': 'eu-west-1' - }, - 'region-default': { - 'key': 'uploads/vids' + 'custom-region-bucket': { + 'key': 'uploads', + 'region': 'cn-north-1', + 'endpoint': 'https://s3.cn-north-1.amazonaws.com.cn' }, - 'key_args': { - 'key': lambda original_filename, args: args + '/' + 'background.jpg', - 'key_args': 'assets/backgrounds' + 'optional-content-disposition-callable': { + 'key': '/', + 'content_disposition': lambda x: 'attachment; filename="{}"'.format(x) }, + 'optional-cache-control-non-callable': { + 'key': '/', + 'cache_control': 'public' + } } ) diff --git a/s3direct/static/s3direct/dist/index.js b/s3direct/static/s3direct/dist/index.js index f0468349..9b91c743 100644 --- a/s3direct/static/s3direct/dist/index.js +++ b/s3direct/static/s3direct/dist/index.js @@ -49,5 +49,5 @@ var t;!function(r){if("object"==typeof exports)module.exports=r();else if("funct },{"./../img/glyphicons-halflings.png":[["glyphicons-halflings.21499243.png","cxU3"],"cxU3"],"./../img/glyphicons-halflings-white.png":[["glyphicons-halflings-white.da002663.png","QCsr"],"QCsr"]}],"/krr":[function(require,module,exports) { },{}],"Focm":[function(require,module,exports) { -"use strict";var e=a(require("js-cookie")),t=a(require("sha.js")),r=a(require("evaporate")),n=a(require("spark-md5"));function a(e){return e&&e.__esModule?e:{default:e}}require("./css/bootstrap.css"),require("./css/styles.css");var o=function(e,t,r,n,a,o){var i=new XMLHttpRequest;i.open(e,t,!0),Object.keys(n).forEach(function(e){i.setRequestHeader(e,n[e])}),i.onload=function(){o(i.status,i.responseText)},i.onerror=i.onabort=function(){d(!1),s(a,"Sorry, failed to upload file.")},i.send(r)},i=function(e){return decodeURIComponent((e+"").replace(/\+/g,"%20"))},c=function(e){var t;try{t=JSON.parse(e)}catch(r){t=null}return t},u=function(e,t){e.querySelector(".bar").style.width=Math.round(100*t)+"%"},s=function(e,t){e.className="s3direct form-active",e.querySelector(".file-input").value="",alert(t)},l=0,d=function(e){var t=document.querySelector(".submit-row");if(t){var r=t.querySelectorAll("input[type=submit],button[type=submit]");!0===e?l++:l--,[].forEach.call(r,function(e){e.disabled=0!==l})}},f=function(e){d(!0),e.className="s3direct progress-active"},p=function(e,t,r){var n=e.querySelector(".file-link"),a=e.querySelector(".file-url");a.value=t+"/"+r,n.setAttribute("href",a.value),n.innerHTML=i(a.value).split("/").pop(),e.className="s3direct link-active",e.querySelector(".bar").style.width="0%",d(!1)},y=function(e){return btoa(n.default.ArrayBuffer.hash(e,!0))},m=function(e){return(0,t.default)("sha256").update(e,"utf-8").digest("hex")},v=function(t){var r=t.querySelector(".csrf-cookie-name"),n=document.querySelector("input[name=csrfmiddlewaretoken]");return n?n.value:e.default.get(r.value)},S=function(e,t,r){var n={};return e&&(n["x-amz-acl"]=e),r&&(n["x-amz-security-token"]=r),t&&(n["x-amz-server-side-encryption"]=t),n},h=function(e){var t={};return e&&(t["x-amz-security-token"]=e),t},q=function(e,t,r){return function(n,a,i,u,s){return new Promise(function(n,a){var s=new FormData,l={"X-CSRFToken":v(e)};s.append("to_sign",i),s.append("datetime",u),s.append("dest",r),o("POST",t,s,l,e,function(e,t){var r=c(t);switch(console.log(r),e){case 200:n(r.s3ObjKey);break;case 403:default:a(r.error)}})})}},g=function(e,t,n,a,o){var i={customAuthMethod:q(e,t,o),aws_key:n.access_key_id,bucket:n.bucket,awsRegion:n.region,computeContentMd5:!0,cryptoMd5Method:y,cryptoHexEncodedHash256:m,partSize:20971520,logging:!0,allowS3ExistenceOptimization:!0,s3FileCacheHoursAgo:12},c={name:n.object_key,file:a,contentType:a.type,xAmzHeadersCommon:h(n.session_token),xAmzHeadersAtInitiate:S(n.acl,n.server_side_encryption,n.session_token),notSignedHeadersAtInitiate:{"Cache-Control":n.cache_control,"Content-Disposition":n.content_disposition},progress:function(t,r){u(e,t)},warn:function(t,r,n){n.includes("InvalidAccessKeyId")&&s(e,n)}};r.default.create(i).then(function(t){f(e),t.add(c).then(function(t){p(e,n.bucket_url,t)},function(t){return s(e,t)})})},k=function(e){var t=e.target.parentElement,r=t.querySelector(".file-input").files[0],n=t.querySelector(".file-dest").value,a=t.getAttribute("data-policy-url"),i=t.getAttribute("data-signing-url"),u=new FormData,l={"X-CSRFToken":v(t)};u.append("dest",n),u.append("name",r.name),u.append("type",r.type),u.append("size",r.size),o("POST",a,u,l,t,function(e,a){var o=c(a);switch(e){case 200:g(t,i,o,r,n);break;case 400:case 403:case 500:s(t,o.error);break;default:s(t,"Sorry, could not get upload URL.")}})},b=function(e){e.preventDefault();var t=e.target.parentElement;t.querySelector(".file-url").value="",t.querySelector(".file-input").value="",t.className="s3direct form-active"},w=function(e){var t=e.querySelector(".file-url"),r=e.querySelector(".file-input"),n=e.querySelector(".file-remove"),a=""===t.value?"form":"link";e.className="s3direct "+a+"-active",n.addEventListener("click",b,!1),r.addEventListener("change",k,!1)};document.addEventListener("DOMContentLoaded",function(e){[].forEach.call(document.querySelectorAll(".s3direct"),w)}),document.addEventListener("DOMNodeInserted",function(e){if(e.target.tagName){var t=e.target.querySelectorAll(".s3direct");[].forEach.call(t,function(e,t,r){w(e)})}}); +"use strict";var e=o(require("js-cookie")),t=o(require("sha.js")),n=o(require("evaporate")),r=o(require("spark-md5"));function o(e){return e&&e.__esModule?e:{default:e}}require("./css/bootstrap.css"),require("./css/styles.css");var a=function(e,t,n,r,o,a){var i=new XMLHttpRequest;i.open(e,t,!0),Object.keys(r).forEach(function(e){i.setRequestHeader(e,r[e])}),i.onload=function(){a(i.status,i.responseText)},i.onerror=i.onabort=function(){d(!1),s(o,"Sorry, failed to upload file.")},i.send(n)},i=function(e){return decodeURIComponent((e+"").replace(/\+/g,"%20"))},c=function(e){var t;try{t=JSON.parse(e)}catch(n){t=null}return t},u=function(e,t){e.querySelector(".bar").style.width=Math.round(100*t)+"%"},s=function(e,t){e.className="s3direct form-active",e.querySelector(".file-input").value="",alert(t)},l=0,d=function(e){var t=document.querySelector(".submit-row");if(t){var n=t.querySelectorAll("input[type=submit],button[type=submit]");!0===e?l++:l--,[].forEach.call(n,function(e){e.disabled=0!==l})}},f=function(e){d(!0),e.className="s3direct progress-active"},p=function(e,t,n,r){var o=e.querySelector(".file-link"),a=e.querySelector(".file-url");a.value=t+"/"+n+"/"+r,o.setAttribute("href",a.value),o.innerHTML=i(a.value).split("/").pop(),e.className="s3direct link-active",e.querySelector(".bar").style.width="0%",d(!1)},y=function(e){return btoa(r.default.ArrayBuffer.hash(e,!0))},m=function(e){return(0,t.default)("sha256").update(e,"utf-8").digest("hex")},v=function(t){var n=t.querySelector(".csrf-cookie-name"),r=document.querySelector("input[name=csrfmiddlewaretoken]");return r?r.value:e.default.get(n.value)},S=function(e,t,n){var r={};return e&&(r["x-amz-acl"]=e),n&&(r["x-amz-security-token"]=n),t&&(r["x-amz-server-side-encryption"]=t),r},h=function(e){var t={};return e&&(t["x-amz-security-token"]=e),t},q=function(e,t,n){return function(r,o,i,u,s){return new Promise(function(r,o){var s=new FormData,l={"X-CSRFToken":v(e)};s.append("to_sign",i),s.append("datetime",u),s.append("dest",n),a("POST",t,s,l,e,function(e,t){var n=c(t);switch(console.log(n),e){case 200:r(n.s3ObjKey);break;case 403:default:o(n.error)}})})}},g=function(e,t,r,o,a){var i={customAuthMethod:q(e,t,a),aws_key:r.access_key_id,bucket:r.bucket,aws_url:r.endpoint,awsRegion:r.region,computeContentMd5:!0,cryptoMd5Method:y,cryptoHexEncodedHash256:m,partSize:20971520,logging:!0,allowS3ExistenceOptimization:!0,s3FileCacheHoursAgo:12},c={name:r.object_key,file:o,contentType:o.type,xAmzHeadersCommon:h(r.session_token),xAmzHeadersAtInitiate:S(r.acl,r.server_side_encryption,r.session_token),notSignedHeadersAtInitiate:{"Cache-Control":r.cache_control,"Content-Disposition":r.content_disposition},progress:function(t,n){u(e,t)},warn:function(t,n,r){r.includes("InvalidAccessKeyId")&&s(e,r)}};n.default.create(i).then(function(t){f(e),t.add(c).then(function(t){p(e,r.endpoint,r.bucket,t)},function(t){return s(e,t)})})},k=function(e){var t=e.target.parentElement,n=t.querySelector(".file-input").files[0],r=t.querySelector(".file-dest").value,o=t.getAttribute("data-policy-url"),i=t.getAttribute("data-signing-url"),u=new FormData,l={"X-CSRFToken":v(t)};u.append("dest",r),u.append("name",n.name),u.append("type",n.type),u.append("size",n.size),a("POST",o,u,l,t,function(e,o){var a=c(o);switch(e){case 200:g(t,i,a,n,r);break;case 400:case 403:case 500:s(t,a.error);break;default:s(t,"Sorry, could not get upload URL.")}})},b=function(e){e.preventDefault();var t=e.target.parentElement;t.querySelector(".file-url").value="",t.querySelector(".file-input").value="",t.className="s3direct form-active"},w=function(e){var t=e.querySelector(".file-url"),n=e.querySelector(".file-input"),r=e.querySelector(".file-remove"),o=""===t.value?"form":"link";e.className="s3direct "+o+"-active",r.addEventListener("click",b,!1),n.addEventListener("change",k,!1)};document.addEventListener("DOMContentLoaded",function(e){[].forEach.call(document.querySelectorAll(".s3direct"),w)}),document.addEventListener("DOMNodeInserted",function(e){if(e.target.tagName){var t=e.target.querySelectorAll(".s3direct");[].forEach.call(t,function(e,t,n){w(e)})}}); },{"js-cookie":"PhdE","sha.js":"t0b9","evaporate":"TlVp","spark-md5":"13nb","./css/bootstrap.css":"MiSu","./css/styles.css":"/krr"}]},{},["Focm"], null) \ No newline at end of file diff --git a/s3direct/tests.py b/s3direct/tests.py index dc0bcee6..42c2a4f3 100644 --- a/s3direct/tests.py +++ b/s3direct/tests.py @@ -49,10 +49,39 @@ def test_widget_html(self): widget = widgets.S3DirectWidget(dest='foo') self.assertEqual(widget.render('filename', None), expected) + def test_missing_dest(self): + data = { + 'name': 'image.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 404) + + def test_incorrectly_named_dest(self): + data = { + 'dest': 'non-existent', + 'name': 'image.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 404) + + def test_missing_key(self): + data = { + 'dest': 'missing-key', + 'name': 'image.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) + def test_get_upload_parameters_logged_in(self): self.client.login(username='admin', password='admin') data = { - 'dest': 'files', + 'dest': 'login-required', 'name': 'image.jpg', 'type': 'image/jpeg', 'size': 1000 @@ -62,7 +91,7 @@ def test_get_upload_parameters_logged_in(self): def test_get_upload_parameters_logged_out(self): data = { - 'dest': 'files', + 'dest': 'login-required', 'name': 'image.jpg', 'type': 'image/jpeg', 'size': 1000 @@ -72,7 +101,7 @@ def test_get_upload_parameters_logged_out(self): def test_allowed_type(self): data = { - 'dest': 'imgs', + 'dest': 'only-images', 'name': 'image.jpg', 'type': 'image/jpeg', 'size': 1000 @@ -82,8 +111,8 @@ def test_allowed_type(self): def test_disallowed_type(self): data = { - 'dest': 'imgs', - 'name': 'image.mp4', + 'dest': 'only-images', + 'name': 'filename.mp4', 'type': 'video/mp4', 'size': 1000 } @@ -92,8 +121,8 @@ def test_disallowed_type(self): def test_allowed_size(self): data = { - 'dest': 'thumbs', - 'name': 'thumbnail.jpg', + 'dest': 'limited-size', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 20000 } @@ -102,39 +131,18 @@ def test_allowed_size(self): def test_disallowed_size(self): data = { - 'dest': 'thumbs', - 'name': 'thumbnail.jpg', + 'dest': 'limited-size', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 200000 } response = self.client.post(reverse('s3direct'), data) self.assertEqual(response.status_code, 400) - def test_allowed_type_logged_in(self): - self.client.login(username='admin', password='admin') + def test_root_object_key(self): data = { - 'dest': 'vids', - 'name': 'video.mp4', - 'type': 'video/mp4', - 'size': 1000 - } - response = self.client.post(reverse('s3direct'), data) - self.assertEqual(response.status_code, 200) - - def test_disallowed_type_logged_out(self): - data = { - u'dest': u'vids', - u'name': u'video.mp4', - u'type': u'video/mp4', - 'size': 1000 - } - response = self.client.post(reverse('s3direct'), data) - self.assertEqual(response.status_code, 403) - - def test_default_upload_key(self): - data = { - 'dest': 'files', - 'name': 'image.jpg', + 'dest': 'generic', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -146,8 +154,8 @@ def test_default_upload_key(self): def test_directory_object_key(self): data = { - 'dest': 'imgs', - 'name': 'image.jpg', + 'dest': 'folder-upload', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -156,13 +164,12 @@ def test_directory_object_key(self): self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) self.assertEqual(policy_dict['object_key'], - 'uploads/imgs/%s' % data['name']) + 'uploads/folder/%s' % data['name']) def test_directory_object_key_with_leading_slash(self): - """Don't want //directory/leading/filename.jpeg""" data = { 'dest': 'accidental-leading-slash', - 'name': 'filename.jpeg', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -171,13 +178,12 @@ def test_directory_object_key_with_leading_slash(self): self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) self.assertEqual(policy_dict['object_key'], - 'directory/leading/filename.jpeg') + 'uploads/folder/filename.jpg') def test_directory_object_key_with_trailing_slash(self): - """Don't want /directory/trailing//filename.jpeg""" data = { 'dest': 'accidental-trailing-slash', - 'name': 'filename.jpeg', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -186,12 +192,12 @@ def test_directory_object_key_with_trailing_slash(self): self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) self.assertEqual(policy_dict['object_key'], - 'directory/trailing/filename.jpeg') + 'uploads/folder/filename.jpg') def test_function_object_key(self): data = { - 'dest': 'misc', - 'name': 'image.jpg', + 'dest': 'function-object-key', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -203,8 +209,8 @@ def test_function_object_key(self): def test_function_object_key_with_args(self): data = { - 'dest': 'key_args', - 'name': 'background.jpg', + 'dest': 'function-object-key-args', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } @@ -214,71 +220,127 @@ def test_function_object_key_with_args(self): policy_dict = json.loads(response.content.decode()) self.assertEqual( policy_dict['object_key'], - settings.S3DIRECT_DESTINATIONS['key_args']['key_args'] + '/' + - data['name']) + settings.S3DIRECT_DESTINATIONS['function-object-key-args'] + ['key_args'] + '/' + data['name']) - def test_function_region_cn_north_1(self): + def test_policy_conditions(self): + self.client.login(username='admin', password='admin') data = { - 'dest': 'region-cn', - 'name': 'background.jpg', + 'dest': 'policy-conditions', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } - self.client.login(username='admin', password='admin') response = self.client.post(reverse('s3direct'), data) self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) - self.assertEqual( - policy_dict['bucket_url'], - 'https://s3.cn-north-1.amazonaws.com.cn/%s' % - settings.AWS_STORAGE_BUCKET_NAME) - - def test_function_region_eu_west_1(self): + self.assertEqual(policy_dict['bucket'], u'astoragebucketname') + self.assertEqual(policy_dict['acl'], u'authenticated-read') + self.assertEqual(policy_dict['cache_control'], u'max-age=2592000') + self.assertEqual(policy_dict['content_disposition'], u'attachment') + self.assertEqual(policy_dict['server_side_encryption'], u'AES256') + + def test_custom_region_bucket(self): data = { - 'dest': 'region-eu', - 'name': 'background.jpg', + 'dest': 'custom-region-bucket', + 'name': 'filename.jpg', 'type': 'image/jpeg', 'size': 1000 } - self.client.login(username='admin', password='admin') response = self.client.post(reverse('s3direct'), data) self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) - self.assertEqual( - policy_dict['bucket_url'], 'https://s3-eu-west-1.amazonaws.com/%s' - % settings.AWS_STORAGE_BUCKET_NAME) + self.assertEqual(policy_dict['endpoint'], + 'https://s3.cn-north-1.amazonaws.com.cn') - def test_function_region_default(self): + def test_optional_param_content_disposition_callable(self): data = { - 'dest': 'region-default', - 'name': 'background.jpg', + 'dest': 'optional-content-disposition-callable', + 'name': 'filename.jpg', 'type': 'image/jpeg', - 'size': 1000 + 'size': 1000, } - self.client.login(username='admin', password='admin') response = self.client.post(reverse('s3direct'), data) self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) - self.assertEqual( - policy_dict['bucket_url'], - 'https://s3.amazonaws.com/%s' % settings.AWS_STORAGE_BUCKET_NAME) + self.assertEqual(policy_dict['content_disposition'], 'attachment; filename="filename.jpg"') - def test_policy_conditions(self): - self.client.login(username='admin', password='admin') + def test_optional_param_cache_control_non_callable(self): data = { - u'dest': u'cached', - u'name': u'video.mp4', - u'type': u'video/mp4', - 'size': 1000 + 'dest': 'optional-cache-control-non-callable', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000, } response = self.client.post(reverse('s3direct'), data) self.assertEqual(response.status_code, 200) policy_dict = json.loads(response.content.decode()) - self.assertEqual(policy_dict['bucket'], u'astoragebucketname') - self.assertEqual(policy_dict['acl'], u'authenticated-read') - self.assertEqual(policy_dict['cache_control'], u'max-age=2592000') - self.assertEqual(policy_dict['content_disposition'], u'attachment') - self.assertEqual(policy_dict['server_side_encryption'], u'AES256') + self.assertEqual(policy_dict['cache_control'], 'public') + + +@override_settings(AWS_STORAGE_BUCKET_NAME=None) +class WidgetTestCaseOverideBucket(TestCase): + def test_missing_bucket(self): + data = { + 'dest': 'generic', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) + + +@override_settings(AWS_S3_REGION_NAME=None) +class WidgetTestCaseOverideRegion(TestCase): + def test_missing_region(self): + data = { + 'dest': 'generic', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) + + +@override_settings(AWS_S3_ENDPOINT_URL=None) +class WidgetTestCaseOverideEndpoint(TestCase): + def test_missing_endpoint(self): + data = { + 'dest': 'generic', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) + + +@override_settings(AWS_ACCESS_KEY_ID=None) +class WidgetTestCaseOverideAccessKey(TestCase): + def test_missing_access_key(self): + data = { + 'dest': 'generic', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) + + +@override_settings(AWS_SECRET_ACCESS_KEY=None) +class WidgetTestCaseOverideSecretAccessKey(TestCase): + def test_missing_secret_key(self): + data = { + 'dest': 'generic', + 'name': 'filename.jpg', + 'type': 'image/jpeg', + 'size': 1000 + } + response = self.client.post(reverse('s3direct'), data) + self.assertEqual(response.status_code, 500) class SignatureViewTestCase(TestCase): @@ -327,7 +389,7 @@ def test_signing(self): data={ 'to_sign': string_to_sign, 'datetime': datetime.strftime(signing_date, '%Y%m%dT%H%M%SZ'), - 'dest': 'not_protected' # auth: not protected + 'dest': 'login-not-required' }, enforce_csrf_checks=True, ) @@ -344,7 +406,7 @@ def test_signing_with_protected(self): data={ 'to_sign': string_to_sign, 'datetime': datetime.strftime(signing_date, '%Y%m%dT%H%M%SZ'), - 'dest': 'protected' # auth: is_staff + 'dest': 'login-required' }, enforce_csrf_checks=True, ) @@ -360,7 +422,7 @@ def test_signing_with_protected_without_valid_auth(self): data={ 'to_sign': string_to_sign, 'datetime': datetime.strftime(signing_date, '%Y%m%dT%H%M%SZ'), - 'dest': 'protected' # auth: is_staff + 'dest': 'login-required' }, enforce_csrf_checks=True, ) diff --git a/s3direct/utils.py b/s3direct/utils.py index bca3558d..8baf08d9 100644 --- a/s3direct/utils.py +++ b/s3direct/utils.py @@ -87,4 +87,8 @@ def get_aws_credentials(): provider = InstanceMetadataProvider( iam_role_fetcher=InstanceMetadataFetcher(timeout=1000, num_attempts=2)) creds = provider.load() - return AWSCredentials(creds.token, creds.secret_key, creds.access_key) + if creds: + return AWSCredentials(creds.token, creds.secret_key, creds.access_key) + else: + # Creds are incorrect + return AWSCredentials(None, None, None) diff --git a/s3direct/views.py b/s3direct/views.py index 1c549596..83b5740e 100644 --- a/s3direct/views.py +++ b/s3direct/views.py @@ -21,55 +21,54 @@ def get_upload_params(request): file_name = request.POST['name'] file_type = request.POST['type'] file_size = int(request.POST['size']) - dest = get_s3direct_destinations().get(request.POST['dest']) + + dest = get_s3direct_destinations().get(request.POST.get('dest', None), None) if not dest: resp = json.dumps({'error': 'File destination does not exist.'}) return HttpResponseNotFound(resp, content_type='application/json') - # Validate request and destination config: - allowed = dest.get('allowed') auth = dest.get('auth') - key = dest.get('key') - cl_range = dest.get('content_length_range') - if auth and not auth(request.user): resp = json.dumps({'error': 'Permission denied.'}) return HttpResponseForbidden(resp, content_type='application/json') + allowed = dest.get('allowed') if (allowed and file_type not in allowed) and allowed != '*': resp = json.dumps({'error': 'Invalid file type (%s).' % file_type}) return HttpResponseBadRequest(resp, content_type='application/json') + cl_range = dest.get('content_length_range') if (cl_range and not cl_range[0] <= file_size <= cl_range[1]): msg = 'Invalid file size (must be between %s and %s bytes).' resp = json.dumps({'error': (msg % cl_range)}) return HttpResponseBadRequest(resp, content_type='application/json') + key = dest.get('key') if not key: resp = json.dumps({'error': 'Missing destination path.'}) return HttpResponseServerError(resp, content_type='application/json') - bucket = dest.get('bucket') + bucket = dest.get('bucket', + getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None)) if not bucket: - bucket = getattr(settings, 'AWS_STORAGE_BUCKET_NAME', None) - if not bucket: - resp = json.dumps({'error': 'Missing S3 bucket config.'}) - return HttpResponseServerError( - resp, content_type='application/json') + resp = json.dumps({'error': 'S3 bucket config missing.'}) + return HttpResponseServerError(resp, content_type='application/json') - region = dest.get('region') + region = dest.get('region', getattr(settings, 'AWS_S3_REGION_NAME', None)) if not region: - region = getattr(settings, 'S3DIRECT_REGION', 'us-east-1') + resp = json.dumps({'error': 'S3 region config missing.'}) + return HttpResponseServerError(resp, content_type='application/json') - if region == 'us-east-1': - endpoint = 's3.amazonaws.com' - elif region in ('cn-north-1', 'cn-northwest-1'): - endpoint = 's3.%s.amazonaws.com.cn' % region - else: - endpoint = 's3-%s.amazonaws.com' % region + endpoint = dest.get('endpoint', + getattr(settings, 'AWS_S3_ENDPOINT_URL', None)) + if not endpoint: + resp = json.dumps({'error': 'S3 endpoint config missing.'}) + return HttpResponseServerError(resp, content_type='application/json') aws_credentials = get_aws_credentials() - bucket_url = 'https://{0}/{1}'.format(endpoint, bucket) + if not aws_credentials.secret_key or not aws_credentials.access_key: + resp = json.dumps({'error': 'AWS credentials config missing.'}) + return HttpResponseServerError(resp, content_type='application/json') upload_data = { 'object_key': get_key(key, file_name, dest), @@ -77,7 +76,7 @@ def get_upload_params(request): 'session_token': aws_credentials.token, 'region': region, 'bucket': bucket, - 'bucket_url': bucket_url, + 'endpoint': endpoint, 'acl': dest.get('acl') or 'public-read', } @@ -100,23 +99,28 @@ def get_upload_params(request): @csrf_protect @require_POST def generate_aws_v4_signature(request): - aws_credentials = get_aws_credentials() message = unquote(request.POST['to_sign']) dest = get_s3direct_destinations().get(unquote(request.POST['dest'])) signing_date = datetime.strptime(request.POST['datetime'], '%Y%m%dT%H%M%SZ') + auth = dest.get('auth') if auth and not auth(request.user): resp = json.dumps({'error': 'Permission denied.'}) return HttpResponseForbidden(resp, content_type='application/json') - if not aws_credentials.secret_key: - resp = json.dumps({'error': 'Invalid AWS credentials.'}) + region = getattr(settings, 'AWS_S3_REGION_NAME', None) + if not region: + resp = json.dumps({'error': 'S3 region config missing.'}) + return HttpResponseServerError(resp, content_type='application/json') + + aws_credentials = get_aws_credentials() + if not aws_credentials.secret_key or not aws_credentials.access_key: + resp = json.dumps({'error': 'AWS credentials config missing.'}) return HttpResponseServerError(resp, content_type='application/json') signing_key = get_aws_v4_signing_key(aws_credentials.secret_key, - signing_date, - settings.S3DIRECT_REGION, 's3') + signing_date, region, 's3') signature = get_aws_v4_signature(signing_key, message) resp = json.dumps({'s3ObjKey': signature}) diff --git a/src/index.js b/src/index.js index fa5b3382..9d284223 100644 --- a/src/index.js +++ b/src/index.js @@ -74,10 +74,10 @@ const beginUpload = (element) => { element.className = 's3direct progress-active'; }; -const finishUpload = (element, awsBucketUrl, objectKey) => { +const finishUpload = (element, endpoint, bucket, objectKey) => { const link = element.querySelector('.file-link'); const url = element.querySelector('.file-url'); - url.value = awsBucketUrl + '/' + objectKey; + url.value = endpoint + '/' + bucket + '/' + objectKey; link.setAttribute('href', url.value); link.innerHTML = parseNameFromUrl(url.value).split('/').pop(); element.className = 's3direct link-active'; @@ -118,11 +118,11 @@ const generateAmzCommonHeaders = (sessionToken) => { const generateCustomAuthMethod = (element, signingUrl, dest) => { const getAwsV4Signature = ( - signParams, - signHeaders, + _signParams, + _signHeaders, stringToSign, signatureDateTime, - canonicalRequest) => { + _canonicalRequest) => { return new Promise((resolve, reject) => { const form = new FormData(); const headers = {'X-CSRFToken': getCsrfToken(element)}; @@ -156,6 +156,7 @@ const initiateUpload = (element, signingUrl, uploadParameters, file, dest) => { customAuthMethod: generateCustomAuthMethod(element, signingUrl, dest), aws_key: uploadParameters.access_key_id, bucket: uploadParameters.bucket, + aws_url: uploadParameters.endpoint, awsRegion: uploadParameters.region, computeContentMd5: true, cryptoMd5Method: computeMd5, @@ -202,7 +203,12 @@ const initiateUpload = (element, signingUrl, uploadParameters, file, dest) => { .add(addConfig) .then( (s3Objkey) => { - finishUpload(element, uploadParameters.bucket_url, s3Objkey); + finishUpload( + element, + uploadParameters.endpoint, + uploadParameters.bucket, + s3Objkey + ); }, (reason) => { return error(element, reason);