diff --git a/INSTALL b/INSTALL index f3cba51..4395f6f 100644 --- a/INSTALL +++ b/INSTALL @@ -60,7 +60,7 @@ OpenSuse Python 2.5 package) or it can be installed using your package manager, e.g. in Debian use - apt-get install python2.4-setuptools + apt-get install python-setuptools Again, consult your distribution documentation on how to find out the actual package name and how to install it then. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8f580df --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include INSTALL README.md NEWS +include s3cmd.1 diff --git a/NEWS b/NEWS index a4b449c..f7c2528 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,57 @@ +s3cmd-1.5.2 - 2015-02-08 +=============== +* Handle unvalidated SSL certificate. Necessary on Ubuntu 14.04 for + SSL to function at all. +* packaging fixes (require python-magic, drop ez_setup) + +s3cmd-1.5.1.2 - 2015-02-04 +=============== +* fix PyPi install + +s3cmd-1.5.1 - 2015-02-04 +=============== + +* Sort s3cmd ls output by bucket name (Andrew Gaul) +* Support relative expiry times in signurl. (Chris Lamb) +* Fixed issue with mixed path separators with s3cmd get --recursive on + Windows. (Luke Winslow) +* fix S3 wildcard certificate checking +* Handle headers with spaces in their values properly (#460) +* Fix lack of SSL certificate checking libraries on older python +* set content-type header for stdin from command line or Config() +* fix uploads from stdin (#464) +* Fix directory exclusions (#467) +* fix signurl +* Don't retry in response to HTTP 405 error (#422) +* Don't crash when a proxy returns an invalid XML error document + +s3cmd-1.5.0 - 2015-01-12 +=============== +* add support for newer regions such as Frankfurt that + require newer authorization signature v4 support + (Vasileios Mitrousis, Michal Ludvig, Matt Domsch) +* drop support for python 2.4 due to signature v4 code. + python 2.6 is now the minimum, and python 3 is still not supported. +* handle redirects to the "right" region for a bucket. +* add --ca-cert=FILE for self-signed certs (Matt Domsch) +* allow proxied SSL connections with python >= 2.7 (Damian Gerow) +* add --remove-headers for [modify] command (Matt Domsch) +* add -s/--ssl and --no-ssl options (Viktor Szakáts) +* add --signature-v2 for backwards compatibility with S3 clones. +* bugfixes by 17 contributors + s3cmd 1.5.0-rc1 - 2014-06-29 =============== -[TODO - extract from: git log --no-merges v1.5.0-beta1..] +* add environment variable S3CMD_CONFIG (Devon Jones), + access key and secre keys (Vasileios Mitrousis) +* added modify command (Francois Gaudin) +* better debug messages (Matt Domsch) +* faster batch deletes (Matt Domsch) +* Added support for restoring files from Glacier storage (Robert Palmer) +* Add and remove full lifecycle policies (Sam Rudge) +* Add support for object expiration (hrchu) +* bugfixes by 26 contributors + s3cmd 1.5.0-beta1 - 2013-12-02 ================= diff --git a/PKG-INFO b/PKG-INFO index f9c3ddc..4137201 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,11 +1,11 @@ Metadata-Version: 1.1 Name: s3cmd -Version: 1.5.0-rc1 +Version: 1.5.2 Summary: Command line tool for managing Amazon S3 and CloudFront services Home-page: http://s3tools.org -Author: Michal Ludvig -Author-email: michal@logix.cz -License: GPL version 2 +Author: github.com/mdomsch, github.com/matteobar +Author-email: s3tools-bugs@lists.sourceforge.net +License: GNU GPL v2+ Description: S3cmd lets you copy files from/to Amazon S3 @@ -20,4 +20,20 @@ Michal Ludvig Platform: UNKNOWN -Requires: dateutil +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: MacOS X +Classifier: Environment :: Win32 (MS Windows) +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) +Classifier: Natural Language :: English +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 2 :: Only +Classifier: Topic :: System :: Archiving +Classifier: Topic :: Utilities diff --git a/README b/README deleted file mode 100644 index 5fed0ee..0000000 --- a/README +++ /dev/null @@ -1,370 +0,0 @@ -S3cmd tool for Amazon Simple Storage Service (S3) -================================================= - -Author: - Michal Ludvig - Copyright (c) TGRMN Software - http://www.tgrmn.com - and contributors - -S3tools / S3cmd project homepage: - http://s3tools.org - -S3tools / S3cmd mailing lists: - - * Announcements of new releases: - s3tools-announce@lists.sourceforge.net - - * General questions and discussion about usage - s3tools-general@lists.sourceforge.net - - * Bug reports - s3tools-bugs@lists.sourceforge.net - -!!! -!!! Please consult INSTALL file for installation instructions! -!!! - -What is S3cmd --------------- -S3cmd is a free command line tool and client for uploading, -retrieving and managing data in Amazon S3 and other cloud -storage service providers that use the S3 protocol, such as -Google Cloud Storage or DreamHost DreamObjects. It is best -suited for power users who are familiar with command line -programs. It is also ideal for batch scripts and automated -backup to S3, triggered from cron, etc. - -S3cmd is written in Python. It's an open source project -available under GNU Public License v2 (GPLv2) and is free -for both commercial and private use. You will only have -to pay Amazon for using their storage. - -Lots of features and options have been added to S3cmd, -since its very first release in 2008.... we recently counted -more than 60 command line options, including multipart -uploads, encryption, incremental backup, s3 sync, ACL and -Metadata management, S3 bucket size, bucket policies, and -more! - -What is Amazon S3 ------------------ -Amazon S3 provides a managed internet-accessible storage -service where anyone can store any amount of data and -retrieve it later again. - -S3 is a paid service operated by Amazon. Before storing -anything into S3 you must sign up for an "AWS" account -(where AWS = Amazon Web Services) to obtain a pair of -identifiers: Access Key and Secret Key. You will need to -give these keys to S3cmd. -Think of them as if they were a username and password for -your S3 account. - -Amazon S3 pricing explained ---------------------------- -At the time of this writing the costs of using S3 are (in USD): - -$0.15 per GB per month of storage space used - -plus - -$0.10 per GB - all data uploaded - -plus - -$0.18 per GB - first 10 TB / month data downloaded -$0.16 per GB - next 40 TB / month data downloaded -$0.13 per GB - data downloaded / month over 50 TB - -plus - -$0.01 per 1,000 PUT or LIST requests -$0.01 per 10,000 GET and all other requests - -If for instance on 1st of January you upload 2GB of -photos in JPEG from your holiday in New Zealand, at the -end of January you will be charged $0.30 for using 2GB of -storage space for a month, $0.20 for uploading 2GB -of data, and a few cents for requests. -That comes to slightly over $0.50 for a complete backup -of your precious holiday pictures. - -In February you don't touch it. Your data are still on S3 -servers so you pay $0.30 for those two gigabytes, but not -a single cent will be charged for any transfer. That comes -to $0.30 as an ongoing cost of your backup. Not too bad. - -In March you allow anonymous read access to some of your -pictures and your friends download, say, 500MB of them. -As the files are owned by you, you are responsible for the -costs incurred. That means at the end of March you'll be -charged $0.30 for storage plus $0.09 for the download traffic -generated by your friends. - -There is no minimum monthly contract or a setup fee. What -you use is what you pay for. At the beginning my bill used -to be like US$0.03 or even nil. - -That's the pricing model of Amazon S3 in a nutshell. Check -Amazon S3 homepage at http://aws.amazon.com/s3 for more -details. - -Needless to say that all these money are charged by Amazon -itself, there is obviously no payment for using S3cmd :-) - -Amazon S3 basics ----------------- -Files stored in S3 are called "objects" and their names are -officially called "keys". Since this is sometimes confusing -for the users we often refer to the objects as "files" or -"remote files". Each object belongs to exactly one "bucket". - -To describe objects in S3 storage we invented a URI-like -schema in the following form: - - s3://BUCKET -or - s3://BUCKET/OBJECT - -Buckets -------- -Buckets are sort of like directories or folders with some -restrictions: -1) each user can only have 100 buckets at the most, -2) bucket names must be unique amongst all users of S3, -3) buckets can not be nested into a deeper hierarchy and -4) a name of a bucket can only consist of basic alphanumeric - characters plus dot (.) and dash (-). No spaces, no accented - or UTF-8 letters, etc. - -It is a good idea to use DNS-compatible bucket names. That -for instance means you should not use upper case characters. -While DNS compliance is not strictly required some features -described below are not available for DNS-incompatible named -buckets. One more step further is using a fully qualified -domain name (FQDN) for a bucket - that has even more benefits. - -* For example "s3://--My-Bucket--" is not DNS compatible. -* On the other hand "s3://my-bucket" is DNS compatible but - is not FQDN. -* Finally "s3://my-bucket.s3tools.org" is DNS compatible - and FQDN provided you own the s3tools.org domain and can - create the domain record for "my-bucket.s3tools.org". - -Look for "Virtual Hosts" later in this text for more details -regarding FQDN named buckets. - -Objects (files stored in Amazon S3) ------------------------------------ -Unlike for buckets there are almost no restrictions on object -names. These can be any UTF-8 strings of up to 1024 bytes long. -Interestingly enough the object name can contain forward -slash character (/) thus a "my/funny/picture.jpg" is a valid -object name. Note that there are not directories nor -buckets called "my" and "funny" - it is really a single object -name called "my/funny/picture.jpg" and S3 does not care at -all that it _looks_ like a directory structure. - -The full URI of such an image could be, for example: - - s3://my-bucket/my/funny/picture.jpg - -Public vs Private files ------------------------ -The files stored in S3 can be either Private or Public. The -Private ones are readable only by the user who uploaded them -while the Public ones can be read by anyone. Additionally the -Public files can be accessed using HTTP protocol, not only -using s3cmd or a similar tool. - -The ACL (Access Control List) of a file can be set at the -time of upload using --acl-public or --acl-private options -with 's3cmd put' or 's3cmd sync' commands (see below). - -Alternatively the ACL can be altered for existing remote files -with 's3cmd setacl --acl-public' (or --acl-private) command. - -Simple s3cmd HowTo ------------------- -1) Register for Amazon AWS / S3 - Go to http://aws.amazon.com/s3, click the "Sign up - for web service" button in the right column and work - through the registration. You will have to supply - your Credit Card details in order to allow Amazon - charge you for S3 usage. - At the end you should have your Access and Secret Keys - -2) Run "s3cmd --configure" - You will be asked for the two keys - copy and paste - them from your confirmation email or from your Amazon - account page. Be careful when copying them! They are - case sensitive and must be entered accurately or you'll - keep getting errors about invalid signatures or similar. - - Remember to add ListAllMyBuckets permissions to the keys - or you will get an AccessDenied error while testing access. - -3) Run "s3cmd ls" to list all your buckets. - As you just started using S3 there are no buckets owned by - you as of now. So the output will be empty. - -4) Make a bucket with "s3cmd mb s3://my-new-bucket-name" - As mentioned above the bucket names must be unique amongst - _all_ users of S3. That means the simple names like "test" - or "asdf" are already taken and you must make up something - more original. To demonstrate as many features as possible - let's create a FQDN-named bucket s3://public.s3tools.org: - - ~$ s3cmd mb s3://public.s3tools.org - Bucket 's3://public.s3tools.org' created - -5) List your buckets again with "s3cmd ls" - Now you should see your freshly created bucket - - ~$ s3cmd ls - 2009-01-28 12:34 s3://public.s3tools.org - -6) List the contents of the bucket - - ~$ s3cmd ls s3://public.s3tools.org - ~$ - - It's empty, indeed. - -7) Upload a single file into the bucket: - - ~$ s3cmd put some-file.xml s3://public.s3tools.org/somefile.xml - some-file.xml -> s3://public.s3tools.org/somefile.xml [1 of 1] - 123456 of 123456 100% in 2s 51.75 kB/s done - - Upload a two directory tree into the bucket's virtual 'directory': - - ~$ s3cmd put --recursive dir1 dir2 s3://public.s3tools.org/somewhere/ - File 'dir1/file1-1.txt' stored as 's3://public.s3tools.org/somewhere/dir1/file1-1.txt' [1 of 5] - File 'dir1/file1-2.txt' stored as 's3://public.s3tools.org/somewhere/dir1/file1-2.txt' [2 of 5] - File 'dir1/file1-3.log' stored as 's3://public.s3tools.org/somewhere/dir1/file1-3.log' [3 of 5] - File 'dir2/file2-1.bin' stored as 's3://public.s3tools.org/somewhere/dir2/file2-1.bin' [4 of 5] - File 'dir2/file2-2.txt' stored as 's3://public.s3tools.org/somewhere/dir2/file2-2.txt' [5 of 5] - - As you can see we didn't have to create the /somewhere - 'directory'. In fact it's only a filename prefix, not - a real directory and it doesn't have to be created in - any way beforehand. - -8) Now list the bucket contents again: - - ~$ s3cmd ls s3://public.s3tools.org - DIR s3://public.s3tools.org/somewhere/ - 2009-02-10 05:10 123456 s3://public.s3tools.org/somefile.xml - - Use --recursive (or -r) to list all the remote files: - - ~$ s3cmd ls --recursive s3://public.s3tools.org - 2009-02-10 05:10 123456 s3://public.s3tools.org/somefile.xml - 2009-02-10 05:13 18 s3://public.s3tools.org/somewhere/dir1/file1-1.txt - 2009-02-10 05:13 8 s3://public.s3tools.org/somewhere/dir1/file1-2.txt - 2009-02-10 05:13 16 s3://public.s3tools.org/somewhere/dir1/file1-3.log - 2009-02-10 05:13 11 s3://public.s3tools.org/somewhere/dir2/file2-1.bin - 2009-02-10 05:13 8 s3://public.s3tools.org/somewhere/dir2/file2-2.txt - -9) Retrieve one of the files back and verify that it hasn't been - corrupted: - - ~$ s3cmd get s3://public.s3tools.org/somefile.xml some-file-2.xml - s3://public.s3tools.org/somefile.xml -> some-file-2.xml [1 of 1] - 123456 of 123456 100% in 3s 35.75 kB/s done - - ~$ md5sum some-file.xml some-file-2.xml - 39bcb6992e461b269b95b3bda303addf some-file.xml - 39bcb6992e461b269b95b3bda303addf some-file-2.xml - - Checksums of the original file matches the one of the - retrieved one. Looks like it worked :-) - - To retrieve a whole 'directory tree' from S3 use recursive get: - - ~$ s3cmd get --recursive s3://public.s3tools.org/somewhere - File s3://public.s3tools.org/somewhere/dir1/file1-1.txt saved as './somewhere/dir1/file1-1.txt' - File s3://public.s3tools.org/somewhere/dir1/file1-2.txt saved as './somewhere/dir1/file1-2.txt' - File s3://public.s3tools.org/somewhere/dir1/file1-3.log saved as './somewhere/dir1/file1-3.log' - File s3://public.s3tools.org/somewhere/dir2/file2-1.bin saved as './somewhere/dir2/file2-1.bin' - File s3://public.s3tools.org/somewhere/dir2/file2-2.txt saved as './somewhere/dir2/file2-2.txt' - - Since the destination directory wasn't specified s3cmd - saved the directory structure in a current working - directory ('.'). - - There is an important difference between: - get s3://public.s3tools.org/somewhere - and - get s3://public.s3tools.org/somewhere/ - (note the trailing slash) - S3cmd always uses the last path part, ie the word - after the last slash, for naming files. - - In the case of s3://.../somewhere the last path part - is 'somewhere' and therefore the recursive get names - the local files as somewhere/dir1, somewhere/dir2, etc. - - On the other hand in s3://.../somewhere/ the last path - part is empty and s3cmd will only create 'dir1' and 'dir2' - without the 'somewhere/' prefix: - - ~$ s3cmd get --recursive s3://public.s3tools.org/somewhere /tmp - File s3://public.s3tools.org/somewhere/dir1/file1-1.txt saved as '/tmp/dir1/file1-1.txt' - File s3://public.s3tools.org/somewhere/dir1/file1-2.txt saved as '/tmp/dir1/file1-2.txt' - File s3://public.s3tools.org/somewhere/dir1/file1-3.log saved as '/tmp/dir1/file1-3.log' - File s3://public.s3tools.org/somewhere/dir2/file2-1.bin saved as '/tmp/dir2/file2-1.bin' - - See? It's /tmp/dir1 and not /tmp/somewhere/dir1 as it - was in the previous example. - -10) Clean up - delete the remote files and remove the bucket: - - Remove everything under s3://public.s3tools.org/somewhere/ - - ~$ s3cmd del --recursive s3://public.s3tools.org/somewhere/ - File s3://public.s3tools.org/somewhere/dir1/file1-1.txt deleted - File s3://public.s3tools.org/somewhere/dir1/file1-2.txt deleted - ... - - Now try to remove the bucket: - - ~$ s3cmd rb s3://public.s3tools.org - ERROR: S3 error: 409 (BucketNotEmpty): The bucket you tried to delete is not empty - - Ouch, we forgot about s3://public.s3tools.org/somefile.xml - We can force the bucket removal anyway: - - ~$ s3cmd rb --force s3://public.s3tools.org/ - WARNING: Bucket is not empty. Removing all the objects from it first. This may take some time... - File s3://public.s3tools.org/somefile.xml deleted - Bucket 's3://public.s3tools.org/' removed - -Hints ------ -The basic usage is as simple as described in the previous -section. - -You can increase the level of verbosity with -v option and -if you're really keen to know what the program does under -its bonet run it with -d to see all 'debugging' output. - -After configuring it with --configure all available options -are spitted into your ~/.s3cfg file. It's a text file ready -to be modified in your favourite text editor. - -For more information refer to: -* S3cmd / S3tools homepage at http://s3tools.org - -=========================================================================== -Copyright (C) 2014 TGRMN Software - http://www.tgrmn.com - and contributors - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..e1d0704 --- /dev/null +++ b/README.md @@ -0,0 +1,328 @@ +## S3cmd tool for Amazon Simple Storage Service (S3) + + +* Author: Michal Ludvig, michal@logix.cz +* [Project homepage](http://s3tools.org) +* (c) [TGRMN Software](http://www.tgrmn.com) and contributors + + +S3tools / S3cmd mailing lists: + +* Announcements of new releases: s3tools-announce@lists.sourceforge.net +* General questions and discussion: s3tools-general@lists.sourceforge.net +* Bug reports: s3tools-bugs@lists.sourceforge.net + +### What is S3cmd + +S3cmd (`s3cmd`) is a free command line tool and client for uploading, retrieving and managing data in Amazon S3 and other cloud storage service providers that use the S3 protocol, such as Google Cloud Storage or DreamHost DreamObjects. It is best suited for power users who are familiar with command line programs. It is also ideal for batch scripts and automated backup to S3, triggered from cron, etc. + +S3cmd is written in Python. It's an open source project available under GNU Public License v2 (GPLv2) and is free for both commercial and private use. You will only have to pay Amazon for using their storage. + +Lots of features and options have been added to S3cmd, since its very first release in 2008.... we recently counted more than 60 command line options, including multipart uploads, encryption, incremental backup, s3 sync, ACL and Metadata management, S3 bucket size, bucket policies, and more! + +### What is Amazon S3 + +Amazon S3 provides a managed internet-accessible storage service where anyone can store any amount of data and retrieve it later again. + +S3 is a paid service operated by Amazon. Before storing anything into S3 you must sign up for an "AWS" account (where AWS = Amazon Web Services) to obtain a pair of identifiers: Access Key and Secret Key. You will need to +give these keys to S3cmd. Think of them as if they were a username and password for your S3 account. + +### Amazon S3 pricing explained + +At the time of this writing the costs of using S3 are (in USD): + +$0.03 per GB per month of storage space used + +plus + +$0.00 per GB - all data uploaded + +plus + +$0.000 per GB - first 1GB / month data downloaded +$0.090 per GB - up to 10 TB / month data downloaded +$0.085 per GB - next 40 TB / month data downloaded +$0.070 per GB - data downloaded / month over 50 TB + +plus + +$0.005 per 1,000 PUT or COPY or LIST requests +$0.004 per 10,000 GET and all other requests + +If for instance on 1st of January you upload 2GB of photos in JPEG from your holiday in New Zealand, at the end of January you will be charged $0.06 for using 2GB of storage space for a month, $0.0 for uploading 2GB of data, and a few cents for requests. That comes to slightly over $0.06 for a complete backup of your precious holiday pictures. + +In February you don't touch it. Your data are still on S3 servers so you pay $0.06 for those two gigabytes, but not a single cent will be charged for any transfer. That comes to $0.06 as an ongoing cost of your backup. Not too bad. + +In March you allow anonymous read access to some of your pictures and your friends download, say, 1500MB of them. As the files are owned by you, you are responsible for the costs incurred. That means at the end of March you'll be charged $0.06 for storage plus $0.045 for the download traffic generated by your friends. + +There is no minimum monthly contract or a setup fee. What you use is what you pay for. At the beginning my bill used to be like US$0.03 or even nil. + +That's the pricing model of Amazon S3 in a nutshell. Check the [Amazon S3 homepage](http://aws.amazon.com/s3/pricing/) for more details. + +Needless to say that all these money are charged by Amazon itself, there is obviously no payment for using S3cmd :-) + +### Amazon S3 basics + +Files stored in S3 are called "objects" and their names are officially called "keys". Since this is sometimes confusing for the users we often refer to the objects as "files" or "remote files". Each object belongs to exactly one "bucket". + +To describe objects in S3 storage we invented a URI-like schema in the following form: + +``` +s3://BUCKET +``` +or + +``` +s3://BUCKET/OBJECT +``` + +### Buckets + +Buckets are sort of like directories or folders with some restrictions: + +1. each user can only have 100 buckets at the most, +2. bucket names must be unique amongst all users of S3, +3. buckets can not be nested into a deeper hierarchy and +4. a name of a bucket can only consist of basic alphanumeric + characters plus dot (.) and dash (-). No spaces, no accented + or UTF-8 letters, etc. + +It is a good idea to use DNS-compatible bucket names. That for instance means you should not use upper case characters. While DNS compliance is not strictly required some features described below are not available for DNS-incompatible named buckets. One more step further is using a fully qualified domain name (FQDN) for a bucket - that has even more benefits. + +* For example "s3://--My-Bucket--" is not DNS compatible. +* On the other hand "s3://my-bucket" is DNS compatible but + is not FQDN. +* Finally "s3://my-bucket.s3tools.org" is DNS compatible + and FQDN provided you own the s3tools.org domain and can + create the domain record for "my-bucket.s3tools.org". + +Look for "Virtual Hosts" later in this text for more details regarding FQDN named buckets. + +### Objects (files stored in Amazon S3) + +Unlike for buckets there are almost no restrictions on object names. These can be any UTF-8 strings of up to 1024 bytes long. Interestingly enough the object name can contain forward slash character (/) thus a `my/funny/picture.jpg` is a valid object name. Note that there are not directories nor buckets called `my` and `funny` - it is really a single object name called `my/funny/picture.jpg` and S3 does not care at all that it _looks_ like a directory structure. + +The full URI of such an image could be, for example: + +``` +s3://my-bucket/my/funny/picture.jpg +``` + +### Public vs Private files + +The files stored in S3 can be either Private or Public. The Private ones are readable only by the user who uploaded them while the Public ones can be read by anyone. Additionally the Public files can be accessed using HTTP protocol, not only using `s3cmd` or a similar tool. + +The ACL (Access Control List) of a file can be set at the time of upload using `--acl-public` or `--acl-private` options with `s3cmd put` or `s3cmd sync` commands (see below). + +Alternatively the ACL can be altered for existing remote files with `s3cmd setacl --acl-public` (or `--acl-private`) command. + +### Simple s3cmd HowTo + +1) Register for Amazon AWS / S3 + +Go to http://aws.amazon.com/s3, click the "Sign up for web service" button in the right column and work through the registration. You will have to supply your Credit Card details in order to allow Amazon charge you for S3 usage. At the end you should have your Access and Secret Keys. + +2) Run `s3cmd --configure` + +You will be asked for the two keys - copy and paste them from your confirmation email or from your Amazon account page. Be careful when copying them! They are case sensitive and must be entered accurately or you'll keep getting errors about invalid signatures or similar. + +Remember to add ListAllMyBuckets permissions to the keys or you will get an AccessDenied error while testing access. + +3) Run `s3cmd ls` to list all your buckets. + +As you just started using S3 there are no buckets owned by you as of now. So the output will be empty. + +4) Make a bucket with `s3cmd mb s3://my-new-bucket-name` + +As mentioned above the bucket names must be unique amongst _all_ users of S3. That means the simple names like "test" or "asdf" are already taken and you must make up something more original. To demonstrate as many features as possible let's create a FQDN-named bucket `s3://public.s3tools.org`: + +``` +$ s3cmd mb s3://public.s3tools.org + +Bucket 's3://public.s3tools.org' created +``` + +5) List your buckets again with `s3cmd ls` + +Now you should see your freshly created bucket: + +``` +$ s3cmd ls + +2009-01-28 12:34 s3://public.s3tools.org +``` + +6) List the contents of the bucket: + +``` +$ s3cmd ls s3://public.s3tools.org +$ +``` + +It's empty, indeed. + +7) Upload a single file into the bucket: + +``` +$ s3cmd put some-file.xml s3://public.s3tools.org/somefile.xml + +some-file.xml -> s3://public.s3tools.org/somefile.xml [1 of 1] + 123456 of 123456 100% in 2s 51.75 kB/s done +``` + +Upload a two-directory tree into the bucket's virtual 'directory': + +``` +$ s3cmd put --recursive dir1 dir2 s3://public.s3tools.org/somewhere/ + +File 'dir1/file1-1.txt' stored as 's3://public.s3tools.org/somewhere/dir1/file1-1.txt' [1 of 5] +File 'dir1/file1-2.txt' stored as 's3://public.s3tools.org/somewhere/dir1/file1-2.txt' [2 of 5] +File 'dir1/file1-3.log' stored as 's3://public.s3tools.org/somewhere/dir1/file1-3.log' [3 of 5] +File 'dir2/file2-1.bin' stored as 's3://public.s3tools.org/somewhere/dir2/file2-1.bin' [4 of 5] +File 'dir2/file2-2.txt' stored as 's3://public.s3tools.org/somewhere/dir2/file2-2.txt' [5 of 5] +``` + +As you can see we didn't have to create the `/somewhere` 'directory'. In fact it's only a filename prefix, not a real directory and it doesn't have to be created in any way beforehand. + +8) Now list the bucket's contents again: + +``` +$ s3cmd ls s3://public.s3tools.org + + DIR s3://public.s3tools.org/somewhere/ +2009-02-10 05:10 123456 s3://public.s3tools.org/somefile.xml +``` + +Use --recursive (or -r) to list all the remote files: + +``` +$ s3cmd ls --recursive s3://public.s3tools.org + +2009-02-10 05:10 123456 s3://public.s3tools.org/somefile.xml +2009-02-10 05:13 18 s3://public.s3tools.org/somewhere/dir1/file1-1.txt +2009-02-10 05:13 8 s3://public.s3tools.org/somewhere/dir1/file1-2.txt +2009-02-10 05:13 16 s3://public.s3tools.org/somewhere/dir1/file1-3.log +2009-02-10 05:13 11 s3://public.s3tools.org/somewhere/dir2/file2-1.bin +2009-02-10 05:13 8 s3://public.s3tools.org/somewhere/dir2/file2-2.txt +``` + +9) Retrieve one of the files back and verify that it hasn't been + corrupted: + +``` +$ s3cmd get s3://public.s3tools.org/somefile.xml some-file-2.xml + +s3://public.s3tools.org/somefile.xml -> some-file-2.xml [1 of 1] + 123456 of 123456 100% in 3s 35.75 kB/s done +``` + +``` +$ md5sum some-file.xml some-file-2.xml + +39bcb6992e461b269b95b3bda303addf some-file.xml +39bcb6992e461b269b95b3bda303addf some-file-2.xml +``` + +Checksums of the original file matches the one of the retrieved ones. Looks like it worked :-) + +To retrieve a whole 'directory tree' from S3 use recursive get: + +``` +$ s3cmd get --recursive s3://public.s3tools.org/somewhere + +File s3://public.s3tools.org/somewhere/dir1/file1-1.txt saved as './somewhere/dir1/file1-1.txt' +File s3://public.s3tools.org/somewhere/dir1/file1-2.txt saved as './somewhere/dir1/file1-2.txt' +File s3://public.s3tools.org/somewhere/dir1/file1-3.log saved as './somewhere/dir1/file1-3.log' +File s3://public.s3tools.org/somewhere/dir2/file2-1.bin saved as './somewhere/dir2/file2-1.bin' +File s3://public.s3tools.org/somewhere/dir2/file2-2.txt saved as './somewhere/dir2/file2-2.txt' +``` + +Since the destination directory wasn't specified, `s3cmd` saved the directory structure in a current working directory ('.'). + +There is an important difference between: + +``` +get s3://public.s3tools.org/somewhere +``` + +and + +``` +get s3://public.s3tools.org/somewhere/ +``` + +(note the trailing slash) + +`s3cmd` always uses the last path part, ie the word after the last slash, for naming files. + +In the case of `s3://.../somewhere` the last path part is 'somewhere' and therefore the recursive get names the local files as somewhere/dir1, somewhere/dir2, etc. + +On the other hand in `s3://.../somewhere/` the last path +part is empty and s3cmd will only create 'dir1' and 'dir2' +without the 'somewhere/' prefix: + +``` +$ s3cmd get --recursive s3://public.s3tools.org/somewhere /tmp + +File s3://public.s3tools.org/somewhere/dir1/file1-1.txt saved as '/tmp/dir1/file1-1.txt' +File s3://public.s3tools.org/somewhere/dir1/file1-2.txt saved as '/tmp/dir1/file1-2.txt' +File s3://public.s3tools.org/somewhere/dir1/file1-3.log saved as '/tmp/dir1/file1-3.log' +File s3://public.s3tools.org/somewhere/dir2/file2-1.bin saved as '/tmp/dir2/file2-1.bin' +``` + +See? It's `/tmp/dir1` and not `/tmp/somewhere/dir1` as it was in the previous example. + +10) Clean up - delete the remote files and remove the bucket: + +Remove everything under s3://public.s3tools.org/somewhere/ + +``` +$ s3cmd del --recursive s3://public.s3tools.org/somewhere/ + +File s3://public.s3tools.org/somewhere/dir1/file1-1.txt deleted +File s3://public.s3tools.org/somewhere/dir1/file1-2.txt deleted +... +``` + +Now try to remove the bucket: + +``` +$ s3cmd rb s3://public.s3tools.org + +ERROR: S3 error: 409 (BucketNotEmpty): The bucket you tried to delete is not empty +``` + +Ouch, we forgot about `s3://public.s3tools.org/somefile.xml`. We can force the bucket removal anyway: + +``` +$ s3cmd rb --force s3://public.s3tools.org/ + +WARNING: Bucket is not empty. Removing all the objects from it first. This may take some time... +File s3://public.s3tools.org/somefile.xml deleted +Bucket 's3://public.s3tools.org/' removed +``` + +### Hints + +The basic usage is as simple as described in the previous section. + +You can increase the level of verbosity with `-v` option and if you're really keen to know what the program does under its bonnet run it with `-d` to see all 'debugging' output. + +After configuring it with `--configure` all available options are spitted into your `~/.s3cfg` file. It's a text file ready to be modified in your favourite text editor. + +For more information refer to the [S3cmd / S3tools homepage](http://s3tools.org). + +### License + +Copyright (C) 2014 TGRMN Software - http://www.tgrmn.com - and contributors + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + diff --git a/S3/ACL.py b/S3/ACL.py index 71a29ae..2cf5372 100644 --- a/S3/ACL.py +++ b/S3/ACL.py @@ -159,11 +159,11 @@ grantee.name = name grantee.permission = permission - if name.find('@') > -1: + if '@' in name: grantee.name = grantee.name.lower() grantee.xsi_type = "AmazonCustomerByEmail" grantee.tag = "EmailAddress" - elif name.find('http://acs.amazonaws.com/groups/') > -1: + elif 'http://acs.amazonaws.com/groups/' in name: grantee.xsi_type = "Group" grantee.tag = "URI" else: diff --git a/S3/CloudFront.py b/S3/CloudFront.py index eb81ea9..2d9c362 100644 --- a/S3/CloudFront.py +++ b/S3/CloudFront.py @@ -19,9 +19,11 @@ from S3 import S3 from Config import Config from Exceptions import * -from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string, getBucketFromHostname, getHostnameFromBucket +from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, getBucketFromHostname, getHostnameFromBucket +from Crypto import sign_string_v2 from S3Uri import S3Uri, S3UriS3 from FileLists import fetch_remote_list +from ConnMan import ConnMan cloudfront_api_version = "2010-11-01" cloudfront_resource = "/%(api_ver)s/distribution" % { 'api_ver' : cloudfront_api_version } @@ -495,14 +497,14 @@ request = self.create_request(operation, dist_id, request_id, headers) conn = self.get_connection() debug("send_request(): %s %s" % (request['method'], request['resource'])) - conn.request(request['method'], request['resource'], body, request['headers']) - http_response = conn.getresponse() + conn.c.request(request['method'], request['resource'], body, request['headers']) + http_response = conn.c.getresponse() response = {} response["status"] = http_response.status response["reason"] = http_response.reason response["headers"] = dict(http_response.getheaders()) response["data"] = http_response.read() - conn.close() + ConnMan.put(conn) debug("CloudFront: response: %r" % response) @@ -553,14 +555,13 @@ def sign_request(self, headers): string_to_sign = headers['x-amz-date'] - signature = sign_string(string_to_sign) + signature = sign_string_v2(string_to_sign) debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature)) return signature def get_connection(self): - if self.config.proxy_host != "": - raise ParameterError("CloudFront commands don't work from behind a HTTP proxy") - return httplib.HTTPSConnection(self.config.cloudfront_host) + conn = ConnMan.get(self.config.cloudfront_host, ssl = True) + return conn def _fail_wait(self, retries): # Wait a few seconds. The more it fails the more we wait. diff --git a/S3/Config.py b/S3/Config.py index 7a55589..aa14a7d 100644 --- a/S3/Config.py +++ b/S3/Config.py @@ -12,6 +12,7 @@ import Progress from SortedDict import SortedDict import httplib +import locale try: import json except ImportError, e: @@ -77,6 +78,8 @@ gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s" use_https = False + ca_certs_file = "" + check_ssl_certificate = True bucket_location = "US" default_mime_type = "binary/octet-stream" guess_mime_type = True @@ -92,7 +95,7 @@ # Dict mapping compiled REGEXPs back to their textual form debug_exclude = {} debug_include = {} - encoding = "utf-8" + encoding = locale.getpreferredencoding() or "UTF-8" urlencoding_mode = "normal" log_target_prefix = "" reduced_redundancy = False @@ -109,10 +112,12 @@ files_from = [] cache_file = "" add_headers = "" + remove_headers = [] ignore_failed_copy = False expiry_days = "" expiry_date = "" expiry_prefix = "" + signature_v2 = False ## Creating a singleton def __new__(self, configfile = None, access_key=None, secret_key=None): @@ -223,7 +228,10 @@ def read_config_file(self, configfile): cp = ConfigParser(configfile) for option in self.option_list(): - self.update_option(option, cp.get(option)) + _option = cp.get(option) + if _option is not None: + _option = _option.strip() + self.update_option(option, _option) if cp.get('add_headers'): for option in cp.get('add_headers').split(","): diff --git a/S3/ConnMan.py b/S3/ConnMan.py index 681b76a..60e6cd1 100644 --- a/S3/ConnMan.py +++ b/S3/ConnMan.py @@ -4,7 +4,9 @@ ## License: GPL Version 2 ## Copyright: TGRMN Software and contributors +import sys import httplib +import ssl from urlparse import urlparse from threading import Semaphore from logging import debug, info, warning, error @@ -12,20 +14,121 @@ from Config import Config from Exceptions import ParameterError +if not 'CertificateError ' in ssl.__dict__: + class CertificateError(Exception): + pass + ssl.CertificateError = CertificateError + __all__ = [ "ConnMan" ] class http_connection(object): + context = None + context_set = False + + @staticmethod + def _ssl_verified_context(cafile): + context = None + try: + context = ssl.create_default_context(cafile=cafile) + except AttributeError: # no ssl.create_default_context + pass + return context + + @staticmethod + def _ssl_context(): + if http_connection.context_set: + return http_connection.context + + cfg = Config() + cafile = cfg.ca_certs_file + if cafile == "": + cafile = None + debug(u"Using ca_certs_file %s" % cafile) + + context = http_connection._ssl_verified_context(cafile) + + if context and not cfg.check_ssl_certificate: + context.check_hostname = False + debug(u'Disabling hostname checking') + + http_connection.context = context + http_connection.context_set = True + return context + + def match_hostname_aws(self, cert, e): + """ + Wildcard matching for *.s3.amazonaws.com and similar per region. + + Per http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html: + "We recommend that all bucket names comply with DNS naming conventions." + + Per http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html: + "When using virtual hosted-style buckets with SSL, the SSL + wild card certificate only matches buckets that do not contain + periods. To work around this, use HTTP or write your own + certificate verification logic." + + Therefore, we need a custom validation routine that allows + mybucket.example.com.s3.amazonaws.com to be considered a valid + hostname for the *.s3.amazonaws.com wildcard cert, and for the + region-specific *.s3-[region].amazonaws.com wildcard cert. + """ + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if value.startswith('*.s3') and value.endswith('.amazonaws.com') and self.c.host.endswith('.amazonaws.com'): + return + raise e + + def match_hostname(self): + cert = self.c.sock.getpeercert() + try: + ssl.match_hostname(cert, self.c.host) + except AttributeError: # old ssl module doesn't have this function + return + except ValueError: # empty SSL cert means underlying SSL library didn't validate it, we don't either. + return + except ssl.CertificateError, e: + self.match_hostname_aws(cert, e) + + @staticmethod + def _https_connection(hostname, port=None): + try: + context = http_connection._ssl_context() + # S3's wildcart certificate doesn't work with DNS-style named buckets. + if hostname.endswith('.amazonaws.com') and context: + # this merely delays running the hostname check until + # after the connection is made and we get control + # back. We then run the same check, relaxed for S3's + # wildcard certificates. + context.check_hostname = False + conn = httplib.HTTPSConnection(hostname, port, context=context) + except TypeError: + conn = httplib.HTTPSConnection(hostname, port) + return conn + def __init__(self, id, hostname, ssl, cfg): - self.hostname = hostname self.ssl = ssl self.id = id self.counter = 0 - if cfg.proxy_host != "": - self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port) - elif not ssl: - self.c = httplib.HTTPConnection(hostname) + + if not ssl: + if cfg.proxy_host != "": + self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port) + debug(u'proxied HTTPConnection(%s, %s)' % (cfg.proxy_host, cfg.proxy_port)) + else: + self.c = httplib.HTTPConnection(hostname) + debug(u'non-proxied HTTPConnection(%s)' % hostname) else: - self.c = httplib.HTTPSConnection(hostname) + if cfg.proxy_host != "": + self.c = http_connection._https_connection(cfg.proxy_host, cfg.proxy_port) + self.c.set_tunnel(hostname) + debug(u'proxied HTTPSConnection(%s, %s)' % (cfg.proxy_host, cfg.proxy_port)) + debug(u'tunnel to %s' % hostname) + else: + self.c = http_connection._https_connection(hostname) + debug(u'non-proxied HTTPSConnection(%s)' % hostname) + class ConnMan(object): conn_pool_sem = Semaphore() @@ -39,8 +142,8 @@ ssl = cfg.use_https conn = None if cfg.proxy_host != "": - if ssl: - raise ParameterError("use_https=True can't be used with proxy") + if ssl and sys.hexversion < 0x02070000: + raise ParameterError("use_https=True can't be used with proxy on Python <2.7") conn_id = "proxy://%s:%s" % (cfg.proxy_host, cfg.proxy_port) else: conn_id = "http%s://%s" % (ssl and "s" or "", hostname) @@ -55,6 +158,8 @@ debug("ConnMan.get(): creating new connection: %s" % conn_id) conn = http_connection(conn_id, hostname, ssl, cfg) conn.c.connect() + if conn.ssl: + conn.match_hostname() conn.counter += 1 return conn diff --git a/S3/Crypto.py b/S3/Crypto.py new file mode 100644 index 0000000..ea08a1b --- /dev/null +++ b/S3/Crypto.py @@ -0,0 +1,169 @@ +## Amazon S3 manager +## Author: Michal Ludvig +## http://www.logix.cz/michal +## License: GPL Version 2 +## Copyright: TGRMN Software and contributors + +import sys +import hmac +import base64 + +import Config +from logging import debug +import Utils + +import os +import datetime +import urllib + +# hashlib backported to python 2.4 / 2.5 is not compatible with hmac! +if sys.version_info[0] == 2 and sys.version_info[1] < 6: + from md5 import md5 + import sha as sha1 + from Crypto.Hash import SHA256 as sha256 +else: + from hashlib import md5, sha1, sha256 + +__all__ = [] + +### AWS Version 2 signing +def sign_string_v2(string_to_sign): + """Sign a string with the secret key, returning base64 encoded results. + By default the configured secret key is used, but may be overridden as + an argument. + + Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html + """ + signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip() + return signature +__all__.append("sign_string_v2") + +def sign_url_v2(url_to_sign, expiry): + """Sign a URL in s3://bucket/object form with the given expiry + time. The object will be accessible via the signed URL until the + AWS key and secret are revoked or the expiry time is reached, even + if the object is otherwise private. + + See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html + """ + return sign_url_base_v2( + bucket = url_to_sign.bucket(), + object = url_to_sign.object(), + expiry = expiry + ) +__all__.append("sign_url_v2") + +def sign_url_base_v2(**parms): + """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args.""" + parms['expiry']=Utils.time_to_epoch(parms['expiry']) + parms['access_key']=Config.Config().access_key + parms['host_base']=Config.Config().host_base + debug("Expiry interpreted as epoch time %s", parms['expiry']) + signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms + debug("Signing plaintext: %r", signtext) + parms['sig'] = urllib.quote_plus(sign_string_v2(signtext)) + debug("Urlencoded signature: %s", parms['sig']) + return "http://%(bucket)s.%(host_base)s/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms + +def sign(key, msg): + return hmac.new(key, msg.encode('utf-8'), sha256).digest() + +def getSignatureKey(key, dateStamp, regionName, serviceName): + kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, 'aws4_request') + return kSigning + +def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='us-east-1', cur_headers={}, body=''): + service = 's3' + + cfg = Config.Config() + access_key = cfg.access_key + secret_key = cfg.secret_key + + t = datetime.datetime.utcnow() + amzdate = t.strftime('%Y%m%dT%H%M%SZ') + datestamp = t.strftime('%Y%m%d') + + canonical_querystring = '&'.join(['%s=%s' % (urllib.quote_plus(p), quote_param(params[p])) for p in sorted(params.keys())]) + + splits = canonical_uri.split('?') + + canonical_uri = quote_param(splits[0], quote_backslashes=False) + canonical_querystring += '&'.join([('%s' if '=' in qs else '%s=') % qs for qs in splits[1:]]) + + if type(body) == type(sha256('')): + payload_hash = body.hexdigest() + else: + payload_hash = sha256(body).hexdigest() + + canonical_headers = {'host' : host, + 'x-amz-content-sha256': payload_hash, + 'x-amz-date' : amzdate + } + signed_headers = 'host;x-amz-content-sha256;x-amz-date' + + for header in cur_headers.keys(): + # avoid duplicate headers and previous Authorization + if header == 'Authorization' or header in signed_headers.split(';'): + continue + canonical_headers[header.strip()] = str(cur_headers[header]).strip() + signed_headers += ';' + header.strip() + + # sort headers into a string + canonical_headers_str = '' + for k, v in sorted(canonical_headers.items()): + canonical_headers_str += k + ":" + v + "\n" + + canonical_headers = canonical_headers_str + debug(u"canonical_headers = %s" % canonical_headers) + signed_headers = ';'.join(sorted(signed_headers.split(';'))) + + canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash + debug('Canonical Request:\n%s\n----------------------' % canonical_request) + + algorithm = 'AWS4-HMAC-SHA256' + credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request' + string_to_sign = algorithm + '\n' + amzdate + '\n' + credential_scope + '\n' + sha256(canonical_request).hexdigest() + signing_key = getSignatureKey(secret_key, datestamp, region, service) + signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), sha256).hexdigest() + authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' + 'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature + headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items()) + debug("signature-v4 headers: %s" % headers) + return headers + +def quote_param(param, quote_backslashes=True): + # As stated by Amazon the '/' in the filename should stay unquoted and %20 should be used for space instead of '+' + quoted = urllib.quote_plus(urllib.unquote_plus(param), safe='~').replace('+', '%20') + if not quote_backslashes: + quoted = quoted.replace('%2F', '/') + return quoted + +def checksum_sha256_file(filename, offset=0, size=None): + try: + hash = sha256() + except: + # fallback to Crypto SHA256 module + hash = sha256.new() + with open(filename,'rb') as f: + if size is None: + for chunk in iter(lambda: f.read(8192), b''): + hash.update(chunk) + else: + f.seek(offset) + chunk = f.read(size) + hash.update(chunk) + return hash + +def checksum_sha256_buffer(buffer, offset=0, size=None): + try: + hash = sha256() + except: + # fallback to Crypto SHA256 module + hash = sha256.new() + if size is None: + hash.update(buffer) + else: + hash.update(buffer[offset:offset+size]) + return hash diff --git a/S3/Exceptions.py b/S3/Exceptions.py index 4486ce1..b2b2ee1 100644 --- a/S3/Exceptions.py +++ b/S3/Exceptions.py @@ -6,11 +6,19 @@ from Utils import getTreeFromXml, unicodise, deunicodise from logging import debug, info, warning, error +import ExitCodes try: import xml.etree.ElementTree as ET except ImportError: + # xml.etree.ElementTree was only added in python 2.5 import elementtree.ElementTree as ET + +try: + from xml.etree.ElementTree import ParseError as XmlParseError +except ImportError: + # ParseError was only added in python2.7, before ET was raising ExpatError + from xml.parsers.expat import ExpatError as XmlParseError class S3Exception(Exception): def __init__(self, message = ""): @@ -48,10 +56,13 @@ if response.has_key("data") and response["data"]: try: tree = getTreeFromXml(response["data"]) - except ET.ParseError: + except XmlParseError: debug("Not an XML response") else: - self.info.update(self.parse_error_xml(tree)) + try: + self.info.update(self.parse_error_xml(tree)) + except Exception, e: + error("Error parsing xml: %s. ErrorXML: %s" % (e, response["data"])) self.code = self.info["Code"] self.message = self.info["Message"] @@ -64,17 +75,39 @@ retval += (u": %s" % self.info["Message"]) return retval + def get_error_code(self): + if self.status in [301, 307]: + return ExitCodes.EX_SERVERMOVED + elif self.status in [400, 405, 411, 416, 501]: + return ExitCodes.EX_SERVERERROR + elif self.status == 403: + return ExitCodes.EX_ACCESSDENIED + elif self.status == 404: + return ExitCodes.EX_NOTFOUND + elif self.status == 409: + return ExitCodes.EX_CONFLICT + elif self.status == 412: + return ExitCodes.EX_PRECONDITION + elif self.status == 500: + return ExitCodes.EX_SOFTWARE + elif self.status == 503: + return ExitCodes.EX_SERVICE + else: + return ExitCodes.EX_SOFTWARE + @staticmethod def parse_error_xml(tree): info = {} error_node = tree if not error_node.tag == "Error": error_node = tree.find(".//Error") - for child in error_node.getchildren(): - if child.text != "": - debug("ErrorXML: " + child.tag + ": " + repr(child.text)) - info[child.tag] = child.text - + if error_node is not None: + for child in error_node.getchildren(): + if child.text != "": + debug("ErrorXML: " + child.tag + ": " + repr(child.text)) + info[child.tag] = child.text + else: + raise S3ResponseError("Malformed error XML returned from remote server.") return info diff --git a/S3/ExitCodes.py b/S3/ExitCodes.py index 7cfb108..e36f4e6 100644 --- a/S3/ExitCodes.py +++ b/S3/ExitCodes.py @@ -1,16 +1,22 @@ # patterned on /usr/include/sysexits.h -EX_OK = 0 -EX_GENERAL = 1 -EX_SOMEFAILED = 2 # some parts of the command succeeded, while others failed -EX_USAGE = 64 # The command was used incorrectly (e.g. bad command line syntax) -EX_SOFTWARE = 70 # internal software error (e.g. S3 error of unknown specificity) -EX_OSERR = 71 # system error (e.g. out of memory) -EX_OSFILE = 72 # OS error (e.g. invalid Python version) -EX_IOERR = 74 # An error occurred while doing I/O on some file. -EX_TEMPFAIL = 75 # temporary failure (S3DownloadError or similar, retry later) -EX_NOPERM = 77 # Insufficient permissions to perform the operation on S3 -EX_CONFIG = 78 # Configuration file error -_EX_SIGNAL = 128 -_EX_SIGINT = 2 -EX_BREAK = _EX_SIGNAL + _EX_SIGINT # Control-C (KeyboardInterrupt raised) +EX_OK = 0 +EX_GENERAL = 1 +EX_PARTIAL = 2 # some parts of the command succeeded, while others failed +EX_SERVERMOVED = 10 # 301: Moved permanantly & 307: Moved temp +EX_SERVERERROR = 11 # 400, 405, 411, 416, 501: Bad request +EX_NOTFOUND = 12 # 404: Not found +EX_CONFLICT = 13 # 409: Conflict (ex: bucket error) +EX_PRECONDITION = 14 # 412: Precondition failed +EX_SERVICE = 15 # 503: Service not available or slow down +EX_USAGE = 64 # The command was used incorrectly (e.g. bad command line syntax) +EX_SOFTWARE = 70 # internal software error (e.g. S3 error of unknown specificity) +EX_OSERR = 71 # system error (e.g. out of memory) +EX_OSFILE = 72 # OS error (e.g. invalid Python version) +EX_IOERR = 74 # An error occurred while doing I/O on some file. +EX_TEMPFAIL = 75 # temporary failure (S3DownloadError or similar, retry later) +EX_ACCESSDENIED = 77 # Insufficient permissions to perform the operation on S3 +EX_CONFIG = 78 # Configuration file error +_EX_SIGNAL = 128 +_EX_SIGINT = 2 +EX_BREAK = _EX_SIGNAL + _EX_SIGINT # Control-C (KeyboardInterrupt raised) diff --git a/S3/FileLists.py b/S3/FileLists.py index f8602e9..fa0ec9a 100644 --- a/S3/FileLists.py +++ b/S3/FileLists.py @@ -98,7 +98,8 @@ debug(u"CHECK: %r" % d) excluded = False for r in cfg.exclude: - if not r.pattern.endswith(u'/'): continue # we only check for directories here + # python versions end their patterns (from globs) differently, test for both styles. + if not (r.pattern.endswith(u'\\/$') or r.pattern.endswith(u'\\/\\Z(?ms)')): continue # we only check for directory patterns here if r.search(d): excluded = True debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r])) @@ -106,7 +107,8 @@ if excluded: ## No need to check for --include if not excluded for r in cfg.include: - if not r.pattern.endswith(u'/'): continue # we only check for directories here + # python versions end their patterns (from globs) differently, test for both styles. + if not (r.pattern.endswith(u'\\/$') or r.pattern.endswith(u'\\/\\Z(?ms)')): continue # we only check for directory patterns here debug(u"INCL-TEST: %s ~ %s" % (d, r.pattern)) if r.search(d): excluded = False @@ -381,14 +383,14 @@ rem_list[key] = { 'size' : int(object['Size']), 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( - 'md5' : object['ETag'][1:-1], + 'md5' : object['ETag'].strip('"\''), 'object_key' : object['Key'], 'object_uri_str' : object_uri_str, 'base_uri' : remote_uri, 'dev' : None, 'inode' : None, } - if rem_list[key]['md5'].find("-") > 0: # always get it for multipart uploads + if '-' in rem_list[key]['md5']: # always get it for multipart uploads _get_remote_attribs(S3Uri(object_uri_str), rem_list[key]) md5 = rem_list[key]['md5'] rem_list.record_md5(key, md5) @@ -468,15 +470,17 @@ return False ## check size first - if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: - debug(u"xfer: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) - attribs_match = False + if 'size' in cfg.sync_checks: + if 'size' in dst_list[file] and 'size' in src_list[file]: + if dst_list[file]['size'] != src_list[file]['size']: + debug(u"xfer: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) + attribs_match = False ## check md5 compare_md5 = 'md5' in cfg.sync_checks # Multipart-uploaded files don't have a valid md5 sum - it ends with "...-nn" if compare_md5: - if (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0): + if (src_remote == True and '-' in src_list[file]['md5']) or (dst_remote == True and '-' in dst_list[file]['md5']): compare_md5 = False info(u"disabled md5 check for %s" % file) if attribs_match and compare_md5: diff --git a/S3/MultiPart.py b/S3/MultiPart.py index 21f0cae..a0ad442 100644 --- a/S3/MultiPart.py +++ b/S3/MultiPart.py @@ -117,7 +117,7 @@ else: while True: buffer = self.file.read(self.chunk_size) - offset = self.chunk_size * (seq - 1) + offset = 0 # send from start of the buffer current_chunk_size = len(buffer) labels = { 'source' : unicodise(self.file.name), @@ -130,7 +130,7 @@ self.upload_part(seq, offset, current_chunk_size, labels, buffer, remote_status = remote_statuses.get(seq)) except: error(u"\nUpload of '%s' part %d failed. Use\n %s abortmp %s %s\nto abort, or\n %s --upload-id %s put ...\nto continue the upload." - % (self.file.name, seq, self.uri, sys.argv[0], self.upload_id, sys.argv[0], self.upload_id)) + % (self.file.name, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id)) raise seq += 1 @@ -147,7 +147,7 @@ if remote_status is not None: if int(remote_status['size']) == chunk_size: checksum = calculateChecksum(buffer, self.file, offset, chunk_size, self.s3.config.send_chunk) - remote_checksum = remote_status['checksum'].strip('"') + remote_checksum = remote_status['checksum'].strip('"\'') if remote_checksum == checksum: warning("MultiPart: size and md5sum match for %s part %d, skipping." % (self.uri, seq)) self.parts[seq] = remote_status['checksum'] @@ -180,8 +180,8 @@ body = "%s" % ("".join(parts_xml)) headers = { "content-length": len(body) } - request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % (self.upload_id)) - response = self.s3.send_request(request, body = body) + request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % (self.upload_id), body = body) + response = self.s3.send_request(request) return response diff --git a/S3/PkgInfo.py b/S3/PkgInfo.py index 0c8970e..4e7d2e1 100644 --- a/S3/PkgInfo.py +++ b/S3/PkgInfo.py @@ -5,9 +5,9 @@ ## Copyright: TGRMN Software and contributors package = "s3cmd" -version = "1.5.0-rc1" +version = "1.5.2" url = "http://s3tools.org" -license = "GPL version 2" +license = "GNU GPL v2+" short_description = "Command line tool for managing Amazon S3 and CloudFront services" long_description = """ S3cmd lets you copy files from/to Amazon S3 diff --git a/S3/S3.py b/S3/S3.py index 1589f7b..614f391 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -33,6 +33,8 @@ from MultiPart import MultiPartUpload from S3Uri import S3Uri from ConnMan import ConnMan +from Crypto import sign_string_v2, sign_string_v4, checksum_sha256_file, checksum_sha256_buffer +from ExitCodes import * try: import magic @@ -61,7 +63,7 @@ return magic_.file(file) except ImportError, e: - if str(e).find("magic") >= 0: + if 'magic' in str(e): magic_message = "Module python-magic is not available." else: magic_message = "Module python-magic can't be used (%s)." % e.message @@ -101,21 +103,18 @@ __all__ = [] class S3Request(object): - def __init__(self, s3, method_string, resource, headers, params = {}): + region_map = {} + + def __init__(self, s3, method_string, resource, headers, body, params = {}): self.s3 = s3 self.headers = SortedDict(headers or {}, ignore_case = True) - # Add in any extra headers from s3 config object - if self.s3.config.extra_headers: - self.headers.update(self.s3.config.extra_headers) if len(self.s3.config.access_token)>0: self.s3.config.role_refresh() self.headers['x-amz-security-token']=self.s3.config.access_token self.resource = resource self.method_string = method_string self.params = params - - self.update_timestamp() - self.sign() + self.body = body def update_timestamp(self): if self.headers.has_key("date"): @@ -137,21 +136,42 @@ param_str += "&%s" % param return param_str and "?" + param_str[1:] + def use_signature_v2(self): + if self.s3.endpoint_requires_signature_v4: + return False + # in case of bad DNS name due to bucket name v2 will be used + # this way we can still use capital letters in bucket names for the older regions + + if self.resource['bucket'] is None or not check_bucket_name_dns_conformity(self.resource['bucket']) or self.s3.config.signature_v2 or self.s3.fallback_to_signature_v2: + return True + return False + def sign(self): h = self.method_string + "\n" h += self.headers.get("content-md5", "")+"\n" h += self.headers.get("content-type", "")+"\n" h += self.headers.get("date", "")+"\n" - for header in self.headers.keys(): + for header in sorted(self.headers.keys()): if header.startswith("x-amz-"): h += header+":"+str(self.headers[header])+"\n" if self.resource['bucket']: h += "/" + self.resource['bucket'] h += self.resource['uri'] - debug("SignHeaders: " + repr(h)) - signature = sign_string(h) - - self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature + + if self.use_signature_v2(): + debug("Using signature v2") + debug("SignHeaders: " + repr(h)) + signature = sign_string_v2(h) + self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature + else: + debug("Using signature v4") + self.headers = sign_string_v4(self.method_string, + self.s3.get_hostname(self.resource['bucket']), + self.resource['uri'], + self.params, + S3Request.region_map.get(self.resource['bucket'], Config().bucket_location), + self.headers, + self.body) def get_triplet(self): self.update_timestamp() @@ -206,9 +226,11 @@ def __init__(self, config): self.config = config + self.fallback_to_signature_v2 = False + self.endpoint_requires_signature_v4 = False def get_hostname(self, bucket): - if bucket and check_bucket_name_dns_conformity(bucket): + if bucket and check_bucket_name_dns_support(self.config.host_bucket, bucket): if self.redir_map.has_key(bucket): host = self.redir_map[bucket] else: @@ -222,7 +244,7 @@ self.redir_map[bucket] = redir_hostname def format_uri(self, resource): - if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']): + if resource['bucket'] and not check_bucket_name_dns_support(self.config.host_bucket, resource['bucket']): uri = "/%s%s" % (resource['bucket'], resource['uri']) else: uri = resource['uri'] @@ -249,6 +271,7 @@ def _get_common_prefixes(data): return getListFromXml(data, "CommonPrefixes") + uri_params = uri_params.copy() truncated = True @@ -302,8 +325,9 @@ check_bucket_name(bucket, dns_strict = False) if self.config.acl_public: headers["x-amz-acl"] = "public-read" - request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers) - response = self.send_request(request, body) + + request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers, body = body) + response = self.send_request(request) return response def bucket_delete(self, bucket): @@ -330,11 +354,10 @@ def website_info(self, uri, bucket_location = None): headers = SortedDict(ignore_case = True) bucket = uri.bucket() - body = "" request = self.create_request("BUCKET_LIST", bucket = bucket, extra="?website") try: - response = self.send_request(request, body) + response = self.send_request(request) response['index_document'] = getTextFromXml(response['data'], ".//IndexDocument//Suffix") response['error_document'] = getTextFromXml(response['data'], ".//ErrorDocument//Key") response['website_endpoint'] = self.config.website_endpoint % { @@ -360,9 +383,8 @@ body += ' ' body += '' - request = self.create_request("BUCKET_CREATE", bucket = bucket, extra="?website") - debug("About to send request '%s' with body '%s'" % (request, body)) - response = self.send_request(request, body) + request = self.create_request("BUCKET_CREATE", bucket = bucket, extra="?website", body = body) + response = self.send_request(request) debug("Received response '%s'" % (response)) return response @@ -370,11 +392,9 @@ def website_delete(self, uri, bucket_location = None): headers = SortedDict(ignore_case = True) bucket = uri.bucket() - body = "" request = self.create_request("BUCKET_DELETE", bucket = bucket, extra="?website") - debug("About to send request '%s' with body '%s'" % (request, body)) - response = self.send_request(request, body) + response = self.send_request(request) debug("Received response '%s'" % (response)) if response['status'] != 204: @@ -385,11 +405,10 @@ def expiration_info(self, uri, bucket_location = None): headers = SortedDict(ignore_case = True) bucket = uri.bucket() - body = "" request = self.create_request("BUCKET_LIST", bucket = bucket, extra="?lifecycle") try: - response = self.send_request(request, body) + response = self.send_request(request) response['prefix'] = getTextFromXml(response['data'], ".//Rule//Prefix") response['date'] = getTextFromXml(response['data'], ".//Rule//Expiration//Date") response['days'] = getTextFromXml(response['data'], ".//Rule//Expiration//Days") @@ -408,12 +427,10 @@ raise ParameterError("Expect either --expiry-day or --expiry-date") debug("del bucket lifecycle") bucket = uri.bucket() - body = "" request = self.create_request("BUCKET_DELETE", bucket = bucket, extra="?lifecycle") else: - request, body = self._expiration_set(uri) - debug("About to send request '%s' with body '%s'" % (request, body)) - response = self.send_request(request, body) + request = self._expiration_set(uri) + response = self.send_request(request) debug("Received response '%s'" % (response)) return response @@ -435,11 +452,53 @@ headers = SortedDict(ignore_case = True) headers['content-md5'] = compute_content_md5(body) bucket = uri.bucket() - request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers, extra="?lifecycle") - return (request, body) + request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers, extra="?lifecycle", body = body) + return (request) + + def _guess_content_type(self, filename): + content_type = self.config.default_mime_type + content_charset = None + + if filename == "-" and not self.config.default_mime_type: + raise ParameterError("You must specify --mime-type or --default-mime-type for files uploaded from stdin.") + + if self.config.guess_mime_type: + if self.config.use_mime_magic: + (content_type, content_charset) = mime_magic(filename) + else: + (content_type, content_charset) = mimetypes.guess_type(filename) + if not content_type: + content_type = self.config.default_mime_type + return (content_type, content_charset) + + def stdin_content_type(self): + content_type = self.config.mime_type + if content_type == '': + content_type = self.config.default_mime_type + + content_type += "; charset=" + self.config.encoding.upper() + return content_type + + def content_type(self, filename=None): + # explicit command line argument always wins + content_type = self.config.mime_type + content_charset = None + + if filename == u'-': + return self.stdin_content_type() + if not content_type: + (content_type, content_charset) = self._guess_content_type(filename) + + ## add charset to content type + if not content_charset: + content_charset = self.config.encoding.upper() + if self.add_encoding(filename, content_type) and content_charset is not None: + content_type = content_type + "; charset=" + content_charset + + return content_type def add_encoding(self, filename, content_type): - if content_type.find("charset=") != -1: + if 'charset=' in content_type: return False exts = self.config.add_encoding_exts.split(',') if exts[0]=='': @@ -480,23 +539,7 @@ headers["x-amz-server-side-encryption"] = "AES256" ## MIME-type handling - content_type = self.config.mime_type - content_charset = None - if filename != "-" and not content_type and self.config.guess_mime_type: - if self.config.use_mime_magic: - (content_type, content_charset) = mime_magic(filename) - else: - (content_type, content_charset) = mimetypes.guess_type(filename) - if not content_type: - content_type = self.config.default_mime_type - if not content_charset: - content_charset = self.config.encoding.upper() - - ## add charset to content type - if self.add_encoding(filename, content_type) and content_charset is not None: - content_type = content_type + "; charset=" + content_charset - - headers["content-type"] = content_type + headers["content-type"] = self.content_type(filename=filename) ## Other Amazon S3 attributes if self.config.acl_public: @@ -528,7 +571,7 @@ if info is not None: remote_size = int(info['headers']['content-length']) - remote_checksum = info['headers']['etag'].strip('"') + remote_checksum = info['headers']['etag'].strip('"\'') if size == remote_size: checksum = calculateChecksum('', file, 0, size, self.config.send_chunk) if remote_checksum == checksum: @@ -561,7 +604,7 @@ for key in key_list: uri = S3Uri(key) if uri.type != "s3": - raise ValueError("Excpected URI type 's3', got '%s'" % uri.type) + raise ValueError("Expected URI type 's3', got '%s'" % uri.type) if not uri.has_object(): raise ValueError("URI '%s' has no object" % key) if uri.bucket() != bucket: @@ -579,9 +622,10 @@ request_body = compose_batch_del_xml(bucket, batch) md5_hash = md5() md5_hash.update(request_body) - headers = {'content-md5': base64.b64encode(md5_hash.digest())} - request = self.create_request("BATCH_DELETE", bucket = bucket, extra = '?delete', headers = headers) - response = self.send_request(request, request_body) + headers = {'content-md5': base64.b64encode(md5_hash.digest()), + 'content-type': 'application/xml'} + request = self.create_request("BATCH_DELETE", bucket = bucket, extra = '?delete', headers = headers, body = request_body) + response = self.send_request(request) return response def object_delete(self, uri): @@ -597,11 +641,32 @@ body = '' body += (' %s' % self.config.restore_days) body += '' - request = self.create_request("OBJECT_POST", uri = uri, extra = "?restore") - debug("About to send request '%s' with body '%s'" % (request, body)) - response = self.send_request(request, body) + request = self.create_request("OBJECT_POST", uri = uri, extra = "?restore", body = body) + response = self.send_request(request) debug("Received response '%s'" % (response)) return response + + def _sanitize_headers(self, headers): + to_remove = [ + # from http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + 'date', + 'content-length', + 'last-modified', + 'content-md5', + 'x-amz-version-id', + 'x-amz-delete-marker', + # other headers returned from object_info() we don't want to send + 'accept-ranges', + 'etag', + 'server', + 'x-amz-id-2', + 'x-amz-request-id', + ] + + for h in to_remove + self.config.remove_headers: + if h.lower() in headers: + del headers[h.lower()] + return headers def object_copy(self, src_uri, dst_uri, extra_headers = None): if src_uri.type != "s3": @@ -610,22 +675,56 @@ raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type) headers = SortedDict(ignore_case = True) headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object())) - ## TODO: For now COPY, later maybe add a switch? headers['x-amz-metadata-directive'] = "COPY" if self.config.acl_public: headers["x-amz-acl"] = "public-read" if self.config.reduced_redundancy: headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY" + else: + headers["x-amz-storage-class"] = "STANDARD" ## Set server side encryption if self.config.server_side_encryption: headers["x-amz-server-side-encryption"] = "AES256" if extra_headers: - headers['x-amz-metadata-directive'] = "REPLACE" headers.update(extra_headers) + request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers) response = self.send_request(request) + return response + + def object_modify(self, src_uri, dst_uri, extra_headers = None): + if src_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type) + if dst_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type) + + info_response = self.object_info(src_uri) + headers = info_response['headers'] + headers = self._sanitize_headers(headers) + acl = self.get_acl(src_uri) + + headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object())) + headers['x-amz-metadata-directive'] = "REPLACE" + + # cannot change between standard and reduced redundancy with a REPLACE. + + ## Set server side encryption + if self.config.server_side_encryption: + headers["x-amz-server-side-encryption"] = "AES256" + + if extra_headers: + headers.update(extra_headers) + + if self.config.mime_type: + headers["content-type"] = self.config.mime_type + + request = self.create_request("OBJECT_PUT", uri = src_uri, headers = headers) + response = self.send_request(request) + + acl_response = self.set_acl(src_uri, acl) + return response def object_move(self, src_uri, dst_uri, extra_headers = None): @@ -652,14 +751,20 @@ return acl def set_acl(self, uri, acl): - if uri.has_object(): - request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl") - else: - request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl") + # dreamhost doesn't support set_acl properly + if 'objects.dreamhost.com' in self.config.host_base: + return { 'status' : 501 } # not implemented body = str(acl) debug(u"set_acl(%s): acl-xml: %s" % (uri, body)) - response = self.send_request(request, body) + + headers = {'content-type': 'application/xml'} + if uri.has_object(): + request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl", body = body) + else: + request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl", body = body) + + response = self.send_request(request) return response def get_policy(self, uri): @@ -672,11 +777,8 @@ # TODO check policy is proper json string headers['content-type'] = 'application/json' request = self.create_request("BUCKET_CREATE", uri = uri, - extra = "?policy", headers=headers) - body = policy - debug(u"set_policy(%s): policy-json: %s" % (uri, body)) - request.sign() - response = self.send_request(request, body=body) + extra = "?policy", headers=headers, body = policy) + response = self.send_request(request) return response def delete_policy(self, uri): @@ -689,11 +791,9 @@ headers = SortedDict(ignore_case = True) headers['content-md5'] = compute_content_md5(policy) request = self.create_request("BUCKET_CREATE", uri = uri, - extra = "?lifecycle", headers=headers) - body = policy - debug(u"set_lifecycle_policy(%s): policy-xml: %s" % (uri, body)) - request.sign() - response = self.send_request(request, body=body) + extra = "?lifecycle", headers=headers, body = policy) + debug(u"set_lifecycle_policy(%s): policy-xml: %s" % (uri)) + response = self.send_request(request) return response def delete_lifecycle_policy(self, uri): @@ -734,22 +834,24 @@ self.set_acl(uri, acl) def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False): - request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging") accesslog = AccessLog() if enable: accesslog.enableLogging(log_target_prefix_uri) accesslog.setAclPublic(acl_public) else: accesslog.disableLogging() + body = str(accesslog) debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body)) + + request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging", body = body) try: - response = self.send_request(request, body) + response = self.send_request(request) except S3Error, e: if e.info['Code'] == "InvalidTargetBucketForLogging": info("Setting up log-delivery ACL for target bucket.") self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket())) - response = self.send_request(request, body) + response = self.send_request(request) else: raise return accesslog, response @@ -806,7 +908,7 @@ debug("String '%s' encoded to '%s'" % (string, encoded)) return encoded - def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params): + def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, body = "", **params): resource = { 'bucket' : None, 'uri' : "/" } if uri and (bucket or object): @@ -825,7 +927,7 @@ method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"]) - request = S3Request(self, method_string, resource, headers, params) + request = S3Request(self, method_string, resource, headers, body, params) debug("CreateRequest: resource[uri]=" + resource['uri']) return request @@ -834,19 +936,68 @@ # Wait a few seconds. The more it fails the more we wait. return (self._max_retries - retries + 1) * 3 - def send_request(self, request, body = None, retries = _max_retries): + def _http_400_handler(self, request, response, fn, *args, **kwargs): + # AWS response AuthorizationHeaderMalformed means we sent the request to the wrong region + # get the right region out of the response and send it there. + message = 'Unknown error' + if 'data' in response and len(response['data']) > 0: + failureCode = getTextFromXml(response['data'], 'Code') + message = getTextFromXml(response['data'], 'Message') + if failureCode == 'AuthorizationHeaderMalformed': # we sent the request to the wrong region + region = getTextFromXml(response['data'], 'Region') + if region is not None: + S3Request.region_map[request.resource['bucket']] = region + info('Forwarding request to %s' % region) + return fn(*args, **kwargs) + else: + message = u'Could not determine bucket location. Please consider using --region parameter.' + + elif failureCode == 'InvalidRequest': + if message == 'The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.': + debug(u'Endpoint requires signature v4') + self.endpoint_requires_signature_v4 = True + return fn(*args, **kwargs) + + elif failureCode == 'InvalidArgument': # returned by DreamObjects on send_request and send_file, + # which doesn't support signature v4. Retry with signature v2 + if not request.use_signature_v2() and not self.fallback_to_signature_v2: # have not tried with v2 yet + debug(u'Falling back to signature v2') + self.fallback_to_signature_v2 = True + return fn(*args, **kwargs) + + else: # returned by DreamObjects on recv_file, which doesn't support signature v4. Retry with signature v2 + if not request.use_signature_v2() and not self.fallback_to_signature_v2: # have not tried with v2 yet + debug(u'Falling back to signature v2') + self.fallback_to_signature_v2 = True + return fn(*args, **kwargs) + + error(u"S3 error: %s" % message) + sys.exit(ExitCodes.EX_GENERAL) + + def _http_403_handler(self, request, response, fn, *args, **kwargs): + message = 'Unknown error' + if 'data' in response and len(response['data']) > 0: + failureCode = getTextFromXml(response['data'], 'Code') + message = getTextFromXml(response['data'], 'Message') + if failureCode == 'AccessDenied': # traditional HTTP 403 + if message == 'AWS authentication requires a valid Date or x-amz-date header': # message from an Eucalyptus walrus server + if not request.use_signature_v2() and not self.fallback_to_signature_v2: # have not tried with v2 yet + debug(u'Falling back to signature v2') + self.fallback_to_signature_v2 = True + return fn(*args, **kwargs) + + error(u"S3 error: %s" % message) + sys.exit(ExitCodes.EX_GENERAL) + + def send_request(self, request, retries = _max_retries): method_string, resource, headers = request.get_triplet() + debug("Processing request, please wait...") - if not headers.has_key('content-length'): - headers['content-length'] = body and len(body) or 0 try: - # "Stringify" all headers - for header in headers.keys(): - headers[header] = str(headers[header]) conn = ConnMan.get(self.get_hostname(resource['bucket'])) uri = self.format_uri(resource) - debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(body or ""))) - conn.c.request(method_string, uri, body, headers) + debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(request.body or ""))) + conn.c.request(method_string, uri, request.body, headers) response = {} http_response = conn.c.getresponse() response["status"] = http_response.status @@ -867,17 +1018,24 @@ warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) - return self.send_request(request, body, retries - 1) + return self.send_request(request, retries - 1) else: raise S3RequestError("Request failed for: %s" % resource['uri']) + + if response["status"] == 400: + return self._http_400_handler(request, response, self.send_request, request) + if response["status"] == 403: + return self._http_403_handler(request, response, self.send_request, request) + if response["status"] == 405: # Method Not Allowed. Don't retry. + raise S3Error(response) if response["status"] == 307: ## RedirectPermanent redir_bucket = getTextFromXml(response['data'], ".//Bucket") redir_hostname = getTextFromXml(response['data'], ".//Endpoint") self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) - return self.send_request(request, body) + info("Redirected to: %s" % (redir_hostname)) + return self.send_request(request) if response["status"] >= 500: e = S3Error(response) @@ -886,7 +1044,7 @@ warning(unicode(e)) warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) - return self.send_request(request, body, retries - 1) + return self.send_request(request, retries - 1) else: raise e @@ -897,12 +1055,25 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = _max_retries, offset = 0, chunk_size = -1): method_string, resource, headers = request.get_triplet() + if S3Request.region_map.get(request.resource['bucket'], None) is None: + s3_uri = S3Uri('s3://' + request.resource['bucket']) + region = self.get_bucket_location(s3_uri) + if region is not None: + S3Request.region_map[request.resource['bucket']] = region + size_left = size_total = headers.get("content-length") if self.config.progress_meter: progress = self.config.progress_class(labels, size_total) else: info("Sending file '%s', please wait..." % file.name) timestamp_start = time.time() + + if buffer: + sha256_hash = checksum_sha256_buffer(buffer, offset, size_total) + else: + sha256_hash = checksum_sha256_file(file.name, offset, size_total) + request.body = sha256_hash + method_string, resource, headers = request.get_triplet() try: conn = ConnMan.get(self.get_hostname(resource['bucket'])) conn.c.putrequest(method_string, self.format_uri(resource)) @@ -986,8 +1157,13 @@ redir_bucket = getTextFromXml(response['data'], ".//Bucket") redir_hostname = getTextFromXml(response['data'], ".//Endpoint") self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) + info("Redirected to: %s" % (redir_hostname)) return self.send_file(request, file, labels, buffer, offset = offset, chunk_size = chunk_size) + + if response["status"] == 400: + return self._http_400_handler(request, response, self.send_file, request, file, labels, buffer, offset = offset, chunk_size = chunk_size) + if response["status"] == 403: + return self._http_403_handler(request, response, self.send_file, request, file, labels, buffer, offset = offset, chunk_size = chunk_size) # S3 from time to time doesn't send ETag back in a response :-( # Force re-upload here. @@ -1089,8 +1265,15 @@ redir_bucket = getTextFromXml(response['data'], ".//Bucket") redir_hostname = getTextFromXml(response['data'], ".//Endpoint") self.set_hostname(redir_bucket, redir_hostname) - warning("Redirected to: %s" % (redir_hostname)) + info("Redirected to: %s" % (redir_hostname)) return self.recv_file(request, stream, labels) + + if response["status"] == 400: + return self._http_400_handler(request, response, self.recv_file, request, stream, labels) + if response["status"] == 403: + return self._http_403_handler(request, response, self.recv_file, request, stream, labels) + if response["status"] == 405: # Method Not Allowed. Don't retry. + raise S3Error(response) if response["status"] < 200 or response["status"] > 299: raise S3Error(response) @@ -1170,7 +1353,7 @@ except KeyError: pass - response["md5match"] = md5_hash.find(response["md5"]) >= 0 + response["md5match"] = response["md5"] in md5_hash response["elapsed"] = timestamp_end - timestamp_start response["size"] = current_position response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) diff --git a/S3/S3Uri.py b/S3/S3Uri.py index 23defe2..283f57c 100644 --- a/S3/S3Uri.py +++ b/S3/S3Uri.py @@ -10,7 +10,7 @@ from BidirMap import BidirMap from logging import debug import S3 -from Utils import unicodise, check_bucket_name_dns_conformity +from Utils import unicodise, check_bucket_name_dns_conformity, check_bucket_name_dns_support import Config class S3Uri(object): @@ -53,7 +53,7 @@ class S3UriS3(S3Uri): type = "s3" - _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE) + _re = re.compile("^s3://([^/]*)/?(.*)", re.IGNORECASE) def __init__(self, string): match = self._re.match(string) if not match: @@ -78,7 +78,7 @@ return u"/".join([u"s3:/", self._bucket, self._object]) def is_dns_compatible(self): - return check_bucket_name_dns_conformity(self._bucket) + return check_bucket_name_dns_support(Config.Config().host_bucket, self._bucket) def public_url(self): if self.is_dns_compatible(): diff --git a/S3/Utils.py b/S3/Utils.py index e67e672..c83b7c0 100644 --- a/S3/Utils.py +++ b/S3/Utils.py @@ -12,8 +12,6 @@ import string import random import rfc822 -import hmac -import base64 import errno import urllib from calendar import timegm @@ -49,8 +47,8 @@ try: import xml.etree.ElementTree as ET except ImportError: + # xml.etree.ElementTree was only added in python 2.5 import elementtree.ElementTree as ET -from xml.parsers.expat import ExpatError __all__ = [] def parseNodes(nodes): @@ -74,7 +72,7 @@ """ removeNameSpace(xml) -- remove top-level AWS namespace """ - r = re.compile('^(]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE) + r = re.compile('^(]+?>\s*)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE) if r.match(xml): xmlns = r.match(xml).groups()[2] xml = r.sub("\\1\\2\\4", xml) @@ -90,11 +88,8 @@ if xmlns: tree.attrib['xmlns'] = xmlns return tree - except ExpatError, e: - error(e) - raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/") except Exception, e: - error(e) + error("Error parsing xml: %s", e) error(xml) raise @@ -343,44 +338,6 @@ return new_string __all__.append("replace_nonprintables") -def sign_string(string_to_sign): - """Sign a string with the secret key, returning base64 encoded results. - By default the configured secret key is used, but may be overridden as - an argument. - - Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html - """ - signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip() - return signature -__all__.append("sign_string") - -def sign_url(url_to_sign, expiry): - """Sign a URL in s3://bucket/object form with the given expiry - time. The object will be accessible via the signed URL until the - AWS key and secret are revoked or the expiry time is reached, even - if the object is otherwise private. - - See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html - """ - return sign_url_base( - bucket = url_to_sign.bucket(), - object = url_to_sign.object(), - expiry = expiry - ) -__all__.append("sign_url") - -def sign_url_base(**parms): - """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args.""" - parms['expiry']=time_to_epoch(parms['expiry']) - parms['access_key']=Config.Config().access_key - parms['host_base']=Config.Config().host_base - debug("Expiry interpreted as epoch time %s", parms['expiry']) - signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms - debug("Signing plaintext: %r", signtext) - parms['sig'] = urllib.quote_plus(sign_string(signtext)) - debug("Urlencoded signature: %s", parms['sig']) - return "http://%(bucket)s.%(host_base)s/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms - def time_to_epoch(t): """Convert time specified in a variety of forms into UNIX epoch time. Accepts datetime.datetime, int, anything that has a strftime() method, and standard time 9-tuples @@ -400,6 +357,9 @@ elif isinstance(t, str) or isinstance(t, unicode): # See if it's a string representation of an epoch try: + # Support relative times (eg. "+60") + if t.startswith('+'): + return time.time() + int(t[1:]) return int(t) except ValueError: # Try to parse it as a timestamp string @@ -447,6 +407,20 @@ return False __all__.append("check_bucket_name_dns_conformity") +def check_bucket_name_dns_support(bucket_host, bucket_name): + """ + Check whether either the host_bucket support buckets and + either bucket name is dns compatible + """ + if "%(bucket)s" not in bucket_host: + return False + + try: + return check_bucket_name(bucket_name, dns_strict = True) + except Exceptions.ParameterError: + return False +__all__.append("check_bucket_name_dns_support") + def getBucketFromHostname(hostname): """ bucket, success = getBucketFromHostname(hostname) diff --git a/s3cmd b/s3cmd index b6abacc..de6422f 100755 --- a/s3cmd +++ b/s3cmd @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 ## -------------------------------------------------------------------- ## s3cmd - S3 client @@ -20,8 +20,8 @@ import sys -if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.4: - sys.stderr.write(u"ERROR: Python 2.4 or higher required, sorry.\n") +if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.6: + sys.stderr.write(u"ERROR: Python 2.6 or higher required, sorry.\n") sys.exit(EX_OSFILE) import logging @@ -124,10 +124,20 @@ if uri.type == "s3" and uri.has_bucket(): subcmd_bucket_list(s3, uri) return EX_OK - subcmd_buckets_list_all(s3) - return EX_OK - -def cmd_buckets_list_all_all(args): + + # If not a s3 type uri or no bucket was provided, list all the buckets + subcmd_all_buckets_list(s3) + return EX_OK + +def subcmd_all_buckets_list(s3): + + response = s3.list_all_buckets() + + for bucket in sorted(response["list"], key=lambda b:b["Name"]): + output(u"%s s3://%s" % (formatDateTime(bucket["CreationDate"]), + bucket["Name"])) + +def cmd_all_buckets_list_all_content(args): s3 = S3(Config()) response = s3.list_all_buckets() @@ -136,14 +146,6 @@ subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"])) output(u"") return EX_OK - -def subcmd_buckets_list_all(s3): - response = s3.list_all_buckets() - for bucket in response["list"]: - output(u"%s s3://%s" % ( - formatDateTime(bucket["CreationDate"]), - bucket["Name"], - )) def subcmd_bucket_list(s3, uri): bucket = uri.bucket() @@ -173,9 +175,9 @@ "uri": uri.compose_uri(bucket, prefix["Prefix"])}) for object in response["list"]: - md5 = object['ETag'].strip('"') + md5 = object['ETag'].strip('"\'') if cfg.list_md5: - if md5.find('-') >= 0: # need to get md5 from the object + if '-' in md5: # need to get md5 from the object object_uri = uri.compose_uri(bucket, object["Key"]) info_response = s3.object_info(S3Uri(object_uri)) try: @@ -465,7 +467,10 @@ if destination_base[-1] != os.path.sep: destination_base += os.path.sep for key in remote_list: - remote_list[key]['local_filename'] = destination_base + key + local_filename = destination_base + key + if os.path.sep != "/": + local_filename = os.path.sep.join(local_filename.split("/")) + remote_list[key]['local_filename'] = deunicodise(local_filename) else: raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base) @@ -753,7 +758,7 @@ def cmd_modify(args): s3 = S3(Config()) - return subcmd_cp_mv(args, s3.object_copy, "modify", u"File %(src)s modified") + return subcmd_cp_mv(args, s3.object_modify, "modify", u"File %(src)s modified") def cmd_mv(args): s3 = S3(Config()) @@ -775,7 +780,7 @@ output(u" File size: %s" % info['headers']['content-length']) output(u" Last mod: %s" % info['headers']['last-modified']) output(u" MIME type: %s" % info['headers']['content-type']) - md5 = info['headers']['etag'].strip('"') + md5 = info['headers']['etag'].strip('"\'') try: md5 = info['s3cmd-attrs']['md5'] except KeyError: @@ -917,9 +922,7 @@ failed_copy_files[key]['target_uri'] = destination_base + key seq = _upload(failed_copy_files, seq, failed_copy_count) - total_elapsed = time.time() - timestamp_start - if total_elapsed == 0.0: - total_elapsed = 1.0 + total_elapsed = max(1.0, time.time() - timestamp_start) outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed) if seq > 0: output(outstr) @@ -1044,6 +1047,10 @@ except OSError, e: if e.errno == errno.EISDIR: warning(u"%s is a directory - skipping over" % unicodise(dst_file)) + continue + elif e.errno == errno.ETXTBSY: + warning(u"%s is currently open for execute, cannot be overwritten. Skipping over." % unicodise(dst_file)) + os.unlink(chkptfname) continue else: raise @@ -1148,7 +1155,7 @@ _set_local_filename(failed_copy_list, destination_base) seq, total_size = _download(failed_copy_list, seq, len(failed_copy_list) + remote_count + update_count, total_size, dir_cache) - total_elapsed = time.time() - timestamp_start + total_elapsed = max(1.0, time.time() - timestamp_start) speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True) # Only print out the result if any work has been done or @@ -1399,13 +1406,13 @@ if cfg.delete_removed and cfg.delete_after and remote_list: subcmd_batch_del(remote_list = remote_list) - total_elapsed = time.time() - timestamp_start + total_elapsed = max(1.0, time.time() - timestamp_start) total_speed = total_elapsed and total_size/total_elapsed or 0.0 speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True) # Only print out the result if any work has been done or # if the user asked for verbose output - outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s. Copied %d files saving %d bytes transfer." % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1], n_copies, saved_bytes) + outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s. Copied %d files saving %d bytes transfer." % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1], n_copies, saved_bytes) if total_size + saved_bytes > 0: output(outstr) else: @@ -1645,7 +1652,7 @@ def cmd_sign(args): string_to_sign = args.pop() debug("string-to-sign: %r" % string_to_sign) - signature = Utils.sign_string(string_to_sign) + signature = Crypto.sign_string_v2(string_to_sign) output("Signature: %s" % signature) return EX_OK @@ -1655,7 +1662,7 @@ if url_to_sign.type != 's3': raise ParameterError("Must be S3Uri. Got: %s" % url_to_sign) debug("url to sign: %r" % url_to_sign) - signed_url = Utils.sign_url(url_to_sign, expiry) + signed_url = Crypto.sign_url_v2(url_to_sign, expiry) output(signed_url) return EX_OK @@ -1735,7 +1742,8 @@ def gpg_command(command, passphrase = ""): debug("GPG command: " + " ".join(command)) - p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, + close_fds = True) p_stdout, p_stderr = p.communicate(passphrase + "\n") debug("GPG output:") for line in p_stdout.split("\n"): @@ -1779,9 +1787,10 @@ options = [ ("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3. Leave them empty for using the env variables."), ("secret_key", "Secret Key"), + ("bucket_location", "Default Region"), ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"), ("gpg_command", "Path to GPG program"), - ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy"), + ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP, and can only be proxied with Python 2.7 or newer"), ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't connect to S3 directly"), ("proxy_port", "HTTP Proxy server port"), ] @@ -1802,7 +1811,7 @@ for option in options: prompt = option[1] ## Option-specific handling - if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True: + if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True and sys.hexversion < 0x02070000: setattr(cfg, option[0], "") continue if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "": @@ -1970,14 +1979,14 @@ {"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1}, {"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1}, {"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0}, - {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0}, + {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_all_buckets_list_all_content, "argc":0}, {"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2}, {"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1}, {"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, {"cmd":"rm", "label":"Delete file from bucket (alias for del)", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1}, #{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1}, {"cmd":"restore", "label":"Restore file from Glacier storage", "param":"s3://BUCKET/OBJECT", "func":cmd_object_restore, "argc":1}, - {"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2}, + {"cmd":"sync", "label":"Synchronize a directory tree to S3 (checks files freshness using size and md5 checksum, unless overriden by options, see below)", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2}, {"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0}, {"cmd":"info", "label":"Get various information about Buckets or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1}, {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, @@ -1988,14 +1997,14 @@ {"cmd":"setpolicy", "label":"Modify Bucket Policy", "param":"FILE s3://BUCKET", "func":cmd_setpolicy, "argc":2}, {"cmd":"delpolicy", "label":"Delete Bucket Policy", "param":"s3://BUCKET", "func":cmd_delpolicy, "argc":1}, - {"cmd":"multipart", "label":"show multipart uploads", "param":"s3://BUCKET [Id]", "func":cmd_multipart, "argc":1}, - {"cmd":"abortmp", "label":"abort a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_abort_multipart, "argc":2}, - - {"cmd":"listmp", "label":"list parts of a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_list_multipart, "argc":2}, + {"cmd":"multipart", "label":"Show multipart uploads", "param":"s3://BUCKET [Id]", "func":cmd_multipart, "argc":1}, + {"cmd":"abortmp", "label":"Abort a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_abort_multipart, "argc":2}, + + {"cmd":"listmp", "label":"List parts of a multipart upload", "param":"s3://BUCKET/OBJECT Id", "func":cmd_list_multipart, "argc":2}, {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1}, {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1}, - {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2}, + {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT ", "func":cmd_signurl, "argc":2}, {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1}, ## Website commands @@ -2122,19 +2131,20 @@ from os.path import expanduser config_file = os.path.join(expanduser("~"), ".s3cfg") - preferred_encoding = locale.getpreferredencoding() or "UTF-8" - - optparser.set_defaults(encoding = preferred_encoding) + autodetected_encoding = locale.getpreferredencoding() or "UTF-8" + optparser.set_defaults(config = config_file) optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://some-bucket' to test access to a specific bucket instead of attempting to list them all.") - optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default") + optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to $HOME/.s3cfg") optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.") optparser.add_option( "--access_key", dest="access_key", help="AWS Access Key") optparser.add_option( "--secret_key", dest="secret_key", help="AWS Secret Key") optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)") + optparser.add_option("-s", "--ssl", dest="use_https", action="store_true", help="Use HTTPS connection when communicating with S3.") + optparser.add_option( "--no-ssl", dest="use_https", action="store_false", help="Don't use HTTPS. (default)") optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.") optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.") optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") @@ -2172,8 +2182,9 @@ optparser.add_option( "--ignore-failed-copy", dest="ignore_failed_copy", action="store_true", help="Don't exit unsuccessfully because of missing keys") optparser.add_option( "--files-from", dest="files_from", action="append", metavar="FILE", help="Read list of source-file names from FILE. Use - to read from stdin.") - optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2") + optparser.add_option( "--region", "--bucket-location", metavar="REGION", dest="bucket_location", help="Region to create bucket in. As of now the regions are: us-east-1, us-west-1, us-west-2, eu-west-1, eu-central-1, ap-northeast-1, ap-southeast-1, ap-southeast-2, sa-east-1") optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]") + optparser.add_option( "--no-reduced-redundancy", "--no-rr", dest="reduced_redundancy", action="store_false", help="Store object without 'Reduced redundancy'. Higher per-GB price. [put, cp, mv]") optparser.add_option( "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)") optparser.add_option( "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)") @@ -2185,10 +2196,11 @@ optparser.add_option("-m", "--mime-type", dest="mime_type", type="mimetype", metavar="MIME/TYPE", help="Force MIME-type. Override both --default-mime-type and --guess-mime-type.") optparser.add_option( "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this option.") - - optparser.add_option( "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects.") - - optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding) + optparser.add_option( "--remove-header", dest="remove_headers", action="append", metavar="NAME", help="Remove a given HTTP header. Can be used multiple times. For instance, remove 'Expires' or 'Cache-Control' headers (or both) using this option. [modify]") + + optparser.add_option( "--server-side-encryption", dest="server_side_encryption", action="store_true", help="Specifies that server-side encryption will be used when putting objects. [put, sync, cp, modify]") + + optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % autodetected_encoding) optparser.add_option( "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )") optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!") @@ -2224,6 +2236,10 @@ optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files") optparser.add_option( "--cache-file", dest="cache_file", action="store", default="", metavar="FILE", help="Cache FILE containing local source MD5 values") optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Silence output on stdout") + optparser.add_option("--ca-certs", dest="ca_certs_file", action="store", default=None, help="Path to SSL CA certificate FILE (instead of system default)") + optparser.add_option("--check-certificate", dest="check_ssl_certificate", action="store_true", help="Check SSL certificate validity") + optparser.add_option("--no-check-certificate", dest="check_ssl_certificate", action="store_false", help="Check SSL certificate validity") + optparser.add_option("--signature-v2", dest="signature_v2", action="store_true", help="Use AWS Signature version 2 instead of newer signature methods. Helpful for S3-like systems that don't have AWS Signature v4 yet.") optparser.set_usage(optparser.usage + " COMMAND [parameters]") optparser.set_description('S3cmd is a tool for managing objects in '+ @@ -2300,8 +2316,12 @@ key_inval = key_inval.replace(" ", "") key_inval = key_inval.replace("\t", "") raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval)) - debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip(), val.strip())) - cfg.extra_headers[key.strip()] = val.strip() + debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip().lower(), val.strip())) + cfg.extra_headers[key.strip().lower()] = val.strip() + + # Process --remove-header + if options.remove_headers: + cfg.remove_headers = options.remove_headers ## --acl-grant/--acl-revoke arguments are pre-parsed by OptionS3ACL() if options.acl_grants: @@ -2437,14 +2457,10 @@ error(u"Not enough parameters for command '%s'" % command) sys.exit(EX_USAGE) - try: - rc = cmd_func(args) - if rc is None: # if we missed any cmd_*() returns - rc = EX_GENERAL - return rc - except S3Error, e: - error(u"S3 error: %s" % e) - sys.exit(EX_SOFTWARE) + rc = cmd_func(args) + if rc is None: # if we missed any cmd_*() returns + rc = EX_GENERAL + return rc def report_exception(e, msg=''): sys.stderr.write(u""" @@ -2462,10 +2478,13 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """ % msg) - s = u' '.join([unicodise(a) for a in sys.argv]) + tb = traceback.format_exc(sys.exc_info()) + try: + s = u' '.join([unicodise(a) for a in sys.argv]) + except NameError: + s = u' '.join([(a) for a in sys.argv]) sys.stderr.write(u"Invoked as: %s\n" % s) - tb = traceback.format_exc(sys.exc_info()) e_class = str(e.__class__) e_class = e_class[e_class.rfind(".")+1 : -2] sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) @@ -2513,13 +2532,19 @@ from S3.FileDict import FileDict from S3.S3Uri import S3Uri from S3 import Utils + from S3 import Crypto from S3.Utils import * from S3.Progress import Progress from S3.CloudFront import Cmd as CfCmd from S3.CloudFront import CloudFront from S3.FileLists import * from S3.MultiPart import MultiPartUpload - + except Exception as e: + report_exception(e, "Error loading some components of s3cmd (Import Error)") + # 1 = EX_GENERAL but be safe in that situation + sys.exit(1) + + try: rc = main() sys.exit(rc) @@ -2535,7 +2560,11 @@ error(u"S3 Temporary Error: %s. Please try again later." % e) sys.exit(EX_TEMPFAIL) - except (S3Error, S3Exception, S3ResponseError, CloudFrontError), e: + except S3Error, e: + error(u"S3 error: %s" % e) + sys.exit(e.get_error_code()) + + except (S3Exception, S3ResponseError, CloudFrontError), e: report_exception(e) sys.exit(EX_SOFTWARE) diff --git a/s3cmd.1 b/s3cmd.1 index 385275b..14a612b 100644 --- a/s3cmd.1 +++ b/s3cmd.1 @@ -50,7 +50,7 @@ Restore file from Glacier storage .TP s3cmd \fBsync\fR \fILOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR\fR -Synchronize a directory tree to S3 +Synchronize a directory tree to S3 (checks files freshness using size and md5 checksum, unless overriden by options, see below) .TP s3cmd \fBdu\fR \fI[s3://BUCKET[/PREFIX]]\fR Disk usage by buckets @@ -77,13 +77,13 @@ Delete Bucket Policy .TP s3cmd \fBmultipart\fR \fIs3://BUCKET [Id]\fR -show multipart uploads +Show multipart uploads .TP s3cmd \fBabortmp\fR \fIs3://BUCKET/OBJECT Id\fR -abort a multipart upload +Abort a multipart upload .TP s3cmd \fBlistmp\fR \fIs3://BUCKET/OBJECT Id\fR -list parts of a multipart upload +List parts of a multipart upload .TP s3cmd \fBaccesslog\fR \fIs3://BUCKET\fR Enable/disable bucket access logging @@ -91,7 +91,7 @@ s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR Sign arbitrary string using the secret key .TP -s3cmd \fBsignurl\fR \fIs3://BUCKET/OBJECT expiry_epoch\fR +s3cmd \fBsignurl\fR \fIs3://BUCKET/OBJECT \fR Sign an S3 URL to provide limited public access with expiry .TP s3cmd \fBfixbucket\fR \fIs3://BUCKET[/PREFIX]\fR @@ -161,7 +161,7 @@ them all. .TP \fB\-c\fR FILE, \fB\-\-config\fR=FILE -Config file name. Defaults to /home/mludvig/.s3cfg +Config file name. Defaults to $HOME/.s3cfg .TP \fB\-\-dump\-config\fR Dump current configuration after parsing config files @@ -178,6 +178,12 @@ don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands) +.TP +\fB\-s\fR, \fB\-\-ssl\fR +Use HTTPS connection when communicating with S3. +.TP +\fB\-\-no\-ssl\fR +Don't use HTTPS. (default) .TP \fB\-e\fR, \fB\-\-encrypt\fR Encrypt files before uploading to S3. @@ -311,14 +317,19 @@ Read list of source-file names from FILE. Use - to read from stdin. .TP -\fB\-\-bucket\-location\fR=BUCKET_LOCATION -Datacentre to create bucket in. As of now the -datacenters are: US (default), EU, ap-northeast-1, ap- -southeast-1, sa-east-1, us-west-1 and us-west-2 +\fB\-\-region\fR=REGION, \fB\-\-bucket\-location\fR=REGION +Region to create bucket in. As of now the regions are: +us-east-1, us-west-1, us-west-2, eu-west-1, eu- +central-1, ap-northeast-1, ap-southeast-1, ap- +southeast-2, sa-east-1 .TP \fB\-\-reduced\-redundancy\fR, \fB\-\-rr\fR Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv] +.TP +\fB\-\-no\-reduced\-redundancy\fR, \fB\-\-no\-rr\fR +Store object without 'Reduced redundancy'. Higher per- +GB price. [put, cp, mv] .TP \fB\-\-access\-logging\-target\-prefix\fR=LOG_TARGET_PREFIX Target prefix for access logs (S3 URI) (for [cfmodify] @@ -353,9 +364,14 @@ used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this option. .TP +\fB\-\-remove\-header\fR=NAME +Remove a given HTTP header. Can be used multiple +times. For instance, remove 'Expires' or 'Cache- +Control' headers (or both) using this option. [modify] +.TP \fB\-\-server\-side\-encryption\fR Specifies that server-side encryption will be used -when putting objects. +when putting objects. [put, sync, cp, modify] .TP \fB\-\-encoding\fR=ENCODING Override autodetected terminal and filesystem encoding @@ -461,7 +477,7 @@ Enable debug output. .TP \fB\-\-version\fR -Show s3cmd version (1.5.0-rc1) and exit. +Show s3cmd version (1.5.2) and exit. .TP \fB\-F\fR, \fB\-\-follow\-symlinks\fR Follow symbolic links as if they are regular files @@ -471,6 +487,21 @@ .TP \fB\-q\fR, \fB\-\-quiet\fR Silence output on stdout +.TP +\fB\-\-ca\-certs\fR=CA_CERTS_FILE +Path to SSL CA certificate FILE (instead of system +default) +.TP +\fB\-\-check\-certificate\fR +Check SSL certificate validity +.TP +\fB\-\-no\-check\-certificate\fR +Check SSL certificate validity +.TP +\fB\-\-signature\-v2\fR +Use AWS Signature version 2 instead of newer signature +methods. Helpful for S3-like systems that don't have +AWS Signature v4 yet. .SH EXAMPLES diff --git a/s3cmd.egg-info/PKG-INFO b/s3cmd.egg-info/PKG-INFO new file mode 100644 index 0000000..4137201 --- /dev/null +++ b/s3cmd.egg-info/PKG-INFO @@ -0,0 +1,39 @@ +Metadata-Version: 1.1 +Name: s3cmd +Version: 1.5.2 +Summary: Command line tool for managing Amazon S3 and CloudFront services +Home-page: http://s3tools.org +Author: github.com/mdomsch, github.com/matteobar +Author-email: s3tools-bugs@lists.sourceforge.net +License: GNU GPL v2+ +Description: + + S3cmd lets you copy files from/to Amazon S3 + (Simple Storage Service) using a simple to use + command line client. Supports rsync-like backup, + GPG encryption, and more. Also supports management + of Amazon's CloudFront content delivery network. + + + Authors: + -------- + Michal Ludvig + +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: MacOS X +Classifier: Environment :: Win32 (MS Windows) +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) +Classifier: Natural Language :: English +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 2 :: Only +Classifier: Topic :: System :: Archiving +Classifier: Topic :: Utilities diff --git a/s3cmd.egg-info/SOURCES.txt b/s3cmd.egg-info/SOURCES.txt new file mode 100644 index 0000000..d86f4df --- /dev/null +++ b/s3cmd.egg-info/SOURCES.txt @@ -0,0 +1,33 @@ +INSTALL +MANIFEST.in +NEWS +README.md +s3cmd +s3cmd.1 +setup.cfg +setup.py +S3/ACL.py +S3/AccessLog.py +S3/BidirMap.py +S3/CloudFront.py +S3/Config.py +S3/ConnMan.py +S3/Crypto.py +S3/Exceptions.py +S3/ExitCodes.py +S3/FileDict.py +S3/FileLists.py +S3/HashCache.py +S3/MultiPart.py +S3/PkgInfo.py +S3/Progress.py +S3/S3.py +S3/S3Uri.py +S3/SortedDict.py +S3/Utils.py +S3/__init__.py +s3cmd.egg-info/PKG-INFO +s3cmd.egg-info/SOURCES.txt +s3cmd.egg-info/dependency_links.txt +s3cmd.egg-info/requires.txt +s3cmd.egg-info/top_level.txt \ No newline at end of file diff --git a/s3cmd.egg-info/dependency_links.txt b/s3cmd.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/s3cmd.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/s3cmd.egg-info/requires.txt b/s3cmd.egg-info/requires.txt new file mode 100644 index 0000000..ffde045 --- /dev/null +++ b/s3cmd.egg-info/requires.txt @@ -0,0 +1,2 @@ +python-dateutil +python-magic diff --git a/s3cmd.egg-info/top_level.txt b/s3cmd.egg-info/top_level.txt new file mode 100644 index 0000000..878cb3c --- /dev/null +++ b/s3cmd.egg-info/top_level.txt @@ -0,0 +1 @@ +S3 diff --git a/setup.cfg b/setup.cfg index 83d33b9..0613883 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,8 @@ [sdist] formats = gztar,zip + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff --git a/setup.py b/setup.py index 6055d60..25457c8 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ -from distutils.core import setup import sys import os +from setuptools import setup, find_packages + import S3.PkgInfo -if float("%d.%d" % sys.version_info[:2]) < 2.4: +if float("%d.%d" % sys.version_info[:2]) < 2.6: sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3]) - sys.stderr.write("S3cmd requires Python 2.4 or newer.\n") + sys.stderr.write("S3cmd requires Python 2.6 or newer.\n") sys.exit(1) try: @@ -47,7 +48,7 @@ man_path = os.getenv("S3CMD_INSTPATH_MAN") or "share/man" doc_path = os.getenv("S3CMD_INSTPATH_DOC") or "share/doc/packages" data_files = [ - (doc_path+"/s3cmd", [ "README", "INSTALL", "NEWS" ]), + (doc_path+"/s3cmd", [ "README.md", "INSTALL", "NEWS" ]), (man_path+"/man1", [ "s3cmd.1" ] ), ] else: @@ -65,6 +66,8 @@ ## Packaging details author = "Michal Ludvig", author_email = "michal@logix.cz", + maintainer = "github.com/mdomsch, github.com/matteobar", + maintainer_email = "s3tools-bugs@lists.sourceforge.net", url = S3.PkgInfo.url, license = S3.PkgInfo.license, description = S3.PkgInfo.short_description, @@ -75,7 +78,28 @@ -------- Michal Ludvig """ % (S3.PkgInfo.long_description), - requires=["dateutil"] + + classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Environment :: MacOS X', + 'Environment :: Win32 (MS Windows)', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', + 'Natural Language :: English', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', + 'Operating System :: POSIX', + 'Operating System :: Unix', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 2 :: Only', + 'Topic :: System :: Archiving', + 'Topic :: Utilities', + ], + + install_requires = ["python-dateutil", "python-magic"] ) # vim:et:ts=4:sts=4:ai