Commit d7ddf9f013349d4c5f2fce3d616c879ca37fbbc6 - s3cmd

Imported Upstream version 0.9.6 Gianfranco Costamagna 8 years ago

9 changed file(s) with 191 addition(s) and 44 deletion(s). Raw diff Collapse all Expand all

+10

-0

NEWS less more

	0	s3cmd 0.9.6 - 2008-02-28
	1	===========
	2	* Support for setting / guessing MIME-type of uploaded file
	3	* Correctly follow redirects when accessing buckets created
	4	in Europe.
	5	* Introduced 'info' command both for buckets and objects
	6	* Correctly display public URL on uploads
	7	* Updated TODO list for everyone to see where we're heading
	8	* Various small fixes. See ChangeLog for details.
	9
0	10	s3cmd 0.9.5 - 2007-11-13
1	11	===========
2	12	* Support for buckets created in Europe

-1

PKG-INFO less more

0	0	Metadata-Version: 1.0
1	1	Name: s3cmd
2		Version: 0.9.5
	2	Version: 0.9.6
3	3	Summary: S3cmd is a tool for managing Amazon S3 storage space.
4	4	Home-page: http://s3tools.logix.cz
5	5	Author: Michal Ludvig

-0

S3/Config.py less more

43	43	gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
44	44	use_https = False
45	45	bucket_location = "US"
	46	default_mime_type = "binary/octet-stream"
	47	guess_mime_type = False
46	48
47	49	## Creating a singleton
48	50	def __new__(self, configfile = None):

-1

S3/PkgInfo.py less more

0	0	package = "s3cmd"
1		version = "0.9.5"
	1	version = "0.9.6"
2	2	url = "http://s3tools.logix.cz"
3	3	license = "GPL version 2"
4	4	short_description = "S3cmd is a tool for managing Amazon S3 storage space."

+82

-8

S3/S3.py less more

10	10	import hmac
11	11	import httplib
12	12	import logging
	13	import mimetypes
13	14	from logging import debug, info, warning, error
14	15	from stat import ST_SIZE
15	16

79	80	"BucketAlreadyExists" : "Bucket '%s' already exists",
80	81	}
81	82
	83	## S3 sometimes sends HTTP-307 response
	84	redir_map = {}
	85
82	86	def __init__(self, config):
83	87	self.config = config
84	88

93	97
94	98	def get_hostname(self, bucket):
95	99	if bucket:
96		host = self.config.host_bucket % { 'bucket' : bucket }
	100	if self.redir_map.has_key(bucket):
	101	host = self.redir_map[bucket]
	102	else:
	103	host = self.config.host_bucket % { 'bucket' : bucket }
97	104	else:
98	105	host = self.config.host_base
99		debug('get_hostname(): ' + host)
	106	debug('get_hostname(%s): %s' % (bucket, host))
100	107	return host
	108
	109	def set_hostname(self, bucket, redir_hostname):
	110	self.redir_map[bucket] = redir_hostname
101	111
102	112	def format_uri(self, resource):
103	113	if self.config.proxy_host != "":

130	140	while _list_truncated(response["data"]):
131	141	marker = list[-1]["Key"]
132	142	info("Listing continues after '%s'" % marker)
133		request = self.create_request("BUCKET_LIST", bucket = bucket, prefix = prefix, marker = marker)
	143	request = self.create_request("BUCKET_LIST", bucket = bucket,
	144	prefix = prefix,
	145	marker = self.urlencode_string(marker))
134	146	response = self.send_request(request)
135	147	list += _get_contents(response["data"])
136	148	response['list'] = list

146	158	body += "</LocationConstraint></CreateBucketConfiguration>"
147	159	debug("bucket_location: " + body)
148	160	headers["content-length"] = len(body)
	161	if self.config.acl_public:
	162	headers["x-amz-acl"] = "public-read"
149	163	request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
150	164	response = self.send_request(request, body)
151	165	return response

155	169	response = self.send_request(request)
156	170	return response
157	171
158		def bucket_info(self, bucket):
159		request = self.create_request("BUCKET_LIST", bucket = bucket, extra = "?location")
	172	def bucket_info(self, uri):
	173	request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location")
160	174	response = self.send_request(request)
161	175	response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"
162	176	return response

165	179	if not os.path.isfile(filename):
166	180	raise ParameterError("%s is not a regular file" % filename)
167	181	try:
168		file = open(filename, "r")
	182	file = open(filename, "rb")
169	183	size = os.stat(filename)[ST_SIZE]
170	184	except IOError, e:
171	185	raise ParameterError("%s: %s" % (filename, e.strerror))

173	187	if extra_headers:
174	188	headers.update(extra_headers)
175	189	headers["content-length"] = size
	190	content_type = None
	191	if self.config.guess_mime_type:
	192	content_type = mimetypes.guess_type(filename)[0]
	193	if not content_type:
	194	content_type = self.config.default_mime_type
	195	debug("Content-Type set to '%s'" % content_type)
	196	headers["content-type"] = content_type
176	197	if self.config.acl_public:
177	198	headers["x-amz-acl"] = "public-read"
178	199	request = self.create_request("OBJECT_PUT", bucket = bucket, object = object, headers = headers)

182	203
183	204	def object_get_file(self, bucket, object, filename):
184	205	try:
185		stream = open(filename, "w")
	206	stream = open(filename, "wb")
186	207	except IOError, e:
187	208	raise ParameterError("%s: %s" % (filename, e.strerror))
188	209	return self.object_get_stream(bucket, object, stream)

214	235	if uri.type != "s3":
215	236	raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
216	237	return self.object_delete(uri.bucket(), uri.object())
	238
	239	def object_info(self, uri):
	240	request = self.create_request("OBJECT_HEAD", bucket = uri.bucket(), object = uri.object())
	241	response = self.send_request(request)
	242	return response
	243
	244	def get_acl(self, uri):
	245	if uri.has_object():
	246	request = self.create_request("OBJECT_GET", bucket = uri.bucket(), object = uri.object(), extra = "?acl")
	247	else:
	248	request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl")
	249	acl = {}
	250	response = self.send_request(request)
	251	grants = getListFromXml(response['data'], "Grant")
	252	for grant in grants:
	253	if grant['Grantee'][0].has_key('DisplayName'):
	254	user = grant['Grantee'][0]['DisplayName']
	255	if grant['Grantee'][0].has_key('URI'):
	256	user = grant['Grantee'][0]['URI']
	257	if user == 'http://acs.amazonaws.com/groups/global/AllUsers':
	258	user = "anon"
	259	perm = grant['Permission']
	260	acl[user] = perm
	261	return acl
217	262
218	263	## Low level methods
219	264	def urlencode_string(self, string):

271	316	del(headers["date"])
272	317
273	318	if not headers.has_key("x-amz-date"):
274		headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime(time.time()))
	319	headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
275	320
276	321	method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"])
277	322	signature = self.sign_headers(method_string, resource, headers)

300	345	response["data"] = http_response.read()
301	346	debug("Response: " + str(response))
302	347	conn.close()
	348
	349	if response["status"] == 307:
	350	## RedirectPermanent
	351	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	352	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	353	self.set_hostname(redir_bucket, redir_hostname)
	354	info("Redirected to: %s" % (redir_hostname))
	355	return self.send_request(request, body)
	356
303	357	if response["status"] < 200 or response["status"] > 299:
304	358	raise S3Error(response)
305	359	return response

313	367	for header in headers.keys():
314	368	conn.putheader(header, str(headers[header]))
315	369	conn.endheaders()
	370	file.seek(0)
316	371	size_left = size_total = headers.get("content-length")
317	372	while (size_left > 0):
318	373	debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))

331	386	response["headers"] = convertTupleListToDict(http_response.getheaders())
332	387	response["data"] = http_response.read()
333	388	conn.close()
	389
	390	if response["status"] == 307:
	391	## RedirectPermanent
	392	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	393	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	394	self.set_hostname(redir_bucket, redir_hostname)
	395	info("Redirected to: %s" % (redir_hostname))
	396	return self.send_file(request, file)
	397
334	398	if response["status"] < 200 or response["status"] > 299:
335	399	raise S3Error(response)
336	400	return response

349	413	response["status"] = http_response.status
350	414	response["reason"] = http_response.reason
351	415	response["headers"] = convertTupleListToDict(http_response.getheaders())
	416
	417	if response["status"] == 307:
	418	## RedirectPermanent
	419	response['data'] = http_response.read()
	420	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	421	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	422	self.set_hostname(redir_bucket, redir_hostname)
	423	info("Redirected to: %s" % (redir_hostname))
	424	return self.recv_file(request, stream)
	425
352	426	if response["status"] < 200 or response["status"] > 299:
353	427	raise S3Error(response)
354	428

-1

S3/S3Uri.py less more

62	62	return "/".join(["s3:/", self._bucket, self._object])
63	63
64	64	def public_url(self):
65		return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object)
	65	return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object)
66	66
67	67	@staticmethod
68	68	def compose_uri(bucket, object = ""):

-3

S3/Utils.py less more

34	34	return retval
35	35
36	36	def parseNodes(nodes, xmlns = ""):
	37	## WARNING: Ignores text nodes from mixed xml/text.
	38	## For instance <tag1>some text<tag2>other text</tag2></tag1>
	39	## will be ignore "some text" node
37	40	retval = []
38	41	for node in nodes:
39	42	retval_item = {}
40	43	for child in node.getchildren():
41	44	name = stripTagXmlns(xmlns, child.tag)
42		retval_item[name] = node.findtext(".//%s" % child.tag)
43
	45	if child.getchildren():
	46	retval_item[name] = parseNodes([child], xmlns)
	47	else:
	48	retval_item[name] = node.findtext(".//%s" % child.tag)
44	49	retval.append(retval_item)
45	50	return retval
46	51

130	135
131	136	def hash_file_md5(filename):
132	137	h = md5.new()
133		f = open(filename, "r")
	138	f = open(filename, "rb")
134	139	h.update(f.read())
135	140	f.close()
136	141	return h.hexdigest()

+50

-24

s3cmd less more

157	157	else:
158	158	raise
159	159	output("Bucket '%s' removed" % uri.bucket())
160
161		def cmd_bucket_info(args):
162		uri = S3Uri(args[0])
163		if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
164		raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % args[0])
165		try:
166		s3 = S3(Config())
167		response = s3.bucket_info(uri.bucket())
168		except S3Error, e:
169		if S3.codes.has_key(e.info["Code"]):
170		error(S3.codes[e.info["Code"]] % uri.bucket())
171		return
172		else:
173		raise
174		output("Bucket '%s':" % uri.bucket())
175		output(" Location: %s" % response['bucket-location'])
176	160
177	161	def cmd_object_put(args):
178	162	s3 = S3(Config())

211	195	seq, total))
212	196	if Config().acl_public:
213	197	output("Public URL of the object is: %s" %
214		(uri.public_url()))
	198	(uri_final.public_url()))
215	199	if Config().encrypt and real_filename != file:
216	200	debug("Removing temporary encrypted file: %s" % real_filename)
217	201	os.remove(real_filename)

276	260	response = s3.object_delete_uri(uri)
277	261	output("Object %s deleted" % uri)
278	262
	263	def cmd_info(args):
	264	s3 = S3(Config())
	265
	266	while (len(args)):
	267	uri_arg = args.pop(0)
	268	uri = S3Uri(uri_arg)
	269	if uri.type != "s3" or not uri.has_bucket():
	270	raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)
	271
	272	try:
	273	if uri.has_object():
	274	info = s3.object_info(uri)
	275	output("%s (object):" % uri.uri())
	276	output(" File size: %s" % info['headers']['content-length'])
	277	output(" Last mod: %s" % info['headers']['last-modified'])
	278	output(" MIME type: %s" % info['headers']['content-type'])
	279	output(" MD5 sum: %s" % info['headers']['etag'].strip('"'))
	280	else:
	281	info = s3.bucket_info(uri)
	282	output("%s (bucket):" % uri.uri())
	283	output(" Location: %s" % info['bucket-location'])
	284	acl = s3.get_acl(uri)
	285	for user in acl.keys():
	286	output(" ACL: %s: %s" % (user, acl[user]))
	287	except S3Error, e:
	288	if S3.codes.has_key(e.info["Code"]):
	289	error(S3.codes[e.info["Code"]] % uri.bucket())
	290	return
	291	else:
	292	raise
	293
279	294	def cmd_sync(args):
280	295	def _build_attr_header(src):
281	296	attrs = {}
282	297	st = os.stat_result(os.stat(src))
283	298	for attr in cfg.preserve_attrs_list:
284	299	if attr == 'uname':
285		val = pwd.getpwuid(st.st_uid).pw_name
	300	try:
	301	val = pwd.getpwuid(st.st_uid).pw_name
	302	except KeyError:
	303	attr = "uid"
	304	val = st.st_uid
	305	warning("%s: Owner username not known. Storing UID=%d instead." % (src, val))
286	306	elif attr == 'gname':
287		val = grp.getgrgid(st.st_gid).gr_name
	307	try:
	308	val = grp.getgrgid(st.st_gid).gr_name
	309	except KeyError:
	310	attr = "gid"
	311	val = st.st_gid
	312	warning("%s: Owner groupname not known. Storing GID=%d instead." % (src, val))
288	313	else:
289	314	val = getattr(st, 'st_' + attr)
290	315	attrs[attr] = val

580	605	commands_list = [
581	606	{"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
582	607	{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
583		{"cmd":"ib", "label":"Bucket information", "param":"s3://BUCKET", "func":cmd_bucket_info, "argc":1},
584	608	{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
585	609	{"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0},
586	610	{"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2},
587	611	{"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1},
588	612	{"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1},
589	613	#{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1},
590		{"cmd":"sync", "label":"Synchronize a directory tree to S3 and back", "param":"LOCAL_DIR s3://BUCKET[/PREFIX]", "func":cmd_sync, "argc":2},
	614	{"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX]", "func":cmd_sync, "argc":2},
591	615	{"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0},
	616	{"cmd":"info", "label":"Get various information about Buckets or Objects", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1},
	617	#{"cmd":"setacl", "label":"Modify Access control list for Bucket or Object", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
592	618	]
593	619
594	620	def format_commands(progname):

635	661	optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
636	662	optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
637	663
638		optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings though.")
	664	optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though.")
639	665
640	666	optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.")
641	667	optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.")
642	668	optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
643		optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read by anyone.")
644		optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access by you only.")
	669	optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.")
	670	optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.")
645	671	optparser.add_option( "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
646	672	optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
647	673	optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")

+36

-6

s3cmd.1 less more

27	27	List all object in all buckets
28	28	.TP
29	29	\fBput\fR \fIFILE [FILE...] s3://BUCKET[/PREFIX]\fR
30		Put file into bucket
	30	Put file into bucket (i.e. upload to S3)
31	31	.TP
32	32	\fBget\fR \fIs3://BUCKET/OBJECT LOCAL_FILE\fR
33		Get file from bucket
	33	Get file from bucket (i.e. download from S3)
34	34	.TP
35	35	\fBdel\fR \fIs3://BUCKET/OBJECT\fR
36	36	Delete file from bucket
	37	.TP
	38	\fBsync\fR \fILOCAL_DIR s3://BUCKET[/PREFIX]\fR
	39	Synchronize a directory tree to S3
	40	.TP
	41	\fBinfo\fR \fIs3://BUCKET[/OBJECT]\fR
	42	Get various information about a Bucket or Object
37	43	.TP
38	44	\fBdu\fR \fI[s3://BUCKET[/PREFIX]]\fR
39	45	Disk usage \- amount of data stored in S3

60	66	Dump current configuration after parsing config files
61	67	and command line options and exit.
62	68	.PP
63		Most of the following options can have a default value set
64		in the above specified config file.
	69	Most options can have a default value set in the above specified config file.
	70	.PP
	71	Options specific to \fBsync\fR command:
	72	.TP
	73	\fB\-\-delete\-removed\fR
	74	Delete remote objects with no corresponding local file
	75	.TP
	76	\fB\-\-no\-delete\-removed\fR
	77	Don't delete remote objects. Default for 'sync' command.
	78	.TP
	79	\fB\-p\fR, \fB\-\-preserve\fR
	80	Preserve filesystem attributes (mode, ownership, timestamps). Default for 'sync' command.
	81	.TP
	82	\fB\-\-no\-preserve\fR
	83	Don't store filesystem attributes with uploaded files.
	84	.TP
	85	\fB\-n\fR, \fB\-\-dry\-run\fR
	86	Only show what would be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though.
	87	.PP
	88	Options common for all commands (where it makes sense indeed):
65	89	.TP
66	90	\fB\-f\fR, \fB\-\-force\fR
67	91	Force overwrite and other dangerous operations.
68	92	.TP
69	93	\fB\-P\fR, \fB\-\-acl\-public\fR
70		Store objects with permissions allowing read by anyone.
	94	Store objects with permissions allowing read for anyone.
	95	.TP
	96	\fB\-\-acl\-private\fR
	97	Store objects with default ACL allowing access for you only.
	98	.TP
	99	\fB\-\-bucket\-location\fR=BUCKET_LOCATION
	100	Specify datacentre where to create the bucket. Possible values are \fIUS\fR (default) or \fIEU\fR.
71	101	.TP
72	102	\fB\-e\fR, \fB\-\-encrypt\fR
73	103	Use GPG encryption to protect stored objects from unauthorized access.

106	136	Report bugs to
107	137	.I s3tools\-general@lists.sourceforge.net
108	138	.SH COPYRIGHT
109		Copyright \(co 2007 Michal Ludvig
	139	Copyright \(co 2007,2008 Michal Ludvig <http://www.logix.cz/michal>
110	140	.br
111	141	This is free software. You may redistribute copies of it under the terms of
112	142	the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>.