Commit 4df819fc95b5fc4fb0b218d2b40bb7637d236210 - s3cmd

Imported Upstream version 1.1.0~beta3 Gianfranco Costamagna 9 years ago

20 changed file(s) with 5178 addition(s) and 4476 deletion(s). Raw diff Collapse all Expand all

+27

-0

NEWS less more

	0	s3cmd 1.1.0 - ???
	1	===========
	2	* MultiPart upload enabled for both [put] and [sync]. Default chunk
	3	size is 15MB.
	4	* CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo].
	5	* Increased socket_timeout from 10 secs to 5 mins.
	6	* Added "Static WebSite" support [ws-create / ws-delete / ws-info]
	7	(contributed by Jens Braeuer)
	8	* Force MIME type with --mime-type=abc/xyz, also --guess-mime-type
	9	is now on by default, -M is no longer shorthand for --guess-mime-type
	10	* Allow parameters in MIME types, for example:
	11	--mime-type="text/plain; charset=utf-8"
	12	* MIME type can be guessed by python-magic which is a lot better than
	13	relying on the extension. Contributed by Karsten Sperling.
	14	* Support for environment variables as config values. For instance
	15	in ~/.s3cmd put "access_key=$S3_ACCESS_KEY". Contributed by Ori Bar.
	16	* Support for --configure checking access to a specific bucket instead
	17	of listing all buckets.
	18	Listing buckets requires the S3 ListAllMyBuckets permission which
	19	is typically not available to delegated IAM accounts. With this change,
	20	s3cmd --configure accepts an (optional) bucket uri as a parameter
	21	and if it's provided, the access check will just verify access to this
	22	bucket individually. Contributed by Mike Repass.
	23	* Allow STDOUT as a destination even for downloading multiple files.
	24	They will be output one after another without any delimiters!
	25	Contributed by Rob Wills.
	26
0	27	s3cmd 1.0.0 - 2011-01-18
1	28	===========
2	29	* [sync] now supports --no-check-md5

-2

PKG-INFO less more

0	0	Metadata-Version: 1.0
1	1	Name: s3cmd
2		Version: 1.0.0
	2	Version: 1.1.0-beta3
3	3	Summary: Command line tool for managing Amazon S3 and CloudFront services
4	4	Home-page: http://s3tools.org
5	5	Author: Michal Ludvig

16	16
17	17	Authors:
18	18	--------
19		Michal Ludvig <michal@logix.cz>
	19	Michal Ludvig <michal@logix.cz>
20	20
21	21	Platform: UNKNOWN

+199

-197

S3/ACL.py less more

5	5	from Utils import getTreeFromXml
6	6
7	7	try:
8		import xml.etree.ElementTree as ET
	8	import xml.etree.ElementTree as ET
9	9	except ImportError:
10		import elementtree.ElementTree as ET
	10	import elementtree.ElementTree as ET
11	11
12	12	class Grantee(object):
13		ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers"
14		LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery"
15
16		def __init__(self):
17		self.xsi_type = None
18		self.tag = None
19		self.name = None
20		self.display_name = None
21		self.permission = None
22
23		def __repr__(self):
24		return 'Grantee("%(tag)s", "%(name)s", "%(permission)s")' % {
25		"tag" : self.tag,
26		"name" : self.name,
27		"permission" : self.permission
28		}
29
30		def isAllUsers(self):
31		return self.tag == "URI" and self.name == Grantee.ALL_USERS_URI
32
33		def isAnonRead(self):
34		return self.isAllUsers() and (self.permission == "READ" or self.permission == "FULL_CONTROL")
35
36		def getElement(self):
37		el = ET.Element("Grant")
38		grantee = ET.SubElement(el, "Grantee", {
39		'xmlns:xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
40		'xsi:type' : self.xsi_type
41		})
42		name = ET.SubElement(grantee, self.tag)
43		name.text = self.name
44		permission = ET.SubElement(el, "Permission")
45		permission.text = self.permission
46		return el
	13	ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers"
	14	LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery"
	15
	16	def __init__(self):
	17	self.xsi_type = None
	18	self.tag = None
	19	self.name = None
	20	self.display_name = None
	21	self.permission = None
	22
	23	def __repr__(self):
	24	return 'Grantee("%(tag)s", "%(name)s", "%(permission)s")' % {
	25	"tag" : self.tag,
	26	"name" : self.name,
	27	"permission" : self.permission
	28	}
	29
	30	def isAllUsers(self):
	31	return self.tag == "URI" and self.name == Grantee.ALL_USERS_URI
	32
	33	def isAnonRead(self):
	34	return self.isAllUsers() and (self.permission == "READ" or self.permission == "FULL_CONTROL")
	35
	36	def getElement(self):
	37	el = ET.Element("Grant")
	38	grantee = ET.SubElement(el, "Grantee", {
	39	'xmlns:xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
	40	'xsi:type' : self.xsi_type
	41	})
	42	name = ET.SubElement(grantee, self.tag)
	43	name.text = self.name
	44	permission = ET.SubElement(el, "Permission")
	45	permission.text = self.permission
	46	return el
47	47
48	48	class GranteeAnonRead(Grantee):
49		def __init__(self):
50		Grantee.__init__(self)
51		self.xsi_type = "Group"
52		self.tag = "URI"
53		self.name = Grantee.ALL_USERS_URI
54		self.permission = "READ"
	49	def __init__(self):
	50	Grantee.__init__(self)
	51	self.xsi_type = "Group"
	52	self.tag = "URI"
	53	self.name = Grantee.ALL_USERS_URI
	54	self.permission = "READ"
55	55
56	56	class GranteeLogDelivery(Grantee):
57		def __init__(self, permission):
58		"""
59		permission must be either READ_ACP or WRITE
60		"""
61		Grantee.__init__(self)
62		self.xsi_type = "Group"
63		self.tag = "URI"
64		self.name = Grantee.LOG_DELIVERY_URI
65		self.permission = permission
	57	def __init__(self, permission):
	58	"""
	59	permission must be either READ_ACP or WRITE
	60	"""
	61	Grantee.__init__(self)
	62	self.xsi_type = "Group"
	63	self.tag = "URI"
	64	self.name = Grantee.LOG_DELIVERY_URI
	65	self.permission = permission
66	66
67	67	class ACL(object):
68		EMPTY_ACL = "<AccessControlPolicy><Owner><ID></ID></Owner><AccessControlList></AccessControlList></AccessControlPolicy>"
69
70		def __init__(self, xml = None):
71		if not xml:
72		xml = ACL.EMPTY_ACL
73
74		self.grantees = []
75		self.owner_id = ""
76		self.owner_nick = ""
77
78		tree = getTreeFromXml(xml)
79		self.parseOwner(tree)
80		self.parseGrants(tree)
81
82		def parseOwner(self, tree):
83		self.owner_id = tree.findtext(".//Owner//ID")
84		self.owner_nick = tree.findtext(".//Owner//DisplayName")
85
86		def parseGrants(self, tree):
87		for grant in tree.findall(".//Grant"):
88		grantee = Grantee()
89		g = grant.find(".//Grantee")
90		grantee.xsi_type = g.attrib['{http://www.w3.org/2001/XMLSchema-instance}type']
91		grantee.permission = grant.find('Permission').text
92		for el in g:
93		if el.tag == "DisplayName":
94		grantee.display_name = el.text
95		else:
96		grantee.tag = el.tag
97		grantee.name = el.text
98		self.grantees.append(grantee)
99
100		def getGrantList(self):
101		acl = []
102		for grantee in self.grantees:
103		if grantee.display_name:
104		user = grantee.display_name
105		elif grantee.isAllUsers():
106		user = "anon"
107		else:
108		user = grantee.name
109		acl.append({'grantee': user, 'permission': grantee.permission})
110		return acl
111
112		def getOwner(self):
113		return { 'id' : self.owner_id, 'nick' : self.owner_nick }
114
115		def isAnonRead(self):
116		for grantee in self.grantees:
117		if grantee.isAnonRead():
118		return True
119		return False
120
121		def grantAnonRead(self):
122		if not self.isAnonRead():
123		self.appendGrantee(GranteeAnonRead())
124
125		def revokeAnonRead(self):
126		self.grantees = [g for g in self.grantees if not g.isAnonRead()]
127
128		def appendGrantee(self, grantee):
129		self.grantees.append(grantee)
130
131		def hasGrant(self, name, permission):
132		name = name.lower()
133		permission = permission.upper()
134
135		for grantee in self.grantees:
136		if grantee.name.lower() == name:
137		if grantee.permission == "FULL_CONTROL":
138		return True
139		elif grantee.permission.upper() == permission:
140		return True
141
142		return False;
143
144		def grant(self, name, permission):
145		if self.hasGrant(name, permission):
146		return
147
148		name = name.lower()
149		permission = permission.upper()
150
151		if "ALL" == permission:
152		permission = "FULL_CONTROL"
153
154		if "FULL_CONTROL" == permission:
155		self.revoke(name, "ALL")
156
157		grantee = Grantee()
158		grantee.name = name
159		grantee.permission = permission
160
161		if name.find('@') <= -1: # ultra lame attempt to differenciate emails id from canonical ids
162		grantee.xsi_type = "CanonicalUser"
163		grantee.tag = "ID"
164		else:
165		grantee.xsi_type = "AmazonCustomerByEmail"
166		grantee.tag = "EmailAddress"
167
168		self.appendGrantee(grantee)
169
170
171		def revoke(self, name, permission):
172		name = name.lower()
173		permission = permission.upper()
174
175		if "ALL" == permission:
176		self.grantees = [g for g in self.grantees if not g.name.lower() == name]
177		else:
178		self.grantees = [g for g in self.grantees if not (g.name.lower() == name and g.permission.upper() == permission)]
179
180
181		def __str__(self):
182		tree = getTreeFromXml(ACL.EMPTY_ACL)
183		tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/"
184		owner = tree.find(".//Owner//ID")
185		owner.text = self.owner_id
186		acl = tree.find(".//AccessControlList")
187		for grantee in self.grantees:
188		acl.append(grantee.getElement())
189		return ET.tostring(tree)
	68	EMPTY_ACL = "<AccessControlPolicy><Owner><ID></ID></Owner><AccessControlList></AccessControlList></AccessControlPolicy>"
	69
	70	def __init__(self, xml = None):
	71	if not xml:
	72	xml = ACL.EMPTY_ACL
	73
	74	self.grantees = []
	75	self.owner_id = ""
	76	self.owner_nick = ""
	77
	78	tree = getTreeFromXml(xml)
	79	self.parseOwner(tree)
	80	self.parseGrants(tree)
	81
	82	def parseOwner(self, tree):
	83	self.owner_id = tree.findtext(".//Owner//ID")
	84	self.owner_nick = tree.findtext(".//Owner//DisplayName")
	85
	86	def parseGrants(self, tree):
	87	for grant in tree.findall(".//Grant"):
	88	grantee = Grantee()
	89	g = grant.find(".//Grantee")
	90	grantee.xsi_type = g.attrib['{http://www.w3.org/2001/XMLSchema-instance}type']
	91	grantee.permission = grant.find('Permission').text
	92	for el in g:
	93	if el.tag == "DisplayName":
	94	grantee.display_name = el.text
	95	else:
	96	grantee.tag = el.tag
	97	grantee.name = el.text
	98	self.grantees.append(grantee)
	99
	100	def getGrantList(self):
	101	acl = []
	102	for grantee in self.grantees:
	103	if grantee.display_name:
	104	user = grantee.display_name
	105	elif grantee.isAllUsers():
	106	user = "anon"
	107	else:
	108	user = grantee.name
	109	acl.append({'grantee': user, 'permission': grantee.permission})
	110	return acl
	111
	112	def getOwner(self):
	113	return { 'id' : self.owner_id, 'nick' : self.owner_nick }
	114
	115	def isAnonRead(self):
	116	for grantee in self.grantees:
	117	if grantee.isAnonRead():
	118	return True
	119	return False
	120
	121	def grantAnonRead(self):
	122	if not self.isAnonRead():
	123	self.appendGrantee(GranteeAnonRead())
	124
	125	def revokeAnonRead(self):
	126	self.grantees = [g for g in self.grantees if not g.isAnonRead()]
	127
	128	def appendGrantee(self, grantee):
	129	self.grantees.append(grantee)
	130
	131	def hasGrant(self, name, permission):
	132	name = name.lower()
	133	permission = permission.upper()
	134
	135	for grantee in self.grantees:
	136	if grantee.name.lower() == name:
	137	if grantee.permission == "FULL_CONTROL":
	138	return True
	139	elif grantee.permission.upper() == permission:
	140	return True
	141
	142	return False;
	143
	144	def grant(self, name, permission):
	145	if self.hasGrant(name, permission):
	146	return
	147
	148	name = name.lower()
	149	permission = permission.upper()
	150
	151	if "ALL" == permission:
	152	permission = "FULL_CONTROL"
	153
	154	if "FULL_CONTROL" == permission:
	155	self.revoke(name, "ALL")
	156
	157	grantee = Grantee()
	158	grantee.name = name
	159	grantee.permission = permission
	160
	161	if name.find('@') <= -1: # ultra lame attempt to differenciate emails id from canonical ids
	162	grantee.xsi_type = "CanonicalUser"
	163	grantee.tag = "ID"
	164	else:
	165	grantee.xsi_type = "AmazonCustomerByEmail"
	166	grantee.tag = "EmailAddress"
	167
	168	self.appendGrantee(grantee)
	169
	170
	171	def revoke(self, name, permission):
	172	name = name.lower()
	173	permission = permission.upper()
	174
	175	if "ALL" == permission:
	176	self.grantees = [g for g in self.grantees if not g.name.lower() == name]
	177	else:
	178	self.grantees = [g for g in self.grantees if not (g.name.lower() == name and g.permission.upper() == permission)]
	179
	180
	181	def __str__(self):
	182	tree = getTreeFromXml(ACL.EMPTY_ACL)
	183	tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/"
	184	owner = tree.find(".//Owner//ID")
	185	owner.text = self.owner_id
	186	acl = tree.find(".//AccessControlList")
	187	for grantee in self.grantees:
	188	acl.append(grantee.getElement())
	189	return ET.tostring(tree)
190	190
191	191	if __name__ == "__main__":
192		xml = """<?xml version="1.0" encoding="UTF-8"?>
	192	xml = """<?xml version="1.0" encoding="UTF-8"?>
193	193	<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
194	194	<Owner>
195		<ID>12345678901234567890</ID>
196		<DisplayName>owner-nickname</DisplayName>
	195	<ID>12345678901234567890</ID>
	196	<DisplayName>owner-nickname</DisplayName>
197	197	</Owner>
198	198	<AccessControlList>
199		<Grant>
200		<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
201		<ID>12345678901234567890</ID>
202		<DisplayName>owner-nickname</DisplayName>
203		</Grantee>
204		<Permission>FULL_CONTROL</Permission>
205		</Grant>
206		<Grant>
207		<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
208		<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
209		</Grantee>
210		<Permission>READ</Permission>
211		</Grant>
	199	<Grant>
	200	<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
	201	<ID>12345678901234567890</ID>
	202	<DisplayName>owner-nickname</DisplayName>
	203	</Grantee>
	204	<Permission>FULL_CONTROL</Permission>
	205	</Grant>
	206	<Grant>
	207	<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
	208	<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
	209	</Grantee>
	210	<Permission>READ</Permission>
	211	</Grant>
212	212	</AccessControlList>
213	213	</AccessControlPolicy>
214		"""
215		acl = ACL(xml)
216		print "Grants:", acl.getGrantList()
217		acl.revokeAnonRead()
218		print "Grants:", acl.getGrantList()
219		acl.grantAnonRead()
220		print "Grants:", acl.getGrantList()
221		print acl
	214	"""
	215	acl = ACL(xml)
	216	print "Grants:", acl.getGrantList()
	217	acl.revokeAnonRead()
	218	print "Grants:", acl.getGrantList()
	219	acl.grantAnonRead()
	220	print "Grants:", acl.getGrantList()
	221	print acl
	222
	223	# vim:et:ts=4:sts=4:ai

+68

-66

S3/AccessLog.py less more

8	8	from ACL import GranteeAnonRead
9	9
10	10	try:
11		import xml.etree.ElementTree as ET
	11	import xml.etree.ElementTree as ET
12	12	except ImportError:
13		import elementtree.ElementTree as ET
	13	import elementtree.ElementTree as ET
14	14
15	15	__all__ = []
16	16	class AccessLog(object):
17		LOG_DISABLED = "<BucketLoggingStatus></BucketLoggingStatus>"
18		LOG_TEMPLATE = "<LoggingEnabled><TargetBucket></TargetBucket><TargetPrefix></TargetPrefix></LoggingEnabled>"
	17	LOG_DISABLED = "<BucketLoggingStatus></BucketLoggingStatus>"
	18	LOG_TEMPLATE = "<LoggingEnabled><TargetBucket></TargetBucket><TargetPrefix></TargetPrefix></LoggingEnabled>"
19	19
20		def __init__(self, xml = None):
21		if not xml:
22		xml = self.LOG_DISABLED
23		self.tree = getTreeFromXml(xml)
24		self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01"
25
26		def isLoggingEnabled(self):
27		return bool(self.tree.find(".//LoggingEnabled"))
	20	def __init__(self, xml = None):
	21	if not xml:
	22	xml = self.LOG_DISABLED
	23	self.tree = getTreeFromXml(xml)
	24	self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01"
28	25
29		def disableLogging(self):
30		el = self.tree.find(".//LoggingEnabled")
31		if el:
32		self.tree.remove(el)
33
34		def enableLogging(self, target_prefix_uri):
35		el = self.tree.find(".//LoggingEnabled")
36		if not el:
37		el = getTreeFromXml(self.LOG_TEMPLATE)
38		self.tree.append(el)
39		el.find(".//TargetBucket").text = target_prefix_uri.bucket()
40		el.find(".//TargetPrefix").text = target_prefix_uri.object()
	26	def isLoggingEnabled(self):
	27	return bool(self.tree.find(".//LoggingEnabled"))
41	28
42		def targetPrefix(self):
43		if self.isLoggingEnabled():
44		el = self.tree.find(".//LoggingEnabled")
45		target_prefix = "s3://%s/%s" % (
46		self.tree.find(".//LoggingEnabled//TargetBucket").text,
47		self.tree.find(".//LoggingEnabled//TargetPrefix").text)
48		return S3Uri.S3Uri(target_prefix)
49		else:
50		return ""
	29	def disableLogging(self):
	30	el = self.tree.find(".//LoggingEnabled")
	31	if el:
	32	self.tree.remove(el)
51	33
52		def setAclPublic(self, acl_public):
53		le = self.tree.find(".//LoggingEnabled")
54		if not le:
55		raise ParameterError("Logging not enabled, can't set default ACL for logs")
56		tg = le.find(".//TargetGrants")
57		if not acl_public:
58		if not tg:
59		## All good, it's not been there
60		return
61		else:
62		le.remove(tg)
63		else: # acl_public == True
64		anon_read = GranteeAnonRead().getElement()
65		if not tg:
66		tg = ET.SubElement(le, "TargetGrants")
67		## What if TargetGrants already exists? We should check if
68		## AnonRead is there before appending a new one. Later...
69		tg.append(anon_read)
	34	def enableLogging(self, target_prefix_uri):
	35	el = self.tree.find(".//LoggingEnabled")
	36	if not el:
	37	el = getTreeFromXml(self.LOG_TEMPLATE)
	38	self.tree.append(el)
	39	el.find(".//TargetBucket").text = target_prefix_uri.bucket()
	40	el.find(".//TargetPrefix").text = target_prefix_uri.object()
70	41
71		def isAclPublic(self):
72		raise NotImplementedError()
	42	def targetPrefix(self):
	43	if self.isLoggingEnabled():
	44	el = self.tree.find(".//LoggingEnabled")
	45	target_prefix = "s3://%s/%s" % (
	46	self.tree.find(".//LoggingEnabled//TargetBucket").text,
	47	self.tree.find(".//LoggingEnabled//TargetPrefix").text)
	48	return S3Uri.S3Uri(target_prefix)
	49	else:
	50	return ""
73	51
74		def __str__(self):
75		return ET.tostring(self.tree)
	52	def setAclPublic(self, acl_public):
	53	le = self.tree.find(".//LoggingEnabled")
	54	if not le:
	55	raise ParameterError("Logging not enabled, can't set default ACL for logs")
	56	tg = le.find(".//TargetGrants")
	57	if not acl_public:
	58	if not tg:
	59	## All good, it's not been there
	60	return
	61	else:
	62	le.remove(tg)
	63	else: # acl_public == True
	64	anon_read = GranteeAnonRead().getElement()
	65	if not tg:
	66	tg = ET.SubElement(le, "TargetGrants")
	67	## What if TargetGrants already exists? We should check if
	68	## AnonRead is there before appending a new one. Later...
	69	tg.append(anon_read)
	70
	71	def isAclPublic(self):
	72	raise NotImplementedError()
	73
	74	def __str__(self):
	75	return ET.tostring(self.tree)
76	76	__all__.append("AccessLog")
77	77
78	78	if __name__ == "__main__":
79		from S3Uri import S3Uri
80		log = AccessLog()
81		print log
82		log.enableLogging(S3Uri("s3://targetbucket/prefix/log-"))
83		print log
84		log.setAclPublic(True)
85		print log
86		log.setAclPublic(False)
87		print log
88		log.disableLogging()
89		print log
	79	from S3Uri import S3Uri
	80	log = AccessLog()
	81	print log
	82	log.enableLogging(S3Uri("s3://targetbucket/prefix/log-"))
	83	print log
	84	log.setAclPublic(True)
	85	print log
	86	log.setAclPublic(False)
	87	print log
	88	log.disableLogging()
	89	print log
	90
	91	# vim:et:ts=4:sts=4:ai

+30

-28

S3/BidirMap.py less more

3	3	## License: GPL Version 2
4	4
5	5	class BidirMap(object):
6		def __init__(self, **map):
7		self.k2v = {}
8		self.v2k = {}
9		for key in map:
10		self.__setitem__(key, map[key])
	6	def __init__(self, **map):
	7	self.k2v = {}
	8	self.v2k = {}
	9	for key in map:
	10	self.__setitem__(key, map[key])
11	11
12		def __setitem__(self, key, value):
13		if self.v2k.has_key(value):
14		if self.v2k[value] != key:
15		raise KeyError("Value '"+str(value)+"' already in use with key '"+str(self.v2k[value])+"'")
16		try:
17		del(self.v2k[self.k2v[key]])
18		except KeyError:
19		pass
20		self.k2v[key] = value
21		self.v2k[value] = key
	12	def __setitem__(self, key, value):
	13	if self.v2k.has_key(value):
	14	if self.v2k[value] != key:
	15	raise KeyError("Value '"+str(value)+"' already in use with key '"+str(self.v2k[value])+"'")
	16	try:
	17	del(self.v2k[self.k2v[key]])
	18	except KeyError:
	19	pass
	20	self.k2v[key] = value
	21	self.v2k[value] = key
22	22
23		def __getitem__(self, key):
24		return self.k2v[key]
	23	def __getitem__(self, key):
	24	return self.k2v[key]
25	25
26		def __str__(self):
27		return self.v2k.__str__()
	26	def __str__(self):
	27	return self.v2k.__str__()
28	28
29		def getkey(self, value):
30		return self.v2k[value]
31
32		def getvalue(self, key):
33		return self.k2v[key]
	29	def getkey(self, value):
	30	return self.v2k[value]
34	31
35		def keys(self):
36		return [key for key in self.k2v]
	32	def getvalue(self, key):
	33	return self.k2v[key]
37	34
38		def values(self):
39		return [value for value in self.v2k]
	35	def keys(self):
	36	return [key for key in self.k2v]
	37
	38	def values(self):
	39	return [value for value in self.v2k]
	40
	41	# vim:et:ts=4:sts=4:ai

+708

-509

S3/CloudFront.py less more

5	5	import sys
6	6	import time
7	7	import httplib
	8	import random
	9	from datetime import datetime
8	10	from logging import debug, info, warning, error
9	11
10	12	try:
11		import xml.etree.ElementTree as ET
	13	import xml.etree.ElementTree as ET
12	14	except ImportError:
13		import elementtree.ElementTree as ET
	15	import elementtree.ElementTree as ET
14	16
15	17	from Config import Config
16	18	from Exceptions import *
17	19	from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string, getBucketFromHostname, getHostnameFromBucket
18	20	from S3Uri import S3Uri, S3UriS3
	21	from FileLists import fetch_remote_list
	22
	23	cloudfront_api_version = "2010-11-01"
	24	cloudfront_resource = "/%(api_ver)s/distribution" % { 'api_ver' : cloudfront_api_version }
19	25
20	26	def output(message):
21		sys.stdout.write(message + "\n")
	27	sys.stdout.write(message + "\n")
22	28
23	29	def pretty_output(label, message):
24		#label = ("%s " % label).ljust(20, ".")
25		label = ("%s:" % label).ljust(15)
26		output("%s %s" % (label, message))
	30	#label = ("%s " % label).ljust(20, ".")
	31	label = ("%s:" % label).ljust(15)
	32	output("%s %s" % (label, message))
27	33
28	34	class DistributionSummary(object):
29		## Example:
30		##
31		## <DistributionSummary>
32		## <Id>1234567890ABC</Id>
33		## <Status>Deployed</Status>
34		## <LastModifiedTime>2009-01-16T11:49:02.189Z</LastModifiedTime>
35		## <DomainName>blahblahblah.cloudfront.net</DomainName>
36		## <Origin>example.bucket.s3.amazonaws.com</Origin>
37		## <Enabled>true</Enabled>
38		## </DistributionSummary>
39
40		def __init__(self, tree):
41		if tree.tag != "DistributionSummary":
42		raise ValueError("Expected <DistributionSummary /> xml, got: <%s />" % tree.tag)
43		self.parse(tree)
44
45		def parse(self, tree):
46		self.info = getDictFromTree(tree)
47		self.info['Enabled'] = (self.info['Enabled'].lower() == "true")
48
49		def uri(self):
50		return S3Uri("cf://%s" % self.info['Id'])
	35	## Example:
	36	##
	37	## <DistributionSummary>
	38	## <Id>1234567890ABC</Id>
	39	## <Status>Deployed</Status>
	40	## <LastModifiedTime>2009-01-16T11:49:02.189Z</LastModifiedTime>
	41	## <DomainName>blahblahblah.cloudfront.net</DomainName>
	42	## <S3Origin>
	43	## <DNSName>example.bucket.s3.amazonaws.com</DNSName>
	44	## </S3Origin>
	45	## <CNAME>cdn.example.com</CNAME>
	46	## <CNAME>img.example.com</CNAME>
	47	## <Comment>What Ever</Comment>
	48	## <Enabled>true</Enabled>
	49	## </DistributionSummary>
	50
	51	def __init__(self, tree):
	52	if tree.tag != "DistributionSummary":
	53	raise ValueError("Expected <DistributionSummary /> xml, got: <%s />" % tree.tag)
	54	self.parse(tree)
	55
	56	def parse(self, tree):
	57	self.info = getDictFromTree(tree)
	58	self.info['Enabled'] = (self.info['Enabled'].lower() == "true")
	59	if self.info.has_key("CNAME") and type(self.info['CNAME']) != list:
	60	self.info['CNAME'] = [self.info['CNAME']]
	61
	62	def uri(self):
	63	return S3Uri("cf://%s" % self.info['Id'])
51	64
52	65	class DistributionList(object):
53		## Example:
54		##
55		## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2010-07-15/">
56		## <Marker />
57		## <MaxItems>100</MaxItems>
58		## <IsTruncated>false</IsTruncated>
59		## <DistributionSummary>
60		## ... handled by DistributionSummary() class ...
61		## </DistributionSummary>
62		## </DistributionList>
63
64		def __init__(self, xml):
65		tree = getTreeFromXml(xml)
66		if tree.tag != "DistributionList":
67		raise ValueError("Expected <DistributionList /> xml, got: <%s />" % tree.tag)
68		self.parse(tree)
69
70		def parse(self, tree):
71		self.info = getDictFromTree(tree)
72		## Normalise some items
73		self.info['IsTruncated'] = (self.info['IsTruncated'].lower() == "true")
74
75		self.dist_summs = []
76		for dist_summ in tree.findall(".//DistributionSummary"):
77		self.dist_summs.append(DistributionSummary(dist_summ))
	66	## Example:
	67	##
	68	## <DistributionList xmlns="http://cloudfront.amazonaws.com/doc/2010-07-15/">
	69	## <Marker />
	70	## <MaxItems>100</MaxItems>
	71	## <IsTruncated>false</IsTruncated>
	72	## <DistributionSummary>
	73	## ... handled by DistributionSummary() class ...
	74	## </DistributionSummary>
	75	## </DistributionList>
	76
	77	def __init__(self, xml):
	78	tree = getTreeFromXml(xml)
	79	if tree.tag != "DistributionList":
	80	raise ValueError("Expected <DistributionList /> xml, got: <%s />" % tree.tag)
	81	self.parse(tree)
	82
	83	def parse(self, tree):
	84	self.info = getDictFromTree(tree)
	85	## Normalise some items
	86	self.info['IsTruncated'] = (self.info['IsTruncated'].lower() == "true")
	87
	88	self.dist_summs = []
	89	for dist_summ in tree.findall(".//DistributionSummary"):
	90	self.dist_summs.append(DistributionSummary(dist_summ))
78	91
79	92	class Distribution(object):
80		## Example:
81		##
82		## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-07-15/">
83		## <Id>1234567890ABC</Id>
84		## <Status>InProgress</Status>
85		## <LastModifiedTime>2009-01-16T13:07:11.319Z</LastModifiedTime>
86		## <DomainName>blahblahblah.cloudfront.net</DomainName>
87		## <DistributionConfig>
88		## ... handled by DistributionConfig() class ...
89		## </DistributionConfig>
90		## </Distribution>
91
92		def __init__(self, xml):
93		tree = getTreeFromXml(xml)
94		if tree.tag != "Distribution":
95		raise ValueError("Expected <Distribution /> xml, got: <%s />" % tree.tag)
96		self.parse(tree)
97
98		def parse(self, tree):
99		self.info = getDictFromTree(tree)
100		## Normalise some items
101		self.info['LastModifiedTime'] = dateS3toPython(self.info['LastModifiedTime'])
102
103		self.info['DistributionConfig'] = DistributionConfig(tree = tree.find(".//DistributionConfig"))
104
105		def uri(self):
106		return S3Uri("cf://%s" % self.info['Id'])
	93	## Example:
	94	##
	95	## <Distribution xmlns="http://cloudfront.amazonaws.com/doc/2010-07-15/">
	96	## <Id>1234567890ABC</Id>
	97	## <Status>InProgress</Status>
	98	## <LastModifiedTime>2009-01-16T13:07:11.319Z</LastModifiedTime>
	99	## <DomainName>blahblahblah.cloudfront.net</DomainName>
	100	## <DistributionConfig>
	101	## ... handled by DistributionConfig() class ...
	102	## </DistributionConfig>
	103	## </Distribution>
	104
	105	def __init__(self, xml):
	106	tree = getTreeFromXml(xml)
	107	if tree.tag != "Distribution":
	108	raise ValueError("Expected <Distribution /> xml, got: <%s />" % tree.tag)
	109	self.parse(tree)
	110
	111	def parse(self, tree):
	112	self.info = getDictFromTree(tree)
	113	## Normalise some items
	114	self.info['LastModifiedTime'] = dateS3toPython(self.info['LastModifiedTime'])
	115
	116	self.info['DistributionConfig'] = DistributionConfig(tree = tree.find(".//DistributionConfig"))
	117
	118	def uri(self):
	119	return S3Uri("cf://%s" % self.info['Id'])
107	120
108	121	class DistributionConfig(object):
109		## Example:
110		##
111		## <DistributionConfig>
112		## <Origin>somebucket.s3.amazonaws.com</Origin>
113		## <CallerReference>s3://somebucket/</CallerReference>
114		## <Comment>http://somebucket.s3.amazonaws.com/</Comment>
115		## <Enabled>true</Enabled>
116		## <Logging>
117		## <Bucket>bu.ck.et</Bucket>
118		## <Prefix>/cf-somebucket/</Prefix>
119		## </Logging>
120		## </DistributionConfig>
121
122		EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"
123		xmlns = "http://cloudfront.amazonaws.com/doc/2010-07-15/"
124		def __init__(self, xml = None, tree = None):
125		if not xml:
126		xml = DistributionConfig.EMPTY_CONFIG
127
128		if not tree:
129		tree = getTreeFromXml(xml)
130
131		if tree.tag != "DistributionConfig":
132		raise ValueError("Expected <DistributionConfig /> xml, got: <%s />" % tree.tag)
133		self.parse(tree)
134
135		def parse(self, tree):
136		self.info = getDictFromTree(tree)
137		self.info['Enabled'] = (self.info['Enabled'].lower() == "true")
138		if not self.info.has_key("CNAME"):
139		self.info['CNAME'] = []
140		if type(self.info['CNAME']) != list:
141		self.info['CNAME'] = [self.info['CNAME']]
142		self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']]
143		if not self.info.has_key("Comment"):
144		self.info['Comment'] = ""
145		if not self.info.has_key("DefaultRootObject"):
146		self.info['DefaultRootObject'] = ""
147		## Figure out logging - complex node not parsed by getDictFromTree()
148		logging_nodes = tree.findall(".//Logging")
149		if logging_nodes:
150		logging_dict = getDictFromTree(logging_nodes[0])
151		logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket'])
152		if not success:
153		warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket'])
154		self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict)
155		else:
156		self.info['Logging'] = None
157
158		def __str__(self):
159		tree = ET.Element("DistributionConfig")
160		tree.attrib['xmlns'] = DistributionConfig.xmlns
161
162		## Retain the order of the following calls!
163		appendXmlTextNode("Origin", self.info['Origin'], tree)
164		appendXmlTextNode("CallerReference", self.info['CallerReference'], tree)
165		for cname in self.info['CNAME']:
166		appendXmlTextNode("CNAME", cname.lower(), tree)
167		if self.info['Comment']:
168		appendXmlTextNode("Comment", self.info['Comment'], tree)
169		appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree)
170		# don't create a empty DefaultRootObject element as it would result in a MalformedXML error
171		if str(self.info['DefaultRootObject']):
172		appendXmlTextNode("DefaultRootObject", str(self.info['DefaultRootObject']), tree)
173		if self.info['Logging']:
174		logging_el = ET.Element("Logging")
175		appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el)
176		appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el)
177		tree.append(logging_el)
178		return ET.tostring(tree)
	122	## Example:
	123	##
	124	## <DistributionConfig>
	125	## <Origin>somebucket.s3.amazonaws.com</Origin>
	126	## <CallerReference>s3://somebucket/</CallerReference>
	127	## <Comment>http://somebucket.s3.amazonaws.com/</Comment>
	128	## <Enabled>true</Enabled>
	129	## <Logging>
	130	## <Bucket>bu.ck.et</Bucket>
	131	## <Prefix>/cf-somebucket/</Prefix>
	132	## </Logging>
	133	## </DistributionConfig>
	134
	135	EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"
	136	xmlns = "http://cloudfront.amazonaws.com/doc/%(api_ver)s/" % { 'api_ver' : cloudfront_api_version }
	137	def __init__(self, xml = None, tree = None):
	138	if xml is None:
	139	xml = DistributionConfig.EMPTY_CONFIG
	140
	141	if tree is None:
	142	tree = getTreeFromXml(xml)
	143
	144	if tree.tag != "DistributionConfig":
	145	raise ValueError("Expected <DistributionConfig /> xml, got: <%s />" % tree.tag)
	146	self.parse(tree)
	147
	148	def parse(self, tree):
	149	self.info = getDictFromTree(tree)
	150	self.info['Enabled'] = (self.info['Enabled'].lower() == "true")
	151	if not self.info.has_key("CNAME"):
	152	self.info['CNAME'] = []
	153	if type(self.info['CNAME']) != list:
	154	self.info['CNAME'] = [self.info['CNAME']]
	155	self.info['CNAME'] = [cname.lower() for cname in self.info['CNAME']]
	156	if not self.info.has_key("Comment"):
	157	self.info['Comment'] = ""
	158	if not self.info.has_key("DefaultRootObject"):
	159	self.info['DefaultRootObject'] = ""
	160	## Figure out logging - complex node not parsed by getDictFromTree()
	161	logging_nodes = tree.findall(".//Logging")
	162	if logging_nodes:
	163	logging_dict = getDictFromTree(logging_nodes[0])
	164	logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket'])
	165	if not success:
	166	warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket'])
	167	self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict)
	168	else:
	169	self.info['Logging'] = None
	170
	171	def __str__(self):
	172	tree = ET.Element("DistributionConfig")
	173	tree.attrib['xmlns'] = DistributionConfig.xmlns
	174
	175	## Retain the order of the following calls!
	176	appendXmlTextNode("Origin", self.info['Origin'], tree)
	177	appendXmlTextNode("CallerReference", self.info['CallerReference'], tree)
	178	for cname in self.info['CNAME']:
	179	appendXmlTextNode("CNAME", cname.lower(), tree)
	180	if self.info['Comment']:
	181	appendXmlTextNode("Comment", self.info['Comment'], tree)
	182	appendXmlTextNode("Enabled", str(self.info['Enabled']).lower(), tree)
	183	# don't create a empty DefaultRootObject element as it would result in a MalformedXML error
	184	if str(self.info['DefaultRootObject']):
	185	appendXmlTextNode("DefaultRootObject", str(self.info['DefaultRootObject']), tree)
	186	if self.info['Logging']:
	187	logging_el = ET.Element("Logging")
	188	appendXmlTextNode("Bucket", getHostnameFromBucket(self.info['Logging'].bucket()), logging_el)
	189	appendXmlTextNode("Prefix", self.info['Logging'].object(), logging_el)
	190	tree.append(logging_el)
	191	return ET.tostring(tree)
	192
	193	class Invalidation(object):
	194	## Example:
	195	##
	196	## <Invalidation xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/">
	197	## <Id>id</Id>
	198	## <Status>status</Status>
	199	## <CreateTime>date</CreateTime>
	200	## <InvalidationBatch>
	201	## <Path>/image1.jpg</Path>
	202	## <Path>/image2.jpg</Path>
	203	## <Path>/videos/movie.flv</Path>
	204	## <CallerReference>my-batch</CallerReference>
	205	## </InvalidationBatch>
	206	## </Invalidation>
	207
	208	def __init__(self, xml):
	209	tree = getTreeFromXml(xml)
	210	if tree.tag != "Invalidation":
	211	raise ValueError("Expected <Invalidation /> xml, got: <%s />" % tree.tag)
	212	self.parse(tree)
	213
	214	def parse(self, tree):
	215	self.info = getDictFromTree(tree)
	216
	217	def __str__(self):
	218	return str(self.info)
	219
	220	class InvalidationList(object):
	221	## Example:
	222	##
	223	## <InvalidationList>
	224	## <Marker/>
	225	## <NextMarker>Invalidation ID</NextMarker>
	226	## <MaxItems>2</MaxItems>
	227	## <IsTruncated>true</IsTruncated>
	228	## <InvalidationSummary>
	229	## <Id>[Second Invalidation ID]</Id>
	230	## <Status>Completed</Status>
	231	## </InvalidationSummary>
	232	## <InvalidationSummary>
	233	## <Id>[First Invalidation ID]</Id>
	234	## <Status>Completed</Status>
	235	## </InvalidationSummary>
	236	## </InvalidationList>
	237
	238	def __init__(self, xml):
	239	tree = getTreeFromXml(xml)
	240	if tree.tag != "InvalidationList":
	241	raise ValueError("Expected <InvalidationList /> xml, got: <%s />" % tree.tag)
	242	self.parse(tree)
	243
	244	def parse(self, tree):
	245	self.info = getDictFromTree(tree)
	246
	247	def __str__(self):
	248	return str(self.info)
	249
	250	class InvalidationBatch(object):
	251	## Example:
	252	##
	253	## <InvalidationBatch>
	254	## <Path>/image1.jpg</Path>
	255	## <Path>/image2.jpg</Path>
	256	## <Path>/videos/movie.flv</Path>
	257	## <Path>/sound%20track.mp3</Path>
	258	## <CallerReference>my-batch</CallerReference>
	259	## </InvalidationBatch>
	260
	261	def __init__(self, reference = None, distribution = None, paths = []):
	262	if reference:
	263	self.reference = reference
	264	else:
	265	if not distribution:
	266	distribution="0"
	267	self.reference = "%s.%s.%s" % (distribution,
	268	datetime.strftime(datetime.now(),"%Y%m%d%H%M%S"),
	269	random.randint(1000,9999))
	270	self.paths = []
	271	self.add_objects(paths)
	272
	273	def add_objects(self, paths):
	274	self.paths.extend(paths)
	275
	276	def get_reference(self):
	277	return self.reference
	278
	279	def __str__(self):
	280	tree = ET.Element("InvalidationBatch")
	281
	282	for path in self.paths:
	283	if path[0] != "/":
	284	path = "/" + path
	285	appendXmlTextNode("Path", path, tree)
	286	appendXmlTextNode("CallerReference", self.reference, tree)
	287	return ET.tostring(tree)
179	288
180	289	class CloudFront(object):
181		operations = {
182		"CreateDist" : { 'method' : "POST", 'resource' : "" },
183		"DeleteDist" : { 'method' : "DELETE", 'resource' : "/%(dist_id)s" },
184		"GetList" : { 'method' : "GET", 'resource' : "" },
185		"GetDistInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s" },
186		"GetDistConfig" : { 'method' : "GET", 'resource' : "/%(dist_id)s/config" },
187		"SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" },
188		}
189
190		## Maximum attempts of re-issuing failed requests
191		_max_retries = 5
192
193		def __init__(self, config):
194		self.config = config
195
196		## --------------------------------------------------
197		## Methods implementing CloudFront API
198		## --------------------------------------------------
199
200		def GetList(self):
201		response = self.send_request("GetList")
202		response['dist_list'] = DistributionList(response['data'])
203		if response['dist_list'].info['IsTruncated']:
204		raise NotImplementedError("List is truncated. Ask s3cmd author to add support.")
205		## TODO: handle Truncated
206		return response
207
208		def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None, default_root_object = None):
209		dist_config = DistributionConfig()
210		dist_config.info['Enabled'] = True
211		dist_config.info['Origin'] = uri.host_name()
212		dist_config.info['CallerReference'] = str(uri)
213		dist_config.info['DefaultRootObject'] = default_root_object
214		if comment == None:
215		dist_config.info['Comment'] = uri.public_url()
216		else:
217		dist_config.info['Comment'] = comment
218		for cname in cnames_add:
219		if dist_config.info['CNAME'].count(cname) == 0:
220		dist_config.info['CNAME'].append(cname)
221		if logging:
222		dist_config.info['Logging'] = S3UriS3(logging)
223		request_body = str(dist_config)
224		debug("CreateDistribution(): request_body: %s" % request_body)
225		response = self.send_request("CreateDist", body = request_body)
226		response['distribution'] = Distribution(response['data'])
227		return response
228
229		def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [],
230		comment = None, enabled = None, logging = None,
	290	operations = {
	291	"CreateDist" : { 'method' : "POST", 'resource' : "" },
	292	"DeleteDist" : { 'method' : "DELETE", 'resource' : "/%(dist_id)s" },
	293	"GetList" : { 'method' : "GET", 'resource' : "" },
	294	"GetDistInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s" },
	295	"GetDistConfig" : { 'method' : "GET", 'resource' : "/%(dist_id)s/config" },
	296	"SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" },
	297	"Invalidate" : { 'method' : "POST", 'resource' : "/%(dist_id)s/invalidation" },
	298	"GetInvalList" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation" },
	299	"GetInvalInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(request_id)s" },
	300	}
	301
	302	## Maximum attempts of re-issuing failed requests
	303	_max_retries = 5
	304	dist_list = None
	305
	306	def __init__(self, config):
	307	self.config = config
	308
	309	## --------------------------------------------------
	310	## Methods implementing CloudFront API
	311	## --------------------------------------------------
	312
	313	def GetList(self):
	314	response = self.send_request("GetList")
	315	response['dist_list'] = DistributionList(response['data'])
	316	if response['dist_list'].info['IsTruncated']:
	317	raise NotImplementedError("List is truncated. Ask s3cmd author to add support.")
	318	## TODO: handle Truncated
	319	return response
	320
	321	def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None, default_root_object = None):
	322	dist_config = DistributionConfig()
	323	dist_config.info['Enabled'] = True
	324	dist_config.info['Origin'] = uri.host_name()
	325	dist_config.info['CallerReference'] = str(uri)
	326	dist_config.info['DefaultRootObject'] = default_root_object
	327	if comment == None:
	328	dist_config.info['Comment'] = uri.public_url()
	329	else:
	330	dist_config.info['Comment'] = comment
	331	for cname in cnames_add:
	332	if dist_config.info['CNAME'].count(cname) == 0:
	333	dist_config.info['CNAME'].append(cname)
	334	if logging:
	335	dist_config.info['Logging'] = S3UriS3(logging)
	336	request_body = str(dist_config)
	337	debug("CreateDistribution(): request_body: %s" % request_body)
	338	response = self.send_request("CreateDist", body = request_body)
	339	response['distribution'] = Distribution(response['data'])
	340	return response
	341
	342	def ModifyDistribution(self, cfuri, cnames_add = [], cnames_remove = [],
	343	comment = None, enabled = None, logging = None,
231	344	default_root_object = None):
232		if cfuri.type != "cf":
233		raise ValueError("Expected CFUri instead of: %s" % cfuri)
234		# Get current dist status (enabled/disabled) and Etag
235		info("Checking current status of %s" % cfuri)
236		response = self.GetDistConfig(cfuri)
237		dc = response['dist_config']
238		if enabled != None:
239		dc.info['Enabled'] = enabled
240		if comment != None:
241		dc.info['Comment'] = comment
242		if default_root_object != None:
243		dc.info['DefaultRootObject'] = default_root_object
244		for cname in cnames_add:
245		if dc.info['CNAME'].count(cname) == 0:
246		dc.info['CNAME'].append(cname)
247		for cname in cnames_remove:
248		while dc.info['CNAME'].count(cname) > 0:
249		dc.info['CNAME'].remove(cname)
250		if logging != None:
251		if logging == False:
252		dc.info['Logging'] = False
253		else:
254		dc.info['Logging'] = S3UriS3(logging)
255		response = self.SetDistConfig(cfuri, dc, response['headers']['etag'])
256		return response
257
258		def DeleteDistribution(self, cfuri):
259		if cfuri.type != "cf":
260		raise ValueError("Expected CFUri instead of: %s" % cfuri)
261		# Get current dist status (enabled/disabled) and Etag
262		info("Checking current status of %s" % cfuri)
263		response = self.GetDistConfig(cfuri)
264		if response['dist_config'].info['Enabled']:
265		info("Distribution is ENABLED. Disabling first.")
266		response['dist_config'].info['Enabled'] = False
267		response = self.SetDistConfig(cfuri, response['dist_config'],
268		response['headers']['etag'])
269		warning("Waiting for Distribution to become disabled.")
270		warning("This may take several minutes, please wait.")
271		while True:
272		response = self.GetDistInfo(cfuri)
273		d = response['distribution']
274		if d.info['Status'] == "Deployed" and d.info['Enabled'] == False:
275		info("Distribution is now disabled")
276		break
277		warning("Still waiting...")
278		time.sleep(10)
279		headers = {}
280		headers['if-match'] = response['headers']['etag']
281		response = self.send_request("DeleteDist", dist_id = cfuri.dist_id(),
282		headers = headers)
283		return response
284
285		def GetDistInfo(self, cfuri):
286		if cfuri.type != "cf":
287		raise ValueError("Expected CFUri instead of: %s" % cfuri)
288		response = self.send_request("GetDistInfo", dist_id = cfuri.dist_id())
289		response['distribution'] = Distribution(response['data'])
290		return response
291
292		def GetDistConfig(self, cfuri):
293		if cfuri.type != "cf":
294		raise ValueError("Expected CFUri instead of: %s" % cfuri)
295		response = self.send_request("GetDistConfig", dist_id = cfuri.dist_id())
296		response['dist_config'] = DistributionConfig(response['data'])
297		return response
298
299		def SetDistConfig(self, cfuri, dist_config, etag = None):
300		if etag == None:
301		debug("SetDistConfig(): Etag not set. Fetching it first.")
302		etag = self.GetDistConfig(cfuri)['headers']['etag']
303		debug("SetDistConfig(): Etag = %s" % etag)
304		request_body = str(dist_config)
305		debug("SetDistConfig(): request_body: %s" % request_body)
306		headers = {}
307		headers['if-match'] = etag
308		response = self.send_request("SetDistConfig", dist_id = cfuri.dist_id(),
309		body = request_body, headers = headers)
310		return response
311
312		## --------------------------------------------------
313		## Low-level methods for handling CloudFront requests
314		## --------------------------------------------------
315
316		def send_request(self, op_name, dist_id = None, body = None, headers = {}, retries = _max_retries):
317		operation = self.operations[op_name]
318		if body:
319		headers['content-type'] = 'text/plain'
320		request = self.create_request(operation, dist_id, headers)
321		conn = self.get_connection()
322		debug("send_request(): %s %s" % (request['method'], request['resource']))
323		conn.request(request['method'], request['resource'], body, request['headers'])
324		http_response = conn.getresponse()
325		response = {}
326		response["status"] = http_response.status
327		response["reason"] = http_response.reason
328		response["headers"] = dict(http_response.getheaders())
329		response["data"] = http_response.read()
330		conn.close()
331
332		debug("CloudFront: response: %r" % response)
333
334		if response["status"] >= 500:
335		e = CloudFrontError(response)
336		if retries:
337		warning(u"Retrying failed request: %s" % op_name)
338		warning(unicode(e))
339		warning("Waiting %d sec..." % self._fail_wait(retries))
340		time.sleep(self._fail_wait(retries))
341		return self.send_request(op_name, dist_id, body, retries - 1)
342		else:
343		raise e
344
345		if response["status"] < 200 or response["status"] > 299:
346		raise CloudFrontError(response)
347
348		return response
349
350		def create_request(self, operation, dist_id = None, headers = None):
351		resource = self.config.cloudfront_resource + (
352		operation['resource'] % { 'dist_id' : dist_id })
353
354		if not headers:
355		headers = {}
356
357		if headers.has_key("date"):
358		if not headers.has_key("x-amz-date"):
359		headers["x-amz-date"] = headers["date"]
360		del(headers["date"])
361
362		if not headers.has_key("x-amz-date"):
363		headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
364
365		signature = self.sign_request(headers)
366		headers["Authorization"] = "AWS "+self.config.access_key+":"+signature
367
368		request = {}
369		request['resource'] = resource
370		request['headers'] = headers
371		request['method'] = operation['method']
372
373		return request
374
375		def sign_request(self, headers):
376		string_to_sign = headers['x-amz-date']
377		signature = sign_string(string_to_sign)
378		debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature))
379		return signature
380
381		def get_connection(self):
382		if self.config.proxy_host != "":
383		raise ParameterError("CloudFront commands don't work from behind a HTTP proxy")
384		return httplib.HTTPSConnection(self.config.cloudfront_host)
385
386		def _fail_wait(self, retries):
387		# Wait a few seconds. The more it fails the more we wait.
388		return (self._max_retries - retries + 1) * 3
	345	if cfuri.type != "cf":
	346	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	347	# Get current dist status (enabled/disabled) and Etag
	348	info("Checking current status of %s" % cfuri)
	349	response = self.GetDistConfig(cfuri)
	350	dc = response['dist_config']
	351	if enabled != None:
	352	dc.info['Enabled'] = enabled
	353	if comment != None:
	354	dc.info['Comment'] = comment
	355	if default_root_object != None:
	356	dc.info['DefaultRootObject'] = default_root_object
	357	for cname in cnames_add:
	358	if dc.info['CNAME'].count(cname) == 0:
	359	dc.info['CNAME'].append(cname)
	360	for cname in cnames_remove:
	361	while dc.info['CNAME'].count(cname) > 0:
	362	dc.info['CNAME'].remove(cname)
	363	if logging != None:
	364	if logging == False:
	365	dc.info['Logging'] = False
	366	else:
	367	dc.info['Logging'] = S3UriS3(logging)
	368	response = self.SetDistConfig(cfuri, dc, response['headers']['etag'])
	369	return response
	370
	371	def DeleteDistribution(self, cfuri):
	372	if cfuri.type != "cf":
	373	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	374	# Get current dist status (enabled/disabled) and Etag
	375	info("Checking current status of %s" % cfuri)
	376	response = self.GetDistConfig(cfuri)
	377	if response['dist_config'].info['Enabled']:
	378	info("Distribution is ENABLED. Disabling first.")
	379	response['dist_config'].info['Enabled'] = False
	380	response = self.SetDistConfig(cfuri, response['dist_config'],
	381	response['headers']['etag'])
	382	warning("Waiting for Distribution to become disabled.")
	383	warning("This may take several minutes, please wait.")
	384	while True:
	385	response = self.GetDistInfo(cfuri)
	386	d = response['distribution']
	387	if d.info['Status'] == "Deployed" and d.info['Enabled'] == False:
	388	info("Distribution is now disabled")
	389	break
	390	warning("Still waiting...")
	391	time.sleep(10)
	392	headers = {}
	393	headers['if-match'] = response['headers']['etag']
	394	response = self.send_request("DeleteDist", dist_id = cfuri.dist_id(),
	395	headers = headers)
	396	return response
	397
	398	def GetDistInfo(self, cfuri):
	399	if cfuri.type != "cf":
	400	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	401	response = self.send_request("GetDistInfo", dist_id = cfuri.dist_id())
	402	response['distribution'] = Distribution(response['data'])
	403	return response
	404
	405	def GetDistConfig(self, cfuri):
	406	if cfuri.type != "cf":
	407	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	408	response = self.send_request("GetDistConfig", dist_id = cfuri.dist_id())
	409	response['dist_config'] = DistributionConfig(response['data'])
	410	return response
	411
	412	def SetDistConfig(self, cfuri, dist_config, etag = None):
	413	if etag == None:
	414	debug("SetDistConfig(): Etag not set. Fetching it first.")
	415	etag = self.GetDistConfig(cfuri)['headers']['etag']
	416	debug("SetDistConfig(): Etag = %s" % etag)
	417	request_body = str(dist_config)
	418	debug("SetDistConfig(): request_body: %s" % request_body)
	419	headers = {}
	420	headers['if-match'] = etag
	421	response = self.send_request("SetDistConfig", dist_id = cfuri.dist_id(),
	422	body = request_body, headers = headers)
	423	return response
	424
	425	def InvalidateObjects(self, uri, paths):
	426	# uri could be either cf:// or s3:// uri
	427	cfuri = self.get_dist_name_for_bucket(uri)
	428	if len(paths) > 999:
	429	try:
	430	tmp_filename = Utils.mktmpfile()
	431	f = open(tmp_filename, "w")
	432	f.write("\n".join(paths)+"\n")
	433	f.close()
	434	warning("Request to invalidate %d paths (max 999 supported)" % len(paths))
	435	warning("All the paths are now saved in: %s" % tmp_filename)
	436	except:
	437	pass
	438	raise ParameterError("Too many paths to invalidate")
	439	invalbatch = InvalidationBatch(distribution = cfuri.dist_id(), paths = paths)
	440	debug("InvalidateObjects(): request_body: %s" % invalbatch)
	441	response = self.send_request("Invalidate", dist_id = cfuri.dist_id(),
	442	body = str(invalbatch))
	443	response['dist_id'] = cfuri.dist_id()
	444	if response['status'] == 201:
	445	inval_info = Invalidation(response['data']).info
	446	response['request_id'] = inval_info['Id']
	447	debug("InvalidateObjects(): response: %s" % response)
	448	return response
	449
	450	def GetInvalList(self, cfuri):
	451	if cfuri.type != "cf":
	452	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	453	response = self.send_request("GetInvalList", dist_id = cfuri.dist_id())
	454	response['inval_list'] = InvalidationList(response['data'])
	455	return response
	456
	457	def GetInvalInfo(self, cfuri):
	458	if cfuri.type != "cf":
	459	raise ValueError("Expected CFUri instead of: %s" % cfuri)
	460	if cfuri.request_id() is None:
	461	raise ValueError("Expected CFUri with Request ID")
	462	response = self.send_request("GetInvalInfo", dist_id = cfuri.dist_id(), request_id = cfuri.request_id())
	463	response['inval_status'] = Invalidation(response['data'])
	464	return response
	465
	466	## --------------------------------------------------
	467	## Low-level methods for handling CloudFront requests
	468	## --------------------------------------------------
	469
	470	def send_request(self, op_name, dist_id = None, request_id = None, body = None, headers = {}, retries = _max_retries):
	471	operation = self.operations[op_name]
	472	if body:
	473	headers['content-type'] = 'text/plain'
	474	request = self.create_request(operation, dist_id, request_id, headers)
	475	conn = self.get_connection()
	476	debug("send_request(): %s %s" % (request['method'], request['resource']))
	477	conn.request(request['method'], request['resource'], body, request['headers'])
	478	http_response = conn.getresponse()
	479	response = {}
	480	response["status"] = http_response.status
	481	response["reason"] = http_response.reason
	482	response["headers"] = dict(http_response.getheaders())
	483	response["data"] = http_response.read()
	484	conn.close()
	485
	486	debug("CloudFront: response: %r" % response)
	487
	488	if response["status"] >= 500:
	489	e = CloudFrontError(response)
	490	if retries:
	491	warning(u"Retrying failed request: %s" % op_name)
	492	warning(unicode(e))
	493	warning("Waiting %d sec..." % self._fail_wait(retries))
	494	time.sleep(self._fail_wait(retries))
	495	return self.send_request(op_name, dist_id, body, retries - 1)
	496	else:
	497	raise e
	498
	499	if response["status"] < 200 or response["status"] > 299:
	500	raise CloudFrontError(response)
	501
	502	return response
	503
	504	def create_request(self, operation, dist_id = None, request_id = None, headers = None):
	505	resource = cloudfront_resource + (
	506	operation['resource'] % { 'dist_id' : dist_id, 'request_id' : request_id })
	507
	508	if not headers:
	509	headers = {}
	510
	511	if headers.has_key("date"):
	512	if not headers.has_key("x-amz-date"):
	513	headers["x-amz-date"] = headers["date"]
	514	del(headers["date"])
	515
	516	if not headers.has_key("x-amz-date"):
	517	headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
	518
	519	signature = self.sign_request(headers)
	520	headers["Authorization"] = "AWS "+self.config.access_key+":"+signature
	521
	522	request = {}
	523	request['resource'] = resource
	524	request['headers'] = headers
	525	request['method'] = operation['method']
	526
	527	return request
	528
	529	def sign_request(self, headers):
	530	string_to_sign = headers['x-amz-date']
	531	signature = sign_string(string_to_sign)
	532	debug(u"CloudFront.sign_request('%s') = %s" % (string_to_sign, signature))
	533	return signature
	534
	535	def get_connection(self):
	536	if self.config.proxy_host != "":
	537	raise ParameterError("CloudFront commands don't work from behind a HTTP proxy")
	538	return httplib.HTTPSConnection(self.config.cloudfront_host)
	539
	540	def _fail_wait(self, retries):
	541	# Wait a few seconds. The more it fails the more we wait.
	542	return (self._max_retries - retries + 1) * 3
	543
	544	def get_dist_name_for_bucket(self, uri):
	545	if (uri.type == "cf"):
	546	return uri
	547	if (uri.type != "s3"):
	548	raise ParameterError("CloudFront or S3 URI required instead of: %s" % arg)
	549
	550	debug("_get_dist_name_for_bucket(%r)" % uri)
	551	if CloudFront.dist_list is None:
	552	response = self.GetList()
	553	CloudFront.dist_list = {}
	554	for d in response['dist_list'].dist_summs:
	555	if d.info.has_key("S3Origin"):
	556	CloudFront.dist_list[getBucketFromHostname(d.info['S3Origin']['DNSName'])[0]] = d.uri()
	557	else:
	558	# Skip over distributions with CustomOrigin
	559	continue
	560	debug("dist_list: %s" % CloudFront.dist_list)
	561	try:
	562	return CloudFront.dist_list[uri.bucket()]
	563	except Exception, e:
	564	debug(e)
	565	raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % arg)
389	566
390	567	class Cmd(object):
391		"""
392		Class that implements CloudFront commands
393		"""
394
395		class Options(object):
396		cf_cnames_add = []
397		cf_cnames_remove = []
398		cf_comment = None
399		cf_enable = None
400		cf_logging = None
401		cf_default_root_object = None
402
403		def option_list(self):
404		return [opt for opt in dir(self) if opt.startswith("cf_")]
405
406		def update_option(self, option, value):
407		setattr(Cmd.options, option, value)
408
409		options = Options()
410		dist_list = None
411
412		@staticmethod
413		def _get_dist_name_for_bucket(uri):
414		cf = CloudFront(Config())
415		debug("_get_dist_name_for_bucket(%r)" % uri)
416		assert(uri.type == "s3")
417		if Cmd.dist_list is None:
418		response = cf.GetList()
419		Cmd.dist_list = {}
420		for d in response['dist_list'].dist_summs:
421		Cmd.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri()
422		debug("dist_list: %s" % Cmd.dist_list)
423		return Cmd.dist_list[uri.bucket()]
424
425		@staticmethod
426		def _parse_args(args):
427		cfuris = []
428		for arg in args:
429		uri = S3Uri(arg)
430		if uri.type == 's3':
431		try:
432		uri = Cmd._get_dist_name_for_bucket(uri)
433		except Exception, e:
434		debug(e)
435		raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % uri)
436		if uri.type != 'cf':
437		raise ParameterError("CloudFront URI required instead of: %s" % arg)
438		cfuris.append(uri)
439		return cfuris
440
441		@staticmethod
442		def info(args):
443		cf = CloudFront(Config())
444		if not args:
445		response = cf.GetList()
446		for d in response['dist_list'].dist_summs:
447		pretty_output("Origin", S3UriS3.httpurl_to_s3uri(d.info['Origin']))
448		pretty_output("DistId", d.uri())
449		pretty_output("DomainName", d.info['DomainName'])
450		pretty_output("Status", d.info['Status'])
451		pretty_output("Enabled", d.info['Enabled'])
452		output("")
453		else:
454		cfuris = Cmd._parse_args(args)
455		for cfuri in cfuris:
456		response = cf.GetDistInfo(cfuri)
457		d = response['distribution']
458		dc = d.info['DistributionConfig']
459		pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
460		pretty_output("DistId", d.uri())
461		pretty_output("DomainName", d.info['DomainName'])
462		pretty_output("Status", d.info['Status'])
463		pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
464		pretty_output("Comment", dc.info['Comment'])
465		pretty_output("Enabled", dc.info['Enabled'])
466		pretty_output("DfltRootObject", dc.info['DefaultRootObject'])
467		pretty_output("Logging", dc.info['Logging'] or "Disabled")
468		pretty_output("Etag", response['headers']['etag'])
469
470		@staticmethod
471		def create(args):
472		cf = CloudFront(Config())
473		buckets = []
474		for arg in args:
475		uri = S3Uri(arg)
476		if uri.type != "s3":
477		raise ParameterError("Bucket can only be created from a s3:// URI instead of: %s" % arg)
478		if uri.object():
479		raise ParameterError("Use s3:// URI with a bucket name only instead of: %s" % arg)
480		if not uri.is_dns_compatible():
481		raise ParameterError("CloudFront can only handle lowercase-named buckets.")
482		buckets.append(uri)
483		if not buckets:
484		raise ParameterError("No valid bucket names found")
485		for uri in buckets:
486		info("Creating distribution from: %s" % uri)
487		response = cf.CreateDistribution(uri, cnames_add = Cmd.options.cf_cnames_add,
488		comment = Cmd.options.cf_comment,
489		logging = Cmd.options.cf_logging,
	568	"""
	569	Class that implements CloudFront commands
	570	"""
	571
	572	class Options(object):
	573	cf_cnames_add = []
	574	cf_cnames_remove = []
	575	cf_comment = None
	576	cf_enable = None
	577	cf_logging = None
	578	cf_default_root_object = None
	579
	580	def option_list(self):
	581	return [opt for opt in dir(self) if opt.startswith("cf_")]
	582
	583	def update_option(self, option, value):
	584	setattr(Cmd.options, option, value)
	585
	586	options = Options()
	587
	588	@staticmethod
	589	def _parse_args(args):
	590	cf = CloudFront(Config())
	591	cfuris = []
	592	for arg in args:
	593	uri = cf.get_dist_name_for_bucket(S3Uri(arg))
	594	cfuris.append(uri)
	595	return cfuris
	596
	597	@staticmethod
	598	def info(args):
	599	cf = CloudFront(Config())
	600	if not args:
	601	response = cf.GetList()
	602	for d in response['dist_list'].dist_summs:
	603	if d.info.has_key("S3Origin"):
	604	origin = S3UriS3.httpurl_to_s3uri(d.info['S3Origin']['DNSName'])
	605	elif d.info.has_key("CustomOrigin"):
	606	origin = "http://%s/" % d.info['CustomOrigin']['DNSName']
	607	else:
	608	origin = "<unknown>"
	609	pretty_output("Origin", origin)
	610	pretty_output("DistId", d.uri())
	611	pretty_output("DomainName", d.info['DomainName'])
	612	if d.info.has_key("CNAME"):
	613	pretty_output("CNAMEs", ", ".join(d.info['CNAME']))
	614	pretty_output("Status", d.info['Status'])
	615	pretty_output("Enabled", d.info['Enabled'])
	616	output("")
	617	else:
	618	cfuris = Cmd._parse_args(args)
	619	for cfuri in cfuris:
	620	response = cf.GetDistInfo(cfuri)
	621	d = response['distribution']
	622	dc = d.info['DistributionConfig']
	623	if dc.info.has_key("S3Origin"):
	624	origin = S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName'])
	625	elif dc.info.has_key("CustomOrigin"):
	626	origin = "http://%s/" % dc.info['CustomOrigin']['DNSName']
	627	else:
	628	origin = "<unknown>"
	629	pretty_output("Origin", origin)
	630	pretty_output("DistId", d.uri())
	631	pretty_output("DomainName", d.info['DomainName'])
	632	if dc.info.has_key("CNAME"):
	633	pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
	634	pretty_output("Status", d.info['Status'])
	635	pretty_output("Comment", dc.info['Comment'])
	636	pretty_output("Enabled", dc.info['Enabled'])
	637	pretty_output("DfltRootObject", dc.info['DefaultRootObject'])
	638	pretty_output("Logging", dc.info['Logging'] or "Disabled")
	639	pretty_output("Etag", response['headers']['etag'])
	640
	641	@staticmethod
	642	def create(args):
	643	cf = CloudFront(Config())
	644	buckets = []
	645	for arg in args:
	646	uri = S3Uri(arg)
	647	if uri.type != "s3":
	648	raise ParameterError("Bucket can only be created from a s3:// URI instead of: %s" % arg)
	649	if uri.object():
	650	raise ParameterError("Use s3:// URI with a bucket name only instead of: %s" % arg)
	651	if not uri.is_dns_compatible():
	652	raise ParameterError("CloudFront can only handle lowercase-named buckets.")
	653	buckets.append(uri)
	654	if not buckets:
	655	raise ParameterError("No valid bucket names found")
	656	for uri in buckets:
	657	info("Creating distribution from: %s" % uri)
	658	response = cf.CreateDistribution(uri, cnames_add = Cmd.options.cf_cnames_add,
	659	comment = Cmd.options.cf_comment,
	660	logging = Cmd.options.cf_logging,
490	661	default_root_object = Cmd.options.cf_default_root_object)
491		d = response['distribution']
492		dc = d.info['DistributionConfig']
493		output("Distribution created:")
494		pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
495		pretty_output("DistId", d.uri())
496		pretty_output("DomainName", d.info['DomainName'])
497		pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
498		pretty_output("Comment", dc.info['Comment'])
499		pretty_output("Status", d.info['Status'])
500		pretty_output("Enabled", dc.info['Enabled'])
501		pretty_output("DefaultRootObject", dc.info['DefaultRootObject'])
502		pretty_output("Etag", response['headers']['etag'])
503
504		@staticmethod
505		def delete(args):
506		cf = CloudFront(Config())
507		cfuris = Cmd._parse_args(args)
508		for cfuri in cfuris:
509		response = cf.DeleteDistribution(cfuri)
510		if response['status'] >= 400:
511		error("Distribution %s could not be deleted: %s" % (cfuri, response['reason']))
512		output("Distribution %s deleted" % cfuri)
513
514		@staticmethod
515		def modify(args):
516		cf = CloudFront(Config())
517		if len(args) > 1:
518		raise ParameterError("Too many parameters. Modify one Distribution at a time.")
519		try:
520		cfuri = Cmd._parse_args(args)[0]
521		except IndexError, e:
522		raise ParameterError("No valid Distribution URI found.")
523		response = cf.ModifyDistribution(cfuri,
524		cnames_add = Cmd.options.cf_cnames_add,
525		cnames_remove = Cmd.options.cf_cnames_remove,
526		comment = Cmd.options.cf_comment,
527		enabled = Cmd.options.cf_enable,
528		logging = Cmd.options.cf_logging,
	662	d = response['distribution']
	663	dc = d.info['DistributionConfig']
	664	output("Distribution created:")
	665	pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
	666	pretty_output("DistId", d.uri())
	667	pretty_output("DomainName", d.info['DomainName'])
	668	pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
	669	pretty_output("Comment", dc.info['Comment'])
	670	pretty_output("Status", d.info['Status'])
	671	pretty_output("Enabled", dc.info['Enabled'])
	672	pretty_output("DefaultRootObject", dc.info['DefaultRootObject'])
	673	pretty_output("Etag", response['headers']['etag'])
	674
	675	@staticmethod
	676	def delete(args):
	677	cf = CloudFront(Config())
	678	cfuris = Cmd._parse_args(args)
	679	for cfuri in cfuris:
	680	response = cf.DeleteDistribution(cfuri)
	681	if response['status'] >= 400:
	682	error("Distribution %s could not be deleted: %s" % (cfuri, response['reason']))
	683	output("Distribution %s deleted" % cfuri)
	684
	685	@staticmethod
	686	def modify(args):
	687	cf = CloudFront(Config())
	688	if len(args) > 1:
	689	raise ParameterError("Too many parameters. Modify one Distribution at a time.")
	690	try:
	691	cfuri = Cmd._parse_args(args)[0]
	692	except IndexError, e:
	693	raise ParameterError("No valid Distribution URI found.")
	694	response = cf.ModifyDistribution(cfuri,
	695	cnames_add = Cmd.options.cf_cnames_add,
	696	cnames_remove = Cmd.options.cf_cnames_remove,
	697	comment = Cmd.options.cf_comment,
	698	enabled = Cmd.options.cf_enable,
	699	logging = Cmd.options.cf_logging,
529	700	default_root_object = Cmd.options.cf_default_root_object)
530		if response['status'] >= 400:
531		error("Distribution %s could not be modified: %s" % (cfuri, response['reason']))
532		output("Distribution modified: %s" % cfuri)
533		response = cf.GetDistInfo(cfuri)
534		d = response['distribution']
535		dc = d.info['DistributionConfig']
536		pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
537		pretty_output("DistId", d.uri())
538		pretty_output("DomainName", d.info['DomainName'])
539		pretty_output("Status", d.info['Status'])
540		pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
541		pretty_output("Comment", dc.info['Comment'])
542		pretty_output("Enabled", dc.info['Enabled'])
543		pretty_output("DefaultRootObject", dc.info['DefaultRootObject'])
544		pretty_output("Etag", response['headers']['etag'])
	701	if response['status'] >= 400:
	702	error("Distribution %s could not be modified: %s" % (cfuri, response['reason']))
	703	output("Distribution modified: %s" % cfuri)
	704	response = cf.GetDistInfo(cfuri)
	705	d = response['distribution']
	706	dc = d.info['DistributionConfig']
	707	pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
	708	pretty_output("DistId", d.uri())
	709	pretty_output("DomainName", d.info['DomainName'])
	710	pretty_output("Status", d.info['Status'])
	711	pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
	712	pretty_output("Comment", dc.info['Comment'])
	713	pretty_output("Enabled", dc.info['Enabled'])
	714	pretty_output("DefaultRootObject", dc.info['DefaultRootObject'])
	715	pretty_output("Etag", response['headers']['etag'])
	716
	717	@staticmethod
	718	def invalinfo(args):
	719	cf = CloudFront(Config())
	720	cfuris = Cmd._parse_args(args)
	721	requests = []
	722	for cfuri in cfuris:
	723	if cfuri.request_id():
	724	requests.append(str(cfuri))
	725	else:
	726	inval_list = cf.GetInvalList(cfuri)
	727	try:
	728	for i in inval_list['inval_list'].info['InvalidationSummary']:
	729	requests.append("/".join(["cf:/", cfuri.dist_id(), i["Id"]]))
	730	except:
	731	continue
	732	for req in requests:
	733	cfuri = S3Uri(req)
	734	inval_info = cf.GetInvalInfo(cfuri)
	735	st = inval_info['inval_status'].info
	736	pretty_output("URI", str(cfuri))
	737	pretty_output("Status", st['Status'])
	738	pretty_output("Created", st['CreateTime'])
	739	pretty_output("Nr of paths", len(st['InvalidationBatch']['Path']))
	740	pretty_output("Reference", st['InvalidationBatch']['CallerReference'])
	741	output("")
	742
	743	# vim:et:ts=4:sts=4:ai

+191

-180

S3/Config.py less more

5	5	import logging
6	6	from logging import debug, info, warning, error
7	7	import re
	8	import os
8	9	import Progress
9	10	from SortedDict import SortedDict
10	11
11	12	class Config(object):
12		_instance = None
13		_parsed_files = []
14		_doc = {}
15		access_key = ""
16		secret_key = ""
17		host_base = "s3.amazonaws.com"
18		host_bucket = "%(bucket)s.s3.amazonaws.com"
19		simpledb_host = "sdb.amazonaws.com"
20		cloudfront_host = "cloudfront.amazonaws.com"
21		cloudfront_resource = "/2010-07-15/distribution"
22		verbosity = logging.WARNING
23		progress_meter = True
24		progress_class = Progress.ProgressCR
25		send_chunk = 4096
26		recv_chunk = 4096
27		list_md5 = False
28		human_readable_sizes = False
29		extra_headers = SortedDict(ignore_case = True)
30		force = False
31		enable = None
32		get_continue = False
33		skip_existing = False
34		recursive = False
35		acl_public = None
36		acl_grants = []
37		acl_revokes = []
38		proxy_host = ""
39		proxy_port = 3128
40		encrypt = False
41		dry_run = False
42		preserve_attrs = True
43		preserve_attrs_list = [
44		'uname', # Verbose owner Name (e.g. 'root')
45		'uid', # Numeric user ID (e.g. 0)
46		'gname', # Group name (e.g. 'users')
47		'gid', # Numeric group ID (e.g. 100)
48		'atime', # Last access timestamp
49		'mtime', # Modification timestamp
50		'ctime', # Creation timestamp
51		'mode', # File mode (e.g. rwxr-xr-x = 755)
52		#'acl', # Full ACL (not yet supported)
53		]
54		delete_removed = False
55		_doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
56		gpg_passphrase = ""
57		gpg_command = ""
58		gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
59		gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
60		use_https = False
61		bucket_location = "US"
62		default_mime_type = "binary/octet-stream"
63		guess_mime_type = True
64		# List of checks to be performed for 'sync'
65		sync_checks = ['size', 'md5'] # 'weak-timestamp'
66		# List of compiled REGEXPs
67		exclude = []
68		include = []
69		# Dict mapping compiled REGEXPs back to their textual form
70		debug_exclude = {}
71		debug_include = {}
72		encoding = "utf-8"
73		urlencoding_mode = "normal"
74		log_target_prefix = ""
75		reduced_redundancy = False
76		follow_symlinks = False
77		socket_timeout = 10
78
79		## Creating a singleton
80		def __new__(self, configfile = None):
81		if self._instance is None:
82		self._instance = object.__new__(self)
83		return self._instance
84
85		def __init__(self, configfile = None):
86		if configfile:
87		self.read_config_file(configfile)
88
89		def option_list(self):
90		retval = []
91		for option in dir(self):
92		## Skip attributes that start with underscore or are not string, int or bool
93		option_type = type(getattr(Config, option))
94		if option.startswith("_") or \
95		not (option_type in (
96		type("string"), # str
97		type(42), # int
98		type(True))): # bool
99		continue
100		retval.append(option)
101		return retval
102
103		def read_config_file(self, configfile):
104		cp = ConfigParser(configfile)
105		for option in self.option_list():
106		self.update_option(option, cp.get(option))
107		self._parsed_files.append(configfile)
108
109		def dump_config(self, stream):
110		ConfigDumper(stream).dump("default", self)
111
112		def update_option(self, option, value):
113		if value is None:
114		return
115		#### Special treatment of some options
116		## verbosity must be known to "logging" module
117		if option == "verbosity":
118		try:
119		setattr(Config, "verbosity", logging._levelNames[value])
120		except KeyError:
121		error("Config: verbosity level '%s' is not valid" % value)
122		## allow yes/no, true/false, on/off and 1/0 for boolean options
123		elif type(getattr(Config, option)) is type(True): # bool
124		if str(value).lower() in ("true", "yes", "on", "1"):
125		setattr(Config, option, True)
126		elif str(value).lower() in ("false", "no", "off", "0"):
127		setattr(Config, option, False)
128		else:
129		error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))
130		elif type(getattr(Config, option)) is type(42): # int
131		try:
132		setattr(Config, option, int(value))
133		except ValueError, e:
134		error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))
135		else: # string
136		setattr(Config, option, value)
	13	_instance = None
	14	_parsed_files = []
	15	_doc = {}
	16	access_key = ""
	17	secret_key = ""
	18	host_base = "s3.amazonaws.com"
	19	host_bucket = "%(bucket)s.s3.amazonaws.com"
	20	simpledb_host = "sdb.amazonaws.com"
	21	cloudfront_host = "cloudfront.amazonaws.com"
	22	verbosity = logging.WARNING
	23	progress_meter = True
	24	progress_class = Progress.ProgressCR
	25	send_chunk = 4096
	26	recv_chunk = 4096
	27	list_md5 = False
	28	human_readable_sizes = False
	29	extra_headers = SortedDict(ignore_case = True)
	30	force = False
	31	enable = None
	32	get_continue = False
	33	skip_existing = False
	34	recursive = False
	35	acl_public = None
	36	acl_grants = []
	37	acl_revokes = []
	38	proxy_host = ""
	39	proxy_port = 3128
	40	encrypt = False
	41	dry_run = False
	42	preserve_attrs = True
	43	preserve_attrs_list = [
	44	'uname', # Verbose owner Name (e.g. 'root')
	45	'uid', # Numeric user ID (e.g. 0)
	46	'gname', # Group name (e.g. 'users')
	47	'gid', # Numeric group ID (e.g. 100)
	48	'atime', # Last access timestamp
	49	'mtime', # Modification timestamp
	50	'ctime', # Creation timestamp
	51	'mode', # File mode (e.g. rwxr-xr-x = 755)
	52	#'acl', # Full ACL (not yet supported)
	53	]
	54	delete_removed = False
	55	_doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
	56	gpg_passphrase = ""
	57	gpg_command = ""
	58	gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
	59	gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
	60	use_https = False
	61	bucket_location = "US"
	62	default_mime_type = "binary/octet-stream"
	63	guess_mime_type = True
	64	mime_type = ""
	65	enable_multipart = True
	66	multipart_chunk_size_mb = 15 # MB
	67	# List of checks to be performed for 'sync'
	68	sync_checks = ['size', 'md5'] # 'weak-timestamp'
	69	# List of compiled REGEXPs
	70	exclude = []
	71	include = []
	72	# Dict mapping compiled REGEXPs back to their textual form
	73	debug_exclude = {}
	74	debug_include = {}
	75	encoding = "utf-8"
	76	urlencoding_mode = "normal"
	77	log_target_prefix = ""
	78	reduced_redundancy = False
	79	follow_symlinks = False
	80	socket_timeout = 300
	81	invalidate_on_cf = False
	82	website_index = "index.html"
	83	website_error = ""
	84	website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"
	85
	86	## Creating a singleton
	87	def __new__(self, configfile = None):
	88	if self._instance is None:
	89	self._instance = object.__new__(self)
	90	return self._instance
	91
	92	def __init__(self, configfile = None):
	93	if configfile:
	94	self.read_config_file(configfile)
	95
	96	def option_list(self):
	97	retval = []
	98	for option in dir(self):
	99	## Skip attributes that start with underscore or are not string, int or bool
	100	option_type = type(getattr(Config, option))
	101	if option.startswith("_") or \
	102	not (option_type in (
	103	type("string"), # str
	104	type(42), # int
	105	type(True))): # bool
	106	continue
	107	retval.append(option)
	108	return retval
	109
	110	def read_config_file(self, configfile):
	111	cp = ConfigParser(configfile)
	112	for option in self.option_list():
	113	self.update_option(option, cp.get(option))
	114	self._parsed_files.append(configfile)
	115
	116	def dump_config(self, stream):
	117	ConfigDumper(stream).dump("default", self)
	118
	119	def update_option(self, option, value):
	120	if value is None:
	121	return
	122	#### Handle environment reference
	123	if str(value).startswith("$"):
	124	return self.update_option(option, os.getenv(str(value)[1:]))
	125	#### Special treatment of some options
	126	## verbosity must be known to "logging" module
	127	if option == "verbosity":
	128	try:
	129	setattr(Config, "verbosity", logging._levelNames[value])
	130	except KeyError:
	131	error("Config: verbosity level '%s' is not valid" % value)
	132	## allow yes/no, true/false, on/off and 1/0 for boolean options
	133	elif type(getattr(Config, option)) is type(True): # bool
	134	if str(value).lower() in ("true", "yes", "on", "1"):
	135	setattr(Config, option, True)
	136	elif str(value).lower() in ("false", "no", "off", "0"):
	137	setattr(Config, option, False)
	138	else:
	139	error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))
	140	elif type(getattr(Config, option)) is type(42): # int
	141	try:
	142	setattr(Config, option, int(value))
	143	except ValueError, e:
	144	error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))
	145	else: # string
	146	setattr(Config, option, value)
137	147
138	148	class ConfigParser(object):
139		def __init__(self, file, sections = []):
140		self.cfg = {}
141		self.parse_file(file, sections)
142
143		def parse_file(self, file, sections = []):
144		debug("ConfigParser: Reading file '%s'" % file)
145		if type(sections) != type([]):
146		sections = [sections]
147		in_our_section = True
148		f = open(file, "r")
149		r_comment = re.compile("^\s#.")
150		r_empty = re.compile("^\s*$")
151		r_section = re.compile("^\[([^\]]+)\]")
152		r_data = re.compile("^\s(?P<key>\w+)\s=\s(?P<value>.)")
153		r_quotes = re.compile("^\"(.)\"\s$")
154		for line in f:
155		if r_comment.match(line) or r_empty.match(line):
156		continue
157		is_section = r_section.match(line)
158		if is_section:
159		section = is_section.groups()[0]
160		in_our_section = (section in sections) or (len(sections) == 0)
161		continue
162		is_data = r_data.match(line)
163		if is_data and in_our_section:
164		data = is_data.groupdict()
165		if r_quotes.match(data["value"]):
166		data["value"] = data["value"][1:-1]
167		self.__setitem__(data["key"], data["value"])
168		if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
169		print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
170		else:
171		print_value = data["value"]
172		debug("ConfigParser: %s->%s" % (data["key"], print_value))
173		continue
174		warning("Ignoring invalid line in '%s': %s" % (file, line))
175
176		def __getitem__(self, name):
177		return self.cfg[name]
178
179		def __setitem__(self, name, value):
180		self.cfg[name] = value
181
182		def get(self, name, default = None):
183		if self.cfg.has_key(name):
184		return self.cfg[name]
185		return default
	149	def __init__(self, file, sections = []):
	150	self.cfg = {}
	151	self.parse_file(file, sections)
	152
	153	def parse_file(self, file, sections = []):
	154	debug("ConfigParser: Reading file '%s'" % file)
	155	if type(sections) != type([]):
	156	sections = [sections]
	157	in_our_section = True
	158	f = open(file, "r")
	159	r_comment = re.compile("^\s#.")
	160	r_empty = re.compile("^\s*$")
	161	r_section = re.compile("^\[([^\]]+)\]")
	162	r_data = re.compile("^\s(?P<key>\w+)\s=\s(?P<value>.)")
	163	r_quotes = re.compile("^\"(.)\"\s$")
	164	for line in f:
	165	if r_comment.match(line) or r_empty.match(line):
	166	continue
	167	is_section = r_section.match(line)
	168	if is_section:
	169	section = is_section.groups()[0]
	170	in_our_section = (section in sections) or (len(sections) == 0)
	171	continue
	172	is_data = r_data.match(line)
	173	if is_data and in_our_section:
	174	data = is_data.groupdict()
	175	if r_quotes.match(data["value"]):
	176	data["value"] = data["value"][1:-1]
	177	self.__setitem__(data["key"], data["value"])
	178	if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
	179	print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
	180	else:
	181	print_value = data["value"]
	182	debug("ConfigParser: %s->%s" % (data["key"], print_value))
	183	continue
	184	warning("Ignoring invalid line in '%s': %s" % (file, line))
	185
	186	def __getitem__(self, name):
	187	return self.cfg[name]
	188
	189	def __setitem__(self, name, value):
	190	self.cfg[name] = value
	191
	192	def get(self, name, default = None):
	193	if self.cfg.has_key(name):
	194	return self.cfg[name]
	195	return default
186	196
187	197	class ConfigDumper(object):
188		def __init__(self, stream):
189		self.stream = stream
190
191		def dump(self, section, config):
192		self.stream.write("[%s]\n" % section)
193		for option in config.option_list():
194		self.stream.write("%s = %s\n" % (option, getattr(config, option)))
195
	198	def __init__(self, stream):
	199	self.stream = stream
	200
	201	def dump(self, section, config):
	202	self.stream.write("[%s]\n" % section)
	203	for option in config.option_list():
	204	self.stream.write("%s = %s\n" % (option, getattr(config, option)))
	205
	206	# vim:et:ts=4:sts=4:ai

+58

-53

S3/Exceptions.py less more

6	6	from logging import debug, info, warning, error
7	7
8	8	try:
9		import xml.etree.ElementTree as ET
	9	import xml.etree.ElementTree as ET
10	10	except ImportError:
11		import elementtree.ElementTree as ET
	11	import elementtree.ElementTree as ET
12	12
13	13	class S3Exception(Exception):
14		def __init__(self, message = ""):
15		self.message = unicodise(message)
	14	def __init__(self, message = ""):
	15	self.message = unicodise(message)
16	16
17		def __str__(self):
18		## Call unicode(self) instead of self.message because
19		## __unicode__() method could be overriden in subclasses!
20		return deunicodise(unicode(self))
	17	def __str__(self):
	18	## Call unicode(self) instead of self.message because
	19	## __unicode__() method could be overriden in subclasses!
	20	return deunicodise(unicode(self))
21	21
22		def __unicode__(self):
23		return self.message
	22	def __unicode__(self):
	23	return self.message
24	24
25		## (Base)Exception.message has been deprecated in Python 2.6
26		def _get_message(self):
27		return self._message
28		def _set_message(self, message):
29		self._message = message
30		message = property(_get_message, _set_message)
	25	## (Base)Exception.message has been deprecated in Python 2.6
	26	def _get_message(self):
	27	return self._message
	28	def _set_message(self, message):
	29	self._message = message
	30	message = property(_get_message, _set_message)
31	31
32	32
33	33	class S3Error (S3Exception):
34		def __init__(self, response):
35		self.status = response["status"]
36		self.reason = response["reason"]
37		self.info = {
38		"Code" : "",
39		"Message" : "",
40		"Resource" : ""
41		}
42		debug("S3Error: %s (%s)" % (self.status, self.reason))
43		if response.has_key("headers"):
44		for header in response["headers"]:
45		debug("HttpHeader: %s: %s" % (header, response["headers"][header]))
46		if response.has_key("data"):
47		tree = getTreeFromXml(response["data"])
48		error_node = tree
49		if not error_node.tag == "Error":
50		error_node = tree.find(".//Error")
51		for child in error_node.getchildren():
52		if child.text != "":
53		debug("ErrorXML: " + child.tag + ": " + repr(child.text))
54		self.info[child.tag] = child.text
55		self.code = self.info["Code"]
56		self.message = self.info["Message"]
57		self.resource = self.info["Resource"]
	34	def __init__(self, response):
	35	self.status = response["status"]
	36	self.reason = response["reason"]
	37	self.info = {
	38	"Code" : "",
	39	"Message" : "",
	40	"Resource" : ""
	41	}
	42	debug("S3Error: %s (%s)" % (self.status, self.reason))
	43	if response.has_key("headers"):
	44	for header in response["headers"]:
	45	debug("HttpHeader: %s: %s" % (header, response["headers"][header]))
	46	if response.has_key("data"):
	47	tree = getTreeFromXml(response["data"])
	48	error_node = tree
	49	if not error_node.tag == "Error":
	50	error_node = tree.find(".//Error")
	51	for child in error_node.getchildren():
	52	if child.text != "":
	53	debug("ErrorXML: " + child.tag + ": " + repr(child.text))
	54	self.info[child.tag] = child.text
	55	self.code = self.info["Code"]
	56	self.message = self.info["Message"]
	57	self.resource = self.info["Resource"]
58	58
59		def __unicode__(self):
60		retval = u"%d " % (self.status)
61		retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason))
62		if self.info.has_key("Message"):
63		retval += (u": %s" % self.info["Message"])
64		return retval
	59	def __unicode__(self):
	60	retval = u"%d " % (self.status)
	61	retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason))
	62	if self.info.has_key("Message"):
	63	retval += (u": %s" % self.info["Message"])
	64	return retval
65	65
66	66	class CloudFrontError(S3Error):
67		pass
68
	67	pass
	68
69	69	class S3UploadError(S3Exception):
70		pass
	70	pass
71	71
72	72	class S3DownloadError(S3Exception):
73		pass
	73	pass
74	74
75	75	class S3RequestError(S3Exception):
76		pass
	76	pass
	77
	78	class S3ResponseError(S3Exception):
	79	pass
77	80
78	81	class InvalidFileError(S3Exception):
79		pass
	82	pass
80	83
81	84	class ParameterError(S3Exception):
82		pass
	85	pass
	86
	87	# vim:et:ts=4:sts=4:ai

+345

-0

S3/FileLists.py less more

	0	## Create and compare lists of files/objects
	1	## Author: Michal Ludvig <michal@logix.cz>
	2	## http://www.logix.cz/michal
	3	## License: GPL Version 2
	4
	5	from S3 import S3
	6	from Config import Config
	7	from S3Uri import S3Uri
	8	from SortedDict import SortedDict
	9	from Utils import *
	10	from Exceptions import ParameterError
	11
	12	from logging import debug, info, warning, error
	13
	14	import os
	15	import glob
	16
	17	__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]
	18
	19	def _fswalk_follow_symlinks(path):
	20	'''
	21	Walk filesystem, following symbolic links (but without recursion), on python2.4 and later
	22
	23	If a recursive directory link is detected, emit a warning and skip.
	24	'''
	25	assert os.path.isdir(path) # only designed for directory argument
	26	walkdirs = set([path])
	27	targets = set()
	28	for dirpath, dirnames, filenames in os.walk(path):
	29	for dirname in dirnames:
	30	current = os.path.join(dirpath, dirname)
	31	target = os.path.realpath(current)
	32	if os.path.islink(current):
	33	if target in targets:
	34	warning("Skipping recursively symlinked directory %s" % dirname)
	35	else:
	36	walkdirs.add(current)
	37	targets.add(target)
	38	for walkdir in walkdirs:
	39	for value in os.walk(walkdir):
	40	yield value
	41
	42	def _fswalk(path, follow_symlinks):
	43	'''
	44	Directory tree generator
	45
	46	path (str) is the root of the directory tree to walk
	47
	48	follow_symlinks (bool) indicates whether to descend into symbolically linked directories
	49	'''
	50	if follow_symlinks:
	51	return _fswalk_follow_symlinks(path)
	52	return os.walk(path)
	53
	54	def filter_exclude_include(src_list):
	55	info(u"Applying --exclude/--include")
	56	cfg = Config()
	57	exclude_list = SortedDict(ignore_case = False)
	58	for file in src_list.keys():
	59	debug(u"CHECK: %s" % file)
	60	excluded = False
	61	for r in cfg.exclude:
	62	if r.search(file):
	63	excluded = True
	64	debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
	65	break
	66	if excluded:
	67	## No need to check for --include if not excluded
	68	for r in cfg.include:
	69	if r.search(file):
	70	excluded = False
	71	debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
	72	break
	73	if excluded:
	74	## Still excluded - ok, action it
	75	debug(u"EXCLUDE: %s" % file)
	76	exclude_list[file] = src_list[file]
	77	del(src_list[file])
	78	continue
	79	else:
	80	debug(u"PASS: %s" % (file))
	81	return src_list, exclude_list
	82
	83	def fetch_local_list(args, recursive = None):
	84	def _get_filelist_local(local_uri):
	85	info(u"Compiling list of local files...")
	86	if local_uri.isdir():
	87	local_base = deunicodise(local_uri.basename())
	88	local_path = deunicodise(local_uri.path())
	89	filelist = _fswalk(local_path, cfg.follow_symlinks)
	90	single_file = False
	91	else:
	92	local_base = ""
	93	local_path = deunicodise(local_uri.dirname())
	94	filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]
	95	single_file = True
	96	loc_list = SortedDict(ignore_case = False)
	97	for root, dirs, files in filelist:
	98	rel_root = root.replace(local_path, local_base, 1)
	99	for f in files:
	100	full_name = os.path.join(root, f)
	101	if not os.path.isfile(full_name):
	102	continue
	103	if os.path.islink(full_name):
	104	if not cfg.follow_symlinks:
	105	continue
	106	relative_file = unicodise(os.path.join(rel_root, f))
	107	if os.path.sep != "/":
	108	# Convert non-unix dir separators to '/'
	109	relative_file = "/".join(relative_file.split(os.path.sep))
	110	if cfg.urlencoding_mode == "normal":
	111	relative_file = replace_nonprintables(relative_file)
	112	if relative_file.startswith('./'):
	113	relative_file = relative_file[2:]
	114	sr = os.stat_result(os.lstat(full_name))
	115	loc_list[relative_file] = {
	116	'full_name_unicode' : unicodise(full_name),
	117	'full_name' : full_name,
	118	'size' : sr.st_size,
	119	'mtime' : sr.st_mtime,
	120	## TODO: Possibly more to save here...
	121	}
	122	return loc_list, single_file
	123
	124	cfg = Config()
	125	local_uris = []
	126	local_list = SortedDict(ignore_case = False)
	127	single_file = False
	128
	129	if type(args) not in (list, tuple):
	130	args = [args]
	131
	132	if recursive == None:
	133	recursive = cfg.recursive
	134
	135	for arg in args:
	136	uri = S3Uri(arg)
	137	if not uri.type == 'file':
	138	raise ParameterError("Expecting filename or directory instead of: %s" % arg)
	139	if uri.isdir() and not recursive:
	140	raise ParameterError("Use --recursive to upload a directory: %s" % arg)
	141	local_uris.append(uri)
	142
	143	for uri in local_uris:
	144	list_for_uri, single_file = _get_filelist_local(uri)
	145	local_list.update(list_for_uri)
	146
	147	## Single file is True if and only if the user
	148	## specified one local URI and that URI represents
	149	## a FILE. Ie it is False if the URI was of a DIR
	150	## and that dir contained only one FILE. That's not
	151	## a case of single_file==True.
	152	if len(local_list) > 1:
	153	single_file = False
	154
	155	return local_list, single_file
	156
	157	def fetch_remote_list(args, require_attribs = False, recursive = None):
	158	def _get_filelist_remote(remote_uri, recursive = True):
	159	## If remote_uri ends with '/' then all remote files will have
	160	## the remote_uri prefix removed in the relative path.
	161	## If, on the other hand, the remote_uri ends with something else
	162	## (probably alphanumeric symbol) we'll use the last path part
	163	## in the relative path.
	164	##
	165	## Complicated, eh? See an example:
	166	## _get_filelist_remote("s3://bckt/abc/def") may yield:
	167	## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
	168	## _get_filelist_remote("s3://bckt/abc/def/") will yield:
	169	## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
	170	## Furthermore a prefix-magic can restrict the return list:
	171	## _get_filelist_remote("s3://bckt/abc/def/x") yields:
	172	## { 'xyz/blah.txt' : {} }
	173
	174	info(u"Retrieving list of remote files for %s ..." % remote_uri)
	175
	176	s3 = S3(Config())
	177	response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive)
	178
	179	rem_base_original = rem_base = remote_uri.object()
	180	remote_uri_original = remote_uri
	181	if rem_base != '' and rem_base[-1] != '/':
	182	rem_base = rem_base[:rem_base.rfind('/')+1]
	183	remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
	184	rem_base_len = len(rem_base)
	185	rem_list = SortedDict(ignore_case = False)
	186	break_now = False
	187	for object in response['list']:
	188	if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep:
	189	## We asked for one file and we got that file :-)
	190	key = os.path.basename(object['Key'])
	191	object_uri_str = remote_uri_original.uri()
	192	break_now = True
	193	rem_list = {} ## Remove whatever has already been put to rem_list
	194	else:
	195	key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
	196	object_uri_str = remote_uri.uri() + key
	197	rem_list[key] = {
	198	'size' : int(object['Size']),
	199	'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
	200	'md5' : object['ETag'][1:-1],
	201	'object_key' : object['Key'],
	202	'object_uri_str' : object_uri_str,
	203	'base_uri' : remote_uri,
	204	}
	205	if break_now:
	206	break
	207	return rem_list
	208
	209	cfg = Config()
	210	remote_uris = []
	211	remote_list = SortedDict(ignore_case = False)
	212
	213	if type(args) not in (list, tuple):
	214	args = [args]
	215
	216	if recursive == None:
	217	recursive = cfg.recursive
	218
	219	for arg in args:
	220	uri = S3Uri(arg)
	221	if not uri.type == 's3':
	222	raise ParameterError("Expecting S3 URI instead of '%s'" % arg)
	223	remote_uris.append(uri)
	224
	225	if recursive:
	226	for uri in remote_uris:
	227	objectlist = _get_filelist_remote(uri)
	228	for key in objectlist:
	229	remote_list[key] = objectlist[key]
	230	else:
	231	for uri in remote_uris:
	232	uri_str = str(uri)
	233	## Wildcards used in remote URI?
	234	## If yes we'll need a bucket listing...
	235	if uri_str.find('*') > -1 or uri_str.find('?') > -1:
	236	first_wildcard = uri_str.find('*')
	237	first_questionmark = uri_str.find('?')
	238	if first_questionmark > -1 and first_questionmark < first_wildcard:
	239	first_wildcard = first_questionmark
	240	prefix = uri_str[:first_wildcard]
	241	rest = uri_str[first_wildcard+1:]
	242	## Only request recursive listing if the 'rest' of the URI,
	243	## i.e. the part after first wildcard, contains '/'
	244	need_recursion = rest.find('/') > -1
	245	objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion)
	246	for key in objectlist:
	247	## Check whether the 'key' matches the requested wildcards
	248	if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str):
	249	remote_list[key] = objectlist[key]
	250	else:
	251	## No wildcards - simply append the given URI to the list
	252	key = os.path.basename(uri.object())
	253	if not key:
	254	raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri())
	255	remote_item = {
	256	'base_uri': uri,
	257	'object_uri_str': unicode(uri),
	258	'object_key': uri.object()
	259	}
	260	if require_attribs:
	261	response = S3(cfg).object_info(uri)
	262	remote_item.update({
	263	'size': int(response['headers']['content-length']),
	264	'md5': response['headers']['etag'].strip('"\''),
	265	'timestamp' : dateRFC822toUnix(response['headers']['date'])
	266	})
	267	remote_list[key] = remote_item
	268	return remote_list
	269
	270	def compare_filelists(src_list, dst_list, src_remote, dst_remote):
	271	def __direction_str(is_remote):
	272	return is_remote and "remote" or "local"
	273
	274	# We don't support local->local sync, use 'rsync' or something like that instead ;-)
	275	assert(not(src_remote == False and dst_remote == False))
	276
	277	info(u"Verifying attributes...")
	278	cfg = Config()
	279	exists_list = SortedDict(ignore_case = False)
	280
	281	debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
	282	debug("src_list.keys: %s" % src_list.keys())
	283	debug("dst_list.keys: %s" % dst_list.keys())
	284
	285	for file in src_list.keys():
	286	debug(u"CHECK: %s" % file)
	287	if dst_list.has_key(file):
	288	## Was --skip-existing requested?
	289	if cfg.skip_existing:
	290	debug(u"IGNR: %s (used --skip-existing)" % (file))
	291	exists_list[file] = src_list[file]
	292	del(src_list[file])
	293	## Remove from destination-list, all that is left there will be deleted
	294	del(dst_list[file])
	295	continue
	296
	297	attribs_match = True
	298	## Check size first
	299	if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:
	300	debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
	301	attribs_match = False
	302
	303	## Check MD5
	304	compare_md5 = 'md5' in cfg.sync_checks
	305	# Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"
	306	if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
	307	compare_md5 = False
	308	info(u"Disabled MD5 check for %s" % file)
	309	if attribs_match and compare_md5:
	310	try:
	311	if src_remote == False and dst_remote == True:
	312	src_md5 = hash_file_md5(src_list[file]['full_name'])
	313	dst_md5 = dst_list[file]['md5']
	314	elif src_remote == True and dst_remote == False:
	315	src_md5 = src_list[file]['md5']
	316	dst_md5 = hash_file_md5(dst_list[file]['full_name'])
	317	elif src_remote == True and dst_remote == True:
	318	src_md5 = src_list[file]['md5']
	319	dst_md5 = dst_list[file]['md5']
	320	except (IOError,OSError), e:
	321	# MD5 sum verification failed - ignore that file altogether
	322	debug(u"IGNR: %s (disappeared)" % (file))
	323	warning(u"%s: file disappeared, ignoring." % (file))
	324	del(src_list[file])
	325	del(dst_list[file])
	326	continue
	327
	328	if src_md5 != dst_md5:
	329	## Checksums are different.
	330	attribs_match = False
	331	debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
	332
	333	if attribs_match:
	334	## Remove from source-list, all that is left there will be transferred
	335	debug(u"IGNR: %s (transfer not needed)" % file)
	336	exists_list[file] = src_list[file]
	337	del(src_list[file])
	338
	339	## Remove from destination-list, all that is left there will be deleted
	340	del(dst_list[file])
	341
	342	return src_list, dst_list, exists_list
	343
	344	# vim:et:ts=4:sts=4:ai

+113

-0

S3/MultiPart.py less more

	0	## Amazon S3 Multipart upload support
	1	## Author: Jerome Leclanche <jerome.leclanche@gmail.com>
	2	## License: GPL Version 2
	3
	4	import os
	5	from stat import ST_SIZE
	6	from logging import debug, info, warning, error
	7	from Utils import getTextFromXml, formatSize, unicodise
	8	from Exceptions import S3UploadError
	9
	10	class MultiPartUpload(object):
	11
	12	MIN_CHUNK_SIZE_MB = 5 # 5MB
	13	MAX_CHUNK_SIZE_MB = 5120 # 5GB
	14	MAX_FILE_SIZE = 42949672960 # 5TB
	15
	16	def __init__(self, s3, file, uri, headers_baseline = {}):
	17	self.s3 = s3
	18	self.file = file
	19	self.uri = uri
	20	self.parts = {}
	21	self.headers_baseline = headers_baseline
	22	self.upload_id = self.initiate_multipart_upload()
	23
	24	def initiate_multipart_upload(self):
	25	"""
	26	Begin a multipart upload
	27	http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html
	28	"""
	29	request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = self.headers_baseline, extra = "?uploads")
	30	response = self.s3.send_request(request)
	31	data = response["data"]
	32	self.upload_id = getTextFromXml(data, "UploadId")
	33	return self.upload_id
	34
	35	def upload_all_parts(self):
	36	"""
	37	Execute a full multipart upload on a file
	38	Returns the seq/etag dict
	39	TODO use num_processes to thread it
	40	"""
	41	if not self.upload_id:
	42	raise RuntimeError("Attempting to use a multipart upload that has not been initiated.")
	43
	44	size_left = file_size = os.stat(self.file.name)[ST_SIZE]
	45	self.chunk_size = self.s3.config.multipart_chunk_size_mb * 1024 * 1024
	46	nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1)
	47	debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts))
	48
	49	seq = 1
	50	while size_left > 0:
	51	offset = self.chunk_size * (seq - 1)
	52	current_chunk_size = min(file_size - offset, self.chunk_size)
	53	size_left -= current_chunk_size
	54	labels = {
	55	'source' : unicodise(self.file.name),
	56	'destination' : unicodise(self.uri.uri()),
	57	'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True))
	58	}
	59	try:
	60	self.upload_part(seq, offset, current_chunk_size, labels)
	61	except:
	62	error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))
	63	self.abort_upload()
	64	raise
	65	seq += 1
	66
	67	debug("MultiPart: Upload finished: %d parts", seq - 1)
	68
	69	def upload_part(self, seq, offset, chunk_size, labels):
	70	"""
	71	Upload a file chunk
	72	http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
	73	"""
	74	# TODO implement Content-MD5
	75	debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))
	76	headers = { "content-length": chunk_size }
	77	query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id)
	78	request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)
	79	response = self.s3.send_file(request, self.file, labels, offset = offset, chunk_size = chunk_size)
	80	self.parts[seq] = response["headers"]["etag"]
	81	return response
	82
	83	def complete_multipart_upload(self):
	84	"""
	85	Finish a multipart upload
	86	http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
	87	"""
	88	debug("MultiPart: Completing upload: %s" % self.upload_id)
	89
	90	parts_xml = []
	91	part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
	92	for seq, etag in self.parts.items():
	93	parts_xml.append(part_xml % (seq, etag))
	94	body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))
	95
	96	headers = { "content-length": len(body) }
	97	request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % (self.upload_id))
	98	response = self.s3.send_request(request, body = body)
	99
	100	return response
	101
	102	def abort_upload(self):
	103	"""
	104	Abort multipart upload
	105	http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadAbort.html
	106	"""
	107	debug("MultiPart: Aborting upload: %s" % self.upload_id)
	108	request = self.s3.create_request("OBJECT_DELETE", uri = self.uri, extra = "?uploadId=%s" % (self.upload_id))
	109	response = self.s3.send_request(request)
	110	return response
	111
	112	# vim:et:ts=4:sts=4:ai

-2

S3/PkgInfo.py less more

0	0	package = "s3cmd"
1		version = "1.0.0"
	1	version = "1.1.0-beta3"
2	2	url = "http://s3tools.org"
3	3	license = "GPL version 2"
4	4	short_description = "Command line tool for managing Amazon S3 and CloudFront services"
5	5	long_description = """
6		S3cmd lets you copy files from/to Amazon S3
	6	S3cmd lets you copy files from/to Amazon S3
7	7	(Simple Storage Service) using a simple to use
8	8	command line client. Supports rsync-like backup,
9	9	GPG encryption, and more. Also supports management
10	10	of Amazon's CloudFront content delivery network.
11	11	"""
12	12
	13	# vim:et:ts=4:sts=4:ai

+125

-123

S3/Progress.py less more

7	7	import Utils
8	8
9	9	class Progress(object):
10		_stdout = sys.stdout
	10	_stdout = sys.stdout
11	11
12		def __init__(self, labels, total_size):
13		self._stdout = sys.stdout
14		self.new_file(labels, total_size)
15
16		def new_file(self, labels, total_size):
17		self.labels = labels
18		self.total_size = total_size
19		# Set initial_position to something in the
20		# case we're not counting from 0. For instance
21		# when appending to a partially downloaded file.
22		# Setting initial_position will let the speed
23		# be computed right.
24		self.initial_position = 0
25		self.current_position = self.initial_position
26		self.time_start = datetime.datetime.now()
27		self.time_last = self.time_start
28		self.time_current = self.time_start
	12	def __init__(self, labels, total_size):
	13	self._stdout = sys.stdout
	14	self.new_file(labels, total_size)
29	15
30		self.display(new_file = True)
31
32		def update(self, current_position = -1, delta_position = -1):
33		self.time_last = self.time_current
34		self.time_current = datetime.datetime.now()
35		if current_position > -1:
36		self.current_position = current_position
37		elif delta_position > -1:
38		self.current_position += delta_position
39		#else:
40		# no update, just call display()
41		self.display()
	16	def new_file(self, labels, total_size):
	17	self.labels = labels
	18	self.total_size = total_size
	19	# Set initial_position to something in the
	20	# case we're not counting from 0. For instance
	21	# when appending to a partially downloaded file.
	22	# Setting initial_position will let the speed
	23	# be computed right.
	24	self.initial_position = 0
	25	self.current_position = self.initial_position
	26	self.time_start = datetime.datetime.now()
	27	self.time_last = self.time_start
	28	self.time_current = self.time_start
42	29
43		def done(self, message):
44		self.display(done_message = message)
	30	self.display(new_file = True)
45	31
46		def output_labels(self):
47		self._stdout.write(u"%(source)s -> %(destination)s %(extra)s\n" % self.labels)
48		self._stdout.flush()
	32	def update(self, current_position = -1, delta_position = -1):
	33	self.time_last = self.time_current
	34	self.time_current = datetime.datetime.now()
	35	if current_position > -1:
	36	self.current_position = current_position
	37	elif delta_position > -1:
	38	self.current_position += delta_position
	39	#else:
	40	# no update, just call display()
	41	self.display()
49	42
50		def display(self, new_file = False, done_message = None):
51		"""
52		display(new_file = False[/True], done = False[/True])
	43	def done(self, message):
	44	self.display(done_message = message)
53	45
54		Override this method to provide a nicer output.
55		"""
56		if new_file:
57		self.output_labels()
58		self.last_milestone = 0
59		return
	46	def output_labels(self):
	47	self._stdout.write(u"%(source)s -> %(destination)s %(extra)s\n" % self.labels)
	48	self._stdout.flush()
60	49
61		if self.current_position == self.total_size:
62		print_size = Utils.formatSize(self.current_position, True)
63		if print_size[1] != "": print_size[1] += "B"
64		timedelta = self.time_current - self.time_start
65		sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
66		print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
67		self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" %
68		(print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1]))
69		self._stdout.flush()
70		return
	50	def display(self, new_file = False, done_message = None):
	51	"""
	52	display(new_file = False[/True], done = False[/True])
71	53
72		rel_position = selfself.current_position * 100 / self.total_size
73		if rel_position >= self.last_milestone:
74		self.last_milestone = (int(rel_position) / 5) * 5
75		self._stdout.write("%d%% ", self.last_milestone)
76		self._stdout.flush()
77		return
	54	Override this method to provide a nicer output.
	55	"""
	56	if new_file:
	57	self.output_labels()
	58	self.last_milestone = 0
	59	return
	60
	61	if self.current_position == self.total_size:
	62	print_size = Utils.formatSize(self.current_position, True)
	63	if print_size[1] != "": print_size[1] += "B"
	64	timedelta = self.time_current - self.time_start
	65	sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
	66	print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
	67	self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" %
	68	(print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1]))
	69	self._stdout.flush()
	70	return
	71
	72	rel_position = selfself.current_position * 100 / self.total_size
	73	if rel_position >= self.last_milestone:
	74	self.last_milestone = (int(rel_position) / 5) * 5
	75	self._stdout.write("%d%% ", self.last_milestone)
	76	self._stdout.flush()
	77	return
78	78
79	79	class ProgressANSI(Progress):
80	80	## http://en.wikipedia.org/wiki/ANSI_escape_code
81		SCI = '\x1b['
82		ANSI_hide_cursor = SCI + "?25l"
83		ANSI_show_cursor = SCI + "?25h"
84		ANSI_save_cursor_pos = SCI + "s"
85		ANSI_restore_cursor_pos = SCI + "u"
86		ANSI_move_cursor_to_column = SCI + "%uG"
87		ANSI_erase_to_eol = SCI + "0K"
88		ANSI_erase_current_line = SCI + "2K"
	81	SCI = '\x1b['
	82	ANSI_hide_cursor = SCI + "?25l"
	83	ANSI_show_cursor = SCI + "?25h"
	84	ANSI_save_cursor_pos = SCI + "s"
	85	ANSI_restore_cursor_pos = SCI + "u"
	86	ANSI_move_cursor_to_column = SCI + "%uG"
	87	ANSI_erase_to_eol = SCI + "0K"
	88	ANSI_erase_current_line = SCI + "2K"
89	89
90		def display(self, new_file = False, done_message = None):
91		"""
92		display(new_file = False[/True], done_message = None)
93		"""
94		if new_file:
95		self.output_labels()
96		self._stdout.write(self.ANSI_save_cursor_pos)
97		self._stdout.flush()
98		return
	90	def display(self, new_file = False, done_message = None):
	91	"""
	92	display(new_file = False[/True], done_message = None)
	93	"""
	94	if new_file:
	95	self.output_labels()
	96	self._stdout.write(self.ANSI_save_cursor_pos)
	97	self._stdout.flush()
	98	return
99	99
100		timedelta = self.time_current - self.time_start
101		sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
102		if (sec_elapsed > 0):
103		print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
104		else:
105		print_speed = (0, "")
106		self._stdout.write(self.ANSI_restore_cursor_pos)
107		self._stdout.write(self.ANSI_erase_to_eol)
108		self._stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % {
109		"current" : str(self.current_position).rjust(len(str(self.total_size))),
110		"total" : self.total_size,
111		"percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
112		"elapsed" : sec_elapsed,
113		"speed" : print_speed[0],
114		"speed_coeff" : print_speed[1]
115		})
	100	timedelta = self.time_current - self.time_start
	101	sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
	102	if (sec_elapsed > 0):
	103	print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
	104	else:
	105	print_speed = (0, "")
	106	self._stdout.write(self.ANSI_restore_cursor_pos)
	107	self._stdout.write(self.ANSI_erase_to_eol)
	108	self._stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % {
	109	"current" : str(self.current_position).rjust(len(str(self.total_size))),
	110	"total" : self.total_size,
	111	"percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
	112	"elapsed" : sec_elapsed,
	113	"speed" : print_speed[0],
	114	"speed_coeff" : print_speed[1]
	115	})
116	116
117		if done_message:
118		self._stdout.write(" %s\n" % done_message)
	117	if done_message:
	118	self._stdout.write(" %s\n" % done_message)
119	119
120		self._stdout.flush()
	120	self._stdout.flush()
121	121
122	122	class ProgressCR(Progress):
123	123	## Uses CR char (Carriage Return) just like other progress bars do.
124		CR_char = chr(13)
	124	CR_char = chr(13)
125	125
126		def display(self, new_file = False, done_message = None):
127		"""
128		display(new_file = False[/True], done_message = None)
129		"""
130		if new_file:
131		self.output_labels()
132		return
	126	def display(self, new_file = False, done_message = None):
	127	"""
	128	display(new_file = False[/True], done_message = None)
	129	"""
	130	if new_file:
	131	self.output_labels()
	132	return
133	133
134		timedelta = self.time_current - self.time_start
135		sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
136		if (sec_elapsed > 0):
137		print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
138		else:
139		print_speed = (0, "")
140		self._stdout.write(self.CR_char)
141		output = " %(current)s of %(total)s %(percent)3d%% in %(elapsed)4ds %(speed)7.2f %(speed_coeff)sB/s" % {
142		"current" : str(self.current_position).rjust(len(str(self.total_size))),
143		"total" : self.total_size,
144		"percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
145		"elapsed" : sec_elapsed,
146		"speed" : print_speed[0],
147		"speed_coeff" : print_speed[1]
148		}
149		self._stdout.write(output)
150		if done_message:
151		self._stdout.write(" %s\n" % done_message)
	134	timedelta = self.time_current - self.time_start
	135	sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
	136	if (sec_elapsed > 0):
	137	print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
	138	else:
	139	print_speed = (0, "")
	140	self._stdout.write(self.CR_char)
	141	output = " %(current)s of %(total)s %(percent)3d%% in %(elapsed)4ds %(speed)7.2f %(speed_coeff)sB/s" % {
	142	"current" : str(self.current_position).rjust(len(str(self.total_size))),
	143	"total" : self.total_size,
	144	"percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
	145	"elapsed" : sec_elapsed,
	146	"speed" : print_speed[0],
	147	"speed_coeff" : print_speed[1]
	148	}
	149	self._stdout.write(output)
	150	if done_message:
	151	self._stdout.write(" %s\n" % done_message)
152	152
153		self._stdout.flush()
	153	self._stdout.flush()
	154
	155	# vim:et:ts=4:sts=4:ai

+853

-729

S3/S3.py less more

13	13	from stat import ST_SIZE
14	14
15	15	try:
16		from hashlib import md5
	16	from hashlib import md5
17	17	except ImportError:
18		from md5 import md5
	18	from md5 import md5
19	19
20	20	from Utils import *
21	21	from SortedDict import SortedDict
	22	from AccessLog import AccessLog
	23	from ACL import ACL, GranteeLogDelivery
22	24	from BidirMap import BidirMap
23	25	from Config import Config
24	26	from Exceptions import *
25		from ACL import ACL, GranteeLogDelivery
26		from AccessLog import AccessLog
	27	from MultiPart import MultiPartUpload
27	28	from S3Uri import S3Uri
	29
	30	try:
	31	import magic
	32	try:
	33	## https://github.com/ahupp/python-magic
	34	magic_ = magic.Magic(mime=True)
	35	def mime_magic(file):
	36	return magic_.from_file(file)
	37	except (TypeError, AttributeError):
	38	## Older python-magic versions
	39	magic_ = magic.open(magic.MAGIC_MIME)
	40	magic_.load()
	41	def mime_magic(file):
	42	return magic_.file(file)
	43	except ImportError, e:
	44	if str(e).find("magic") >= 0:
	45	magic_message = "Module python-magic is not available."
	46	else:
	47	magic_message = "Module python-magic can't be used (%s)." % e.message
	48	magic_message += " Guessing MIME types based on file extensions."
	49	magic_warned = False
	50	def mime_magic(file):
	51	global magic_warned
	52	if (not magic_warned):
	53	warning(magic_message)
	54	magic_warned = True
	55	return mimetypes.guess_type(file)[0]
28	56
29	57	__all__ = []
30	58	class S3Request(object):
31		def __init__(self, s3, method_string, resource, headers, params = {}):
32		self.s3 = s3
33		self.headers = SortedDict(headers or {}, ignore_case = True)
34		self.resource = resource
35		self.method_string = method_string
36		self.params = params
37
38		self.update_timestamp()
39		self.sign()
40
41		def update_timestamp(self):
42		if self.headers.has_key("date"):
43		del(self.headers["date"])
44		self.headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
45
46		def format_param_str(self):
47		"""
48		Format URL parameters from self.params and returns
49		?parm1=val1&parm2=val2 or an empty string if there
50		are no parameters. Output of this function should
51		be appended directly to self.resource['uri']
52		"""
53		param_str = ""
54		for param in self.params:
55		if self.params[param] not in (None, ""):
56		param_str += "&%s=%s" % (param, self.params[param])
57		else:
58		param_str += "&%s" % param
59		return param_str and "?" + param_str[1:]
60
61		def sign(self):
62		h = self.method_string + "\n"
63		h += self.headers.get("content-md5", "")+"\n"
64		h += self.headers.get("content-type", "")+"\n"
65		h += self.headers.get("date", "")+"\n"
66		for header in self.headers.keys():
67		if header.startswith("x-amz-"):
68		h += header+":"+str(self.headers[header])+"\n"
69		if self.resource['bucket']:
70		h += "/" + self.resource['bucket']
71		h += self.resource['uri']
72		debug("SignHeaders: " + repr(h))
73		signature = sign_string(h)
74
75		self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature
76
77		def get_triplet(self):
78		self.update_timestamp()
79		self.sign()
80		resource = dict(self.resource) ## take a copy
81		resource['uri'] += self.format_param_str()
82		return (self.method_string, resource, self.headers)
	59	def __init__(self, s3, method_string, resource, headers, params = {}):
	60	self.s3 = s3
	61	self.headers = SortedDict(headers or {}, ignore_case = True)
	62	self.resource = resource
	63	self.method_string = method_string
	64	self.params = params
	65
	66	self.update_timestamp()
	67	self.sign()
	68
	69	def update_timestamp(self):
	70	if self.headers.has_key("date"):
	71	del(self.headers["date"])
	72	self.headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
	73
	74	def format_param_str(self):
	75	"""
	76	Format URL parameters from self.params and returns
	77	?parm1=val1&parm2=val2 or an empty string if there
	78	are no parameters. Output of this function should
	79	be appended directly to self.resource['uri']
	80	"""
	81	param_str = ""
	82	for param in self.params:
	83	if self.params[param] not in (None, ""):
	84	param_str += "&%s=%s" % (param, self.params[param])
	85	else:
	86	param_str += "&%s" % param
	87	return param_str and "?" + param_str[1:]
	88
	89	def sign(self):
	90	h = self.method_string + "\n"
	91	h += self.headers.get("content-md5", "")+"\n"
	92	h += self.headers.get("content-type", "")+"\n"
	93	h += self.headers.get("date", "")+"\n"
	94	for header in self.headers.keys():
	95	if header.startswith("x-amz-"):
	96	h += header+":"+str(self.headers[header])+"\n"
	97	if self.resource['bucket']:
	98	h += "/" + self.resource['bucket']
	99	h += self.resource['uri']
	100	debug("SignHeaders: " + repr(h))
	101	signature = sign_string(h)
	102
	103	self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature
	104
	105	def get_triplet(self):
	106	self.update_timestamp()
	107	self.sign()
	108	resource = dict(self.resource) ## take a copy
	109	resource['uri'] += self.format_param_str()
	110	return (self.method_string, resource, self.headers)
83	111
84	112	class S3(object):
85		http_methods = BidirMap(
86		GET = 0x01,
87		PUT = 0x02,
88		HEAD = 0x04,
89		DELETE = 0x08,
90		MASK = 0x0F,
91		)
92
93		targets = BidirMap(
94		SERVICE = 0x0100,
95		BUCKET = 0x0200,
96		OBJECT = 0x0400,
97		MASK = 0x0700,
98		)
99
100		operations = BidirMap(
101		UNDFINED = 0x0000,
102		LIST_ALL_BUCKETS = targets["SERVICE"] \| http_methods["GET"],
103		BUCKET_CREATE = targets["BUCKET"] \| http_methods["PUT"],
104		BUCKET_LIST = targets["BUCKET"] \| http_methods["GET"],
105		BUCKET_DELETE = targets["BUCKET"] \| http_methods["DELETE"],
106		OBJECT_PUT = targets["OBJECT"] \| http_methods["PUT"],
107		OBJECT_GET = targets["OBJECT"] \| http_methods["GET"],
108		OBJECT_HEAD = targets["OBJECT"] \| http_methods["HEAD"],
109		OBJECT_DELETE = targets["OBJECT"] \| http_methods["DELETE"],
110		)
111
112		codes = {
113		"NoSuchBucket" : "Bucket '%s' does not exist",
114		"AccessDenied" : "Access to bucket '%s' was denied",
115		"BucketAlreadyExists" : "Bucket '%s' already exists",
116		}
117
118		## S3 sometimes sends HTTP-307 response
119		redir_map = {}
120
121		## Maximum attempts of re-issuing failed requests
122		_max_retries = 5
123
124		def __init__(self, config):
125		self.config = config
126
127		def get_connection(self, bucket):
128		if self.config.proxy_host != "":
129		return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
130		else:
131		if self.config.use_https:
132		return httplib.HTTPSConnection(self.get_hostname(bucket))
133		else:
134		return httplib.HTTPConnection(self.get_hostname(bucket))
135
136		def get_hostname(self, bucket):
137		if bucket and check_bucket_name_dns_conformity(bucket):
138		if self.redir_map.has_key(bucket):
139		host = self.redir_map[bucket]
140		else:
141		host = getHostnameFromBucket(bucket)
142		else:
143		host = self.config.host_base
144		debug('get_hostname(%s): %s' % (bucket, host))
145		return host
146
147		def set_hostname(self, bucket, redir_hostname):
148		self.redir_map[bucket] = redir_hostname
149
150		def format_uri(self, resource):
151		if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):
152		uri = "/%s%s" % (resource['bucket'], resource['uri'])
153		else:
154		uri = resource['uri']
155		if self.config.proxy_host != "":
156		uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri)
157		debug('format_uri(): ' + uri)
158		return uri
159
160		## Commands / Actions
161		def list_all_buckets(self):
162		request = self.create_request("LIST_ALL_BUCKETS")
163		response = self.send_request(request)
164		response["list"] = getListFromXml(response["data"], "Bucket")
165		return response
166
167		def bucket_list(self, bucket, prefix = None, recursive = None):
168		def _list_truncated(data):
169		## <IsTruncated> can either be "true" or "false" or be missing completely
170		is_truncated = getTextFromXml(data, ".//IsTruncated") or "false"
171		return is_truncated.lower() != "false"
172
173		def _get_contents(data):
174		return getListFromXml(data, "Contents")
175
176		def _get_common_prefixes(data):
177		return getListFromXml(data, "CommonPrefixes")
178
179		uri_params = {}
180		truncated = True
181		list = []
182		prefixes = []
183
184		while truncated:
185		response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
186		current_list = _get_contents(response["data"])
187		current_prefixes = _get_common_prefixes(response["data"])
188		truncated = _list_truncated(response["data"])
189		if truncated:
190		if current_list:
191		uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])
192		else:
193		uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"])
194		debug("Listing continues after '%s'" % uri_params['marker'])
195
196		list += current_list
197		prefixes += current_prefixes
198
199		response['list'] = list
200		response['common_prefixes'] = prefixes
201		return response
202
203		def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):
204		if prefix:
205		uri_params['prefix'] = self.urlencode_string(prefix)
206		if not self.config.recursive and not recursive:
207		uri_params['delimiter'] = "/"
208		request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)
209		response = self.send_request(request)
210		#debug(response)
211		return response
212
213		def bucket_create(self, bucket, bucket_location = None):
214		headers = SortedDict(ignore_case = True)
215		body = ""
216		if bucket_location and bucket_location.strip().upper() != "US":
217		bucket_location = bucket_location.strip()
218		if bucket_location.upper() == "EU":
219		bucket_location = bucket_location.upper()
220		else:
221		bucket_location = bucket_location.lower()
222		body = "<CreateBucketConfiguration><LocationConstraint>"
223		body += bucket_location
224		body += "</LocationConstraint></CreateBucketConfiguration>"
225		debug("bucket_location: " + body)
226		check_bucket_name(bucket, dns_strict = True)
227		else:
228		check_bucket_name(bucket, dns_strict = False)
229		if self.config.acl_public:
230		headers["x-amz-acl"] = "public-read"
231		request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
232		response = self.send_request(request, body)
233		return response
234
235		def bucket_delete(self, bucket):
236		request = self.create_request("BUCKET_DELETE", bucket = bucket)
237		response = self.send_request(request)
238		return response
239
240		def bucket_info(self, uri):
241		request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location")
242		response = self.send_request(request)
243		response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"
244		return response
245
246		def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
247		# TODO TODO
248		# Make it consistent with stream-oriented object_get()
249		if uri.type != "s3":
250		raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
251
252		if not os.path.isfile(filename):
253		raise InvalidFileError(u"%s is not a regular file" % unicodise(filename))
254		try:
255		file = open(filename, "rb")
256		size = os.stat(filename)[ST_SIZE]
257		except (IOError, OSError), e:
258		raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))
259		headers = SortedDict(ignore_case = True)
260		if extra_headers:
261		headers.update(extra_headers)
262		headers["content-length"] = size
263		content_type = None
264		if self.config.guess_mime_type:
265		content_type = mimetypes.guess_type(filename)[0]
266		if not content_type:
267		content_type = self.config.default_mime_type
268		debug("Content-Type set to '%s'" % content_type)
269		headers["content-type"] = content_type
270		if self.config.acl_public:
271		headers["x-amz-acl"] = "public-read"
272		if self.config.reduced_redundancy:
273		headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
274		request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)
275		labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label }
276		response = self.send_file(request, file, labels)
277		return response
278
279		def object_get(self, uri, stream, start_position = 0, extra_label = ""):
280		if uri.type != "s3":
281		raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
282		request = self.create_request("OBJECT_GET", uri = uri)
283		labels = { 'source' : unicodise(uri.uri()), 'destination' : unicodise(stream.name), 'extra' : extra_label }
284		response = self.recv_file(request, stream, labels, start_position)
285		return response
286
287		def object_delete(self, uri):
288		if uri.type != "s3":
289		raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
290		request = self.create_request("OBJECT_DELETE", uri = uri)
291		response = self.send_request(request)
292		return response
293
294		def object_copy(self, src_uri, dst_uri, extra_headers = None):
295		if src_uri.type != "s3":
296		raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type)
297		if dst_uri.type != "s3":
298		raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type)
299		headers = SortedDict(ignore_case = True)
300		headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object()))
301		## TODO: For now COPY, later maybe add a switch?
302		headers['x-amz-metadata-directive'] = "COPY"
303		if self.config.acl_public:
304		headers["x-amz-acl"] = "public-read"
305		if self.config.reduced_redundancy:
306		headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
307		# if extra_headers:
308		# headers.update(extra_headers)
309		request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers)
310		response = self.send_request(request)
311		return response
312
313		def object_move(self, src_uri, dst_uri, extra_headers = None):
314		response_copy = self.object_copy(src_uri, dst_uri, extra_headers)
315		debug("Object %s copied to %s" % (src_uri, dst_uri))
316		if getRootTagName(response_copy["data"]) == "CopyObjectResult":
317		response_delete = self.object_delete(src_uri)
318		debug("Object %s deleted" % src_uri)
319		return response_copy
320
321		def object_info(self, uri):
322		request = self.create_request("OBJECT_HEAD", uri = uri)
323		response = self.send_request(request)
324		return response
325
326		def get_acl(self, uri):
327		if uri.has_object():
328		request = self.create_request("OBJECT_GET", uri = uri, extra = "?acl")
329		else:
330		request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl")
331
332		response = self.send_request(request)
333		acl = ACL(response['data'])
334		return acl
335
336		def set_acl(self, uri, acl):
337		if uri.has_object():
338		request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl")
339		else:
340		request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl")
341
342		body = str(acl)
343		debug(u"set_acl(%s): acl-xml: %s" % (uri, body))
344		response = self.send_request(request, body)
345		return response
346
347		def get_accesslog(self, uri):
348		request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging")
349		response = self.send_request(request)
350		accesslog = AccessLog(response['data'])
351		return accesslog
352
353		def set_accesslog_acl(self, uri):
354		acl = self.get_acl(uri)
355		debug("Current ACL(%s): %s" % (uri.uri(), str(acl)))
356		acl.appendGrantee(GranteeLogDelivery("READ_ACP"))
357		acl.appendGrantee(GranteeLogDelivery("WRITE"))
358		debug("Updated ACL(%s): %s" % (uri.uri(), str(acl)))
359		self.set_acl(uri, acl)
360
361		def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False):
362		request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging")
363		accesslog = AccessLog()
364		if enable:
365		accesslog.enableLogging(log_target_prefix_uri)
366		accesslog.setAclPublic(acl_public)
367		else:
368		accesslog.disableLogging()
369		body = str(accesslog)
370		debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body))
371		try:
372		response = self.send_request(request, body)
373		except S3Error, e:
374		if e.info['Code'] == "InvalidTargetBucketForLogging":
375		info("Setting up log-delivery ACL for target bucket.")
376		self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket()))
377		response = self.send_request(request, body)
378		else:
379		raise
380		return accesslog, response
381
382		## Low level methods
383		def urlencode_string(self, string, urlencoding_mode = None):
384		if type(string) == unicode:
385		string = string.encode("utf-8")
386
387		if urlencoding_mode is None:
388		urlencoding_mode = self.config.urlencoding_mode
389
390		if urlencoding_mode == "verbatim":
391		## Don't do any pre-processing
392		return string
393
394		encoded = ""
395		## List of characters that must be escaped for S3
396		## Haven't found this in any official docs
397		## but my tests show it's more less correct.
398		## If you start getting InvalidSignature errors
399		## from S3 check the error headers returned
400		## from S3 to see whether the list hasn't
401		## changed.
402		for c in string: # I'm not sure how to know in what encoding
403		# 'object' is. Apparently "type(object)==str"
404		# but the contents is a string of unicode
405		# bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k'
406		# Don't know what it will do on non-utf8
407		# systems.
408		# [hope that sounds reassuring ;-)]
409		o = ord(c)
410		if (o < 0x20 or o == 0x7f):
411		if urlencoding_mode == "fixbucket":
412		encoded += "%%%02X" % o
413		else:
414		error(u"Non-printable character 0x%02x in: %s" % (o, string))
415		error(u"Please report it to s3tools-bugs@lists.sourceforge.net")
416		encoded += replace_nonprintables(c)
417		elif (o == 0x20 or # Space and below
418		o == 0x22 or # "
419		o == 0x23 or # #
420		o == 0x25 or # % (escape character)
421		o == 0x26 or # &
422		o == 0x2B or # + (or it would become <space>)
423		o == 0x3C or # <
424		o == 0x3E or # >
425		o == 0x3F or # ?
426		o == 0x60 or # `
427		o >= 123): # { and above, including >= 128 for UTF-8
428		encoded += "%%%02X" % o
429		else:
430		encoded += c
431		debug("String '%s' encoded to '%s'" % (string, encoded))
432		return encoded
433
434		def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params):
435		resource = { 'bucket' : None, 'uri' : "/" }
436
437		if uri and (bucket or object):
438		raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied")
439		## If URI is given use that instead of bucket/object parameters
440		if uri:
441		bucket = uri.bucket()
442		object = uri.has_object() and uri.object() or None
443
444		if bucket:
445		resource['bucket'] = str(bucket)
446		if object:
447		resource['uri'] = "/" + self.urlencode_string(object)
448		if extra:
449		resource['uri'] += extra
450
451		method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"])
452
453		request = S3Request(self, method_string, resource, headers, params)
454
455		debug("CreateRequest: resource[uri]=" + resource['uri'])
456		return request
457
458		def _fail_wait(self, retries):
459		# Wait a few seconds. The more it fails the more we wait.
460		return (self._max_retries - retries + 1) * 3
461
462		def send_request(self, request, body = None, retries = _max_retries):
463		method_string, resource, headers = request.get_triplet()
464		debug("Processing request, please wait...")
465		if not headers.has_key('content-length'):
466		headers['content-length'] = body and len(body) or 0
467		try:
468		# "Stringify" all headers
469		for header in headers.keys():
470		headers[header] = str(headers[header])
471		conn = self.get_connection(resource['bucket'])
472		conn.request(method_string, self.format_uri(resource), body, headers)
473		response = {}
474		http_response = conn.getresponse()
475		response["status"] = http_response.status
476		response["reason"] = http_response.reason
477		response["headers"] = convertTupleListToDict(http_response.getheaders())
478		response["data"] = http_response.read()
479		debug("Response: " + str(response))
480		conn.close()
481		except Exception, e:
482		if retries:
483		warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
484		warning("Waiting %d sec..." % self._fail_wait(retries))
485		time.sleep(self._fail_wait(retries))
486		return self.send_request(request, body, retries - 1)
487		else:
488		raise S3RequestError("Request failed for: %s" % resource['uri'])
489
490		if response["status"] == 307:
491		## RedirectPermanent
492		redir_bucket = getTextFromXml(response['data'], ".//Bucket")
493		redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
494		self.set_hostname(redir_bucket, redir_hostname)
495		warning("Redirected to: %s" % (redir_hostname))
496		return self.send_request(request, body)
497
498		if response["status"] >= 500:
499		e = S3Error(response)
500		if retries:
501		warning(u"Retrying failed request: %s" % resource['uri'])
502		warning(unicode(e))
503		warning("Waiting %d sec..." % self._fail_wait(retries))
504		time.sleep(self._fail_wait(retries))
505		return self.send_request(request, body, retries - 1)
506		else:
507		raise e
508
509		if response["status"] < 200 or response["status"] > 299:
510		raise S3Error(response)
511
512		return response
513
514		def send_file(self, request, file, labels, throttle = 0, retries = _max_retries):
515		method_string, resource, headers = request.get_triplet()
516		size_left = size_total = headers.get("content-length")
517		if self.config.progress_meter:
518		progress = self.config.progress_class(labels, size_total)
519		else:
520		info("Sending file '%s', please wait..." % file.name)
521		timestamp_start = time.time()
522		try:
523		conn = self.get_connection(resource['bucket'])
524		conn.connect()
525		conn.putrequest(method_string, self.format_uri(resource))
526		for header in headers.keys():
527		conn.putheader(header, str(headers[header]))
528		conn.endheaders()
529		except Exception, e:
530		if self.config.progress_meter:
531		progress.done("failed")
532		if retries:
533		warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
534		warning("Waiting %d sec..." % self._fail_wait(retries))
535		time.sleep(self._fail_wait(retries))
536		# Connection error -> same throttle value
537		return self.send_file(request, file, labels, throttle, retries - 1)
538		else:
539		raise S3UploadError("Upload failed for: %s" % resource['uri'])
540		file.seek(0)
541		md5_hash = md5()
542		try:
543		while (size_left > 0):
544		#debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))
545		data = file.read(self.config.send_chunk)
546		md5_hash.update(data)
547		conn.send(data)
548		if self.config.progress_meter:
549		progress.update(delta_position = len(data))
550		size_left -= len(data)
551		if throttle:
552		time.sleep(throttle)
553		md5_computed = md5_hash.hexdigest()
554		response = {}
555		http_response = conn.getresponse()
556		response["status"] = http_response.status
557		response["reason"] = http_response.reason
558		response["headers"] = convertTupleListToDict(http_response.getheaders())
559		response["data"] = http_response.read()
560		response["size"] = size_total
561		conn.close()
562		debug(u"Response: %s" % response)
563		except Exception, e:
564		if self.config.progress_meter:
565		progress.done("failed")
566		if retries:
567		if retries < self._max_retries:
568		throttle = throttle and throttle * 5 or 0.01
569		warning("Upload failed: %s (%s)" % (resource['uri'], e))
570		warning("Retrying on lower speed (throttle=%0.2f)" % throttle)
571		warning("Waiting %d sec..." % self._fail_wait(retries))
572		time.sleep(self._fail_wait(retries))
573		# Connection error -> same throttle value
574		return self.send_file(request, file, labels, throttle, retries - 1)
575		else:
576		debug("Giving up on '%s' %s" % (file.name, e))
577		raise S3UploadError("Upload failed for: %s" % resource['uri'])
578
579		timestamp_end = time.time()
580		response["elapsed"] = timestamp_end - timestamp_start
581		response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
582
583		if self.config.progress_meter:
584		## The above conn.close() takes some time -> update() progress meter
585		## to correct the average speed. Otherwise people will complain that
586		## 'progress' and response["speed"] are inconsistent ;-)
587		progress.update()
588		progress.done("done")
589
590		if response["status"] == 307:
591		## RedirectPermanent
592		redir_bucket = getTextFromXml(response['data'], ".//Bucket")
593		redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
594		self.set_hostname(redir_bucket, redir_hostname)
595		warning("Redirected to: %s" % (redir_hostname))
596		return self.send_file(request, file, labels)
597
598		# S3 from time to time doesn't send ETag back in a response :-(
599		# Force re-upload here.
600		if not response['headers'].has_key('etag'):
601		response['headers']['etag'] = ''
602
603		if response["status"] < 200 or response["status"] > 299:
604		try_retry = False
605		if response["status"] >= 500:
606		## AWS internal error - retry
607		try_retry = True
608		elif response["status"] >= 400:
609		err = S3Error(response)
610		## Retriable client error?
611		if err.code in [ 'BadDigest', 'OperationAborted', 'TokenRefreshRequired', 'RequestTimeout' ]:
612		try_retry = True
613
614		if try_retry:
615		if retries:
616		warning("Upload failed: %s (%s)" % (resource['uri'], S3Error(response)))
617		warning("Waiting %d sec..." % self._fail_wait(retries))
618		time.sleep(self._fail_wait(retries))
619		return self.send_file(request, file, labels, throttle, retries - 1)
620		else:
621		warning("Too many failures. Giving up on '%s'" % (file.name))
622		raise S3UploadError
623
624		## Non-recoverable error
625		raise S3Error(response)
626
627		debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"]))
628		if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest():
629		warning("MD5 Sums don't match!")
630		if retries:
631		warning("Retrying upload of %s" % (file.name))
632		return self.send_file(request, file, labels, throttle, retries - 1)
633		else:
634		warning("Too many failures. Giving up on '%s'" % (file.name))
635		raise S3UploadError
636
637		return response
638
639		def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
640		method_string, resource, headers = request.get_triplet()
641		if self.config.progress_meter:
642		progress = self.config.progress_class(labels, 0)
643		else:
644		info("Receiving file '%s', please wait..." % stream.name)
645		timestamp_start = time.time()
646		try:
647		conn = self.get_connection(resource['bucket'])
648		conn.connect()
649		conn.putrequest(method_string, self.format_uri(resource))
650		for header in headers.keys():
651		conn.putheader(header, str(headers[header]))
652		if start_position > 0:
653		debug("Requesting Range: %d .. end" % start_position)
654		conn.putheader("Range", "bytes=%d-" % start_position)
655		conn.endheaders()
656		response = {}
657		http_response = conn.getresponse()
658		response["status"] = http_response.status
659		response["reason"] = http_response.reason
660		response["headers"] = convertTupleListToDict(http_response.getheaders())
661		debug("Response: %s" % response)
662		except Exception, e:
663		if self.config.progress_meter:
664		progress.done("failed")
665		if retries:
666		warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
667		warning("Waiting %d sec..." % self._fail_wait(retries))
668		time.sleep(self._fail_wait(retries))
669		# Connection error -> same throttle value
670		return self.recv_file(request, stream, labels, start_position, retries - 1)
671		else:
672		raise S3DownloadError("Download failed for: %s" % resource['uri'])
673
674		if response["status"] == 307:
675		## RedirectPermanent
676		response['data'] = http_response.read()
677		redir_bucket = getTextFromXml(response['data'], ".//Bucket")
678		redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
679		self.set_hostname(redir_bucket, redir_hostname)
680		warning("Redirected to: %s" % (redir_hostname))
681		return self.recv_file(request, stream, labels)
682
683		if response["status"] < 200 or response["status"] > 299:
684		raise S3Error(response)
685
686		if start_position == 0:
687		# Only compute MD5 on the fly if we're downloading from beginning
688		# Otherwise we'd get a nonsense.
689		md5_hash = md5()
690		size_left = int(response["headers"]["content-length"])
691		size_total = start_position + size_left
692		current_position = start_position
693
694		if self.config.progress_meter:
695		progress.total_size = size_total
696		progress.initial_position = current_position
697		progress.current_position = current_position
698
699		try:
700		while (current_position < size_total):
701		this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left
702		data = http_response.read(this_chunk)
703		stream.write(data)
704		if start_position == 0:
705		md5_hash.update(data)
706		current_position += len(data)
707		## Call progress meter from here...
708		if self.config.progress_meter:
709		progress.update(delta_position = len(data))
710		conn.close()
711		except Exception, e:
712		if self.config.progress_meter:
713		progress.done("failed")
714		if retries:
715		warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
716		warning("Waiting %d sec..." % self._fail_wait(retries))
717		time.sleep(self._fail_wait(retries))
718		# Connection error -> same throttle value
719		return self.recv_file(request, stream, labels, current_position, retries - 1)
720		else:
721		raise S3DownloadError("Download failed for: %s" % resource['uri'])
722
723		stream.flush()
724		timestamp_end = time.time()
725
726		if self.config.progress_meter:
727		## The above stream.flush() may take some time -> update() progress meter
728		## to correct the average speed. Otherwise people will complain that
729		## 'progress' and response["speed"] are inconsistent ;-)
730		progress.update()
731		progress.done("done")
732
733		if start_position == 0:
734		# Only compute MD5 on the fly if we were downloading from the beginning
735		response["md5"] = md5_hash.hexdigest()
736		else:
737		# Otherwise try to compute MD5 of the output file
738		try:
739		response["md5"] = hash_file_md5(stream.name)
740		except IOError, e:
741		if e.errno != errno.ENOENT:
742		warning("Unable to open file: %s: %s" % (stream.name, e))
743		warning("Unable to verify MD5. Assume it matches.")
744		response["md5"] = response["headers"]["etag"]
745
746		response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0
747		response["elapsed"] = timestamp_end - timestamp_start
748		response["size"] = current_position
749		response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
750		if response["size"] != start_position + long(response["headers"]["content-length"]):
751		warning("Reported size (%s) does not match received size (%s)" % (
752		start_position + response["headers"]["content-length"], response["size"]))
753		debug("ReceiveFile: Computed MD5 = %s" % response["md5"])
754		if not response["md5match"]:
755		warning("MD5 signatures do not match: computed=%s, received=%s" % (
756		response["md5"], response["headers"]["etag"]))
757		return response
	113	http_methods = BidirMap(
	114	GET = 0x01,
	115	PUT = 0x02,
	116	HEAD = 0x04,
	117	DELETE = 0x08,
	118	POST = 0x10,
	119	MASK = 0x1F,
	120	)
	121
	122	targets = BidirMap(
	123	SERVICE = 0x0100,
	124	BUCKET = 0x0200,
	125	OBJECT = 0x0400,
	126	MASK = 0x0700,
	127	)
	128
	129	operations = BidirMap(
	130	UNDFINED = 0x0000,
	131	LIST_ALL_BUCKETS = targets["SERVICE"] \| http_methods["GET"],
	132	BUCKET_CREATE = targets["BUCKET"] \| http_methods["PUT"],
	133	BUCKET_LIST = targets["BUCKET"] \| http_methods["GET"],
	134	BUCKET_DELETE = targets["BUCKET"] \| http_methods["DELETE"],
	135	OBJECT_PUT = targets["OBJECT"] \| http_methods["PUT"],
	136	OBJECT_GET = targets["OBJECT"] \| http_methods["GET"],
	137	OBJECT_HEAD = targets["OBJECT"] \| http_methods["HEAD"],
	138	OBJECT_DELETE = targets["OBJECT"] \| http_methods["DELETE"],
	139	OBJECT_POST = targets["OBJECT"] \| http_methods["POST"],
	140	)
	141
	142	codes = {
	143	"NoSuchBucket" : "Bucket '%s' does not exist",
	144	"AccessDenied" : "Access to bucket '%s' was denied",
	145	"BucketAlreadyExists" : "Bucket '%s' already exists",
	146	}
	147
	148	## S3 sometimes sends HTTP-307 response
	149	redir_map = {}
	150
	151	## Maximum attempts of re-issuing failed requests
	152	_max_retries = 5
	153
	154	def __init__(self, config):
	155	self.config = config
	156
	157	def get_connection(self, bucket):
	158	if self.config.proxy_host != "":
	159	return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
	160	else:
	161	if self.config.use_https:
	162	return httplib.HTTPSConnection(self.get_hostname(bucket))
	163	else:
	164	return httplib.HTTPConnection(self.get_hostname(bucket))
	165
	166	def get_hostname(self, bucket):
	167	if bucket and check_bucket_name_dns_conformity(bucket):
	168	if self.redir_map.has_key(bucket):
	169	host = self.redir_map[bucket]
	170	else:
	171	host = getHostnameFromBucket(bucket)
	172	else:
	173	host = self.config.host_base
	174	debug('get_hostname(%s): %s' % (bucket, host))
	175	return host
	176
	177	def set_hostname(self, bucket, redir_hostname):
	178	self.redir_map[bucket] = redir_hostname
	179
	180	def format_uri(self, resource):
	181	if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):
	182	uri = "/%s%s" % (resource['bucket'], resource['uri'])
	183	else:
	184	uri = resource['uri']
	185	if self.config.proxy_host != "":
	186	uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri)
	187	debug('format_uri(): ' + uri)
	188	return uri
	189
	190	## Commands / Actions
	191	def list_all_buckets(self):
	192	request = self.create_request("LIST_ALL_BUCKETS")
	193	response = self.send_request(request)
	194	response["list"] = getListFromXml(response["data"], "Bucket")
	195	return response
	196
	197	def bucket_list(self, bucket, prefix = None, recursive = None):
	198	def _list_truncated(data):
	199	## <IsTruncated> can either be "true" or "false" or be missing completely
	200	is_truncated = getTextFromXml(data, ".//IsTruncated") or "false"
	201	return is_truncated.lower() != "false"
	202
	203	def _get_contents(data):
	204	return getListFromXml(data, "Contents")
	205
	206	def _get_common_prefixes(data):
	207	return getListFromXml(data, "CommonPrefixes")
	208
	209	uri_params = {}
	210	truncated = True
	211	list = []
	212	prefixes = []
	213
	214	while truncated:
	215	response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
	216	current_list = _get_contents(response["data"])
	217	current_prefixes = _get_common_prefixes(response["data"])
	218	truncated = _list_truncated(response["data"])
	219	if truncated:
	220	if current_list:
	221	uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])
	222	else:
	223	uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"])
	224	debug("Listing continues after '%s'" % uri_params['marker'])
	225
	226	list += current_list
	227	prefixes += current_prefixes
	228
	229	response['list'] = list
	230	response['common_prefixes'] = prefixes
	231	return response
	232
	233	def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):
	234	if prefix:
	235	uri_params['prefix'] = self.urlencode_string(prefix)
	236	if not self.config.recursive and not recursive:
	237	uri_params['delimiter'] = "/"
	238	request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)
	239	response = self.send_request(request)
	240	#debug(response)
	241	return response
	242
	243	def bucket_create(self, bucket, bucket_location = None):
	244	headers = SortedDict(ignore_case = True)
	245	body = ""
	246	if bucket_location and bucket_location.strip().upper() != "US":
	247	bucket_location = bucket_location.strip()
	248	if bucket_location.upper() == "EU":
	249	bucket_location = bucket_location.upper()
	250	else:
	251	bucket_location = bucket_location.lower()
	252	body = "<CreateBucketConfiguration><LocationConstraint>"
	253	body += bucket_location
	254	body += "</LocationConstraint></CreateBucketConfiguration>"
	255	debug("bucket_location: " + body)
	256	check_bucket_name(bucket, dns_strict = True)
	257	else:
	258	check_bucket_name(bucket, dns_strict = False)
	259	if self.config.acl_public:
	260	headers["x-amz-acl"] = "public-read"
	261	request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
	262	response = self.send_request(request, body)
	263	return response
	264
	265	def bucket_delete(self, bucket):
	266	request = self.create_request("BUCKET_DELETE", bucket = bucket)
	267	response = self.send_request(request)
	268	return response
	269
	270	def get_bucket_location(self, uri):
	271	request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location")
	272	response = self.send_request(request)
	273	location = getTextFromXml(response['data'], "LocationConstraint")
	274	if not location or location in [ "", "US" ]:
	275	location = "us-east-1"
	276	elif location == "EU":
	277	location = "eu-west-1"
	278	return location
	279
	280	def bucket_info(self, uri):
	281	# For now reports only "Location". One day perhaps more.
	282	response = {}
	283	response['bucket-location'] = self.get_bucket_location(uri)
	284	return response
	285
	286	def website_info(self, uri, bucket_location = None):
	287	headers = SortedDict(ignore_case = True)
	288	bucket = uri.bucket()
	289	body = ""
	290
	291	request = self.create_request("BUCKET_LIST", bucket = bucket, extra="?website")
	292	try:
	293	response = self.send_request(request, body)
	294	response['index_document'] = getTextFromXml(response['data'], ".//IndexDocument//Suffix")
	295	response['error_document'] = getTextFromXml(response['data'], ".//ErrorDocument//Key")
	296	response['website_endpoint'] = self.config.website_endpoint % {
	297	"bucket" : uri.bucket(),
	298	"location" : self.get_bucket_location(uri)}
	299	return response
	300	except S3Error, e:
	301	if e.status == 404:
	302	debug("Could not get /?website - website probably not configured for this bucket")
	303	return None
	304	raise
	305
	306	def website_create(self, uri, bucket_location = None):
	307	headers = SortedDict(ignore_case = True)
	308	bucket = uri.bucket()
	309	body = '<WebsiteConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'
	310	body += ' <IndexDocument>'
	311	body += (' <Suffix>%s</Suffix>' % self.config.website_index)
	312	body += ' </IndexDocument>'
	313	if self.config.website_error:
	314	body += ' <ErrorDocument>'
	315	body += (' <Key>%s</Key>' % self.config.website_error)
	316	body += ' </ErrorDocument>'
	317	body += '</WebsiteConfiguration>'
	318
	319	request = self.create_request("BUCKET_CREATE", bucket = bucket, extra="?website")
	320	debug("About to send request '%s' with body '%s'" % (request, body))
	321	response = self.send_request(request, body)
	322	debug("Received response '%s'" % (response))
	323
	324	return response
	325
	326	def website_delete(self, uri, bucket_location = None):
	327	headers = SortedDict(ignore_case = True)
	328	bucket = uri.bucket()
	329	body = ""
	330
	331	request = self.create_request("BUCKET_DELETE", bucket = bucket, extra="?website")
	332	debug("About to send request '%s' with body '%s'" % (request, body))
	333	response = self.send_request(request, body)
	334	debug("Received response '%s'" % (response))
	335
	336	if response['status'] != 204:
	337	raise S3ResponseError("Expected status 204: %s" % response)
	338
	339	return response
	340
	341	def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
	342	# TODO TODO
	343	# Make it consistent with stream-oriented object_get()
	344	if uri.type != "s3":
	345	raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
	346
	347	if not os.path.isfile(filename):
	348	raise InvalidFileError(u"%s is not a regular file" % unicodise(filename))
	349	try:
	350	file = open(filename, "rb")
	351	size = os.stat(filename)[ST_SIZE]
	352	except (IOError, OSError), e:
	353	raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))
	354
	355	headers = SortedDict(ignore_case = True)
	356	if extra_headers:
	357	headers.update(extra_headers)
	358
	359	## MIME-type handling
	360	content_type = self.config.mime_type
	361	if not content_type and self.config.guess_mime_type:
	362	content_type = mime_magic(filename)
	363	if not content_type:
	364	content_type = self.config.default_mime_type
	365	debug("Content-Type set to '%s'" % content_type)
	366	headers["content-type"] = content_type
	367
	368	## Other Amazon S3 attributes
	369	if self.config.acl_public:
	370	headers["x-amz-acl"] = "public-read"
	371	if self.config.reduced_redundancy:
	372	headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
	373
	374	## Multipart decision
	375	multipart = False
	376	if self.config.enable_multipart:
	377	if size > self.config.multipart_chunk_size_mb * 1024 * 1024:
	378	multipart = True
	379	if multipart:
	380	# Multipart requests are quite different... drop here
	381	return self.send_file_multipart(file, headers, uri, size)
	382
	383	## Not multipart...
	384	headers["content-length"] = size
	385	request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)
	386	labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label }
	387	response = self.send_file(request, file, labels)
	388	return response
	389
	390	def object_get(self, uri, stream, start_position = 0, extra_label = ""):
	391	if uri.type != "s3":
	392	raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
	393	request = self.create_request("OBJECT_GET", uri = uri)
	394	labels = { 'source' : unicodise(uri.uri()), 'destination' : unicodise(stream.name), 'extra' : extra_label }
	395	response = self.recv_file(request, stream, labels, start_position)
	396	return response
	397
	398	def object_delete(self, uri):
	399	if uri.type != "s3":
	400	raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
	401	request = self.create_request("OBJECT_DELETE", uri = uri)
	402	response = self.send_request(request)
	403	return response
	404
	405	def object_copy(self, src_uri, dst_uri, extra_headers = None):
	406	if src_uri.type != "s3":
	407	raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type)
	408	if dst_uri.type != "s3":
	409	raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type)
	410	headers = SortedDict(ignore_case = True)
	411	headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object()))
	412	## TODO: For now COPY, later maybe add a switch?
	413	headers['x-amz-metadata-directive'] = "COPY"
	414	if self.config.acl_public:
	415	headers["x-amz-acl"] = "public-read"
	416	if self.config.reduced_redundancy:
	417	headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
	418	# if extra_headers:
	419	# headers.update(extra_headers)
	420	request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers)
	421	response = self.send_request(request)
	422	return response
	423
	424	def object_move(self, src_uri, dst_uri, extra_headers = None):
	425	response_copy = self.object_copy(src_uri, dst_uri, extra_headers)
	426	debug("Object %s copied to %s" % (src_uri, dst_uri))
	427	if getRootTagName(response_copy["data"]) == "CopyObjectResult":
	428	response_delete = self.object_delete(src_uri)
	429	debug("Object %s deleted" % src_uri)
	430	return response_copy
	431
	432	def object_info(self, uri):
	433	request = self.create_request("OBJECT_HEAD", uri = uri)
	434	response = self.send_request(request)
	435	return response
	436
	437	def get_acl(self, uri):
	438	if uri.has_object():
	439	request = self.create_request("OBJECT_GET", uri = uri, extra = "?acl")
	440	else:
	441	request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl")
	442
	443	response = self.send_request(request)
	444	acl = ACL(response['data'])
	445	return acl
	446
	447	def set_acl(self, uri, acl):
	448	if uri.has_object():
	449	request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl")
	450	else:
	451	request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl")
	452
	453	body = str(acl)
	454	debug(u"set_acl(%s): acl-xml: %s" % (uri, body))
	455	response = self.send_request(request, body)
	456	return response
	457
	458	def get_accesslog(self, uri):
	459	request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging")
	460	response = self.send_request(request)
	461	accesslog = AccessLog(response['data'])
	462	return accesslog
	463
	464	def set_accesslog_acl(self, uri):
	465	acl = self.get_acl(uri)
	466	debug("Current ACL(%s): %s" % (uri.uri(), str(acl)))
	467	acl.appendGrantee(GranteeLogDelivery("READ_ACP"))
	468	acl.appendGrantee(GranteeLogDelivery("WRITE"))
	469	debug("Updated ACL(%s): %s" % (uri.uri(), str(acl)))
	470	self.set_acl(uri, acl)
	471
	472	def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False):
	473	request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging")
	474	accesslog = AccessLog()
	475	if enable:
	476	accesslog.enableLogging(log_target_prefix_uri)
	477	accesslog.setAclPublic(acl_public)
	478	else:
	479	accesslog.disableLogging()
	480	body = str(accesslog)
	481	debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body))
	482	try:
	483	response = self.send_request(request, body)
	484	except S3Error, e:
	485	if e.info['Code'] == "InvalidTargetBucketForLogging":
	486	info("Setting up log-delivery ACL for target bucket.")
	487	self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket()))
	488	response = self.send_request(request, body)
	489	else:
	490	raise
	491	return accesslog, response
	492
	493	## Low level methods
	494	def urlencode_string(self, string, urlencoding_mode = None):
	495	if type(string) == unicode:
	496	string = string.encode("utf-8")
	497
	498	if urlencoding_mode is None:
	499	urlencoding_mode = self.config.urlencoding_mode
	500
	501	if urlencoding_mode == "verbatim":
	502	## Don't do any pre-processing
	503	return string
	504
	505	encoded = ""
	506	## List of characters that must be escaped for S3
	507	## Haven't found this in any official docs
	508	## but my tests show it's more less correct.
	509	## If you start getting InvalidSignature errors
	510	## from S3 check the error headers returned
	511	## from S3 to see whether the list hasn't
	512	## changed.
	513	for c in string: # I'm not sure how to know in what encoding
	514	# 'object' is. Apparently "type(object)==str"
	515	# but the contents is a string of unicode
	516	# bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k'
	517	# Don't know what it will do on non-utf8
	518	# systems.
	519	# [hope that sounds reassuring ;-)]
	520	o = ord(c)
	521	if (o < 0x20 or o == 0x7f):
	522	if urlencoding_mode == "fixbucket":
	523	encoded += "%%%02X" % o
	524	else:
	525	error(u"Non-printable character 0x%02x in: %s" % (o, string))
	526	error(u"Please report it to s3tools-bugs@lists.sourceforge.net")
	527	encoded += replace_nonprintables(c)
	528	elif (o == 0x20 or # Space and below
	529	o == 0x22 or # "
	530	o == 0x23 or # #
	531	o == 0x25 or # % (escape character)
	532	o == 0x26 or # &
	533	o == 0x2B or # + (or it would become <space>)
	534	o == 0x3C or # <
	535	o == 0x3E or # >
	536	o == 0x3F or # ?
	537	o == 0x60 or # `
	538	o >= 123): # { and above, including >= 128 for UTF-8
	539	encoded += "%%%02X" % o
	540	else:
	541	encoded += c
	542	debug("String '%s' encoded to '%s'" % (string, encoded))
	543	return encoded
	544
	545	def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params):
	546	resource = { 'bucket' : None, 'uri' : "/" }
	547
	548	if uri and (bucket or object):
	549	raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied")
	550	## If URI is given use that instead of bucket/object parameters
	551	if uri:
	552	bucket = uri.bucket()
	553	object = uri.has_object() and uri.object() or None
	554
	555	if bucket:
	556	resource['bucket'] = str(bucket)
	557	if object:
	558	resource['uri'] = "/" + self.urlencode_string(object)
	559	if extra:
	560	resource['uri'] += extra
	561
	562	method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"])
	563
	564	request = S3Request(self, method_string, resource, headers, params)
	565
	566	debug("CreateRequest: resource[uri]=" + resource['uri'])
	567	return request
	568
	569	def _fail_wait(self, retries):
	570	# Wait a few seconds. The more it fails the more we wait.
	571	return (self._max_retries - retries + 1) * 3
	572
	573	def send_request(self, request, body = None, retries = _max_retries):
	574	method_string, resource, headers = request.get_triplet()
	575	debug("Processing request, please wait...")
	576	if not headers.has_key('content-length'):
	577	headers['content-length'] = body and len(body) or 0
	578	try:
	579	# "Stringify" all headers
	580	for header in headers.keys():
	581	headers[header] = str(headers[header])
	582	conn = self.get_connection(resource['bucket'])
	583	uri = self.format_uri(resource)
	584	debug("Sending request method_string=%r, uri=%r, headers=%r, body=(%i bytes)" % (method_string, uri, headers, len(body or "")))
	585	conn.request(method_string, uri, body, headers)
	586	response = {}
	587	http_response = conn.getresponse()
	588	response["status"] = http_response.status
	589	response["reason"] = http_response.reason
	590	response["headers"] = convertTupleListToDict(http_response.getheaders())
	591	response["data"] = http_response.read()
	592	debug("Response: " + str(response))
	593	conn.close()
	594	except Exception, e:
	595	if retries:
	596	warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
	597	warning("Waiting %d sec..." % self._fail_wait(retries))
	598	time.sleep(self._fail_wait(retries))
	599	return self.send_request(request, body, retries - 1)
	600	else:
	601	raise S3RequestError("Request failed for: %s" % resource['uri'])
	602
	603	if response["status"] == 307:
	604	## RedirectPermanent
	605	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	606	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	607	self.set_hostname(redir_bucket, redir_hostname)
	608	warning("Redirected to: %s" % (redir_hostname))
	609	return self.send_request(request, body)
	610
	611	if response["status"] >= 500:
	612	e = S3Error(response)
	613	if retries:
	614	warning(u"Retrying failed request: %s" % resource['uri'])
	615	warning(unicode(e))
	616	warning("Waiting %d sec..." % self._fail_wait(retries))
	617	time.sleep(self._fail_wait(retries))
	618	return self.send_request(request, body, retries - 1)
	619	else:
	620	raise e
	621
	622	if response["status"] < 200 or response["status"] > 299:
	623	raise S3Error(response)
	624
	625	return response
	626
	627	def send_file(self, request, file, labels, throttle = 0, retries = _max_retries, offset = 0, chunk_size = -1):
	628	method_string, resource, headers = request.get_triplet()
	629	size_left = size_total = headers.get("content-length")
	630	if self.config.progress_meter:
	631	progress = self.config.progress_class(labels, size_total)
	632	else:
	633	info("Sending file '%s', please wait..." % file.name)
	634	timestamp_start = time.time()
	635	try:
	636	conn = self.get_connection(resource['bucket'])
	637	conn.connect()
	638	conn.putrequest(method_string, self.format_uri(resource))
	639	for header in headers.keys():
	640	conn.putheader(header, str(headers[header]))
	641	conn.endheaders()
	642	except Exception, e:
	643	if self.config.progress_meter:
	644	progress.done("failed")
	645	if retries:
	646	warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
	647	warning("Waiting %d sec..." % self._fail_wait(retries))
	648	time.sleep(self._fail_wait(retries))
	649	# Connection error -> same throttle value
	650	return self.send_file(request, file, labels, throttle, retries - 1, offset, chunk_size)
	651	else:
	652	raise S3UploadError("Upload failed for: %s" % resource['uri'])
	653	file.seek(offset)
	654	md5_hash = md5()
	655	try:
	656	while (size_left > 0):
	657	#debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))
	658	data = file.read(min(self.config.send_chunk, size_left))
	659	md5_hash.update(data)
	660	conn.send(data)
	661	if self.config.progress_meter:
	662	progress.update(delta_position = len(data))
	663	size_left -= len(data)
	664	if throttle:
	665	time.sleep(throttle)
	666	md5_computed = md5_hash.hexdigest()
	667	response = {}
	668	http_response = conn.getresponse()
	669	response["status"] = http_response.status
	670	response["reason"] = http_response.reason
	671	response["headers"] = convertTupleListToDict(http_response.getheaders())
	672	response["data"] = http_response.read()
	673	response["size"] = size_total
	674	conn.close()
	675	debug(u"Response: %s" % response)
	676	except Exception, e:
	677	if self.config.progress_meter:
	678	progress.done("failed")
	679	if retries:
	680	if retries < self._max_retries:
	681	throttle = throttle and throttle * 5 or 0.01
	682	warning("Upload failed: %s (%s)" % (resource['uri'], e))
	683	warning("Retrying on lower speed (throttle=%0.2f)" % throttle)
	684	warning("Waiting %d sec..." % self._fail_wait(retries))
	685	time.sleep(self._fail_wait(retries))
	686	# Connection error -> same throttle value
	687	return self.send_file(request, file, labels, throttle, retries - 1, offset, chunk_size)
	688	else:
	689	debug("Giving up on '%s' %s" % (file.name, e))
	690	raise S3UploadError("Upload failed for: %s" % resource['uri'])
	691
	692	timestamp_end = time.time()
	693	response["elapsed"] = timestamp_end - timestamp_start
	694	response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
	695
	696	if self.config.progress_meter:
	697	## The above conn.close() takes some time -> update() progress meter
	698	## to correct the average speed. Otherwise people will complain that
	699	## 'progress' and response["speed"] are inconsistent ;-)
	700	progress.update()
	701	progress.done("done")
	702
	703	if response["status"] == 307:
	704	## RedirectPermanent
	705	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	706	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	707	self.set_hostname(redir_bucket, redir_hostname)
	708	warning("Redirected to: %s" % (redir_hostname))
	709	return self.send_file(request, file, labels, offset = offset, chunk_size = chunk_size)
	710
	711	# S3 from time to time doesn't send ETag back in a response :-(
	712	# Force re-upload here.
	713	if not response['headers'].has_key('etag'):
	714	response['headers']['etag'] = ''
	715
	716	if response["status"] < 200 or response["status"] > 299:
	717	try_retry = False
	718	if response["status"] >= 500:
	719	## AWS internal error - retry
	720	try_retry = True
	721	elif response["status"] >= 400:
	722	err = S3Error(response)
	723	## Retriable client error?
	724	if err.code in [ 'BadDigest', 'OperationAborted', 'TokenRefreshRequired', 'RequestTimeout' ]:
	725	try_retry = True
	726
	727	if try_retry:
	728	if retries:
	729	warning("Upload failed: %s (%s)" % (resource['uri'], S3Error(response)))
	730	warning("Waiting %d sec..." % self._fail_wait(retries))
	731	time.sleep(self._fail_wait(retries))
	732	return self.send_file(request, file, labels, throttle, retries - 1, offset, chunk_size)
	733	else:
	734	warning("Too many failures. Giving up on '%s'" % (file.name))
	735	raise S3UploadError
	736
	737	## Non-recoverable error
	738	raise S3Error(response)
	739
	740	debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"]))
	741	if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest():
	742	warning("MD5 Sums don't match!")
	743	if retries:
	744	warning("Retrying upload of %s" % (file.name))
	745	return self.send_file(request, file, labels, throttle, retries - 1, offset, chunk_size)
	746	else:
	747	warning("Too many failures. Giving up on '%s'" % (file.name))
	748	raise S3UploadError
	749
	750	return response
	751
	752	def send_file_multipart(self, file, headers, uri, size):
	753	chunk_size = self.config.multipart_chunk_size_mb * 1024 * 1024
	754	upload = MultiPartUpload(self, file, uri, headers)
	755	upload.upload_all_parts()
	756	response = upload.complete_multipart_upload()
	757	response["speed"] = 0 # XXX
	758	response["size"] = size
	759	return response
	760
	761	def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
	762	method_string, resource, headers = request.get_triplet()
	763	if self.config.progress_meter:
	764	progress = self.config.progress_class(labels, 0)
	765	else:
	766	info("Receiving file '%s', please wait..." % stream.name)
	767	timestamp_start = time.time()
	768	try:
	769	conn = self.get_connection(resource['bucket'])
	770	conn.connect()
	771	conn.putrequest(method_string, self.format_uri(resource))
	772	for header in headers.keys():
	773	conn.putheader(header, str(headers[header]))
	774	if start_position > 0:
	775	debug("Requesting Range: %d .. end" % start_position)
	776	conn.putheader("Range", "bytes=%d-" % start_position)
	777	conn.endheaders()
	778	response = {}
	779	http_response = conn.getresponse()
	780	response["status"] = http_response.status
	781	response["reason"] = http_response.reason
	782	response["headers"] = convertTupleListToDict(http_response.getheaders())
	783	debug("Response: %s" % response)
	784	except Exception, e:
	785	if self.config.progress_meter:
	786	progress.done("failed")
	787	if retries:
	788	warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
	789	warning("Waiting %d sec..." % self._fail_wait(retries))
	790	time.sleep(self._fail_wait(retries))
	791	# Connection error -> same throttle value
	792	return self.recv_file(request, stream, labels, start_position, retries - 1)
	793	else:
	794	raise S3DownloadError("Download failed for: %s" % resource['uri'])
	795
	796	if response["status"] == 307:
	797	## RedirectPermanent
	798	response['data'] = http_response.read()
	799	redir_bucket = getTextFromXml(response['data'], ".//Bucket")
	800	redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
	801	self.set_hostname(redir_bucket, redir_hostname)
	802	warning("Redirected to: %s" % (redir_hostname))
	803	return self.recv_file(request, stream, labels)
	804
	805	if response["status"] < 200 or response["status"] > 299:
	806	raise S3Error(response)
	807
	808	if start_position == 0:
	809	# Only compute MD5 on the fly if we're downloading from beginning
	810	# Otherwise we'd get a nonsense.
	811	md5_hash = md5()
	812	size_left = int(response["headers"]["content-length"])
	813	size_total = start_position + size_left
	814	current_position = start_position
	815
	816	if self.config.progress_meter:
	817	progress.total_size = size_total
	818	progress.initial_position = current_position
	819	progress.current_position = current_position
	820
	821	try:
	822	while (current_position < size_total):
	823	this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left
	824	data = http_response.read(this_chunk)
	825	stream.write(data)
	826	if start_position == 0:
	827	md5_hash.update(data)
	828	current_position += len(data)
	829	## Call progress meter from here...
	830	if self.config.progress_meter:
	831	progress.update(delta_position = len(data))
	832	conn.close()
	833	except Exception, e:
	834	if self.config.progress_meter:
	835	progress.done("failed")
	836	if retries:
	837	warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
	838	warning("Waiting %d sec..." % self._fail_wait(retries))
	839	time.sleep(self._fail_wait(retries))
	840	# Connection error -> same throttle value
	841	return self.recv_file(request, stream, labels, current_position, retries - 1)
	842	else:
	843	raise S3DownloadError("Download failed for: %s" % resource['uri'])
	844
	845	stream.flush()
	846	timestamp_end = time.time()
	847
	848	if self.config.progress_meter:
	849	## The above stream.flush() may take some time -> update() progress meter
	850	## to correct the average speed. Otherwise people will complain that
	851	## 'progress' and response["speed"] are inconsistent ;-)
	852	progress.update()
	853	progress.done("done")
	854
	855	if start_position == 0:
	856	# Only compute MD5 on the fly if we were downloading from the beginning
	857	response["md5"] = md5_hash.hexdigest()
	858	else:
	859	# Otherwise try to compute MD5 of the output file
	860	try:
	861	response["md5"] = hash_file_md5(stream.name)
	862	except IOError, e:
	863	if e.errno != errno.ENOENT:
	864	warning("Unable to open file: %s: %s" % (stream.name, e))
	865	warning("Unable to verify MD5. Assume it matches.")
	866	response["md5"] = response["headers"]["etag"]
	867
	868	response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0
	869	response["elapsed"] = timestamp_end - timestamp_start
	870	response["size"] = current_position
	871	response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
	872	if response["size"] != start_position + long(response["headers"]["content-length"]):
	873	warning("Reported size (%s) does not match received size (%s)" % (
	874	start_position + response["headers"]["content-length"], response["size"]))
	875	debug("ReceiveFile: Computed MD5 = %s" % response["md5"])
	876	if not response["md5match"]:
	877	warning("MD5 signatures do not match: computed=%s, received=%s" % (
	878	response["md5"], response["headers"]["etag"]))
	879	return response
758	880	__all__.append("S3")
	881
	882	# vim:et:ts=4:sts=4:ai

+198

-187

S3/S3Uri.py less more

11	11	from Utils import unicodise, check_bucket_name_dns_conformity
12	12
13	13	class S3Uri(object):
14		type = None
15		_subclasses = None
16
17		def __new__(self, string):
18		if not self._subclasses:
19		## Generate a list of all subclasses of S3Uri
20		self._subclasses = []
21		dict = sys.modules[__name__].__dict__
22		for something in dict:
23		if type(dict[something]) is not type(self):
24		continue
25		if issubclass(dict[something], self) and dict[something] != self:
26		self._subclasses.append(dict[something])
27		for subclass in self._subclasses:
28		try:
29		instance = object.__new__(subclass)
30		instance.__init__(string)
31		return instance
32		except ValueError, e:
33		continue
34		raise ValueError("%s: not a recognized URI" % string)
35
36		def __str__(self):
37		return self.uri()
38
39		def __unicode__(self):
40		return self.uri()
41
42		def public_url(self):
43		raise ValueError("This S3 URI does not have Anonymous URL representation")
44
45		def basename(self):
46		return self.__unicode__().split("/")[-1]
	14	type = None
	15	_subclasses = None
	16
	17	def __new__(self, string):
	18	if not self._subclasses:
	19	## Generate a list of all subclasses of S3Uri
	20	self._subclasses = []
	21	dict = sys.modules[__name__].__dict__
	22	for something in dict:
	23	if type(dict[something]) is not type(self):
	24	continue
	25	if issubclass(dict[something], self) and dict[something] != self:
	26	self._subclasses.append(dict[something])
	27	for subclass in self._subclasses:
	28	try:
	29	instance = object.__new__(subclass)
	30	instance.__init__(string)
	31	return instance
	32	except ValueError, e:
	33	continue
	34	raise ValueError("%s: not a recognized URI" % string)
	35
	36	def __str__(self):
	37	return self.uri()
	38
	39	def __unicode__(self):
	40	return self.uri()
	41
	42	def __repr__(self):
	43	return "<%s: %s>" % (self.__class__.__name__, self.__unicode__())
	44
	45	def public_url(self):
	46	raise ValueError("This S3 URI does not have Anonymous URL representation")
	47
	48	def basename(self):
	49	return self.__unicode__().split("/")[-1]
47	50
48	51	class S3UriS3(S3Uri):
49		type = "s3"
50		_re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE)
51		def __init__(self, string):
52		match = self._re.match(string)
53		if not match:
54		raise ValueError("%s: not a S3 URI" % string)
55		groups = match.groups()
56		self._bucket = groups[0]
57		self._object = unicodise(groups[1])
58
59		def bucket(self):
60		return self._bucket
61
62		def object(self):
63		return self._object
64
65		def has_bucket(self):
66		return bool(self._bucket)
67
68		def has_object(self):
69		return bool(self._object)
70
71		def uri(self):
72		return "/".join(["s3:/", self._bucket, self._object])
73
74		def is_dns_compatible(self):
75		return check_bucket_name_dns_conformity(self._bucket)
76
77		def public_url(self):
78		if self.is_dns_compatible():
79		return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object)
80		else:
81		return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object)
82
83		def host_name(self):
84		if self.is_dns_compatible():
85		return "%s.s3.amazonaws.com" % (self._bucket)
86		else:
87		return "s3.amazonaws.com"
88
89		@staticmethod
90		def compose_uri(bucket, object = ""):
91		return "s3://%s/%s" % (bucket, object)
92
93		@staticmethod
94		def httpurl_to_s3uri(http_url):
95		m=re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE)
96		hostname, object = m.groups()[1:]
97		hostname = hostname.lower()
98		if hostname == "s3.amazonaws.com":
99		## old-style url: http://s3.amazonaws.com/bucket/object
100		if object.count("/") == 0:
101		## no object given
102		bucket = object
103		object = ""
104		else:
105		## bucket/object
106		bucket, object = object.split("/", 1)
107		elif hostname.endswith(".s3.amazonaws.com"):
108		## new-style url: http://bucket.s3.amazonaws.com/object
109		bucket = hostname[:-(len(".s3.amazonaws.com"))]
110		else:
111		raise ValueError("Unable to parse URL: %s" % http_url)
112		return S3Uri("s3://%(bucket)s/%(object)s" % {
113		'bucket' : bucket,
114		'object' : object })
	52	type = "s3"
	53	_re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE)
	54	def __init__(self, string):
	55	match = self._re.match(string)
	56	if not match:
	57	raise ValueError("%s: not a S3 URI" % string)
	58	groups = match.groups()
	59	self._bucket = groups[0]
	60	self._object = unicodise(groups[1])
	61
	62	def bucket(self):
	63	return self._bucket
	64
	65	def object(self):
	66	return self._object
	67
	68	def has_bucket(self):
	69	return bool(self._bucket)
	70
	71	def has_object(self):
	72	return bool(self._object)
	73
	74	def uri(self):
	75	return "/".join(["s3:/", self._bucket, self._object])
	76
	77	def is_dns_compatible(self):
	78	return check_bucket_name_dns_conformity(self._bucket)
	79
	80	def public_url(self):
	81	if self.is_dns_compatible():
	82	return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object)
	83	else:
	84	return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object)
	85
	86	def host_name(self):
	87	if self.is_dns_compatible():
	88	return "%s.s3.amazonaws.com" % (self._bucket)
	89	else:
	90	return "s3.amazonaws.com"
	91
	92	@staticmethod
	93	def compose_uri(bucket, object = ""):
	94	return "s3://%s/%s" % (bucket, object)
	95
	96	@staticmethod
	97	def httpurl_to_s3uri(http_url):
	98	m=re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE)
	99	hostname, object = m.groups()[1:]
	100	hostname = hostname.lower()
	101	if hostname == "s3.amazonaws.com":
	102	## old-style url: http://s3.amazonaws.com/bucket/object
	103	if object.count("/") == 0:
	104	## no object given
	105	bucket = object
	106	object = ""
	107	else:
	108	## bucket/object
	109	bucket, object = object.split("/", 1)
	110	elif hostname.endswith(".s3.amazonaws.com"):
	111	## new-style url: http://bucket.s3.amazonaws.com/object
	112	bucket = hostname[:-(len(".s3.amazonaws.com"))]
	113	else:
	114	raise ValueError("Unable to parse URL: %s" % http_url)
	115	return S3Uri("s3://%(bucket)s/%(object)s" % {
	116	'bucket' : bucket,
	117	'object' : object })
115	118
116	119	class S3UriS3FS(S3Uri):
117		type = "s3fs"
118		_re = re.compile("^s3fs://([^/])/?(.)", re.IGNORECASE)
119		def __init__(self, string):
120		match = self._re.match(string)
121		if not match:
122		raise ValueError("%s: not a S3fs URI" % string)
123		groups = match.groups()
124		self._fsname = groups[0]
125		self._path = unicodise(groups[1]).split("/")
126
127		def fsname(self):
128		return self._fsname
129
130		def path(self):
131		return "/".join(self._path)
132
133		def uri(self):
134		return "/".join(["s3fs:/", self._fsname, self.path()])
	120	type = "s3fs"
	121	_re = re.compile("^s3fs://([^/])/?(.)", re.IGNORECASE)
	122	def __init__(self, string):
	123	match = self._re.match(string)
	124	if not match:
	125	raise ValueError("%s: not a S3fs URI" % string)
	126	groups = match.groups()
	127	self._fsname = groups[0]
	128	self._path = unicodise(groups[1]).split("/")
	129
	130	def fsname(self):
	131	return self._fsname
	132
	133	def path(self):
	134	return "/".join(self._path)
	135
	136	def uri(self):
	137	return "/".join(["s3fs:/", self._fsname, self.path()])
135	138
136	139	class S3UriFile(S3Uri):
137		type = "file"
138		_re = re.compile("^(\w+://)?(.*)")
139		def __init__(self, string):
140		match = self._re.match(string)
141		groups = match.groups()
142		if groups[0] not in (None, "file://"):
143		raise ValueError("%s: not a file:// URI" % string)
144		self._path = unicodise(groups[1]).split("/")
145
146		def path(self):
147		return "/".join(self._path)
148
149		def uri(self):
150		return "/".join(["file:/", self.path()])
151
152		def isdir(self):
153		return os.path.isdir(self.path())
154
155		def dirname(self):
156		return os.path.dirname(self.path())
	140	type = "file"
	141	_re = re.compile("^(\w+://)?(.*)")
	142	def __init__(self, string):
	143	match = self._re.match(string)
	144	groups = match.groups()
	145	if groups[0] not in (None, "file://"):
	146	raise ValueError("%s: not a file:// URI" % string)
	147	self._path = unicodise(groups[1]).split("/")
	148
	149	def path(self):
	150	return "/".join(self._path)
	151
	152	def uri(self):
	153	return "/".join(["file:/", self.path()])
	154
	155	def isdir(self):
	156	return os.path.isdir(self.path())
	157
	158	def dirname(self):
	159	return os.path.dirname(self.path())
157	160
158	161	class S3UriCloudFront(S3Uri):
159		type = "cf"
160		_re = re.compile("^cf://([^/]*)/?", re.IGNORECASE)
161		def __init__(self, string):
162		match = self._re.match(string)
163		if not match:
164		raise ValueError("%s: not a CloudFront URI" % string)
165		groups = match.groups()
166		self._dist_id = groups[0]
167
168		def dist_id(self):
169		return self._dist_id
170
171		def uri(self):
172		return "/".join(["cf:/", self.dist_id()])
	162	type = "cf"
	163	_re = re.compile("^cf://([^/])/(.*)", re.IGNORECASE)
	164	def __init__(self, string):
	165	match = self._re.match(string)
	166	if not match:
	167	raise ValueError("%s: not a CloudFront URI" % string)
	168	groups = match.groups()
	169	self._dist_id = groups[0]
	170	self._request_id = groups[1] != "/" and groups[1] or None
	171
	172	def dist_id(self):
	173	return self._dist_id
	174
	175	def request_id(self):
	176	return self._request_id
	177
	178	def uri(self):
	179	uri = "cf://" + self.dist_id()
	180	if self.request_id():
	181	uri += "/" + self.request_id()
	182	return uri
173	183
174	184	if __name__ == "__main__":
175		uri = S3Uri("s3://bucket/object")
176		print "type() =", type(uri)
177		print "uri =", uri
178		print "uri.type=", uri.type
179		print "bucket =", uri.bucket()
180		print "object =", uri.object()
181		print
182
183		uri = S3Uri("s3://bucket")
184		print "type() =", type(uri)
185		print "uri =", uri
186		print "uri.type=", uri.type
187		print "bucket =", uri.bucket()
188		print
189
190		uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt")
191		print "type() =", type(uri)
192		print "uri =", uri
193		print "uri.type=", uri.type
194		print "path =", uri.path()
195		print
196
197		uri = S3Uri("/path/to/local/file.txt")
198		print "type() =", type(uri)
199		print "uri =", uri
200		print "uri.type=", uri.type
201		print "path =", uri.path()
202		print
203
204		uri = S3Uri("cf://1234567890ABCD/")
205		print "type() =", type(uri)
206		print "uri =", uri
207		print "uri.type=", uri.type
208		print "dist_id =", uri.dist_id()
209		print
210
	185	uri = S3Uri("s3://bucket/object")
	186	print "type() =", type(uri)
	187	print "uri =", uri
	188	print "uri.type=", uri.type
	189	print "bucket =", uri.bucket()
	190	print "object =", uri.object()
	191	print
	192
	193	uri = S3Uri("s3://bucket")
	194	print "type() =", type(uri)
	195	print "uri =", uri
	196	print "uri.type=", uri.type
	197	print "bucket =", uri.bucket()
	198	print
	199
	200	uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt")
	201	print "type() =", type(uri)
	202	print "uri =", uri
	203	print "uri.type=", uri.type
	204	print "path =", uri.path()
	205	print
	206
	207	uri = S3Uri("/path/to/local/file.txt")
	208	print "type() =", type(uri)
	209	print "uri =", uri
	210	print "uri.type=", uri.type
	211	print "path =", uri.path()
	212	print
	213
	214	uri = S3Uri("cf://1234567890ABCD/")
	215	print "type() =", type(uri)
	216	print "uri =", uri
	217	print "uri.type=", uri.type
	218	print "dist_id =", uri.dist_id()
	219	print
	220
	221	# vim:et:ts=4:sts=4:ai

+136

-134

S3/SimpleDB.py less more

19	19	from Exceptions import *
20	20
21	21	class SimpleDB(object):
22		# API Version
23		# See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/
24		Version = "2007-11-07"
25		SignatureVersion = 1
	22	# API Version
	23	# See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/
	24	Version = "2007-11-07"
	25	SignatureVersion = 1
26	26
27		def __init__(self, config):
28		self.config = config
	27	def __init__(self, config):
	28	self.config = config
29	29
30		## ------------------------------------------------
31		## Methods implementing SimpleDB API
32		## ------------------------------------------------
	30	## ------------------------------------------------
	31	## Methods implementing SimpleDB API
	32	## ------------------------------------------------
33	33
34		def ListDomains(self, MaxNumberOfDomains = 100):
35		'''
36		Lists all domains associated with our Access Key. Returns
37		domain names up to the limit set by MaxNumberOfDomains.
38		'''
39		parameters = SortedDict()
40		parameters['MaxNumberOfDomains'] = MaxNumberOfDomains
41		return self.send_request("ListDomains", DomainName = None, parameters = parameters)
	34	def ListDomains(self, MaxNumberOfDomains = 100):
	35	'''
	36	Lists all domains associated with our Access Key. Returns
	37	domain names up to the limit set by MaxNumberOfDomains.
	38	'''
	39	parameters = SortedDict()
	40	parameters['MaxNumberOfDomains'] = MaxNumberOfDomains
	41	return self.send_request("ListDomains", DomainName = None, parameters = parameters)
42	42
43		def CreateDomain(self, DomainName):
44		return self.send_request("CreateDomain", DomainName = DomainName)
	43	def CreateDomain(self, DomainName):
	44	return self.send_request("CreateDomain", DomainName = DomainName)
45	45
46		def DeleteDomain(self, DomainName):
47		return self.send_request("DeleteDomain", DomainName = DomainName)
	46	def DeleteDomain(self, DomainName):
	47	return self.send_request("DeleteDomain", DomainName = DomainName)
48	48
49		def PutAttributes(self, DomainName, ItemName, Attributes):
50		parameters = SortedDict()
51		parameters['ItemName'] = ItemName
52		seq = 0
53		for attrib in Attributes:
54		if type(Attributes[attrib]) == type(list()):
55		for value in Attributes[attrib]:
56		parameters['Attribute.%d.Name' % seq] = attrib
57		parameters['Attribute.%d.Value' % seq] = unicode(value)
58		seq += 1
59		else:
60		parameters['Attribute.%d.Name' % seq] = attrib
61		parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
62		seq += 1
63		## TODO:
64		## - support for Attribute.N.Replace
65		## - support for multiple values for one attribute
66		return self.send_request("PutAttributes", DomainName = DomainName, parameters = parameters)
	49	def PutAttributes(self, DomainName, ItemName, Attributes):
	50	parameters = SortedDict()
	51	parameters['ItemName'] = ItemName
	52	seq = 0
	53	for attrib in Attributes:
	54	if type(Attributes[attrib]) == type(list()):
	55	for value in Attributes[attrib]:
	56	parameters['Attribute.%d.Name' % seq] = attrib
	57	parameters['Attribute.%d.Value' % seq] = unicode(value)
	58	seq += 1
	59	else:
	60	parameters['Attribute.%d.Name' % seq] = attrib
	61	parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
	62	seq += 1
	63	## TODO:
	64	## - support for Attribute.N.Replace
	65	## - support for multiple values for one attribute
	66	return self.send_request("PutAttributes", DomainName = DomainName, parameters = parameters)
67	67
68		def GetAttributes(self, DomainName, ItemName, Attributes = []):
69		parameters = SortedDict()
70		parameters['ItemName'] = ItemName
71		seq = 0
72		for attrib in Attributes:
73		parameters['AttributeName.%d' % seq] = attrib
74		seq += 1
75		return self.send_request("GetAttributes", DomainName = DomainName, parameters = parameters)
	68	def GetAttributes(self, DomainName, ItemName, Attributes = []):
	69	parameters = SortedDict()
	70	parameters['ItemName'] = ItemName
	71	seq = 0
	72	for attrib in Attributes:
	73	parameters['AttributeName.%d' % seq] = attrib
	74	seq += 1
	75	return self.send_request("GetAttributes", DomainName = DomainName, parameters = parameters)
76	76
77		def DeleteAttributes(self, DomainName, ItemName, Attributes = {}):
78		"""
79		Remove specified Attributes from ItemName.
80		Attributes parameter can be either:
81		- not specified, in which case the whole Item is removed
82		- list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed
83		- dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the
84		specified values are removed from multi-value attributes.
85		"""
86		parameters = SortedDict()
87		parameters['ItemName'] = ItemName
88		seq = 0
89		for attrib in Attributes:
90		parameters['Attribute.%d.Name' % seq] = attrib
91		if type(Attributes) == type(dict()):
92		parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
93		seq += 1
94		return self.send_request("DeleteAttributes", DomainName = DomainName, parameters = parameters)
	77	def DeleteAttributes(self, DomainName, ItemName, Attributes = {}):
	78	"""
	79	Remove specified Attributes from ItemName.
	80	Attributes parameter can be either:
	81	- not specified, in which case the whole Item is removed
	82	- list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed
	83	- dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the
	84	specified values are removed from multi-value attributes.
	85	"""
	86	parameters = SortedDict()
	87	parameters['ItemName'] = ItemName
	88	seq = 0
	89	for attrib in Attributes:
	90	parameters['Attribute.%d.Name' % seq] = attrib
	91	if type(Attributes) == type(dict()):
	92	parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
	93	seq += 1
	94	return self.send_request("DeleteAttributes", DomainName = DomainName, parameters = parameters)
95	95
96		def Query(self, DomainName, QueryExpression = None, MaxNumberOfItems = None, NextToken = None):
97		parameters = SortedDict()
98		if QueryExpression:
99		parameters['QueryExpression'] = QueryExpression
100		if MaxNumberOfItems:
101		parameters['MaxNumberOfItems'] = MaxNumberOfItems
102		if NextToken:
103		parameters['NextToken'] = NextToken
104		return self.send_request("Query", DomainName = DomainName, parameters = parameters)
105		## Handle NextToken? Or maybe not - let the upper level do it
	96	def Query(self, DomainName, QueryExpression = None, MaxNumberOfItems = None, NextToken = None):
	97	parameters = SortedDict()
	98	if QueryExpression:
	99	parameters['QueryExpression'] = QueryExpression
	100	if MaxNumberOfItems:
	101	parameters['MaxNumberOfItems'] = MaxNumberOfItems
	102	if NextToken:
	103	parameters['NextToken'] = NextToken
	104	return self.send_request("Query", DomainName = DomainName, parameters = parameters)
	105	## Handle NextToken? Or maybe not - let the upper level do it
106	106
107		## ------------------------------------------------
108		## Low-level methods for handling SimpleDB requests
109		## ------------------------------------------------
	107	## ------------------------------------------------
	108	## Low-level methods for handling SimpleDB requests
	109	## ------------------------------------------------
110	110
111		def send_request(self, args, *kwargs):
112		request = self.create_request(args, *kwargs)
113		#debug("Request: %s" % repr(request))
114		conn = self.get_connection()
115		conn.request("GET", self.format_uri(request['uri_params']))
116		http_response = conn.getresponse()
117		response = {}
118		response["status"] = http_response.status
119		response["reason"] = http_response.reason
120		response["headers"] = convertTupleListToDict(http_response.getheaders())
121		response["data"] = http_response.read()
122		conn.close()
	111	def send_request(self, args, *kwargs):
	112	request = self.create_request(args, *kwargs)
	113	#debug("Request: %s" % repr(request))
	114	conn = self.get_connection()
	115	conn.request("GET", self.format_uri(request['uri_params']))
	116	http_response = conn.getresponse()
	117	response = {}
	118	response["status"] = http_response.status
	119	response["reason"] = http_response.reason
	120	response["headers"] = convertTupleListToDict(http_response.getheaders())
	121	response["data"] = http_response.read()
	122	conn.close()
123	123
124		if response["status"] < 200 or response["status"] > 299:
125		debug("Response: " + str(response))
126		raise S3Error(response)
	124	if response["status"] < 200 or response["status"] > 299:
	125	debug("Response: " + str(response))
	126	raise S3Error(response)
127	127
128		return response
	128	return response
129	129
130		def create_request(self, Action, DomainName, parameters = None):
131		if not parameters:
132		parameters = SortedDict()
133		parameters['AWSAccessKeyId'] = self.config.access_key
134		parameters['Version'] = self.Version
135		parameters['SignatureVersion'] = self.SignatureVersion
136		parameters['Action'] = Action
137		parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
138		if DomainName:
139		parameters['DomainName'] = DomainName
140		parameters['Signature'] = self.sign_request(parameters)
141		parameters.keys_return_lowercase = False
142		uri_params = urllib.urlencode(parameters)
143		request = {}
144		request['uri_params'] = uri_params
145		request['parameters'] = parameters
146		return request
	130	def create_request(self, Action, DomainName, parameters = None):
	131	if not parameters:
	132	parameters = SortedDict()
	133	parameters['AWSAccessKeyId'] = self.config.access_key
	134	parameters['Version'] = self.Version
	135	parameters['SignatureVersion'] = self.SignatureVersion
	136	parameters['Action'] = Action
	137	parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
	138	if DomainName:
	139	parameters['DomainName'] = DomainName
	140	parameters['Signature'] = self.sign_request(parameters)
	141	parameters.keys_return_lowercase = False
	142	uri_params = urllib.urlencode(parameters)
	143	request = {}
	144	request['uri_params'] = uri_params
	145	request['parameters'] = parameters
	146	return request
147	147
148		def sign_request(self, parameters):
149		h = ""
150		parameters.keys_sort_lowercase = True
151		parameters.keys_return_lowercase = False
152		for key in parameters:
153		h += "%s%s" % (key, parameters[key])
154		#debug("SignRequest: %s" % h)
155		return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
	148	def sign_request(self, parameters):
	149	h = ""
	150	parameters.keys_sort_lowercase = True
	151	parameters.keys_return_lowercase = False
	152	for key in parameters:
	153	h += "%s%s" % (key, parameters[key])
	154	#debug("SignRequest: %s" % h)
	155	return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
156	156
157		def get_connection(self):
158		if self.config.proxy_host != "":
159		return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
160		else:
161		if self.config.use_https:
162		return httplib.HTTPSConnection(self.config.simpledb_host)
163		else:
164		return httplib.HTTPConnection(self.config.simpledb_host)
	157	def get_connection(self):
	158	if self.config.proxy_host != "":
	159	return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
	160	else:
	161	if self.config.use_https:
	162	return httplib.HTTPSConnection(self.config.simpledb_host)
	163	else:
	164	return httplib.HTTPConnection(self.config.simpledb_host)
165	165
166		def format_uri(self, uri_params):
167		if self.config.proxy_host != "":
168		uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params)
169		else:
170		uri = "/?%s" % uri_params
171		#debug('format_uri(): ' + uri)
172		return uri
	166	def format_uri(self, uri_params):
	167	if self.config.proxy_host != "":
	168	uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params)
	169	else:
	170	uri = "/?%s" % uri_params
	171	#debug('format_uri(): ' + uri)
	172	return uri
	173
	174	# vim:et:ts=4:sts=4:ai

+48

-46

S3/SortedDict.py less more

5	5	from BidirMap import BidirMap
6	6
7	7	class SortedDictIterator(object):
8		def __init__(self, sorted_dict, keys):
9		self.sorted_dict = sorted_dict
10		self.keys = keys
	8	def __init__(self, sorted_dict, keys):
	9	self.sorted_dict = sorted_dict
	10	self.keys = keys
11	11
12		def next(self):
13		try:
14		return self.keys.pop(0)
15		except IndexError:
16		raise StopIteration
	12	def next(self):
	13	try:
	14	return self.keys.pop(0)
	15	except IndexError:
	16	raise StopIteration
17	17
18	18	class SortedDict(dict):
19		def __init__(self, mapping = {}, ignore_case = True, **kwargs):
20		"""
21		WARNING: SortedDict() with ignore_case==True will
22		drop entries differing only in capitalisation!
23		Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland']
24		With ignore_case==False it's all right
25		"""
26		dict.__init__(self, mapping, **kwargs)
27		self.ignore_case = ignore_case
	19	def __init__(self, mapping = {}, ignore_case = True, **kwargs):
	20	"""
	21	WARNING: SortedDict() with ignore_case==True will
	22	drop entries differing only in capitalisation!
	23	Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland']
	24	With ignore_case==False it's all right
	25	"""
	26	dict.__init__(self, mapping, **kwargs)
	27	self.ignore_case = ignore_case
28	28
29		def keys(self):
30		keys = dict.keys(self)
31		if self.ignore_case:
32		# Translation map
33		xlat_map = BidirMap()
34		for key in keys:
35		xlat_map[key.lower()] = key
36		# Lowercase keys
37		lc_keys = xlat_map.keys()
38		lc_keys.sort()
39		return [xlat_map[k] for k in lc_keys]
40		else:
41		keys.sort()
42		return keys
	29	def keys(self):
	30	keys = dict.keys(self)
	31	if self.ignore_case:
	32	# Translation map
	33	xlat_map = BidirMap()
	34	for key in keys:
	35	xlat_map[key.lower()] = key
	36	# Lowercase keys
	37	lc_keys = xlat_map.keys()
	38	lc_keys.sort()
	39	return [xlat_map[k] for k in lc_keys]
	40	else:
	41	keys.sort()
	42	return keys
43	43
44		def __iter__(self):
45		return SortedDictIterator(self, self.keys())
	44	def __iter__(self):
	45	return SortedDictIterator(self, self.keys())
46	46
47	47	if __name__ == "__main__":
48		d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
49		sd = SortedDict(d)
50		print "Wanted: Action, america, Auckland, AWS, [ignore case]"
51		print "Got: ",
52		for key in sd:
53		print "%s," % key,
54		print " [used: __iter__()]"
55		d = SortedDict(d, ignore_case = False)
56		print "Wanted: AWS, Action, Auckland, america, [case sensitive]"
57		print "Got: ",
58		for key in d.keys():
59		print "%s," % key,
60		print " [used: keys()]"
	48	d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
	49	sd = SortedDict(d)
	50	print "Wanted: Action, america, Auckland, AWS, [ignore case]"
	51	print "Got: ",
	52	for key in sd:
	53	print "%s," % key,
	54	print " [used: __iter__()]"
	55	d = SortedDict(d, ignore_case = False)
	56	print "Wanted: AWS, Action, Auckland, america, [case sensitive]"
	57	print "Got: ",
	58	for key in d.keys():
	59	print "%s," % key,
	60	print " [used: keys()]"
	61
	62	# vim:et:ts=4:sts=4:ai

+267

-260

S3/Utils.py less more

3	3	## License: GPL Version 2
4	4
5	5	import os
	6	import sys
6	7	import time
7	8	import re
8	9	import string
9	10	import random
10	11	import rfc822
11		try:
12		from hashlib import md5, sha1
13		except ImportError:
14		from md5 import md5
15		import sha as sha1
16	12	import hmac
17	13	import base64
18	14	import errno

22	18	import Config
23	19	import Exceptions
24	20
	21	# hashlib backported to python 2.4 / 2.5 is not compatible with hmac!
	22	if sys.version_info[0] == 2 and sys.version_info[1] < 6:
	23	from md5 import md5
	24	import sha as sha1
	25	else:
	26	from hashlib import md5, sha1
	27
25	28	try:
26		import xml.etree.ElementTree as ET
	29	import xml.etree.ElementTree as ET
27	30	except ImportError:
28		import elementtree.ElementTree as ET
	31	import elementtree.ElementTree as ET
29	32	from xml.parsers.expat import ExpatError
30	33
31	34	__all__ = []
32	35	def parseNodes(nodes):
33		## WARNING: Ignores text nodes from mixed xml/text.
34		## For instance <tag1>some text<tag2>other text</tag2></tag1>
35		## will be ignore "some text" node
36		retval = []
37		for node in nodes:
38		retval_item = {}
39		for child in node.getchildren():
40		name = child.tag
41		if child.getchildren():
42		retval_item[name] = parseNodes([child])
43		else:
44		retval_item[name] = node.findtext(".//%s" % child.tag)
45		retval.append(retval_item)
46		return retval
	36	## WARNING: Ignores text nodes from mixed xml/text.
	37	## For instance <tag1>some text<tag2>other text</tag2></tag1>
	38	## will be ignore "some text" node
	39	retval = []
	40	for node in nodes:
	41	retval_item = {}
	42	for child in node.getchildren():
	43	name = child.tag
	44	if child.getchildren():
	45	retval_item[name] = parseNodes([child])
	46	else:
	47	retval_item[name] = node.findtext(".//%s" % child.tag)
	48	retval.append(retval_item)
	49	return retval
47	50	__all__.append("parseNodes")
48	51
49	52	def stripNameSpace(xml):
50		"""
51		removeNameSpace(xml) -- remove top-level AWS namespace
52		"""
53		r = re.compile('^(<?[^>]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE)
54		if r.match(xml):
55		xmlns = r.match(xml).groups()[2]
56		xml = r.sub("\\1\\2\\4", xml)
57		else:
58		xmlns = None
59		return xml, xmlns
	53	"""
	54	removeNameSpace(xml) -- remove top-level AWS namespace
	55	"""
	56	r = re.compile('^(<?[^>]+?>\s?)(<\w+) xmlns=[\'"](http://[^\'"]+)[\'"](.*)', re.MULTILINE)
	57	if r.match(xml):
	58	xmlns = r.match(xml).groups()[2]
	59	xml = r.sub("\\1\\2\\4", xml)
	60	else:
	61	xmlns = None
	62	return xml, xmlns
60	63	__all__.append("stripNameSpace")
61	64
62	65	def getTreeFromXml(xml):
63		xml, xmlns = stripNameSpace(xml)
64		try:
65		tree = ET.fromstring(xml)
66		if xmlns:
67		tree.attrib['xmlns'] = xmlns
68		return tree
69		except ExpatError, e:
70		error(e)
71		raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/")
	66	xml, xmlns = stripNameSpace(xml)
	67	try:
	68	tree = ET.fromstring(xml)
	69	if xmlns:
	70	tree.attrib['xmlns'] = xmlns
	71	return tree
	72	except ExpatError, e:
	73	error(e)
	74	raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/")
72	75	__all__.append("getTreeFromXml")
73
	76
74	77	def getListFromXml(xml, node):
75		tree = getTreeFromXml(xml)
76		nodes = tree.findall('.//%s' % (node))
77		return parseNodes(nodes)
	78	tree = getTreeFromXml(xml)
	79	nodes = tree.findall('.//%s' % (node))
	80	return parseNodes(nodes)
78	81	__all__.append("getListFromXml")
79	82
80	83	def getDictFromTree(tree):
81		ret_dict = {}
82		for child in tree.getchildren():
83		if child.getchildren():
84		## Complex-type child. We're not interested
85		continue
86		if ret_dict.has_key(child.tag):
87		if not type(ret_dict[child.tag]) == list:
88		ret_dict[child.tag] = [ret_dict[child.tag]]
89		ret_dict[child.tag].append(child.text or "")
90		else:
91		ret_dict[child.tag] = child.text or ""
92		return ret_dict
	84	ret_dict = {}
	85	for child in tree.getchildren():
	86	if child.getchildren():
	87	## Complex-type child. Recurse
	88	content = getDictFromTree(child)
	89	else:
	90	content = child.text
	91	if ret_dict.has_key(child.tag):
	92	if not type(ret_dict[child.tag]) == list:
	93	ret_dict[child.tag] = [ret_dict[child.tag]]
	94	ret_dict[child.tag].append(content or "")
	95	else:
	96	ret_dict[child.tag] = content or ""
	97	return ret_dict
93	98	__all__.append("getDictFromTree")
94	99
95	100	def getTextFromXml(xml, xpath):
96		tree = getTreeFromXml(xml)
97		if tree.tag.endswith(xpath):
98		return tree.text
99		else:
100		return tree.findtext(xpath)
	101	tree = getTreeFromXml(xml)
	102	if tree.tag.endswith(xpath):
	103	return tree.text
	104	else:
	105	return tree.findtext(xpath)
101	106	__all__.append("getTextFromXml")
102	107
103	108	def getRootTagName(xml):
104		tree = getTreeFromXml(xml)
105		return tree.tag
	109	tree = getTreeFromXml(xml)
	110	return tree.tag
106	111	__all__.append("getRootTagName")
107	112
108	113	def xmlTextNode(tag_name, text):
109		el = ET.Element(tag_name)
110		el.text = unicode(text)
111		return el
	114	el = ET.Element(tag_name)
	115	el.text = unicode(text)
	116	return el
112	117	__all__.append("xmlTextNode")
113	118
114	119	def appendXmlTextNode(tag_name, text, parent):
115		"""
116		Creates a new <tag_name> Node and sets
117		its content to 'text'. Then appends the
118		created Node to 'parent' element if given.
119		Returns the newly created Node.
120		"""
121		el = xmlTextNode(tag_name, text)
122		parent.append(el)
123		return el
	120	"""
	121	Creates a new <tag_name> Node and sets
	122	its content to 'text'. Then appends the
	123	created Node to 'parent' element if given.
	124	Returns the newly created Node.
	125	"""
	126	el = xmlTextNode(tag_name, text)
	127	parent.append(el)
	128	return el
124	129	__all__.append("appendXmlTextNode")
125	130
126	131	def dateS3toPython(date):
127		date = re.compile("(\.\d*)?Z").sub(".000Z", date)
128		return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z")
	132	date = re.compile("(\.\d*)?Z").sub(".000Z", date)
	133	return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z")
129	134	__all__.append("dateS3toPython")
130	135
131	136	def dateS3toUnix(date):
132		## FIXME: This should be timezone-aware.
133		## Currently the argument to strptime() is GMT but mktime()
134		## treats it as "localtime". Anyway...
135		return time.mktime(dateS3toPython(date))
	137	## FIXME: This should be timezone-aware.
	138	## Currently the argument to strptime() is GMT but mktime()
	139	## treats it as "localtime". Anyway...
	140	return time.mktime(dateS3toPython(date))
136	141	__all__.append("dateS3toUnix")
137	142
138	143	def dateRFC822toPython(date):
139		return rfc822.parsedate(date)
	144	return rfc822.parsedate(date)
140	145	__all__.append("dateRFC822toPython")
141	146
142	147	def dateRFC822toUnix(date):
143		return time.mktime(dateRFC822toPython(date))
	148	return time.mktime(dateRFC822toPython(date))
144	149	__all__.append("dateRFC822toUnix")
145	150
146	151	def formatSize(size, human_readable = False, floating_point = False):
147		size = floating_point and float(size) or int(size)
148		if human_readable:
149		coeffs = ['k', 'M', 'G', 'T']
150		coeff = ""
151		while size > 2048:
152		size /= 1024
153		coeff = coeffs.pop(0)
154		return (size, coeff)
155		else:
156		return (size, "")
	152	size = floating_point and float(size) or int(size)
	153	if human_readable:
	154	coeffs = ['k', 'M', 'G', 'T']
	155	coeff = ""
	156	while size > 2048:
	157	size /= 1024
	158	coeff = coeffs.pop(0)
	159	return (size, coeff)
	160	else:
	161	return (size, "")
157	162	__all__.append("formatSize")
158	163
159	164	def formatDateTime(s3timestamp):
160		return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp))
	165	return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp))
161	166	__all__.append("formatDateTime")
162	167
163	168	def convertTupleListToDict(list):
164		retval = {}
165		for tuple in list:
166		retval[tuple[0]] = tuple[1]
167		return retval
	169	retval = {}
	170	for tuple in list:
	171	retval[tuple[0]] = tuple[1]
	172	return retval
168	173	__all__.append("convertTupleListToDict")
169	174
170	175	_rnd_chars = string.ascii_letters+string.digits
171	176	_rnd_chars_len = len(_rnd_chars)
172	177	def rndstr(len):
173		retval = ""
174		while len > 0:
175		retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)]
176		len -= 1
177		return retval
	178	retval = ""
	179	while len > 0:
	180	retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)]
	181	len -= 1
	182	return retval
178	183	__all__.append("rndstr")
179	184
180	185	def mktmpsomething(prefix, randchars, createfunc):
181		old_umask = os.umask(0077)
182		tries = 5
183		while tries > 0:
184		dirname = prefix + rndstr(randchars)
185		try:
186		createfunc(dirname)
187		break
188		except OSError, e:
189		if e.errno != errno.EEXIST:
190		os.umask(old_umask)
191		raise
192		tries -= 1
193
194		os.umask(old_umask)
195		return dirname
	186	old_umask = os.umask(0077)
	187	tries = 5
	188	while tries > 0:
	189	dirname = prefix + rndstr(randchars)
	190	try:
	191	createfunc(dirname)
	192	break
	193	except OSError, e:
	194	if e.errno != errno.EEXIST:
	195	os.umask(old_umask)
	196	raise
	197	tries -= 1
	198
	199	os.umask(old_umask)
	200	return dirname
196	201	__all__.append("mktmpsomething")
197	202
198	203	def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10):
199		return mktmpsomething(prefix, randchars, os.mkdir)
	204	return mktmpsomething(prefix, randchars, os.mkdir)
200	205	__all__.append("mktmpdir")
201	206
202	207	def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20):
203		createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT \| os.O_EXCL))
204		return mktmpsomething(prefix, randchars, createfunc)
	208	createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT \| os.O_EXCL))
	209	return mktmpsomething(prefix, randchars, createfunc)
205	210	__all__.append("mktmpfile")
206	211
207	212	def hash_file_md5(filename):
208		h = md5()
209		f = open(filename, "rb")
210		while True:
211		# Hash 32kB chunks
212		data = f.read(32*1024)
213		if not data:
214		break
215		h.update(data)
216		f.close()
217		return h.hexdigest()
	213	h = md5()
	214	f = open(filename, "rb")
	215	while True:
	216	# Hash 32kB chunks
	217	data = f.read(32*1024)
	218	if not data:
	219	break
	220	h.update(data)
	221	f.close()
	222	return h.hexdigest()
218	223	__all__.append("hash_file_md5")
219	224
220	225	def mkdir_with_parents(dir_name):
221		"""
222		mkdir_with_parents(dst_dir)
223
224		Create directory 'dir_name' with all parent directories
225
226		Returns True on success, False otherwise.
227		"""
228		pathmembers = dir_name.split(os.sep)
229		tmp_stack = []
230		while pathmembers and not os.path.isdir(os.sep.join(pathmembers)):
231		tmp_stack.append(pathmembers.pop())
232		while tmp_stack:
233		pathmembers.append(tmp_stack.pop())
234		cur_dir = os.sep.join(pathmembers)
235		try:
236		debug("mkdir(%s)" % cur_dir)
237		os.mkdir(cur_dir)
238		except (OSError, IOError), e:
239		warning("%s: can not make directory: %s" % (cur_dir, e.strerror))
240		return False
241		except Exception, e:
242		warning("%s: %s" % (cur_dir, e))
243		return False
244		return True
	226	"""
	227	mkdir_with_parents(dst_dir)
	228
	229	Create directory 'dir_name' with all parent directories
	230
	231	Returns True on success, False otherwise.
	232	"""
	233	pathmembers = dir_name.split(os.sep)
	234	tmp_stack = []
	235	while pathmembers and not os.path.isdir(os.sep.join(pathmembers)):
	236	tmp_stack.append(pathmembers.pop())
	237	while tmp_stack:
	238	pathmembers.append(tmp_stack.pop())
	239	cur_dir = os.sep.join(pathmembers)
	240	try:
	241	debug("mkdir(%s)" % cur_dir)
	242	os.mkdir(cur_dir)
	243	except (OSError, IOError), e:
	244	warning("%s: can not make directory: %s" % (cur_dir, e.strerror))
	245	return False
	246	except Exception, e:
	247	warning("%s: %s" % (cur_dir, e))
	248	return False
	249	return True
245	250	__all__.append("mkdir_with_parents")
246	251
247	252	def unicodise(string, encoding = None, errors = "replace"):
248		"""
249		Convert 'string' to Unicode or raise an exception.
250		"""
251
252		if not encoding:
253		encoding = Config.Config().encoding
254
255		if type(string) == unicode:
256		return string
257		debug("Unicodising %r using %s" % (string, encoding))
258		try:
259		return string.decode(encoding, errors)
260		except UnicodeDecodeError:
261		raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)
	253	"""
	254	Convert 'string' to Unicode or raise an exception.
	255	"""
	256
	257	if not encoding:
	258	encoding = Config.Config().encoding
	259
	260	if type(string) == unicode:
	261	return string
	262	debug("Unicodising %r using %s" % (string, encoding))
	263	try:
	264	return string.decode(encoding, errors)
	265	except UnicodeDecodeError:
	266	raise UnicodeDecodeError("Conversion to unicode failed: %r" % string)
262	267	__all__.append("unicodise")
263	268
264	269	def deunicodise(string, encoding = None, errors = "replace"):
265		"""
266		Convert unicode 'string' to <type str>, by default replacing
267		all invalid characters with '?' or raise an exception.
268		"""
269
270		if not encoding:
271		encoding = Config.Config().encoding
272
273		if type(string) != unicode:
274		return str(string)
275		debug("DeUnicodising %r using %s" % (string, encoding))
276		try:
277		return string.encode(encoding, errors)
278		except UnicodeEncodeError:
279		raise UnicodeEncodeError("Conversion from unicode failed: %r" % string)
	270	"""
	271	Convert unicode 'string' to <type str>, by default replacing
	272	all invalid characters with '?' or raise an exception.
	273	"""
	274
	275	if not encoding:
	276	encoding = Config.Config().encoding
	277
	278	if type(string) != unicode:
	279	return str(string)
	280	debug("DeUnicodising %r using %s" % (string, encoding))
	281	try:
	282	return string.encode(encoding, errors)
	283	except UnicodeEncodeError:
	284	raise UnicodeEncodeError("Conversion from unicode failed: %r" % string)
280	285	__all__.append("deunicodise")
281	286
282	287	def unicodise_safe(string, encoding = None):
283		"""
284		Convert 'string' to Unicode according to current encoding
285		and replace all invalid characters with '?'
286		"""
287
288		return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')
	288	"""
	289	Convert 'string' to Unicode according to current encoding
	290	and replace all invalid characters with '?'
	291	"""
	292
	293	return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')
289	294	__all__.append("unicodise_safe")
290	295
291	296	def replace_nonprintables(string):
292		"""
293		replace_nonprintables(string)
294
295		Replaces all non-printable characters 'ch' in 'string'
296		where ord(ch) <= 26 with ^@, ^A, ... ^Z
297		"""
298		new_string = ""
299		modified = 0
300		for c in string:
301		o = ord(c)
302		if (o <= 31):
303		new_string += "^" + chr(ord('@') + o)
304		modified += 1
305		elif (o == 127):
306		new_string += "^?"
307		modified += 1
308		else:
309		new_string += c
310		if modified and Config.Config().urlencoding_mode != "fixbucket":
311		warning("%d non-printable characters replaced in: %s" % (modified, new_string))
312		return new_string
	297	"""
	298	replace_nonprintables(string)
	299
	300	Replaces all non-printable characters 'ch' in 'string'
	301	where ord(ch) <= 26 with ^@, ^A, ... ^Z
	302	"""
	303	new_string = ""
	304	modified = 0
	305	for c in string:
	306	o = ord(c)
	307	if (o <= 31):
	308	new_string += "^" + chr(ord('@') + o)
	309	modified += 1
	310	elif (o == 127):
	311	new_string += "^?"
	312	modified += 1
	313	else:
	314	new_string += c
	315	if modified and Config.Config().urlencoding_mode != "fixbucket":
	316	warning("%d non-printable characters replaced in: %s" % (modified, new_string))
	317	return new_string
313	318	__all__.append("replace_nonprintables")
314	319
315	320	def sign_string(string_to_sign):
316		#debug("string_to_sign: %s" % string_to_sign)
317		signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
318		#debug("signature: %s" % signature)
319		return signature
	321	#debug("string_to_sign: %s" % string_to_sign)
	322	signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
	323	#debug("signature: %s" % signature)
	324	return signature
320	325	__all__.append("sign_string")
321	326
322	327	def check_bucket_name(bucket, dns_strict = True):
323		if dns_strict:
324		invalid = re.search("([^a-z0-9\.-])", bucket)
325		if invalid:
326		raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
327		else:
328		invalid = re.search("([^A-Za-z0-9\._-])", bucket)
329		if invalid:
330		raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
331
332		if len(bucket) < 3:
333		raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
334		if len(bucket) > 255:
335		raise Exceptions.ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
336		if dns_strict:
337		if len(bucket) > 63:
338		raise Exceptions.ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
339		if re.search("-\.", bucket):
340		raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
341		if re.search("\.\.", bucket):
342		raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
343		if not re.search("^[0-9a-z]", bucket):
344		raise Exceptions.ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
345		if not re.search("[0-9a-z]$", bucket):
346		raise Exceptions.ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
347		return True
	328	if dns_strict:
	329	invalid = re.search("([^a-z0-9\.-])", bucket)
	330	if invalid:
	331	raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: lowercase us-ascii letters (a-z), digits (0-9), dot (.) and hyphen (-)." % (bucket, invalid.groups()[0]))
	332	else:
	333	invalid = re.search("([^A-Za-z0-9\._-])", bucket)
	334	if invalid:
	335	raise Exceptions.ParameterError("Bucket name '%s' contains disallowed character '%s'. The only supported ones are: us-ascii letters (a-z, A-Z), digits (0-9), dot (.), hyphen (-) and underscore (_)." % (bucket, invalid.groups()[0]))
	336
	337	if len(bucket) < 3:
	338	raise Exceptions.ParameterError("Bucket name '%s' is too short (min 3 characters)" % bucket)
	339	if len(bucket) > 255:
	340	raise Exceptions.ParameterError("Bucket name '%s' is too long (max 255 characters)" % bucket)
	341	if dns_strict:
	342	if len(bucket) > 63:
	343	raise Exceptions.ParameterError("Bucket name '%s' is too long (max 63 characters)" % bucket)
	344	if re.search("-\.", bucket):
	345	raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '-.' for DNS compatibility" % bucket)
	346	if re.search("\.\.", bucket):
	347	raise Exceptions.ParameterError("Bucket name '%s' must not contain sequence '..' for DNS compatibility" % bucket)
	348	if not re.search("^[0-9a-z]", bucket):
	349	raise Exceptions.ParameterError("Bucket name '%s' must start with a letter or a digit" % bucket)
	350	if not re.search("[0-9a-z]$", bucket):
	351	raise Exceptions.ParameterError("Bucket name '%s' must end with a letter or a digit" % bucket)
	352	return True
348	353	__all__.append("check_bucket_name")
349	354
350	355	def check_bucket_name_dns_conformity(bucket):
351		try:
352		return check_bucket_name(bucket, dns_strict = True)
353		except Exceptions.ParameterError:
354		return False
	356	try:
	357	return check_bucket_name(bucket, dns_strict = True)
	358	except Exceptions.ParameterError:
	359	return False
355	360	__all__.append("check_bucket_name_dns_conformity")
356	361
357	362	def getBucketFromHostname(hostname):
358		"""
359		bucket, success = getBucketFromHostname(hostname)
360
361		Only works for hostnames derived from bucket names
362		using Config.host_bucket pattern.
363
364		Returns bucket name and a boolean success flag.
365		"""
366
367		# Create RE pattern from Config.host_bucket
368		pattern = Config.Config().host_bucket % { 'bucket' : '(?P<bucket>.*)' }
369		m = re.match(pattern, hostname)
370		if not m:
371		return (hostname, False)
372		return m.groups()[0], True
	363	"""
	364	bucket, success = getBucketFromHostname(hostname)
	365
	366	Only works for hostnames derived from bucket names
	367	using Config.host_bucket pattern.
	368
	369	Returns bucket name and a boolean success flag.
	370	"""
	371
	372	# Create RE pattern from Config.host_bucket
	373	pattern = Config.Config().host_bucket % { 'bucket' : '(?P<bucket>.*)' }
	374	m = re.match(pattern, hostname)
	375	if not m:
	376	return (hostname, False)
	377	return m.groups()[0], True
373	378	__all__.append("getBucketFromHostname")
374	379
375	380	def getHostnameFromBucket(bucket):
376		return Config.Config().host_bucket % { 'bucket' : bucket }
	381	return Config.Config().host_bucket % { 'bucket' : bucket }
377	382	__all__.append("getHostnameFromBucket")
	383
	384	# vim:et:ts=4:sts=4:ai

+1698

-1902

s3cmd less more

0		#!/usr/bin/env python
	0	#!/usr/bin/python
1	1
2	2	## Amazon S3 manager
3	3	## Author: Michal Ludvig <michal@logix.cz>

7	7	import sys
8	8
9	9	if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.4:
10		sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n")
11		sys.exit(1)
	10	sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n")
	11	sys.exit(1)
12	12
13	13	import logging
14	14	import time

29	29	from distutils.spawn import find_executable
30	30
31	31	def output(message):
32		sys.stdout.write(message + "\n")
	32	sys.stdout.write(message + "\n")
33	33
34	34	def check_args_type(args, type, verbose_type):
35		for arg in args:
36		if S3Uri(arg).type != type:
37		raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))
38
39		def _fswalk_follow_symlinks(path):
40		'''
41		Walk filesystem, following symbolic links (but without recursion), on python2.4 and later
42
43		If a recursive directory link is detected, emit a warning and skip.
44		'''
45		assert os.path.isdir(path) # only designed for directory argument
46		walkdirs = set([path])
47		targets = set()
48		for dirpath, dirnames, filenames in os.walk(path):
49		for dirname in dirnames:
50		current = os.path.join(dirpath, dirname)
51		target = os.path.realpath(current)
52		if os.path.islink(current):
53		if target in targets:
54		warning("Skipping recursively symlinked directory %s" % dirname)
55		else:
56		walkdirs.add(current)
57		targets.add(target)
58		for walkdir in walkdirs:
59		for value in os.walk(walkdir):
60		yield value
61
62		def fswalk(path, follow_symlinks):
63		'''
64		Directory tree generator
65
66		path (str) is the root of the directory tree to walk
67
68		follow_symlinks (bool) indicates whether to descend into symbolically linked directories
69		'''
70		if follow_symlinks:
71		return _fswalk_follow_symlinks(path)
72		return os.walk(path)
	35	for arg in args:
	36	if S3Uri(arg).type != type:
	37	raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))
73	38
74	39	def cmd_du(args):
75		s3 = S3(Config())
76		if len(args) > 0:
77		uri = S3Uri(args[0])
78		if uri.type == "s3" and uri.has_bucket():
79		subcmd_bucket_usage(s3, uri)
80		return
81		subcmd_bucket_usage_all(s3)
	40	s3 = S3(Config())
	41	if len(args) > 0:
	42	uri = S3Uri(args[0])
	43	if uri.type == "s3" and uri.has_bucket():
	44	subcmd_bucket_usage(s3, uri)
	45	return
	46	subcmd_bucket_usage_all(s3)
82	47
83	48	def subcmd_bucket_usage_all(s3):
84		response = s3.list_all_buckets()
85
86		buckets_size = 0
87		for bucket in response["list"]:
88		size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"]))
89		if size != None:
90		buckets_size += size
91		total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes)
92		total_size_str = str(total_size) + size_coeff
93		output(u"".rjust(8, "-"))
94		output(u"%s Total" % (total_size_str.ljust(8)))
	49	response = s3.list_all_buckets()
	50
	51	buckets_size = 0
	52	for bucket in response["list"]:
	53	size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"]))
	54	if size != None:
	55	buckets_size += size
	56	total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes)
	57	total_size_str = str(total_size) + size_coeff
	58	output(u"".rjust(8, "-"))
	59	output(u"%s Total" % (total_size_str.ljust(8)))
95	60
96	61	def subcmd_bucket_usage(s3, uri):
97		bucket = uri.bucket()
98		object = uri.object()
99
100		if object.endswith('*'):
101		object = object[:-1]
102		try:
103		response = s3.bucket_list(bucket, prefix = object, recursive = True)
104		except S3Error, e:
105		if S3.codes.has_key(e.info["Code"]):
106		error(S3.codes[e.info["Code"]] % bucket)
107		return
108		else:
109		raise
110		bucket_size = 0
111		for object in response["list"]:
112		size, size_coeff = formatSize(object["Size"], False)
113		bucket_size += size
114		total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes)
115		total_size_str = str(total_size) + size_coeff
116		output(u"%s %s" % (total_size_str.ljust(8), uri))
117		return bucket_size
	62	bucket = uri.bucket()
	63	object = uri.object()
	64
	65	if object.endswith('*'):
	66	object = object[:-1]
	67	try:
	68	response = s3.bucket_list(bucket, prefix = object, recursive = True)
	69	except S3Error, e:
	70	if S3.codes.has_key(e.info["Code"]):
	71	error(S3.codes[e.info["Code"]] % bucket)
	72	return
	73	else:
	74	raise
	75	bucket_size = 0
	76	for object in response["list"]:
	77	size, size_coeff = formatSize(object["Size"], False)
	78	bucket_size += size
	79	total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes)
	80	total_size_str = str(total_size) + size_coeff
	81	output(u"%s %s" % (total_size_str.ljust(8), uri))
	82	return bucket_size
118	83
119	84	def cmd_ls(args):
120		s3 = S3(Config())
121		if len(args) > 0:
122		uri = S3Uri(args[0])
123		if uri.type == "s3" and uri.has_bucket():
124		subcmd_bucket_list(s3, uri)
125		return
126		subcmd_buckets_list_all(s3)
	85	s3 = S3(Config())
	86	if len(args) > 0:
	87	uri = S3Uri(args[0])
	88	if uri.type == "s3" and uri.has_bucket():
	89	subcmd_bucket_list(s3, uri)
	90	return
	91	subcmd_buckets_list_all(s3)
127	92
128	93	def cmd_buckets_list_all_all(args):
129		s3 = S3(Config())
130
131		response = s3.list_all_buckets()
132
133		for bucket in response["list"]:
134		subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]))
135		output(u"")
	94	s3 = S3(Config())
	95
	96	response = s3.list_all_buckets()
	97
	98	for bucket in response["list"]:
	99	subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]))
	100	output(u"")
136	101
137	102
138	103	def subcmd_buckets_list_all(s3):
139		response = s3.list_all_buckets()
140		for bucket in response["list"]:
141		output(u"%s s3://%s" % (
142		formatDateTime(bucket["CreationDate"]),
143		bucket["Name"],
144		))
	104	response = s3.list_all_buckets()
	105	for bucket in response["list"]:
	106	output(u"%s s3://%s" % (
	107	formatDateTime(bucket["CreationDate"]),
	108	bucket["Name"],
	109	))
145	110
146	111	def subcmd_bucket_list(s3, uri):
147		bucket = uri.bucket()
148		prefix = uri.object()
149
150		debug(u"Bucket 's3://%s':" % bucket)
151		if prefix.endswith('*'):
152		prefix = prefix[:-1]
153		try:
154		response = s3.bucket_list(bucket, prefix = prefix)
155		except S3Error, e:
156		if S3.codes.has_key(e.info["Code"]):
157		error(S3.codes[e.info["Code"]] % bucket)
158		return
159		else:
160		raise
161
162		if cfg.list_md5:
163		format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s"
164		else:
165		format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s"
166
167		for prefix in response['common_prefixes']:
168		output(format_string % {
169		"timestamp": "",
170		"size": "DIR",
171		"coeff": "",
172		"md5": "",
173		"uri": uri.compose_uri(bucket, prefix["Prefix"])})
174
175		for object in response["list"]:
176		size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes)
177		output(format_string % {
178		"timestamp": formatDateTime(object["LastModified"]),
179		"size" : str(size),
180		"coeff": size_coeff,
181		"md5" : object['ETag'].strip('"'),
182		"uri": uri.compose_uri(bucket, object["Key"]),
183		})
	112	bucket = uri.bucket()
	113	prefix = uri.object()
	114
	115	debug(u"Bucket 's3://%s':" % bucket)
	116	if prefix.endswith('*'):
	117	prefix = prefix[:-1]
	118	try:
	119	response = s3.bucket_list(bucket, prefix = prefix)
	120	except S3Error, e:
	121	if S3.codes.has_key(e.info["Code"]):
	122	error(S3.codes[e.info["Code"]] % bucket)
	123	return
	124	else:
	125	raise
	126
	127	if cfg.list_md5:
	128	format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s"
	129	else:
	130	format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s"
	131
	132	for prefix in response['common_prefixes']:
	133	output(format_string % {
	134	"timestamp": "",
	135	"size": "DIR",
	136	"coeff": "",
	137	"md5": "",
	138	"uri": uri.compose_uri(bucket, prefix["Prefix"])})
	139
	140	for object in response["list"]:
	141	size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes)
	142	output(format_string % {
	143	"timestamp": formatDateTime(object["LastModified"]),
	144	"size" : str(size),
	145	"coeff": size_coeff,
	146	"md5" : object['ETag'].strip('"'),
	147	"uri": uri.compose_uri(bucket, object["Key"]),
	148	})
184	149
185	150	def cmd_bucket_create(args):
186		s3 = S3(Config())
187		for arg in args:
188		uri = S3Uri(arg)
189		if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
190		raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
191		try:
192		response = s3.bucket_create(uri.bucket(), cfg.bucket_location)
193		output(u"Bucket '%s' created" % uri.uri())
194		except S3Error, e:
195		if S3.codes.has_key(e.info["Code"]):
196		error(S3.codes[e.info["Code"]] % uri.bucket())
197		return
198		else:
199		raise
	151	s3 = S3(Config())
	152	for arg in args:
	153	uri = S3Uri(arg)
	154	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
	155	raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
	156	try:
	157	response = s3.bucket_create(uri.bucket(), cfg.bucket_location)
	158	output(u"Bucket '%s' created" % uri.uri())
	159	except S3Error, e:
	160	if S3.codes.has_key(e.info["Code"]):
	161	error(S3.codes[e.info["Code"]] % uri.bucket())
	162	return
	163	else:
	164	raise
	165
	166	def cmd_website_info(args):
	167	s3 = S3(Config())
	168	for arg in args:
	169	uri = S3Uri(arg)
	170	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
	171	raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
	172	try:
	173	response = s3.website_info(uri, cfg.bucket_location)
	174	if response:
	175	output(u"Bucket %s: Website configuration" % uri.uri())
	176	output(u"Website endpoint: %s" % response['website_endpoint'])
	177	output(u"Index document: %s" % response['index_document'])
	178	output(u"Error document: %s" % response['error_document'])
	179	else:
	180	output(u"Bucket %s: Unable to receive website configuration." % (uri.uri()))
	181	except S3Error, e:
	182	if S3.codes.has_key(e.info["Code"]):
	183	error(S3.codes[e.info["Code"]] % uri.bucket())
	184	return
	185	else:
	186	raise
	187
	188	def cmd_website_create(args):
	189	s3 = S3(Config())
	190	for arg in args:
	191	uri = S3Uri(arg)
	192	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
	193	raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
	194	try:
	195	response = s3.website_create(uri, cfg.bucket_location)
	196	output(u"Bucket '%s': website configuration created." % (uri.uri()))
	197	except S3Error, e:
	198	if S3.codes.has_key(e.info["Code"]):
	199	error(S3.codes[e.info["Code"]] % uri.bucket())
	200	return
	201	else:
	202	raise
	203
	204	def cmd_website_delete(args):
	205	s3 = S3(Config())
	206	for arg in args:
	207	uri = S3Uri(arg)
	208	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
	209	raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
	210	try:
	211	response = s3.website_delete(uri, cfg.bucket_location)
	212	output(u"Bucket '%s': website configuration deleted." % (uri.uri()))
	213	except S3Error, e:
	214	if S3.codes.has_key(e.info["Code"]):
	215	error(S3.codes[e.info["Code"]] % uri.bucket())
	216	return
	217	else:
	218	raise
200	219
201	220	def cmd_bucket_delete(args):
202		def _bucket_delete_one(uri):
203		try:
204		response = s3.bucket_delete(uri.bucket())
205		except S3Error, e:
206		if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive):
207		warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")
208		subcmd_object_del_uri(uri.uri(), recursive = True)
209		return _bucket_delete_one(uri)
210		elif S3.codes.has_key(e.info["Code"]):
211		error(S3.codes[e.info["Code"]] % uri.bucket())
212		return
213		else:
214		raise
215
216		s3 = S3(Config())
217		for arg in args:
218		uri = S3Uri(arg)
219		if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
220		raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
221		_bucket_delete_one(uri)
222		output(u"Bucket '%s' removed" % uri.uri())
223
224		def fetch_local_list(args, recursive = None):
225		local_uris = []
226		local_list = SortedDict(ignore_case = False)
227		single_file = False
228
229		if type(args) not in (list, tuple):
230		args = [args]
231
232		if recursive == None:
233		recursive = cfg.recursive
234
235		for arg in args:
236		uri = S3Uri(arg)
237		if not uri.type == 'file':
238		raise ParameterError("Expecting filename or directory instead of: %s" % arg)
239		if uri.isdir() and not recursive:
240		raise ParameterError("Use --recursive to upload a directory: %s" % arg)
241		local_uris.append(uri)
242
243		for uri in local_uris:
244		list_for_uri, single_file = _get_filelist_local(uri)
245		local_list.update(list_for_uri)
246
247		## Single file is True if and only if the user
248		## specified one local URI and that URI represents
249		## a FILE. Ie it is False if the URI was of a DIR
250		## and that dir contained only one FILE. That's not
251		## a case of single_file==True.
252		if len(local_list) > 1:
253		single_file = False
254
255		return local_list, single_file
256
257		def fetch_remote_list(args, require_attribs = False, recursive = None):
258		remote_uris = []
259		remote_list = SortedDict(ignore_case = False)
260
261		if type(args) not in (list, tuple):
262		args = [args]
263
264		if recursive == None:
265		recursive = cfg.recursive
266
267		for arg in args:
268		uri = S3Uri(arg)
269		if not uri.type == 's3':
270		raise ParameterError("Expecting S3 URI instead of '%s'" % arg)
271		remote_uris.append(uri)
272
273		if recursive:
274		for uri in remote_uris:
275		objectlist = _get_filelist_remote(uri)
276		for key in objectlist:
277		remote_list[key] = objectlist[key]
278		else:
279		for uri in remote_uris:
280		uri_str = str(uri)
281		## Wildcards used in remote URI?
282		## If yes we'll need a bucket listing...
283		if uri_str.find('*') > -1 or uri_str.find('?') > -1:
284		first_wildcard = uri_str.find('*')
285		first_questionmark = uri_str.find('?')
286		if first_questionmark > -1 and first_questionmark < first_wildcard:
287		first_wildcard = first_questionmark
288		prefix = uri_str[:first_wildcard]
289		rest = uri_str[first_wildcard+1:]
290		## Only request recursive listing if the 'rest' of the URI,
291		## i.e. the part after first wildcard, contains '/'
292		need_recursion = rest.find('/') > -1
293		objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion)
294		for key in objectlist:
295		## Check whether the 'key' matches the requested wildcards
296		if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str):
297		remote_list[key] = objectlist[key]
298		else:
299		## No wildcards - simply append the given URI to the list
300		key = os.path.basename(uri.object())
301		if not key:
302		raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri())
303		remote_item = {
304		'base_uri': uri,
305		'object_uri_str': unicode(uri),
306		'object_key': uri.object()
307		}
308		if require_attribs:
309		response = S3(cfg).object_info(uri)
310		remote_item.update({
311		'size': int(response['headers']['content-length']),
312		'md5': response['headers']['etag'].strip('"\''),
313		'timestamp' : Utils.dateRFC822toUnix(response['headers']['date'])
314		})
315		remote_list[key] = remote_item
316		return remote_list
	221	def _bucket_delete_one(uri):
	222	try:
	223	response = s3.bucket_delete(uri.bucket())
	224	except S3Error, e:
	225	if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive):
	226	warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")
	227	subcmd_object_del_uri(uri.uri(), recursive = True)
	228	return _bucket_delete_one(uri)
	229	elif S3.codes.has_key(e.info["Code"]):
	230	error(S3.codes[e.info["Code"]] % uri.bucket())
	231	return
	232	else:
	233	raise
	234
	235	s3 = S3(Config())
	236	for arg in args:
	237	uri = S3Uri(arg)
	238	if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
	239	raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
	240	_bucket_delete_one(uri)
	241	output(u"Bucket '%s' removed" % uri.uri())
317	242
318	243	def cmd_object_put(args):
319		cfg = Config()
320		s3 = S3(cfg)
321
322		if len(args) == 0:
323		raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.")
324
325		## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
326		destination_base_uri = S3Uri(args.pop())
327		if destination_base_uri.type != 's3':
328		raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
329		destination_base = str(destination_base_uri)
330
331		if len(args) == 0:
332		raise ParameterError("Nothing to upload. Expecting a local file or directory.")
333
334		local_list, single_file_local = fetch_local_list(args)
335
336		local_list, exclude_list = _filelist_filter_exclude_include(local_list)
337
338		local_count = len(local_list)
339
340		info(u"Summary: %d local files to upload" % local_count)
341
342		if local_count > 0:
343		if not destination_base.endswith("/"):
344		if not single_file_local:
345		raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
346		local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
347		else:
348		for key in local_list:
349		local_list[key]['remote_uri'] = unicodise(destination_base + key)
350
351		if cfg.dry_run:
352		for key in exclude_list:
353		output(u"exclude: %s" % unicodise(key))
354		for key in local_list:
355		output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
356
357		warning(u"Exitting now because of --dry-run")
358		return
359
360		seq = 0
361		for key in local_list:
362		seq += 1
363
364		uri_final = S3Uri(local_list[key]['remote_uri'])
365
366		extra_headers = copy(cfg.extra_headers)
367		full_name_orig = local_list[key]['full_name']
368		full_name = full_name_orig
369		seq_label = "[%d of %d]" % (seq, local_count)
370		if Config().encrypt:
371		exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)
372		try:
373		response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)
374		except S3UploadError, e:
375		error(u"Upload of '%s' failed too many times. Skipping that file." % full_name_orig)
376		continue
377		except InvalidFileError, e:
378		warning(u"File can not be uploaded: %s" % e)
379		continue
380		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
381		if not Config().progress_meter:
382		output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
383		(unicodise(full_name_orig), uri_final, response["size"], response["elapsed"],
384		speed_fmt[0], speed_fmt[1], seq_label))
385		if Config().acl_public:
386		output(u"Public URL of the object is: %s" %
387		(uri_final.public_url()))
388		if Config().encrypt and full_name != full_name_orig:
389		debug(u"Removing temporary encrypted file: %s" % unicodise(full_name))
390		os.remove(full_name)
	244	cfg = Config()
	245	s3 = S3(cfg)
	246
	247	if len(args) == 0:
	248	raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.")
	249
	250	## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
	251	destination_base_uri = S3Uri(args.pop())
	252	if destination_base_uri.type != 's3':
	253	raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
	254	destination_base = str(destination_base_uri)
	255
	256	if len(args) == 0:
	257	raise ParameterError("Nothing to upload. Expecting a local file or directory.")
	258
	259	local_list, single_file_local = fetch_local_list(args)
	260
	261	local_list, exclude_list = filter_exclude_include(local_list)
	262
	263	local_count = len(local_list)
	264
	265	info(u"Summary: %d local files to upload" % local_count)
	266
	267	if local_count > 0:
	268	if not destination_base.endswith("/"):
	269	if not single_file_local:
	270	raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
	271	local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
	272	else:
	273	for key in local_list:
	274	local_list[key]['remote_uri'] = unicodise(destination_base + key)
	275
	276	if cfg.dry_run:
	277	for key in exclude_list:
	278	output(u"exclude: %s" % unicodise(key))
	279	for key in local_list:
	280	output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
	281
	282	warning(u"Exitting now because of --dry-run")
	283	return
	284
	285	seq = 0
	286	for key in local_list:
	287	seq += 1
	288
	289	uri_final = S3Uri(local_list[key]['remote_uri'])
	290
	291	extra_headers = copy(cfg.extra_headers)
	292	full_name_orig = local_list[key]['full_name']
	293	full_name = full_name_orig
	294	seq_label = "[%d of %d]" % (seq, local_count)
	295	if Config().encrypt:
	296	exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)
	297	try:
	298	response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)
	299	except S3UploadError, e:
	300	error(u"Upload of '%s' failed too many times. Skipping that file." % full_name_orig)
	301	continue
	302	except InvalidFileError, e:
	303	warning(u"File can not be uploaded: %s" % e)
	304	continue
	305	speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
	306	if not Config().progress_meter:
	307	output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
	308	(unicodise(full_name_orig), uri_final, response["size"], response["elapsed"],
	309	speed_fmt[0], speed_fmt[1], seq_label))
	310	if Config().acl_public:
	311	output(u"Public URL of the object is: %s" %
	312	(uri_final.public_url()))
	313	if Config().encrypt and full_name != full_name_orig:
	314	debug(u"Removing temporary encrypted file: %s" % unicodise(full_name))
	315	os.remove(full_name)
391	316
392	317	def cmd_object_get(args):
393		cfg = Config()
394		s3 = S3(cfg)
395
396		## Check arguments:
397		## if not --recursive:
398		## - first N arguments must be S3Uri
399		## - if the last one is S3 make current dir the destination_base
400		## - if the last one is a directory:
401		## - take all 'basenames' of the remote objects and
402		## make the destination name be 'destination_base'+'basename'
403		## - if the last one is a file or not existing:
404		## - if the number of sources (N, above) == 1 treat it
405		## as a filename and save the object there.
406		## - if there's more sources -> Error
407		## if --recursive:
408		## - first N arguments must be S3Uri
409		## - for each Uri get a list of remote objects with that Uri as a prefix
410		## - apply exclude/include rules
411		## - each list item will have MD5sum, Timestamp and pointer to S3Uri
412		## used as a prefix.
413		## - the last arg may be a local directory - destination_base
414		## - if the last one is S3 make current dir the destination_base
415		## - if the last one doesn't exist check remote list:
416		## - if there is only one item and its_prefix==its_name
417		## download that item to the name given in last arg.
418		## - if there are more remote items use the last arg as a destination_base
419		## and try to create the directory (incl. all parents).
420		##
421		## In both cases we end up with a list mapping remote object names (keys) to local file names.
422
423		## Each item will be a dict with the following attributes
424		# {'remote_uri', 'local_filename'}
425		download_list = []
426
427		if len(args) == 0:
428		raise ParameterError("Nothing to download. Expecting S3 URI.")
429
430		if S3Uri(args[-1]).type == 'file':
431		destination_base = args.pop()
432		else:
433		destination_base = "."
434
435		if len(args) == 0:
436		raise ParameterError("Nothing to download. Expecting S3 URI.")
437
438		remote_list = fetch_remote_list(args, require_attribs = False)
439		remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
440
441		remote_count = len(remote_list)
442
443		info(u"Summary: %d remote files to download" % remote_count)
444
445		if remote_count > 0:
446		if not os.path.isdir(destination_base) or destination_base == '-':
447		## We were either given a file name (existing or not) or want STDOUT
448		if remote_count > 1:
449		raise ParameterError("Destination must be a directory when downloading multiple sources.")
450		remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
451		elif os.path.isdir(destination_base):
452		if destination_base[-1] != os.path.sep:
453		destination_base += os.path.sep
454		for key in remote_list:
455		remote_list[key]['local_filename'] = destination_base + key
456		else:
457		raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base)
458
459		if cfg.dry_run:
460		for key in exclude_list:
461		output(u"exclude: %s" % unicodise(key))
462		for key in remote_list:
463		output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
464
465		warning(u"Exitting now because of --dry-run")
466		return
467
468		seq = 0
469		for key in remote_list:
470		seq += 1
471		item = remote_list[key]
472		uri = S3Uri(item['object_uri_str'])
473		## Encode / Decode destination with "replace" to make sure it's compatible with current encoding
474		destination = unicodise_safe(item['local_filename'])
475		seq_label = "[%d of %d]" % (seq, remote_count)
476
477		start_position = 0
478
479		if destination == "-":
480		## stdout
481		dst_stream = sys.__stdout__
482		else:
483		## File
484		try:
485		file_exists = os.path.exists(destination)
486		try:
487		dst_stream = open(destination, "ab")
488		except IOError, e:
489		if e.errno == errno.ENOENT:
490		basename = destination[:destination.rindex(os.path.sep)]
491		info(u"Creating directory: %s" % basename)
492		os.makedirs(basename)
493		dst_stream = open(destination, "ab")
494		else:
495		raise
496		if file_exists:
497		if Config().get_continue:
498		start_position = dst_stream.tell()
499		elif Config().force:
500		start_position = 0L
501		dst_stream.seek(0L)
502		dst_stream.truncate()
503		elif Config().skip_existing:
504		info(u"Skipping over existing file: %s" % (destination))
505		continue
506		else:
507		dst_stream.close()
508		raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination)
509		except IOError, e:
510		error(u"Skipping %s: %s" % (destination, e.strerror))
511		continue
512		response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label)
513		if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"):
514		gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])
515		response["size"] = os.stat(destination)[6]
516		if not Config().progress_meter and destination != "-":
517		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
518		output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %
519		(uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))
	318	cfg = Config()
	319	s3 = S3(cfg)
	320
	321	## Check arguments:
	322	## if not --recursive:
	323	## - first N arguments must be S3Uri
	324	## - if the last one is S3 make current dir the destination_base
	325	## - if the last one is a directory:
	326	## - take all 'basenames' of the remote objects and
	327	## make the destination name be 'destination_base'+'basename'
	328	## - if the last one is a file or not existing:
	329	## - if the number of sources (N, above) == 1 treat it
	330	## as a filename and save the object there.
	331	## - if there's more sources -> Error
	332	## if --recursive:
	333	## - first N arguments must be S3Uri
	334	## - for each Uri get a list of remote objects with that Uri as a prefix
	335	## - apply exclude/include rules
	336	## - each list item will have MD5sum, Timestamp and pointer to S3Uri
	337	## used as a prefix.
	338	## - the last arg may be '-' (stdout)
	339	## - the last arg may be a local directory - destination_base
	340	## - if the last one is S3 make current dir the destination_base
	341	## - if the last one doesn't exist check remote list:
	342	## - if there is only one item and its_prefix==its_name
	343	## download that item to the name given in last arg.
	344	## - if there are more remote items use the last arg as a destination_base
	345	## and try to create the directory (incl. all parents).
	346	##
	347	## In both cases we end up with a list mapping remote object names (keys) to local file names.
	348
	349	## Each item will be a dict with the following attributes
	350	# {'remote_uri', 'local_filename'}
	351	download_list = []
	352
	353	if len(args) == 0:
	354	raise ParameterError("Nothing to download. Expecting S3 URI.")
	355
	356	if S3Uri(args[-1]).type == 'file':
	357	destination_base = args.pop()
	358	else:
	359	destination_base = "."
	360
	361	if len(args) == 0:
	362	raise ParameterError("Nothing to download. Expecting S3 URI.")
	363
	364	remote_list = fetch_remote_list(args, require_attribs = False)
	365	remote_list, exclude_list = filter_exclude_include(remote_list)
	366
	367	remote_count = len(remote_list)
	368
	369	info(u"Summary: %d remote files to download" % remote_count)
	370
	371	if remote_count > 0:
	372	if destination_base == "-":
	373	## stdout is ok for multiple remote files!
	374	for key in remote_list:
	375	remote_list[key]['local_filename'] = "-"
	376	elif not os.path.isdir(destination_base):
	377	## We were either given a file name (existing or not)
	378	if remote_count > 1:
	379	raise ParameterError("Destination must be a directory or stdout when downloading multiple sources.")
	380	remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
	381	elif os.path.isdir(destination_base):
	382	if destination_base[-1] != os.path.sep:
	383	destination_base += os.path.sep
	384	for key in remote_list:
	385	remote_list[key]['local_filename'] = destination_base + key
	386	else:
	387	raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base)
	388
	389	if cfg.dry_run:
	390	for key in exclude_list:
	391	output(u"exclude: %s" % unicodise(key))
	392	for key in remote_list:
	393	output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
	394
	395	warning(u"Exitting now because of --dry-run")
	396	return
	397
	398	seq = 0
	399	for key in remote_list:
	400	seq += 1
	401	item = remote_list[key]
	402	uri = S3Uri(item['object_uri_str'])
	403	## Encode / Decode destination with "replace" to make sure it's compatible with current encoding
	404	destination = unicodise_safe(item['local_filename'])
	405	seq_label = "[%d of %d]" % (seq, remote_count)
	406
	407	start_position = 0
	408
	409	if destination == "-":
	410	## stdout
	411	dst_stream = sys.__stdout__
	412	else:
	413	## File
	414	try:
	415	file_exists = os.path.exists(destination)
	416	try:
	417	dst_stream = open(destination, "ab")
	418	except IOError, e:
	419	if e.errno == errno.ENOENT:
	420	basename = destination[:destination.rindex(os.path.sep)]
	421	info(u"Creating directory: %s" % basename)
	422	os.makedirs(basename)
	423	dst_stream = open(destination, "ab")
	424	else:
	425	raise
	426	if file_exists:
	427	if Config().get_continue:
	428	start_position = dst_stream.tell()
	429	elif Config().force:
	430	start_position = 0L
	431	dst_stream.seek(0L)
	432	dst_stream.truncate()
	433	elif Config().skip_existing:
	434	info(u"Skipping over existing file: %s" % (destination))
	435	continue
	436	else:
	437	dst_stream.close()
	438	raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination)
	439	except IOError, e:
	440	error(u"Skipping %s: %s" % (destination, e.strerror))
	441	continue
	442	response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label)
	443	if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"):
	444	gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])
	445	response["size"] = os.stat(destination)[6]
	446	if not Config().progress_meter and destination != "-":
	447	speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
	448	output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %
	449	(uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))
520	450
521	451	def cmd_object_del(args):
522		for uri_str in args:
523		uri = S3Uri(uri_str)
524		if uri.type != "s3":
525		raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str)
526		if not uri.has_object():
527		if Config().recursive and not Config().force:
528		raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str)
529		elif not Config().recursive:
530		raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")
531		subcmd_object_del_uri(uri_str)
	452	for uri_str in args:
	453	uri = S3Uri(uri_str)
	454	if uri.type != "s3":
	455	raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str)
	456	if not uri.has_object():
	457	if Config().recursive and not Config().force:
	458	raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str)
	459	elif not Config().recursive:
	460	raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")
	461	subcmd_object_del_uri(uri_str)
532	462
533	463	def subcmd_object_del_uri(uri_str, recursive = None):
534		s3 = S3(cfg)
535
536		if recursive is None:
537		recursive = cfg.recursive
538
539		remote_list = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive)
540		remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
541
542		remote_count = len(remote_list)
543
544		info(u"Summary: %d remote files to delete" % remote_count)
545
546		if cfg.dry_run:
547		for key in exclude_list:
548		output(u"exclude: %s" % unicodise(key))
549		for key in remote_list:
550		output(u"delete: %s" % remote_list[key]['object_uri_str'])
551
552		warning(u"Exitting now because of --dry-run")
553		return
554
555		for key in remote_list:
556		item = remote_list[key]
557		response = s3.object_delete(S3Uri(item['object_uri_str']))
558		output(u"File %s deleted" % item['object_uri_str'])
	464	s3 = S3(cfg)
	465
	466	if recursive is None:
	467	recursive = cfg.recursive
	468
	469	remote_list = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive)
	470	remote_list, exclude_list = filter_exclude_include(remote_list)
	471
	472	remote_count = len(remote_list)
	473
	474	info(u"Summary: %d remote files to delete" % remote_count)
	475
	476	if cfg.dry_run:
	477	for key in exclude_list:
	478	output(u"exclude: %s" % unicodise(key))
	479	for key in remote_list:
	480	output(u"delete: %s" % remote_list[key]['object_uri_str'])
	481
	482	warning(u"Exitting now because of --dry-run")
	483	return
	484
	485	for key in remote_list:
	486	item = remote_list[key]
	487	response = s3.object_delete(S3Uri(item['object_uri_str']))
	488	output(u"File %s deleted" % item['object_uri_str'])
559	489
560	490	def subcmd_cp_mv(args, process_fce, action_str, message):
561		if len(args) < 2:
562		raise ParameterError("Expecting two or more S3 URIs for " + action_str)
563		dst_base_uri = S3Uri(args.pop())
564		if dst_base_uri.type != "s3":
565		raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.")
566		destination_base = dst_base_uri.uri()
567
568		remote_list = fetch_remote_list(args, require_attribs = False)
569		remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
570
571		remote_count = len(remote_list)
572
573		info(u"Summary: %d remote files to %s" % (remote_count, action_str))
574
575		if cfg.recursive:
576		if not destination_base.endswith("/"):
577		destination_base += "/"
578		for key in remote_list:
579		remote_list[key]['dest_name'] = destination_base + key
580		else:
581		key = remote_list.keys()[0]
582		if destination_base.endswith("/"):
583		remote_list[key]['dest_name'] = destination_base + key
584		else:
585		remote_list[key]['dest_name'] = destination_base
586
587		if cfg.dry_run:
588		for key in exclude_list:
589		output(u"exclude: %s" % unicodise(key))
590		for key in remote_list:
591		output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name']))
592
593		warning(u"Exitting now because of --dry-run")
594		return
595
596		seq = 0
597		for key in remote_list:
598		seq += 1
599		seq_label = "[%d of %d]" % (seq, remote_count)
600
601		item = remote_list[key]
602		src_uri = S3Uri(item['object_uri_str'])
603		dst_uri = S3Uri(item['dest_name'])
604
605		extra_headers = copy(cfg.extra_headers)
606		response = process_fce(src_uri, dst_uri, extra_headers)
607		output(message % { "src" : src_uri, "dst" : dst_uri })
608		if Config().acl_public:
609		info(u"Public URL is: %s" % dst_uri.public_url())
	491	if len(args) < 2:
	492	raise ParameterError("Expecting two or more S3 URIs for " + action_str)
	493	dst_base_uri = S3Uri(args.pop())
	494	if dst_base_uri.type != "s3":
	495	raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.")
	496	destination_base = dst_base_uri.uri()
	497
	498	remote_list = fetch_remote_list(args, require_attribs = False)
	499	remote_list, exclude_list = filter_exclude_include(remote_list)
	500
	501	remote_count = len(remote_list)
	502
	503	info(u"Summary: %d remote files to %s" % (remote_count, action_str))
	504
	505	if cfg.recursive:
	506	if not destination_base.endswith("/"):
	507	destination_base += "/"
	508	for key in remote_list:
	509	remote_list[key]['dest_name'] = destination_base + key
	510	else:
	511	for key in remote_list:
	512	if destination_base.endswith("/"):
	513	remote_list[key]['dest_name'] = destination_base + key
	514	else:
	515	remote_list[key]['dest_name'] = destination_base
	516
	517	if cfg.dry_run:
	518	for key in exclude_list:
	519	output(u"exclude: %s" % unicodise(key))
	520	for key in remote_list:
	521	output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name']))
	522
	523	warning(u"Exitting now because of --dry-run")
	524	return
	525
	526	seq = 0
	527	for key in remote_list:
	528	seq += 1
	529	seq_label = "[%d of %d]" % (seq, remote_count)
	530
	531	item = remote_list[key]
	532	src_uri = S3Uri(item['object_uri_str'])
	533	dst_uri = S3Uri(item['dest_name'])
	534
	535	extra_headers = copy(cfg.extra_headers)
	536	response = process_fce(src_uri, dst_uri, extra_headers)
	537	output(message % { "src" : src_uri, "dst" : dst_uri })
	538	if Config().acl_public:
	539	info(u"Public URL is: %s" % dst_uri.public_url())
610	540
611	541	def cmd_cp(args):
612		s3 = S3(Config())
613		subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s")
	542	s3 = S3(Config())
	543	subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s")
614	544
615	545	def cmd_mv(args):
616		s3 = S3(Config())
617		subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s")
	546	s3 = S3(Config())
	547	subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s")
618	548
619	549	def cmd_info(args):
620		s3 = S3(Config())
621
622		while (len(args)):
623		uri_arg = args.pop(0)
624		uri = S3Uri(uri_arg)
625		if uri.type != "s3" or not uri.has_bucket():
626		raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)
627
628		try:
629		if uri.has_object():
630		info = s3.object_info(uri)
631		output(u"%s (object):" % uri.uri())
632		output(u" File size: %s" % info['headers']['content-length'])
633		output(u" Last mod: %s" % info['headers']['last-modified'])
634		output(u" MIME type: %s" % info['headers']['content-type'])
635		output(u" MD5 sum: %s" % info['headers']['etag'].strip('"'))
636		else:
637		info = s3.bucket_info(uri)
638		output(u"%s (bucket):" % uri.uri())
639		output(u" Location: %s" % info['bucket-location'])
640		acl = s3.get_acl(uri)
641		acl_grant_list = acl.getGrantList()
642		for grant in acl_grant_list:
643		output(u" ACL: %s: %s" % (grant['grantee'], grant['permission']))
644		if acl.isAnonRead():
645		output(u" URL: %s" % uri.public_url())
646		except S3Error, e:
647		if S3.codes.has_key(e.info["Code"]):
648		error(S3.codes[e.info["Code"]] % uri.bucket())
649		return
650		else:
651		raise
652
653		def _get_filelist_local(local_uri):
654		info(u"Compiling list of local files...")
655		if local_uri.isdir():
656		local_base = deunicodise(local_uri.basename())
657		local_path = deunicodise(local_uri.path())
658		filelist = fswalk(local_path, cfg.follow_symlinks)
659		single_file = False
660		else:
661		local_base = ""
662		local_path = deunicodise(local_uri.dirname())
663		filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]
664		single_file = True
665		loc_list = SortedDict(ignore_case = False)
666		for root, dirs, files in filelist:
667		rel_root = root.replace(local_path, local_base, 1)
668		for f in files:
669		full_name = os.path.join(root, f)
670		if not os.path.isfile(full_name):
671		continue
672		if os.path.islink(full_name):
673		if not cfg.follow_symlinks:
674		continue
675		relative_file = unicodise(os.path.join(rel_root, f))
676		if os.path.sep != "/":
677		# Convert non-unix dir separators to '/'
678		relative_file = "/".join(relative_file.split(os.path.sep))
679		if cfg.urlencoding_mode == "normal":
680		relative_file = replace_nonprintables(relative_file)
681		if relative_file.startswith('./'):
682		relative_file = relative_file[2:]
683		sr = os.stat_result(os.lstat(full_name))
684		loc_list[relative_file] = {
685		'full_name_unicode' : unicodise(full_name),
686		'full_name' : full_name,
687		'size' : sr.st_size,
688		'mtime' : sr.st_mtime,
689		## TODO: Possibly more to save here...
690		}
691		return loc_list, single_file
692
693		def _get_filelist_remote(remote_uri, recursive = True):
694		## If remote_uri ends with '/' then all remote files will have
695		## the remote_uri prefix removed in the relative path.
696		## If, on the other hand, the remote_uri ends with something else
697		## (probably alphanumeric symbol) we'll use the last path part
698		## in the relative path.
699		##
700		## Complicated, eh? See an example:
701		## _get_filelist_remote("s3://bckt/abc/def") may yield:
702		## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} }
703		## _get_filelist_remote("s3://bckt/abc/def/") will yield:
704		## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} }
705		## Furthermore a prefix-magic can restrict the return list:
706		## _get_filelist_remote("s3://bckt/abc/def/x") yields:
707		## { 'xyz/blah.txt' : {} }
708
709		info(u"Retrieving list of remote files for %s ..." % remote_uri)
710
711		s3 = S3(Config())
712		response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive)
713
714		rem_base_original = rem_base = remote_uri.object()
715		remote_uri_original = remote_uri
716		if rem_base != '' and rem_base[-1] != '/':
717		rem_base = rem_base[:rem_base.rfind('/')+1]
718		remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
719		rem_base_len = len(rem_base)
720		rem_list = SortedDict(ignore_case = False)
721		break_now = False
722		for object in response['list']:
723		if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep:
724		## We asked for one file and we got that file :-)
725		key = os.path.basename(object['Key'])
726		object_uri_str = remote_uri_original.uri()
727		break_now = True
728		rem_list = {} ## Remove whatever has already been put to rem_list
729		else:
730		key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !!
731		object_uri_str = remote_uri.uri() + key
732		rem_list[key] = {
733		'size' : int(object['Size']),
734		'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
735		'md5' : object['ETag'][1:-1],
736		'object_key' : object['Key'],
737		'object_uri_str' : object_uri_str,
738		'base_uri' : remote_uri,
739		}
740		if break_now:
741		break
742		return rem_list
743
744		def _filelist_filter_exclude_include(src_list):
745		info(u"Applying --exclude/--include")
746		cfg = Config()
747		exclude_list = SortedDict(ignore_case = False)
748		for file in src_list.keys():
749		debug(u"CHECK: %s" % file)
750		excluded = False
751		for r in cfg.exclude:
752		if r.search(file):
753		excluded = True
754		debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
755		break
756		if excluded:
757		## No need to check for --include if not excluded
758		for r in cfg.include:
759		if r.search(file):
760		excluded = False
761		debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
762		break
763		if excluded:
764		## Still excluded - ok, action it
765		debug(u"EXCLUDE: %s" % file)
766		exclude_list[file] = src_list[file]
767		del(src_list[file])
768		continue
769		else:
770		debug(u"PASS: %s" % (file))
771		return src_list, exclude_list
772
773		def _compare_filelists(src_list, dst_list, src_remote, dst_remote):
774		def __direction_str(is_remote):
775		return is_remote and "remote" or "local"
776
777		# We don't support local->local sync, use 'rsync' or something like that instead ;-)
778		assert(not(src_remote == False and dst_remote == False))
779
780		info(u"Verifying attributes...")
781		cfg = Config()
782		exists_list = SortedDict(ignore_case = False)
783
784		debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
785		debug("src_list.keys: %s" % src_list.keys())
786		debug("dst_list.keys: %s" % dst_list.keys())
787
788		for file in src_list.keys():
789		debug(u"CHECK: %s" % file)
790		if dst_list.has_key(file):
791		## Was --skip-existing requested?
792		if cfg.skip_existing:
793		debug(u"IGNR: %s (used --skip-existing)" % (file))
794		exists_list[file] = src_list[file]
795		del(src_list[file])
796		## Remove from destination-list, all that is left there will be deleted
797		del(dst_list[file])
798		continue
799
800		attribs_match = True
801		## Check size first
802		if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:
803		debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
804		attribs_match = False
805
806		if attribs_match and 'md5' in cfg.sync_checks:
807		## ... same size, check MD5
808		try:
809		if src_remote == False and dst_remote == True:
810		src_md5 = Utils.hash_file_md5(src_list[file]['full_name'])
811		dst_md5 = dst_list[file]['md5']
812		elif src_remote == True and dst_remote == False:
813		src_md5 = src_list[file]['md5']
814		dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name'])
815		elif src_remote == True and dst_remote == True:
816		src_md5 = src_list[file]['md5']
817		dst_md5 = dst_list[file]['md5']
818		except (IOError,OSError), e:
819		# MD5 sum verification failed - ignore that file altogether
820		debug(u"IGNR: %s (disappeared)" % (file))
821		warning(u"%s: file disappeared, ignoring." % (file))
822		del(src_list[file])
823		del(dst_list[file])
824		continue
825
826		if src_md5 != dst_md5:
827		## Checksums are different.
828		attribs_match = False
829		debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
830
831		if attribs_match:
832		## Remove from source-list, all that is left there will be transferred
833		debug(u"IGNR: %s (transfer not needed)" % file)
834		exists_list[file] = src_list[file]
835		del(src_list[file])
836
837		## Remove from destination-list, all that is left there will be deleted
838		del(dst_list[file])
839
840		return src_list, dst_list, exists_list
	550	s3 = S3(Config())
	551
	552	while (len(args)):
	553	uri_arg = args.pop(0)
	554	uri = S3Uri(uri_arg)
	555	if uri.type != "s3" or not uri.has_bucket():
	556	raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)
	557
	558	try:
	559	if uri.has_object():
	560	info = s3.object_info(uri)
	561	output(u"%s (object):" % uri.uri())
	562	output(u" File size: %s" % info['headers']['content-length'])
	563	output(u" Last mod: %s" % info['headers']['last-modified'])
	564	output(u" MIME type: %s" % info['headers']['content-type'])
	565	output(u" MD5 sum: %s" % info['headers']['etag'].strip('"'))
	566	else:
	567	info = s3.bucket_info(uri)
	568	output(u"%s (bucket):" % uri.uri())
	569	output(u" Location: %s" % info['bucket-location'])
	570	acl = s3.get_acl(uri)
	571	acl_grant_list = acl.getGrantList()
	572	for grant in acl_grant_list:
	573	output(u" ACL: %s: %s" % (grant['grantee'], grant['permission']))
	574	if acl.isAnonRead():
	575	output(u" URL: %s" % uri.public_url())
	576	except S3Error, e:
	577	if S3.codes.has_key(e.info["Code"]):
	578	error(S3.codes[e.info["Code"]] % uri.bucket())
	579	return
	580	else:
	581	raise
841	582
842	583	def cmd_sync_remote2remote(args):
843		s3 = S3(Config())
844
845		# Normalise s3://uri (e.g. assert trailing slash)
846		destination_base = unicode(S3Uri(args[-1]))
847
848		src_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
849		dst_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
850
851		src_count = len(src_list)
852		dst_count = len(dst_list)
853
854		info(u"Found %d source files, %d destination files" % (src_count, dst_count))
855
856		src_list, exclude_list = _filelist_filter_exclude_include(src_list)
857
858		src_list, dst_list, existing_list = _compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True)
859
860		src_count = len(src_list)
861		dst_count = len(dst_list)
862
863		print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count))
864
865		if src_count > 0:
866		### Populate 'remote_uri' only if we've got something to sync from src to dst
867		for key in src_list:
868		src_list[key]['target_uri'] = destination_base + key
869
870		if cfg.dry_run:
871		for key in exclude_list:
872		output(u"exclude: %s" % unicodise(key))
873		if cfg.delete_removed:
874		for key in dst_list:
875		output(u"delete: %s" % dst_list[key]['object_uri_str'])
876		for key in src_list:
877		output(u"Sync: %s -> %s" % (src_list[key]['object_uri_str'], src_list[key]['target_uri']))
878		warning(u"Exitting now because of --dry-run")
879		return
880
881		# Delete items in destination that are not in source
882		if cfg.delete_removed:
883		if cfg.dry_run:
884		for key in dst_list:
885		output(u"delete: %s" % dst_list[key]['object_uri_str'])
886		else:
887		for key in dst_list:
888		uri = S3Uri(dst_list[key]['object_uri_str'])
889		s3.object_delete(uri)
890		output(u"deleted: '%s'" % uri)
891
892		# Perform the synchronization of files
893		timestamp_start = time.time()
894		seq = 0
895		file_list = src_list.keys()
896		file_list.sort()
897		for file in file_list:
898		seq += 1
899		item = src_list[file]
900		src_uri = S3Uri(item['object_uri_str'])
901		dst_uri = S3Uri(item['target_uri'])
902		seq_label = "[%d of %d]" % (seq, src_count)
903		extra_headers = copy(cfg.extra_headers)
904		try:
905		response = s3.object_copy(src_uri, dst_uri, extra_headers)
906		output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri })
907		except S3Error, e:
908		error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e })
909		total_elapsed = time.time() - timestamp_start
910		outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
911		if seq > 0:
912		output(outstr)
913		else:
914		info(outstr)
	584	s3 = S3(Config())
	585
	586	# Normalise s3://uri (e.g. assert trailing slash)
	587	destination_base = unicode(S3Uri(args[-1]))
	588
	589	src_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
	590	dst_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
	591
	592	src_count = len(src_list)
	593	dst_count = len(dst_list)
	594
	595	info(u"Found %d source files, %d destination files" % (src_count, dst_count))
	596
	597	src_list, exclude_list = filter_exclude_include(src_list)
	598
	599	src_list, dst_list, existing_list = compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True)
	600
	601	src_count = len(src_list)
	602	dst_count = len(dst_list)
	603
	604	print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count))
	605
	606	if src_count > 0:
	607	### Populate 'remote_uri' only if we've got something to sync from src to dst
	608	for key in src_list:
	609	src_list[key]['target_uri'] = destination_base + key
	610
	611	if cfg.dry_run:
	612	for key in exclude_list:
	613	output(u"exclude: %s" % unicodise(key))
	614	if cfg.delete_removed:
	615	for key in dst_list:
	616	output(u"delete: %s" % dst_list[key]['object_uri_str'])
	617	for key in src_list:
	618	output(u"Sync: %s -> %s" % (src_list[key]['object_uri_str'], src_list[key]['target_uri']))
	619	warning(u"Exitting now because of --dry-run")
	620	return
	621
	622	# Delete items in destination that are not in source
	623	if cfg.delete_removed:
	624	if cfg.dry_run:
	625	for key in dst_list:
	626	output(u"delete: %s" % dst_list[key]['object_uri_str'])
	627	else:
	628	for key in dst_list:
	629	uri = S3Uri(dst_list[key]['object_uri_str'])
	630	s3.object_delete(uri)
	631	output(u"deleted: '%s'" % uri)
	632
	633	# Perform the synchronization of files
	634	timestamp_start = time.time()
	635	seq = 0
	636	file_list = src_list.keys()
	637	file_list.sort()
	638	for file in file_list:
	639	seq += 1
	640	item = src_list[file]
	641	src_uri = S3Uri(item['object_uri_str'])
	642	dst_uri = S3Uri(item['target_uri'])
	643	seq_label = "[%d of %d]" % (seq, src_count)
	644	extra_headers = copy(cfg.extra_headers)
	645	try:
	646	response = s3.object_copy(src_uri, dst_uri, extra_headers)
	647	output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri })
	648	except S3Error, e:
	649	error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e })
	650	total_elapsed = time.time() - timestamp_start
	651	outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
	652	if seq > 0:
	653	output(outstr)
	654	else:
	655	info(outstr)
915	656
916	657	def cmd_sync_remote2local(args):
917		def _parse_attrs_header(attrs_header):
918		attrs = {}
919		for attr in attrs_header.split("/"):
920		key, val = attr.split(":")
921		attrs[key] = val
922		return attrs
923
924		s3 = S3(Config())
925
926		destination_base = args[-1]
927		local_list, single_file_local = fetch_local_list(destination_base, recursive = True)
928		remote_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
929
930		local_count = len(local_list)
931		remote_count = len(remote_list)
932
933		info(u"Found %d remote files, %d local files" % (remote_count, local_count))
934
935		remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
936
937		remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False)
938
939		local_count = len(local_list)
940		remote_count = len(remote_list)
941
942		info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count))
943
944		if not os.path.isdir(destination_base):
945		## We were either given a file name (existing or not) or want STDOUT
946		if remote_count > 1:
947		raise ParameterError("Destination must be a directory when downloading multiple sources.")
948		remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
949		else:
950		if destination_base[-1] != os.path.sep:
951		destination_base += os.path.sep
952		for key in remote_list:
953		local_filename = destination_base + key
954		if os.path.sep != "/":
955		local_filename = os.path.sep.join(local_filename.split("/"))
956		remote_list[key]['local_filename'] = deunicodise(local_filename)
957
958		if cfg.dry_run:
959		for key in exclude_list:
960		output(u"exclude: %s" % unicodise(key))
961		if cfg.delete_removed:
962		for key in local_list:
963		output(u"delete: %s" % local_list[key]['full_name_unicode'])
964		for key in remote_list:
965		output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
966
967		warning(u"Exitting now because of --dry-run")
968		return
969
970		if cfg.delete_removed:
971		for key in local_list:
972		os.unlink(local_list[key]['full_name'])
973		output(u"deleted: %s" % local_list[key]['full_name_unicode'])
974
975		total_size = 0
976		total_elapsed = 0.0
977		timestamp_start = time.time()
978		seq = 0
979		dir_cache = {}
980		file_list = remote_list.keys()
981		file_list.sort()
982		for file in file_list:
983		seq += 1
984		item = remote_list[file]
985		uri = S3Uri(item['object_uri_str'])
986		dst_file = item['local_filename']
987		seq_label = "[%d of %d]" % (seq, remote_count)
988		try:
989		dst_dir = os.path.dirname(dst_file)
990		if not dir_cache.has_key(dst_dir):
991		dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir)
992		if dir_cache[dst_dir] == False:
993		warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
994		continue
995		try:
996		open_flags = os.O_CREAT
997		open_flags \|= os.O_TRUNC
998		# open_flags \|= os.O_EXCL
999
1000		debug(u"dst_file=%s" % unicodise(dst_file))
1001		# This will have failed should the file exist
1002		os.close(os.open(dst_file, open_flags))
1003		# Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode.
1004		dst_stream = open(dst_file, "wb")
1005		response = s3.object_get(uri, dst_stream, extra_label = seq_label)
1006		dst_stream.close()
1007		if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
1008		attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
1009		if attrs.has_key('mode'):
1010		os.chmod(dst_file, int(attrs['mode']))
1011		if attrs.has_key('mtime') or attrs.has_key('atime'):
1012		mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())
1013		atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())
1014		os.utime(dst_file, (atime, mtime))
1015		## FIXME: uid/gid / uname/gname handling comes here! TODO
1016		except OSError, e:
1017		try: dst_stream.close()
1018		except: pass
1019		if e.errno == errno.EEXIST:
1020		warning(u"%s exists - not overwriting" % (dst_file))
1021		continue
1022		if e.errno in (errno.EPERM, errno.EACCES):
1023		warning(u"%s not writable: %s" % (dst_file, e.strerror))
1024		continue
1025		if e.errno == errno.EISDIR:
1026		warning(u"%s is a directory - skipping over" % dst_file)
1027		continue
1028		raise e
1029		except KeyboardInterrupt:
1030		try: dst_stream.close()
1031		except: pass
1032		warning(u"Exiting after keyboard interrupt")
1033		return
1034		except Exception, e:
1035		try: dst_stream.close()
1036		except: pass
1037		error(u"%s: %s" % (file, e))
1038		continue
1039		# We have to keep repeating this call because
1040		# Python 2.4 doesn't support try/except/finally
1041		# construction :-(
1042		try: dst_stream.close()
1043		except: pass
1044		except S3DownloadError, e:
1045		error(u"%s: download failed too many times. Skipping that file." % file)
1046		continue
1047		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
1048		if not Config().progress_meter:
1049		output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
1050		(uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],
1051		seq_label))
1052		total_size += response["size"]
1053
1054		total_elapsed = time.time() - timestamp_start
1055		speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True)
1056
1057		# Only print out the result if any work has been done or
1058		# if the user asked for verbose output
1059		outstr = "Done. Downloaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
1060		if total_size > 0:
1061		output(outstr)
1062		else:
1063		info(outstr)
	658	def _parse_attrs_header(attrs_header):
	659	attrs = {}
	660	for attr in attrs_header.split("/"):
	661	key, val = attr.split(":")
	662	attrs[key] = val
	663	return attrs
	664
	665	s3 = S3(Config())
	666
	667	destination_base = args[-1]
	668	local_list, single_file_local = fetch_local_list(destination_base, recursive = True)
	669	remote_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
	670
	671	local_count = len(local_list)
	672	remote_count = len(remote_list)
	673
	674	info(u"Found %d remote files, %d local files" % (remote_count, local_count))
	675
	676	remote_list, exclude_list = filter_exclude_include(remote_list)
	677
	678	remote_list, local_list, existing_list = compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False)
	679
	680	local_count = len(local_list)
	681	remote_count = len(remote_list)
	682
	683	info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count))
	684
	685	if not os.path.isdir(destination_base):
	686	## We were either given a file name (existing or not) or want STDOUT
	687	if remote_count > 1:
	688	raise ParameterError("Destination must be a directory when downloading multiple sources.")
	689	remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
	690	else:
	691	if destination_base[-1] != os.path.sep:
	692	destination_base += os.path.sep
	693	for key in remote_list:
	694	local_filename = destination_base + key
	695	if os.path.sep != "/":
	696	local_filename = os.path.sep.join(local_filename.split("/"))
	697	remote_list[key]['local_filename'] = deunicodise(local_filename)
	698
	699	if cfg.dry_run:
	700	for key in exclude_list:
	701	output(u"exclude: %s" % unicodise(key))
	702	if cfg.delete_removed:
	703	for key in local_list:
	704	output(u"delete: %s" % local_list[key]['full_name_unicode'])
	705	for key in remote_list:
	706	output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
	707
	708	warning(u"Exitting now because of --dry-run")
	709	return
	710
	711	if cfg.delete_removed:
	712	for key in local_list:
	713	os.unlink(local_list[key]['full_name'])
	714	output(u"deleted: %s" % local_list[key]['full_name_unicode'])
	715
	716	total_size = 0
	717	total_elapsed = 0.0
	718	timestamp_start = time.time()
	719	seq = 0
	720	dir_cache = {}
	721	file_list = remote_list.keys()
	722	file_list.sort()
	723	for file in file_list:
	724	seq += 1
	725	item = remote_list[file]
	726	uri = S3Uri(item['object_uri_str'])
	727	dst_file = item['local_filename']
	728	seq_label = "[%d of %d]" % (seq, remote_count)
	729	try:
	730	dst_dir = os.path.dirname(dst_file)
	731	if not dir_cache.has_key(dst_dir):
	732	dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir)
	733	if dir_cache[dst_dir] == False:
	734	warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
	735	continue
	736	try:
	737	open_flags = os.O_CREAT
	738	open_flags \|= os.O_TRUNC
	739	# open_flags \|= os.O_EXCL
	740
	741	debug(u"dst_file=%s" % unicodise(dst_file))
	742	# This will have failed should the file exist
	743	os.close(os.open(dst_file, open_flags))
	744	# Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode.
	745	dst_stream = open(dst_file, "wb")
	746	response = s3.object_get(uri, dst_stream, extra_label = seq_label)
	747	dst_stream.close()
	748	if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
	749	attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
	750	if attrs.has_key('mode'):
	751	os.chmod(dst_file, int(attrs['mode']))
	752	if attrs.has_key('mtime') or attrs.has_key('atime'):
	753	mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())
	754	atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())
	755	os.utime(dst_file, (atime, mtime))
	756	## FIXME: uid/gid / uname/gname handling comes here! TODO
	757	except OSError, e:
	758	try: dst_stream.close()
	759	except: pass
	760	if e.errno == errno.EEXIST:
	761	warning(u"%s exists - not overwriting" % (dst_file))
	762	continue
	763	if e.errno in (errno.EPERM, errno.EACCES):
	764	warning(u"%s not writable: %s" % (dst_file, e.strerror))
	765	continue
	766	if e.errno == errno.EISDIR:
	767	warning(u"%s is a directory - skipping over" % dst_file)
	768	continue
	769	raise e
	770	except KeyboardInterrupt:
	771	try: dst_stream.close()
	772	except: pass
	773	warning(u"Exiting after keyboard interrupt")
	774	return
	775	except Exception, e:
	776	try: dst_stream.close()
	777	except: pass
	778	error(u"%s: %s" % (file, e))
	779	continue
	780	# We have to keep repeating this call because
	781	# Python 2.4 doesn't support try/except/finally
	782	# construction :-(
	783	try: dst_stream.close()
	784	except: pass
	785	except S3DownloadError, e:
	786	error(u"%s: download failed too many times. Skipping that file." % file)
	787	continue
	788	speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
	789	if not Config().progress_meter:
	790	output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
	791	(uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],
	792	seq_label))
	793	total_size += response["size"]
	794
	795	total_elapsed = time.time() - timestamp_start
	796	speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True)
	797
	798	# Only print out the result if any work has been done or
	799	# if the user asked for verbose output
	800	outstr = "Done. Downloaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
	801	if total_size > 0:
	802	output(outstr)
	803	else:
	804	info(outstr)
1064	805
1065	806	def cmd_sync_local2remote(args):
1066		def _build_attr_header(src):
1067		import pwd, grp
1068		attrs = {}
1069		src = deunicodise(src)
1070		try:
1071		st = os.stat_result(os.stat(src))
1072		except OSError, e:
1073		raise InvalidFileError(u"%s: %s" % (unicodise(src), e.strerror))
1074		for attr in cfg.preserve_attrs_list:
1075		if attr == 'uname':
1076		try:
1077		val = pwd.getpwuid(st.st_uid).pw_name
1078		except KeyError:
1079		attr = "uid"
1080		val = st.st_uid
1081		warning(u"%s: Owner username not known. Storing UID=%d instead." % (unicodise(src), val))
1082		elif attr == 'gname':
1083		try:
1084		val = grp.getgrgid(st.st_gid).gr_name
1085		except KeyError:
1086		attr = "gid"
1087		val = st.st_gid
1088		warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (unicodise(src), val))
1089		else:
1090		val = getattr(st, 'st_' + attr)
1091		attrs[attr] = val
1092		result = ""
1093		for k in attrs: result += "%s:%s/" % (k, attrs[k])
1094		return { 'x-amz-meta-s3cmd-attrs' : result[:-1] }
1095
1096		s3 = S3(cfg)
1097
1098		if cfg.encrypt:
1099		error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
1100		error(u"Either use unconditional 's3cmd put --recursive'")
1101		error(u"or disable encryption with --no-encrypt parameter.")
1102		sys.exit(1)
1103
1104		## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
1105		destination_base_uri = S3Uri(args[-1])
1106		if destination_base_uri.type != 's3':
1107		raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
1108		destination_base = str(destination_base_uri)
1109
1110		local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
1111		remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
1112
1113		local_count = len(local_list)
1114		remote_count = len(remote_list)
1115
1116		info(u"Found %d local files, %d remote files" % (local_count, remote_count))
1117
1118		local_list, exclude_list = _filelist_filter_exclude_include(local_list)
1119
1120		if single_file_local and len(local_list) == 1 and len(remote_list) == 1:
1121		## Make remote_key same as local_key for comparison if we're dealing with only one file
1122		remote_list_entry = remote_list[remote_list.keys()[0]]
1123		# Flush remote_list, by the way
1124		remote_list = { local_list.keys()[0] : remote_list_entry }
1125
1126		local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True)
1127
1128		local_count = len(local_list)
1129		remote_count = len(remote_list)
1130
1131		info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count))
1132
1133		if local_count > 0:
1134		## Populate 'remote_uri' only if we've got something to upload
1135		if not destination_base.endswith("/"):
1136		if not single_file_local:
1137		raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
1138		local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
1139		else:
1140		for key in local_list:
1141		local_list[key]['remote_uri'] = unicodise(destination_base + key)
1142
1143		if cfg.dry_run:
1144		for key in exclude_list:
1145		output(u"exclude: %s" % unicodise(key))
1146		if cfg.delete_removed:
1147		for key in remote_list:
1148		output(u"delete: %s" % remote_list[key]['object_uri_str'])
1149		for key in local_list:
1150		output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
1151
1152		warning(u"Exitting now because of --dry-run")
1153		return
1154
1155		if cfg.delete_removed:
1156		for key in remote_list:
1157		uri = S3Uri(remote_list[key]['object_uri_str'])
1158		s3.object_delete(uri)
1159		output(u"deleted: '%s'" % uri)
1160
1161		total_size = 0
1162		total_elapsed = 0.0
1163		timestamp_start = time.time()
1164		seq = 0
1165		file_list = local_list.keys()
1166		file_list.sort()
1167		for file in file_list:
1168		seq += 1
1169		item = local_list[file]
1170		src = item['full_name']
1171		uri = S3Uri(item['remote_uri'])
1172		seq_label = "[%d of %d]" % (seq, local_count)
1173		extra_headers = copy(cfg.extra_headers)
1174		try:
1175		if cfg.preserve_attrs:
1176		attr_header = _build_attr_header(src)
1177		debug(u"attr_header: %s" % attr_header)
1178		extra_headers.update(attr_header)
1179		response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
1180		except InvalidFileError, e:
1181		warning(u"File can not be uploaded: %s" % e)
1182		continue
1183		except S3UploadError, e:
1184		error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
1185		continue
1186		speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
1187		if not cfg.progress_meter:
1188		output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
1189		(item['full_name_unicode'], uri, response["size"], response["elapsed"],
1190		speed_fmt[0], speed_fmt[1], seq_label))
1191		total_size += response["size"]
1192
1193		total_elapsed = time.time() - timestamp_start
1194		total_speed = total_elapsed and total_size/total_elapsed or 0.0
1195		speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)
1196
1197		# Only print out the result if any work has been done or
1198		# if the user asked for verbose output
1199		outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
1200		if total_size > 0:
1201		output(outstr)
1202		else:
1203		info(outstr)
	807	def _build_attr_header(src):
	808	import pwd, grp
	809	attrs = {}
	810	src = deunicodise(src)
	811	try:
	812	st = os.stat_result(os.stat(src))
	813	except OSError, e:
	814	raise InvalidFileError(u"%s: %s" % (unicodise(src), e.strerror))
	815	for attr in cfg.preserve_attrs_list:
	816	if attr == 'uname':
	817	try:
	818	val = pwd.getpwuid(st.st_uid).pw_name
	819	except KeyError:
	820	attr = "uid"
	821	val = st.st_uid
	822	warning(u"%s: Owner username not known. Storing UID=%d instead." % (unicodise(src), val))
	823	elif attr == 'gname':
	824	try:
	825	val = grp.getgrgid(st.st_gid).gr_name
	826	except KeyError:
	827	attr = "gid"
	828	val = st.st_gid
	829	warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (unicodise(src), val))
	830	else:
	831	val = getattr(st, 'st_' + attr)
	832	attrs[attr] = val
	833	result = ""
	834	for k in attrs: result += "%s:%s/" % (k, attrs[k])
	835	return { 'x-amz-meta-s3cmd-attrs' : result[:-1] }
	836
	837	s3 = S3(cfg)
	838
	839	if cfg.encrypt:
	840	error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
	841	error(u"Either use unconditional 's3cmd put --recursive'")
	842	error(u"or disable encryption with --no-encrypt parameter.")
	843	sys.exit(1)
	844
	845	## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
	846	destination_base_uri = S3Uri(args[-1])
	847	if destination_base_uri.type != 's3':
	848	raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
	849	destination_base = str(destination_base_uri)
	850
	851	local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
	852	remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
	853
	854	local_count = len(local_list)
	855	remote_count = len(remote_list)
	856
	857	info(u"Found %d local files, %d remote files" % (local_count, remote_count))
	858
	859	local_list, exclude_list = filter_exclude_include(local_list)
	860
	861	if single_file_local and len(local_list) == 1 and len(remote_list) == 1:
	862	## Make remote_key same as local_key for comparison if we're dealing with only one file
	863	remote_list_entry = remote_list[remote_list.keys()[0]]
	864	# Flush remote_list, by the way
	865	remote_list = { local_list.keys()[0] : remote_list_entry }
	866
	867	local_list, remote_list, existing_list = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True)
	868
	869	local_count = len(local_list)
	870	remote_count = len(remote_list)
	871
	872	info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count))
	873
	874	if local_count > 0:
	875	## Populate 'remote_uri' only if we've got something to upload
	876	if not destination_base.endswith("/"):
	877	if not single_file_local:
	878	raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
	879	local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
	880	else:
	881	for key in local_list:
	882	local_list[key]['remote_uri'] = unicodise(destination_base + key)
	883
	884	if cfg.dry_run:
	885	for key in exclude_list:
	886	output(u"exclude: %s" % unicodise(key))
	887	if cfg.delete_removed:
	888	for key in remote_list:
	889	output(u"delete: %s" % remote_list[key]['object_uri_str'])
	890	for key in local_list:
	891	output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
	892
	893	warning(u"Exitting now because of --dry-run")
	894	return
	895
	896	if cfg.delete_removed:
	897	for key in remote_list:
	898	uri = S3Uri(remote_list[key]['object_uri_str'])
	899	s3.object_delete(uri)
	900	output(u"deleted: '%s'" % uri)
	901
	902	uploaded_objects_list = []
	903	total_size = 0
	904	total_elapsed = 0.0
	905	timestamp_start = time.time()
	906	seq = 0
	907	file_list = local_list.keys()
	908	file_list.sort()
	909	for file in file_list:
	910	seq += 1
	911	item = local_list[file]
	912	src = item['full_name']
	913	uri = S3Uri(item['remote_uri'])
	914	seq_label = "[%d of %d]" % (seq, local_count)
	915	extra_headers = copy(cfg.extra_headers)
	916	try:
	917	if cfg.preserve_attrs:
	918	attr_header = _build_attr_header(src)
	919	debug(u"attr_header: %s" % attr_header)
	920	extra_headers.update(attr_header)
	921	response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
	922	except InvalidFileError, e:
	923	warning(u"File can not be uploaded: %s" % e)
	924	continue
	925	except S3UploadError, e:
	926	error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
	927	continue
	928	speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
	929	if not cfg.progress_meter:
	930	output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
	931	(item['full_name_unicode'], uri, response["size"], response["elapsed"],
	932	speed_fmt[0], speed_fmt[1], seq_label))
	933	total_size += response["size"]
	934	uploaded_objects_list.append(uri.object())
	935
	936	total_elapsed = time.time() - timestamp_start
	937	total_speed = total_elapsed and total_size/total_elapsed or 0.0
	938	speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)
	939
	940	# Only print out the result if any work has been done or
	941	# if the user asked for verbose output
	942	outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
	943	if total_size > 0:
	944	output(outstr)
	945	else:
	946	info(outstr)
	947
	948	if cfg.invalidate_on_cf:
	949	if len(uploaded_objects_list) == 0:
	950	info("Nothing to invalidate in CloudFront")
	951	else:
	952	# 'uri' from the last iteration is still valid at this point
	953	cf = CloudFront(cfg)
	954	result = cf.InvalidateObjects(uri, uploaded_objects_list)
	955	if result['status'] == 201:
	956	output("Created invalidation request for %d paths" % len(uploaded_objects_list))
	957	output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
1204	958
1205	959	def cmd_sync(args):
1206		if (len(args) < 2):
1207		raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param'])
1208
1209		if S3Uri(args[0]).type == "file" and S3Uri(args[-1]).type == "s3":
1210		return cmd_sync_local2remote(args)
1211		if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file":
1212		return cmd_sync_remote2local(args)
1213		if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3":
1214		return cmd_sync_remote2remote(args)
1215		raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args))
	960	if (len(args) < 2):
	961	raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param'])
	962
	963	if S3Uri(args[0]).type == "file" and S3Uri(args[-1]).type == "s3":
	964	return cmd_sync_local2remote(args)
	965	if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file":
	966	return cmd_sync_remote2local(args)
	967	if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3":
	968	return cmd_sync_remote2remote(args)
	969	raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args))
1216	970
1217	971	def cmd_setacl(args):
1218		def _update_acl(uri, seq_label = ""):
1219		something_changed = False
1220		acl = s3.get_acl(uri)
1221		debug(u"acl: %s - %r" % (uri, acl.grantees))
1222		if cfg.acl_public == True:
1223		if acl.isAnonRead():
1224		info(u"%s: already Public, skipping %s" % (uri, seq_label))
1225		else:
1226		acl.grantAnonRead()
1227		something_changed = True
1228		elif cfg.acl_public == False: # we explicitely check for False, because it could be None
1229		if not acl.isAnonRead():
1230		info(u"%s: already Private, skipping %s" % (uri, seq_label))
1231		else:
1232		acl.revokeAnonRead()
1233		something_changed = True
1234
1235		# update acl with arguments
1236		# grant first and revoke later, because revoke has priority
1237		if cfg.acl_grants:
1238		something_changed = True
1239		for grant in cfg.acl_grants:
1240		acl.grant(**grant);
1241
1242		if cfg.acl_revokes:
1243		something_changed = True
1244		for revoke in cfg.acl_revokes:
1245		acl.revoke(**revoke);
1246
1247		if not something_changed:
1248		return
1249
1250		retsponse = s3.set_acl(uri, acl)
1251		if retsponse['status'] == 200:
1252		if cfg.acl_public in (True, False):
1253		output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label))
1254		else:
1255		output(u"%s: ACL updated" % uri)
1256
1257		s3 = S3(cfg)
1258
1259		set_to_acl = cfg.acl_public and "Public" or "Private"
1260
1261		if not cfg.recursive:
1262		old_args = args
1263		args = []
1264		for arg in old_args:
1265		uri = S3Uri(arg)
1266		if not uri.has_object():
1267		if cfg.acl_public != None:
1268		info("Setting bucket-level ACL for %s to %s" % (uri.uri(), set_to_acl))
1269		else:
1270		info("Setting bucket-level ACL for %s" % (uri.uri()))
1271		if not cfg.dry_run:
1272		_update_acl(uri)
1273		else:
1274		args.append(arg)
1275
1276		remote_list = fetch_remote_list(args)
1277		remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
1278
1279		remote_count = len(remote_list)
1280
1281		info(u"Summary: %d remote files to update" % remote_count)
1282
1283		if cfg.dry_run:
1284		for key in exclude_list:
1285		output(u"exclude: %s" % unicodise(key))
1286		for key in remote_list:
1287		output(u"setacl: %s" % remote_list[key]['object_uri_str'])
1288
1289		warning(u"Exitting now because of --dry-run")
1290		return
1291
1292		seq = 0
1293		for key in remote_list:
1294		seq += 1
1295		seq_label = "[%d of %d]" % (seq, remote_count)
1296		uri = S3Uri(remote_list[key]['object_uri_str'])
1297		_update_acl(uri, seq_label)
	972	def _update_acl(uri, seq_label = ""):
	973	something_changed = False
	974	acl = s3.get_acl(uri)
	975	debug(u"acl: %s - %r" % (uri, acl.grantees))
	976	if cfg.acl_public == True:
	977	if acl.isAnonRead():
	978	info(u"%s: already Public, skipping %s" % (uri, seq_label))
	979	else:
	980	acl.grantAnonRead()
	981	something_changed = True
	982	elif cfg.acl_public == False: # we explicitely check for False, because it could be None
	983	if not acl.isAnonRead():
	984	info(u"%s: already Private, skipping %s" % (uri, seq_label))
	985	else:
	986	acl.revokeAnonRead()
	987	something_changed = True
	988
	989	# update acl with arguments
	990	# grant first and revoke later, because revoke has priority
	991	if cfg.acl_grants:
	992	something_changed = True
	993	for grant in cfg.acl_grants:
	994	acl.grant(**grant);
	995
	996	if cfg.acl_revokes:
	997	something_changed = True
	998	for revoke in cfg.acl_revokes:
	999	acl.revoke(**revoke);
	1000
	1001	if not something_changed:
	1002	return
	1003
	1004	retsponse = s3.set_acl(uri, acl)
	1005	if retsponse['status'] == 200:
	1006	if cfg.acl_public in (True, False):
	1007	output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label))
	1008	else:
	1009	output(u"%s: ACL updated" % uri)
	1010
	1011	s3 = S3(cfg)
	1012
	1013	set_to_acl = cfg.acl_public and "Public" or "Private"
	1014
	1015	if not cfg.recursive:
	1016	old_args = args
	1017	args = []
	1018	for arg in old_args:
	1019	uri = S3Uri(arg)
	1020	if not uri.has_object():
	1021	if cfg.acl_public != None:
	1022	info("Setting bucket-level ACL for %s to %s" % (uri.uri(), set_to_acl))
	1023	else:
	1024	info("Setting bucket-level ACL for %s" % (uri.uri()))
	1025	if not cfg.dry_run:
	1026	_update_acl(uri)
	1027	else:
	1028	args.append(arg)
	1029
	1030	remote_list = fetch_remote_list(args)
	1031	remote_list, exclude_list = filter_exclude_include(remote_list)
	1032
	1033	remote_count = len(remote_list)
	1034
	1035	info(u"Summary: %d remote files to update" % remote_count)
	1036
	1037	if cfg.dry_run:
	1038	for key in exclude_list:
	1039	output(u"exclude: %s" % unicodise(key))
	1040	for key in remote_list:
	1041	output(u"setacl: %s" % remote_list[key]['object_uri_str'])
	1042
	1043	warning(u"Exitting now because of --dry-run")
	1044	return
	1045
	1046	seq = 0
	1047	for key in remote_list:
	1048	seq += 1
	1049	seq_label = "[%d of %d]" % (seq, remote_count)
	1050	uri = S3Uri(remote_list[key]['object_uri_str'])
	1051	_update_acl(uri, seq_label)
1298	1052
1299	1053	def cmd_accesslog(args):
1300		s3 = S3(cfg)
1301		bucket_uri = S3Uri(args.pop())
1302		if bucket_uri.object():
1303		raise ParameterError("Only bucket name is required for [accesslog] command")
1304		if cfg.log_target_prefix == False:
1305		accesslog, response = s3.set_accesslog(bucket_uri, enable = False)
1306		elif cfg.log_target_prefix:
1307		log_target_prefix_uri = S3Uri(cfg.log_target_prefix)
1308		if log_target_prefix_uri.type != "s3":
1309		raise ParameterError("--log-target-prefix must be a S3 URI")
1310		accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public)
1311		else: # cfg.log_target_prefix == None
1312		accesslog = s3.get_accesslog(bucket_uri)
1313
1314		output(u"Access logging for: %s" % bucket_uri.uri())
1315		output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled())
1316		if accesslog.isLoggingEnabled():
1317		output(u" Target prefix: %s" % accesslog.targetPrefix().uri())
1318		#output(u" Public Access: %s" % accesslog.isAclPublic())
1319
	1054	s3 = S3(cfg)
	1055	bucket_uri = S3Uri(args.pop())
	1056	if bucket_uri.object():
	1057	raise ParameterError("Only bucket name is required for [accesslog] command")
	1058	if cfg.log_target_prefix == False:
	1059	accesslog, response = s3.set_accesslog(bucket_uri, enable = False)
	1060	elif cfg.log_target_prefix:
	1061	log_target_prefix_uri = S3Uri(cfg.log_target_prefix)
	1062	if log_target_prefix_uri.type != "s3":
	1063	raise ParameterError("--log-target-prefix must be a S3 URI")
	1064	accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public)
	1065	else: # cfg.log_target_prefix == None
	1066	accesslog = s3.get_accesslog(bucket_uri)
	1067
	1068	output(u"Access logging for: %s" % bucket_uri.uri())
	1069	output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled())
	1070	if accesslog.isLoggingEnabled():
	1071	output(u" Target prefix: %s" % accesslog.targetPrefix().uri())
	1072	#output(u" Public Access: %s" % accesslog.isAclPublic())
	1073
1320	1074	def cmd_sign(args):
1321		string_to_sign = args.pop()
1322		debug("string-to-sign: %r" % string_to_sign)
1323		signature = Utils.sign_string(string_to_sign)
1324		output("Signature: %s" % signature)
	1075	string_to_sign = args.pop()
	1076	debug("string-to-sign: %r" % string_to_sign)
	1077	signature = Utils.sign_string(string_to_sign)
	1078	output("Signature: %s" % signature)
1325	1079
1326	1080	def cmd_fixbucket(args):
1327		def _unescape(text):
1328		##
1329		# Removes HTML or XML character references and entities from a text string.
1330		#
1331		# @param text The HTML (or XML) source text.
1332		# @return The plain text, as a Unicode string, if necessary.
1333		#
1334		# From: http://effbot.org/zone/re-sub.htm#unescape-html
1335		def _unescape_fixup(m):
1336		text = m.group(0)
1337		if not htmlentitydefs.name2codepoint.has_key('apos'):
1338		htmlentitydefs.name2codepoint['apos'] = ord("'")
1339		if text[:2] == "&#":
1340		# character reference
1341		try:
1342		if text[:3] == "&#x":
1343		return unichr(int(text[3:-1], 16))
1344		else:
1345		return unichr(int(text[2:-1]))
1346		except ValueError:
1347		pass
1348		else:
1349		# named entity
1350		try:
1351		text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
1352		except KeyError:
1353		pass
1354		return text # leave as is
1355		text = text.encode('ascii', 'xmlcharrefreplace')
1356		return re.sub("&#?\w+;", _unescape_fixup, text)
1357
1358		cfg.urlencoding_mode = "fixbucket"
1359		s3 = S3(cfg)
1360
1361		count = 0
1362		for arg in args:
1363		culprit = S3Uri(arg)
1364		if culprit.type != "s3":
1365		raise ParameterError("Expecting S3Uri instead of: %s" % arg)
1366		response = s3.bucket_list_noparse(culprit.bucket(), culprit.object(), recursive = True)
1367		r_xent = re.compile("&#x[\da-fA-F]+;")
1368		response['data'] = unicode(response['data'], 'UTF-8')
1369		keys = re.findall("<Key>(.*?)</Key>", response['data'], re.MULTILINE)
1370		debug("Keys: %r" % keys)
1371		for key in keys:
1372		if r_xent.search(key):
1373		info("Fixing: %s" % key)
1374		debug("Step 1: Transforming %s" % key)
1375		key_bin = _unescape(key)
1376		debug("Step 2: ... to %s" % key_bin)
1377		key_new = replace_nonprintables(key_bin)
1378		debug("Step 3: ... then to %s" % key_new)
1379		src = S3Uri("s3://%s/%s" % (culprit.bucket(), key_bin))
1380		dst = S3Uri("s3://%s/%s" % (culprit.bucket(), key_new))
1381		resp_move = s3.object_move(src, dst)
1382		if resp_move['status'] == 200:
1383		output("File %r renamed to %s" % (key_bin, key_new))
1384		count += 1
1385		else:
1386		error("Something went wrong for: %r" % key)
1387		error("Please report the problem to s3tools-bugs@lists.sourceforge.net")
1388		if count > 0:
1389		warning("Fixed %d files' names. Their ACL were reset to Private." % count)
1390		warning("Use 's3cmd setacl --acl-public s3://...' to make")
1391		warning("them publicly readable if required.")
	1081	def _unescape(text):
	1082	##
	1083	# Removes HTML or XML character references and entities from a text string.
	1084	#
	1085	# @param text The HTML (or XML) source text.
	1086	# @return The plain text, as a Unicode string, if necessary.
	1087	#
	1088	# From: http://effbot.org/zone/re-sub.htm#unescape-html
	1089	def _unescape_fixup(m):
	1090	text = m.group(0)
	1091	if not htmlentitydefs.name2codepoint.has_key('apos'):
	1092	htmlentitydefs.name2codepoint['apos'] = ord("'")
	1093	if text[:2] == "&#":
	1094	# character reference
	1095	try:
	1096	if text[:3] == "&#x":
	1097	return unichr(int(text[3:-1], 16))
	1098	else:
	1099	return unichr(int(text[2:-1]))
	1100	except ValueError:
	1101	pass
	1102	else:
	1103	# named entity
	1104	try:
	1105	text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
	1106	except KeyError:
	1107	pass
	1108	return text # leave as is
	1109	text = text.encode('ascii', 'xmlcharrefreplace')
	1110	return re.sub("&#?\w+;", _unescape_fixup, text)
	1111
	1112	cfg.urlencoding_mode = "fixbucket"
	1113	s3 = S3(cfg)
	1114
	1115	count = 0
	1116	for arg in args:
	1117	culprit = S3Uri(arg)
	1118	if culprit.type != "s3":
	1119	raise ParameterError("Expecting S3Uri instead of: %s" % arg)
	1120	response = s3.bucket_list_noparse(culprit.bucket(), culprit.object(), recursive = True)
	1121	r_xent = re.compile("&#x[\da-fA-F]+;")
	1122	response['data'] = unicode(response['data'], 'UTF-8')
	1123	keys = re.findall("<Key>(.*?)</Key>", response['data'], re.MULTILINE)
	1124	debug("Keys: %r" % keys)
	1125	for key in keys:
	1126	if r_xent.search(key):
	1127	info("Fixing: %s" % key)
	1128	debug("Step 1: Transforming %s" % key)
	1129	key_bin = _unescape(key)
	1130	debug("Step 2: ... to %s" % key_bin)
	1131	key_new = replace_nonprintables(key_bin)
	1132	debug("Step 3: ... then to %s" % key_new)
	1133	src = S3Uri("s3://%s/%s" % (culprit.bucket(), key_bin))
	1134	dst = S3Uri("s3://%s/%s" % (culprit.bucket(), key_new))
	1135	resp_move = s3.object_move(src, dst)
	1136	if resp_move['status'] == 200:
	1137	output("File %r renamed to %s" % (key_bin, key_new))
	1138	count += 1
	1139	else:
	1140	error("Something went wrong for: %r" % key)
	1141	error("Please report the problem to s3tools-bugs@lists.sourceforge.net")
	1142	if count > 0:
	1143	warning("Fixed %d files' names. Their ACL were reset to Private." % count)
	1144	warning("Use 's3cmd setacl --acl-public s3://...' to make")
	1145	warning("them publicly readable if required.")
1392	1146
1393	1147	def resolve_list(lst, args):
1394		retval = []
1395		for item in lst:
1396		retval.append(item % args)
1397		return retval
	1148	retval = []
	1149	for item in lst:
	1150	retval.append(item % args)
	1151	return retval
1398	1152
1399	1153	def gpg_command(command, passphrase = ""):
1400		debug("GPG command: " + " ".join(command))
1401		p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
1402		p_stdout, p_stderr = p.communicate(passphrase + "\n")
1403		debug("GPG output:")
1404		for line in p_stdout.split("\n"):
1405		debug("GPG: " + line)
1406		p_exitcode = p.wait()
1407		return p_exitcode
	1154	debug("GPG command: " + " ".join(command))
	1155	p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
	1156	p_stdout, p_stderr = p.communicate(passphrase + "\n")
	1157	debug("GPG output:")
	1158	for line in p_stdout.split("\n"):
	1159	debug("GPG: " + line)
	1160	p_exitcode = p.wait()
	1161	return p_exitcode
1408	1162
1409	1163	def gpg_encrypt(filename):
1410		tmp_filename = Utils.mktmpfile()
1411		args = {
1412		"gpg_command" : cfg.gpg_command,
1413		"passphrase_fd" : "0",
1414		"input_file" : filename,
1415		"output_file" : tmp_filename,
1416		}
1417		info(u"Encrypting file %(input_file)s to %(output_file)s..." % args)
1418		command = resolve_list(cfg.gpg_encrypt.split(" "), args)
1419		code = gpg_command(command, cfg.gpg_passphrase)
1420		return (code, tmp_filename, "gpg")
	1164	tmp_filename = Utils.mktmpfile()
	1165	args = {
	1166	"gpg_command" : cfg.gpg_command,
	1167	"passphrase_fd" : "0",
	1168	"input_file" : filename,
	1169	"output_file" : tmp_filename,
	1170	}
	1171	info(u"Encrypting file %(input_file)s to %(output_file)s..." % args)
	1172	command = resolve_list(cfg.gpg_encrypt.split(" "), args)
	1173	code = gpg_command(command, cfg.gpg_passphrase)
	1174	return (code, tmp_filename, "gpg")
1421	1175
1422	1176	def gpg_decrypt(filename, gpgenc_header = "", in_place = True):
1423		tmp_filename = Utils.mktmpfile(filename)
1424		args = {
1425		"gpg_command" : cfg.gpg_command,
1426		"passphrase_fd" : "0",
1427		"input_file" : filename,
1428		"output_file" : tmp_filename,
1429		}
1430		info(u"Decrypting file %(input_file)s to %(output_file)s..." % args)
1431		command = resolve_list(cfg.gpg_decrypt.split(" "), args)
1432		code = gpg_command(command, cfg.gpg_passphrase)
1433		if code == 0 and in_place:
1434		debug(u"Renaming %s to %s" % (tmp_filename, filename))
1435		os.unlink(filename)
1436		os.rename(tmp_filename, filename)
1437		tmp_filename = filename
1438		return (code, tmp_filename)
1439
1440		def run_configure(config_file):
1441		cfg = Config()
1442		options = [
1443		("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3"),
1444		("secret_key", "Secret Key"),
1445		("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"),
1446		("gpg_command", "Path to GPG program"),
1447		("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy"),
1448		("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't conect to S3 directly"),
1449		("proxy_port", "HTTP Proxy server port"),
1450		]
1451		## Option-specfic defaults
1452		if getattr(cfg, "gpg_command") == "":
1453		setattr(cfg, "gpg_command", find_executable("gpg"))
1454
1455		if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"):
1456		re_match=re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy"))
1457		if re_match:
1458		setattr(cfg, "proxy_host", re_match.groups()[1])
1459		setattr(cfg, "proxy_port", re_match.groups()[2])
1460
1461		try:
1462		while 1:
1463		output(u"\nEnter new values or accept defaults in brackets with Enter.")
1464		output(u"Refer to user manual for detailed description of all options.")
1465		for option in options:
1466		prompt = option[1]
1467		## Option-specific handling
1468		if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True:
1469		setattr(cfg, option[0], "")
1470		continue
1471		if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "":
1472		setattr(cfg, option[0], 0)
1473		continue
1474
1475		try:
1476		val = getattr(cfg, option[0])
1477		if type(val) is bool:
1478		val = val and "Yes" or "No"
1479		if val not in (None, ""):
1480		prompt += " [%s]" % val
1481		except AttributeError:
1482		pass
1483
1484		if len(option) >= 3:
1485		output(u"\n%s" % option[2])
1486
1487		val = raw_input(prompt + ": ")
1488		if val != "":
1489		if type(getattr(cfg, option[0])) is bool:
1490		# Turn 'Yes' into True, everything else into False
1491		val = val.lower().startswith('y')
1492		setattr(cfg, option[0], val)
1493		output(u"\nNew settings:")
1494		for option in options:
1495		output(u" %s: %s" % (option[1], getattr(cfg, option[0])))
1496		val = raw_input("\nTest access with supplied credentials? [Y/n] ")
1497		if val.lower().startswith("y") or val == "":
1498		try:
1499		output(u"Please wait...")
1500		S3(Config()).bucket_list("", "")
1501		output(u"Success. Your access key and secret key worked fine :-)")
1502
1503		output(u"\nNow verifying that encryption works...")
1504		if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"):
1505		output(u"Not configured. Never mind.")
1506		else:
1507		if not getattr(cfg, "gpg_command"):
1508		raise Exception("Path to GPG program not set")
1509		if not os.path.isfile(getattr(cfg, "gpg_command")):
1510		raise Exception("GPG program not found")
1511		filename = Utils.mktmpfile()
1512		f = open(filename, "w")
1513		f.write(os.sys.copyright)
1514		f.close()
1515		ret_enc = gpg_encrypt(filename)
1516		ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False)
1517		hash = [
1518		Utils.hash_file_md5(filename),
1519		Utils.hash_file_md5(ret_enc[1]),
1520		Utils.hash_file_md5(ret_dec[1]),
1521		]
1522		os.unlink(filename)
1523		os.unlink(ret_enc[1])
1524		os.unlink(ret_dec[1])
1525		if hash[0] == hash[2] and hash[0] != hash[1]:
1526		output ("Success. Encryption and decryption worked fine :-)")
1527		else:
1528		raise Exception("Encryption verification error.")
1529
1530		except Exception, e:
1531		error(u"Test failed: %s" % (e))
1532		val = raw_input("\nRetry configuration? [Y/n] ")
1533		if val.lower().startswith("y") or val == "":
1534		continue
1535
1536
1537		val = raw_input("\nSave settings? [y/N] ")
1538		if val.lower().startswith("y"):
1539		break
1540		val = raw_input("Retry configuration? [Y/n] ")
1541		if val.lower().startswith("n"):
1542		raise EOFError()
1543
1544		## Overwrite existing config file, make it user-readable only
1545		old_mask = os.umask(0077)
1546		try:
1547		os.remove(config_file)
1548		except OSError, e:
1549		if e.errno != errno.ENOENT:
1550		raise
1551		f = open(config_file, "w")
1552		os.umask(old_mask)
1553		cfg.dump_config(f)
1554		f.close()
1555		output(u"Configuration saved to '%s'" % config_file)
1556
1557		except (EOFError, KeyboardInterrupt):
1558		output(u"\nConfiguration aborted. Changes were NOT saved.")
1559		return
1560
1561		except IOError, e:
1562		error(u"Writing config file failed: %s: %s" % (config_file, e.strerror))
1563		sys.exit(1)
	1177	tmp_filename = Utils.mktmpfile(filename)
	1178	args = {
	1179	"gpg_command" : cfg.gpg_command,
	1180	"passphrase_fd" : "0",
	1181	"input_file" : filename,
	1182	"output_file" : tmp_filename,
	1183	}
	1184	info(u"Decrypting file %(input_file)s to %(output_file)s..." % args)
	1185	command = resolve_list(cfg.gpg_decrypt.split(" "), args)
	1186	code = gpg_command(command, cfg.gpg_passphrase)
	1187	if code == 0 and in_place:
	1188	debug(u"Renaming %s to %s" % (tmp_filename, filename))
	1189	os.unlink(filename)
	1190	os.rename(tmp_filename, filename)
	1191	tmp_filename = filename
	1192	return (code, tmp_filename)
	1193
	1194	def run_configure(config_file, args):
	1195	cfg = Config()
	1196	options = [
	1197	("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3"),
	1198	("secret_key", "Secret Key"),
	1199	("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"),
	1200	("gpg_command", "Path to GPG program"),
	1201	("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy"),
	1202	("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't conect to S3 directly"),
	1203	("proxy_port", "HTTP Proxy server port"),
	1204	]
	1205	## Option-specfic defaults
	1206	if getattr(cfg, "gpg_command") == "":
	1207	setattr(cfg, "gpg_command", find_executable("gpg"))
	1208
	1209	if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"):
	1210	re_match=re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy"))
	1211	if re_match:
	1212	setattr(cfg, "proxy_host", re_match.groups()[1])
	1213	setattr(cfg, "proxy_port", re_match.groups()[2])
	1214
	1215	try:
	1216	while 1:
	1217	output(u"\nEnter new values or accept defaults in brackets with Enter.")
	1218	output(u"Refer to user manual for detailed description of all options.")
	1219	for option in options:
	1220	prompt = option[1]
	1221	## Option-specific handling
	1222	if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True:
	1223	setattr(cfg, option[0], "")
	1224	continue
	1225	if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "":
	1226	setattr(cfg, option[0], 0)
	1227	continue
	1228
	1229	try:
	1230	val = getattr(cfg, option[0])
	1231	if type(val) is bool:
	1232	val = val and "Yes" or "No"
	1233	if val not in (None, ""):
	1234	prompt += " [%s]" % val
	1235	except AttributeError:
	1236	pass
	1237
	1238	if len(option) >= 3:
	1239	output(u"\n%s" % option[2])
	1240
	1241	val = raw_input(prompt + ": ")
	1242	if val != "":
	1243	if type(getattr(cfg, option[0])) is bool:
	1244	# Turn 'Yes' into True, everything else into False
	1245	val = val.lower().startswith('y')
	1246	setattr(cfg, option[0], val)
	1247	output(u"\nNew settings:")
	1248	for option in options:
	1249	output(u" %s: %s" % (option[1], getattr(cfg, option[0])))
	1250	val = raw_input("\nTest access with supplied credentials? [Y/n] ")
	1251	if val.lower().startswith("y") or val == "":
	1252	try:
	1253	# Default, we try to list 'all' buckets which requires
	1254	# ListAllMyBuckets permission
	1255	if len(args) == 0:
	1256	output(u"Please wait, attempting to list all buckets...")
	1257	S3(Config()).bucket_list("", "")
	1258	else:
	1259	# If user specified a bucket name directly, we check it and only it.
	1260	# Thus, access check can succeed even if user only has access to
	1261	# to a single bucket and not ListAllMyBuckets permission.
	1262	output(u"Please wait, attempting to list bucket: " + args[0])
	1263	uri = S3Uri(args[0])
	1264	if uri.type == "s3" and uri.has_bucket():
	1265	S3(Config()).bucket_list(uri.bucket(), "")
	1266	else:
	1267	raise Exception(u"Invalid bucket uri: " + args[0])
	1268
	1269	output(u"Success. Your access key and secret key worked fine :-)")
	1270
	1271	output(u"\nNow verifying that encryption works...")
	1272	if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"):
	1273	output(u"Not configured. Never mind.")
	1274	else:
	1275	if not getattr(cfg, "gpg_command"):
	1276	raise Exception("Path to GPG program not set")
	1277	if not os.path.isfile(getattr(cfg, "gpg_command")):
	1278	raise Exception("GPG program not found")
	1279	filename = Utils.mktmpfile()
	1280	f = open(filename, "w")
	1281	f.write(os.sys.copyright)
	1282	f.close()
	1283	ret_enc = gpg_encrypt(filename)
	1284	ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False)
	1285	hash = [
	1286	Utils.hash_file_md5(filename),
	1287	Utils.hash_file_md5(ret_enc[1]),
	1288	Utils.hash_file_md5(ret_dec[1]),
	1289	]
	1290	os.unlink(filename)
	1291	os.unlink(ret_enc[1])
	1292	os.unlink(ret_dec[1])
	1293	if hash[0] == hash[2] and hash[0] != hash[1]:
	1294	output ("Success. Encryption and decryption worked fine :-)")
	1295	else:
	1296	raise Exception("Encryption verification error.")
	1297
	1298	except Exception, e:
	1299	error(u"Test failed: %s" % (e))
	1300	val = raw_input("\nRetry configuration? [Y/n] ")
	1301	if val.lower().startswith("y") or val == "":
	1302	continue
	1303
	1304
	1305	val = raw_input("\nSave settings? [y/N] ")
	1306	if val.lower().startswith("y"):
	1307	break
	1308	val = raw_input("Retry configuration? [Y/n] ")
	1309	if val.lower().startswith("n"):
	1310	raise EOFError()
	1311
	1312	## Overwrite existing config file, make it user-readable only
	1313	old_mask = os.umask(0077)
	1314	try:
	1315	os.remove(config_file)
	1316	except OSError, e:
	1317	if e.errno != errno.ENOENT:
	1318	raise
	1319	f = open(config_file, "w")
	1320	os.umask(old_mask)
	1321	cfg.dump_config(f)
	1322	f.close()
	1323	output(u"Configuration saved to '%s'" % config_file)
	1324
	1325	except (EOFError, KeyboardInterrupt):
	1326	output(u"\nConfiguration aborted. Changes were NOT saved.")
	1327	return
	1328
	1329	except IOError, e:
	1330	error(u"Writing config file failed: %s: %s" % (config_file, e.strerror))
	1331	sys.exit(1)
1564	1332
1565	1333	def process_patterns_from_file(fname, patterns_list):
1566		try:
1567		fn = open(fname, "rt")
1568		except IOError, e:
1569		error(e)
1570		sys.exit(1)
1571		for pattern in fn:
1572		pattern = pattern.strip()
1573		if re.match("^#", pattern) or re.match("^\s*$", pattern):
1574		continue
1575		debug(u"%s: adding rule: %s" % (fname, pattern))
1576		patterns_list.append(pattern)
1577
1578		return patterns_list
	1334	try:
	1335	fn = open(fname, "rt")
	1336	except IOError, e:
	1337	error(e)
	1338	sys.exit(1)
	1339	for pattern in fn:
	1340	pattern = pattern.strip()
	1341	if re.match("^#", pattern) or re.match("^\s*$", pattern):
	1342	continue
	1343	debug(u"%s: adding rule: %s" % (fname, pattern))
	1344	patterns_list.append(pattern)
	1345
	1346	return patterns_list
1579	1347
1580	1348	def process_patterns(patterns_list, patterns_from, is_glob, option_txt = ""):
1581		"""
1582		process_patterns(patterns, patterns_from, is_glob, option_txt = "")
1583		Process --exclude / --include GLOB and REGEXP patterns.
1584		'option_txt' is 'exclude' / 'include' / 'rexclude' / 'rinclude'
1585		Returns: patterns_compiled, patterns_text
1586		"""
1587
1588		patterns_compiled = []
1589		patterns_textual = {}
1590
1591		if patterns_list is None:
1592		patterns_list = []
1593
1594		if patterns_from:
1595		## Append patterns from glob_from
1596		for fname in patterns_from:
1597		debug(u"processing --%s-from %s" % (option_txt, fname))
1598		patterns_list = process_patterns_from_file(fname, patterns_list)
1599
1600		for pattern in patterns_list:
1601		debug(u"processing %s rule: %s" % (option_txt, patterns_list))
1602		if is_glob:
1603		pattern = glob.fnmatch.translate(pattern)
1604		r = re.compile(pattern)
1605		patterns_compiled.append(r)
1606		patterns_textual[r] = pattern
1607
1608		return patterns_compiled, patterns_textual
	1349	"""
	1350	process_patterns(patterns, patterns_from, is_glob, option_txt = "")
	1351	Process --exclude / --include GLOB and REGEXP patterns.
	1352	'option_txt' is 'exclude' / 'include' / 'rexclude' / 'rinclude'
	1353	Returns: patterns_compiled, patterns_text
	1354	"""
	1355
	1356	patterns_compiled = []
	1357	patterns_textual = {}
	1358
	1359	if patterns_list is None:
	1360	patterns_list = []
	1361
	1362	if patterns_from:
	1363	## Append patterns from glob_from
	1364	for fname in patterns_from:
	1365	debug(u"processing --%s-from %s" % (option_txt, fname))
	1366	patterns_list = process_patterns_from_file(fname, patterns_list)
	1367
	1368	for pattern in patterns_list:
	1369	debug(u"processing %s rule: %s" % (option_txt, patterns_list))
	1370	if is_glob:
	1371	pattern = glob.fnmatch.translate(pattern)
	1372	r = re.compile(pattern)
	1373	patterns_compiled.append(r)
	1374	patterns_textual[r] = pattern
	1375
	1376	return patterns_compiled, patterns_textual
1609	1377
1610	1378	def get_commands_list():
1611		return [
1612		{"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
1613		{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
1614		{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
1615		{"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0},
1616		{"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2},
1617		{"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1},
1618		{"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1},
1619		#{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1},
1620		{"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2},
1621		{"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0},
1622		{"cmd":"info", "label":"Get various information about Buckets or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1},
1623		{"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},
1624		{"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},
1625		{"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
1626		{"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
1627		{"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
1628		{"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
1629
1630		## CloudFront commands
1631		{"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.info, "argc":0},
1632		{"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"[cf://DIST_ID]", "func":CfCmd.info, "argc":0},
1633		{"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1},
1634		{"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1},
1635		{"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1},
1636		]
	1379	return [
	1380	{"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
	1381	{"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
	1382	{"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
	1383	{"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0},
	1384	{"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2},
	1385	{"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1},
	1386	{"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1},
	1387	#{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1},
	1388	{"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2},
	1389	{"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0},
	1390	{"cmd":"info", "label":"Get various information about Buckets or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1},
	1391	{"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},
	1392	{"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},
	1393	{"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
	1394	{"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
	1395	{"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
	1396	{"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
	1397
	1398	## Website commands
	1399	{"cmd":"ws-create", "label":"Create Website from bucket", "param":"s3://BUCKET", "func":cmd_website_create, "argc":1},
	1400	{"cmd":"ws-delete", "label":"Delete Website", "param":"s3://BUCKET", "func":cmd_website_delete, "argc":1},
	1401	{"cmd":"ws-info", "label":"Info about Website", "param":"s3://BUCKET", "func":cmd_website_info, "argc":1},
	1402
	1403	## CloudFront commands
	1404	{"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.info, "argc":0},
	1405	{"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"[cf://DIST_ID]", "func":CfCmd.info, "argc":0},
	1406	{"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1},
	1407	{"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1},
	1408	{"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1},
	1409	#{"cmd":"cfinval", "label":"Invalidate CloudFront objects", "param":"s3://BUCKET/OBJECT [s3://BUCKET/OBJECT ...]", "func":CfCmd.invalidate, "argc":1},
	1410	{"cmd":"cfinvalinfo", "label":"Display CloudFront invalidation request(s) status", "param":"cf://DIST_ID[/INVAL_ID]", "func":CfCmd.invalinfo, "argc":1},
	1411	]
1637	1412
1638	1413	def format_commands(progname, commands_list):
1639		help = "Commands:\n"
1640		for cmd in commands_list:
1641		help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
1642		return help
	1414	help = "Commands:\n"
	1415	for cmd in commands_list:
	1416	help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
	1417	return help
1643	1418
1644	1419	class OptionMimeType(Option):
1645		def check_mimetype(option, opt, value):
1646		if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+$", re.IGNORECASE).match(value):
1647		return value
1648		raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value))
	1420	def check_mimetype(option, opt, value):
	1421	if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+(;.*)?$", re.IGNORECASE).match(value):
	1422	return value
	1423	raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value))
1649	1424
1650	1425	class OptionS3ACL(Option):
1651		def check_s3acl(option, opt, value):
1652		permissions = ('read', 'write', 'read_acp', 'write_acp', 'full_control', 'all')
1653		try:
1654		permission, grantee = re.compile("^(\w+):(.+)$", re.IGNORECASE).match(value).groups()
1655		if not permission or not grantee:
1656		raise
1657		if permission in permissions:
1658		return { 'name' : grantee, 'permission' : permission.upper() }
1659		else:
1660		raise OptionValueError("option %s: invalid S3 ACL permission: %s (valid values: %s)" %
1661		(opt, permission, ", ".join(permissions)))
1662		except:
1663		raise OptionValueError("option %s: invalid S3 ACL format: %r" % (opt, value))
	1426	def check_s3acl(option, opt, value):
	1427	permissions = ('read', 'write', 'read_acp', 'write_acp', 'full_control', 'all')
	1428	try:
	1429	permission, grantee = re.compile("^(\w+):(.+)$", re.IGNORECASE).match(value).groups()
	1430	if not permission or not grantee:
	1431	raise
	1432	if permission in permissions:
	1433	return { 'name' : grantee, 'permission' : permission.upper() }
	1434	else:
	1435	raise OptionValueError("option %s: invalid S3 ACL permission: %s (valid values: %s)" %
	1436	(opt, permission, ", ".join(permissions)))
	1437	except:
	1438	raise OptionValueError("option %s: invalid S3 ACL format: %r" % (opt, value))
1664	1439
1665	1440	class OptionAll(OptionMimeType, OptionS3ACL):
1666		TYPE_CHECKER = copy(Option.TYPE_CHECKER)
1667		TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype
1668		TYPE_CHECKER["s3acl"] = OptionS3ACL.check_s3acl
1669		TYPES = Option.TYPES + ("mimetype", "s3acl")
	1441	TYPE_CHECKER = copy(Option.TYPE_CHECKER)
	1442	TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype
	1443	TYPE_CHECKER["s3acl"] = OptionS3ACL.check_s3acl
	1444	TYPES = Option.TYPES + ("mimetype", "s3acl")
1670	1445
1671	1446	class MyHelpFormatter(IndentedHelpFormatter):
1672		def format_epilog(self, epilog):
1673		if epilog:
1674		return "\n" + epilog + "\n"
1675		else:
1676		return ""
	1447	def format_epilog(self, epilog):
	1448	if epilog:
	1449	return "\n" + epilog + "\n"
	1450	else:
	1451	return ""
1677	1452
1678	1453	def main():
1679		global cfg
1680
1681		commands_list = get_commands_list()
1682		commands = {}
1683
1684		## Populate "commands" from "commands_list"
1685		for cmd in commands_list:
1686		if cmd.has_key("cmd"):
1687		commands[cmd["cmd"]] = cmd
1688
1689		default_verbosity = Config().verbosity
1690		optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter())
1691		#optparser.disable_interspersed_args()
1692
1693		config_file = None
1694		if os.getenv("HOME"):
1695		config_file = os.path.join(os.getenv("HOME"), ".s3cfg")
1696		elif os.name == "nt" and os.getenv("USERPROFILE"):
1697		config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), "Application Data", "s3cmd.ini")
1698
1699		preferred_encoding = locale.getpreferredencoding() or "UTF-8"
1700
1701		optparser.set_defaults(encoding = preferred_encoding)
1702		optparser.set_defaults(config = config_file)
1703		optparser.set_defaults(verbosity = default_verbosity)
1704
1705		optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.")
1706		optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
1707		optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
1708
1709		optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)")
1710
1711		optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.")
1712		optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.")
1713		optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
1714		optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).")
1715		optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).")
1716		optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.")
1717		optparser.add_option( "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)")
1718		optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size will be compared. May significantly speed up transfer but may also miss some changed files.")
1719		optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.")
1720		optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.")
1721		optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all")
1722		optparser.add_option( "--acl-revoke", dest="acl_revokes", type="s3acl", action="append", metavar="PERMISSION:USER_CANONICAL_ID", help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr ite_acp, full_control, all")
1723
1724		optparser.add_option( "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
1725		optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
1726		optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
1727		optparser.add_option( "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
1728		optparser.add_option( "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync")
1729		optparser.add_option( "--exclude-from", dest="exclude_from", action="append", metavar="FILE", help="Read --exclude GLOBs from FILE")
1730		optparser.add_option( "--rexclude", dest="rexclude", action="append", metavar="REGEXP", help="Filenames and paths matching REGEXP (regular expression) will be excluded from sync")
1731		optparser.add_option( "--rexclude-from", dest="rexclude_from", action="append", metavar="FILE", help="Read --rexclude REGEXPs from FILE")
1732		optparser.add_option( "--include", dest="include", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns")
1733		optparser.add_option( "--include-from", dest="include_from", action="append", metavar="FILE", help="Read --include GLOBs from FILE")
1734		optparser.add_option( "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
1735		optparser.add_option( "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
1736
1737		optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1")
1738		optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
1739
1740		optparser.add_option( "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")
1741		optparser.add_option( "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)")
1742
1743		optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.")
1744		optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option")
1745
1746		optparser.add_option( "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.")
1747
1748		optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)
1749		optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")
1750
1751		optparser.add_option( "--list-md5", dest="list_md5", action="store_true", help="Include MD5 sums in bucket listings (only for 'ls' command).")
1752		optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form (eg 1kB instead of 1234).")
1753
1754		optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")
1755		optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")
1756		optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")
1757		optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")
1758		optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
1759		optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)")
1760		optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
1761		optparser.add_option( "--cf-default-root-object", dest="cf_default_root_object", action="store", metavar="DEFAULT_ROOT_OBJECT", help="Set the default root object to return when no object is specified in the URL. Use a relative path, i.e. default/index.html instead of /default/index.html or s3://bucket/default/index.html (only for [cfcreate] and [cfmodify] commands)")
1762		optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.")
1763		optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
1764		optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
1765		optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
1766
1767		optparser.set_usage(optparser.usage + " COMMAND [parameters]")
1768		optparser.set_description('S3cmd is a tool for managing objects in '+
1769		'Amazon S3 storage. It allows for making and removing '+
1770		'"buckets" and uploading, downloading and removing '+
1771		'"objects" from these buckets.')
1772		optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)
1773		optparser.epilog += ("\nFor more informations see the progect homepage:\n%s\n" % PkgInfo.url)
1774		optparser.epilog += ("\nConsider a donation if you have found s3cmd useful:\n%s/donate\n" % PkgInfo.url)
1775
1776		(options, args) = optparser.parse_args()
1777
1778		## Some mucking with logging levels to enable
1779		## debugging/verbose output for config file parser on request
1780		logging.basicConfig(level=options.verbosity,
1781		format='%(levelname)s: %(message)s',
1782		stream = sys.stderr)
1783
1784		if options.show_version:
1785		output(u"s3cmd version %s" % PkgInfo.version)
1786		sys.exit(0)
1787
1788		## Now finally parse the config file
1789		if not options.config:
1790		error(u"Can't find a config file. Please use --config option.")
1791		sys.exit(1)
1792
1793		try:
1794		cfg = Config(options.config)
1795		except IOError, e:
1796		if options.run_configure:
1797		cfg = Config()
1798		else:
1799		error(u"%s: %s" % (options.config, e.strerror))
1800		error(u"Configuration file not available.")
1801		error(u"Consider using --configure parameter to create one.")
1802		sys.exit(1)
1803
1804		## And again some logging level adjustments
1805		## according to configfile and command line parameters
1806		if options.verbosity != default_verbosity:
1807		cfg.verbosity = options.verbosity
1808		logging.root.setLevel(cfg.verbosity)
1809
1810		## Default to --progress on TTY devices, --no-progress elsewhere
1811		## Can be overriden by actual --(no-)progress parameter
1812		cfg.update_option('progress_meter', sys.stdout.isatty())
1813
1814		## Unsupported features on Win32 platform
1815		if os.name == "nt":
1816		if cfg.preserve_attrs:
1817		error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.")
1818		cfg.preserve_attrs = False
1819		if cfg.progress_meter:
1820		error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.")
1821		cfg.progress_meter = False
1822
1823		## Pre-process --add-header's and put them to Config.extra_headers SortedDict()
1824		if options.add_header:
1825		for hdr in options.add_header:
1826		try:
1827		key, val = hdr.split(":", 1)
1828		except ValueError:
1829		raise ParameterError("Invalid header format: %s" % hdr)
1830		key_inval = re.sub("[a-zA-Z0-9-.]", "", key)
1831		if key_inval:
1832		key_inval = key_inval.replace(" ", "<space>")
1833		key_inval = key_inval.replace("\t", "<tab>")
1834		raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval))
1835		debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip(), val.strip()))
1836		cfg.extra_headers[key.strip()] = val.strip()
1837
1838		## --acl-grant/--acl-revoke arguments are pre-parsed by OptionS3ACL()
1839		if options.acl_grants:
1840		for grant in options.acl_grants:
1841		cfg.acl_grants.append(grant)
1842
1843		if options.acl_revokes:
1844		for grant in options.acl_revokes:
1845		cfg.acl_revokes.append(grant)
1846
1847		## Process --(no-)check-md5
1848		if options.check_md5 == False:
1849		try:
1850		cfg.sync_checks.remove("md5")
1851		except:
1852		pass
1853		if options.check_md5 == True and cfg.sync_checks.count("md5") == 0:
1854		cfg.sync_checks.append("md5")
1855
1856		## Update Config with other parameters
1857		for option in cfg.option_list():
1858		try:
1859		if getattr(options, option) != None:
1860		debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option)))
1861		cfg.update_option(option, getattr(options, option))
1862		except AttributeError:
1863		## Some Config() options are not settable from command line
1864		pass
1865
1866		## Special handling for tri-state options (True, False, None)
1867		cfg.update_option("enable", options.enable)
1868		cfg.update_option("acl_public", options.acl_public)
1869
1870		## CloudFront's cf_enable and Config's enable share the same --enable switch
1871		options.cf_enable = options.enable
1872
1873		## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch
1874		options.cf_logging = options.log_target_prefix
1875
1876		## Update CloudFront options if some were set
1877		for option in CfCmd.options.option_list():
1878		try:
1879		if getattr(options, option) != None:
1880		debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option)))
1881		CfCmd.options.update_option(option, getattr(options, option))
1882		except AttributeError:
1883		## Some CloudFront.Cmd.Options() options are not settable from command line
1884		pass
1885
1886		## Set output and filesystem encoding for printing out filenames.
1887		sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")
1888		sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace")
1889
1890		## Process --exclude and --exclude-from
1891		patterns_list, patterns_textual = process_patterns(options.exclude, options.exclude_from, is_glob = True, option_txt = "exclude")
1892		cfg.exclude.extend(patterns_list)
1893		cfg.debug_exclude.update(patterns_textual)
1894
1895		## Process --rexclude and --rexclude-from
1896		patterns_list, patterns_textual = process_patterns(options.rexclude, options.rexclude_from, is_glob = False, option_txt = "rexclude")
1897		cfg.exclude.extend(patterns_list)
1898		cfg.debug_exclude.update(patterns_textual)
1899
1900		## Process --include and --include-from
1901		patterns_list, patterns_textual = process_patterns(options.include, options.include_from, is_glob = True, option_txt = "include")
1902		cfg.include.extend(patterns_list)
1903		cfg.debug_include.update(patterns_textual)
1904
1905		## Process --rinclude and --rinclude-from
1906		patterns_list, patterns_textual = process_patterns(options.rinclude, options.rinclude_from, is_glob = False, option_txt = "rinclude")
1907		cfg.include.extend(patterns_list)
1908		cfg.debug_include.update(patterns_textual)
1909
1910		## Set socket read()/write() timeout
1911		socket.setdefaulttimeout(cfg.socket_timeout)
1912
1913		if cfg.encrypt and cfg.gpg_passphrase == "":
1914		error(u"Encryption requested but no passphrase set in config file.")
1915		error(u"Please re-run 's3cmd --configure' and supply it.")
1916		sys.exit(1)
1917
1918		if options.dump_config:
1919		cfg.dump_config(sys.stdout)
1920		sys.exit(0)
1921
1922		if options.run_configure:
1923		run_configure(options.config)
1924		sys.exit(0)
1925
1926		if len(args) < 1:
1927		error(u"Missing command. Please run with --help for more information.")
1928		sys.exit(1)
1929
1930		## Unicodise all remaining arguments:
1931		args = [unicodise(arg) for arg in args]
1932
1933		command = args.pop(0)
1934		try:
1935		debug(u"Command: %s" % commands[command]["cmd"])
1936		## We must do this lookup in extra step to
1937		## avoid catching all KeyError exceptions
1938		## from inner functions.
1939		cmd_func = commands[command]["func"]
1940		except KeyError, e:
1941		error(u"Invalid command: %s" % e)
1942		sys.exit(1)
1943
1944		if len(args) < commands[command]["argc"]:
1945		error(u"Not enough paramters for command '%s'" % command)
1946		sys.exit(1)
1947
1948		try:
1949		cmd_func(args)
1950		except S3Error, e:
1951		error(u"S3 error: %s" % e)
1952		sys.exit(1)
	1454	global cfg
	1455
	1456	commands_list = get_commands_list()
	1457	commands = {}
	1458
	1459	## Populate "commands" from "commands_list"
	1460	for cmd in commands_list:
	1461	if cmd.has_key("cmd"):
	1462	commands[cmd["cmd"]] = cmd
	1463
	1464	default_verbosity = Config().verbosity
	1465	optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter())
	1466	#optparser.disable_interspersed_args()
	1467
	1468	config_file = None
	1469	if os.getenv("HOME"):
	1470	config_file = os.path.join(os.getenv("HOME"), ".s3cfg")
	1471	elif os.name == "nt" and os.getenv("USERPROFILE"):
	1472	config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), "Application Data", "s3cmd.ini")
	1473
	1474	preferred_encoding = locale.getpreferredencoding() or "UTF-8"
	1475
	1476	optparser.set_defaults(encoding = preferred_encoding)
	1477	optparser.set_defaults(config = config_file)
	1478	optparser.set_defaults(verbosity = default_verbosity)
	1479
	1480	optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://come-bucket' to test access to a specific bucket instead of attempting to list them all.")
	1481	optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
	1482	optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
	1483
	1484	optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)")
	1485
	1486	optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.")
	1487	optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.")
	1488	optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
	1489	optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).")
	1490	optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).")
	1491	optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.")
	1492	optparser.add_option( "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)")
	1493	optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size will be compared. May significantly speed up transfer but may also miss some changed files.")
	1494	optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.")
	1495	optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.")
	1496	optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all")
	1497	optparser.add_option( "--acl-revoke", dest="acl_revokes", type="s3acl", action="append", metavar="PERMISSION:USER_CANONICAL_ID", help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr ite_acp, full_control, all")
	1498
	1499	optparser.add_option( "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
	1500	optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
	1501	optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
	1502	optparser.add_option( "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
	1503	optparser.add_option( "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync")
	1504	optparser.add_option( "--exclude-from", dest="exclude_from", action="append", metavar="FILE", help="Read --exclude GLOBs from FILE")
	1505	optparser.add_option( "--rexclude", dest="rexclude", action="append", metavar="REGEXP", help="Filenames and paths matching REGEXP (regular expression) will be excluded from sync")
	1506	optparser.add_option( "--rexclude-from", dest="rexclude_from", action="append", metavar="FILE", help="Read --rexclude REGEXPs from FILE")
	1507	optparser.add_option( "--include", dest="include", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns")
	1508	optparser.add_option( "--include-from", dest="include_from", action="append", metavar="FILE", help="Read --include GLOBs from FILE")
	1509	optparser.add_option( "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
	1510	optparser.add_option( "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
	1511
	1512	optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1")
	1513	optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
	1514
	1515	optparser.add_option( "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")
	1516	optparser.add_option( "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)")
	1517
	1518	optparser.add_option( "--default-mime-type", dest="default_mime_type", action="store_true", help="Default MIME-type for stored objects. Application default is binary/octet-stream.")
	1519	optparser.add_option( "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by --default-mime-type option")
	1520	optparser.add_option( "--no-guess-mime-type", dest="guess_mime_type", action="store_false", help="Don't guess MIME-type and use the default type instead.")
	1521	optparser.add_option("-m", "--mime-type", dest="mime_type", type="mimetype", metavar="MIME/TYPE", help="Force MIME-type. Override both --default-mime-type and --guess-mime-type.")
	1522
	1523	optparser.add_option( "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.")
	1524
	1525	optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)
	1526	optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")
	1527
	1528	optparser.add_option( "--disable-multipart", dest="enable_multipart", action="store_false", help="Disable multipart upload on files bigger than --multipart-chunk-size-mb")
	1529	optparser.add_option( "--multipart-chunk-size-mb", dest="multipart_chunk_size_mb", type="int", action="store", metavar="SIZE", help="Size of each chunk of a multipart upload. Files bigger than SIZE are automatically uploaded as multithreaded-multipart, smaller files are uploaded using the traditional method. SIZE is in Mega-Bytes, default chunk size is %defaultMB, minimum allowed chunk size is 5MB, maximum is 5GB.")
	1530
	1531	optparser.add_option( "--list-md5", dest="list_md5", action="store_true", help="Include MD5 sums in bucket listings (only for 'ls' command).")
	1532	optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form (eg 1kB instead of 1234).")
	1533
	1534	optparser.add_option( "--ws-index", dest="website_index", action="store", help="Name of error-document (only for [ws-create] command)")
	1535	optparser.add_option( "--ws-error", dest="website_error", action="store", help="Name of index-document (only for [ws-create] command)")
	1536
	1537	optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")
	1538	optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")
	1539	optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")
	1540	optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")
	1541	optparser.add_option( "--cf-invalidate", dest="invalidate_on_cf", action="store_true", help="Invalidate the uploaded filed in CloudFront. Also see [cfinval] command.")
	1542	optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
	1543	optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)")
	1544	optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
	1545	optparser.add_option( "--cf-default-root-object", dest="cf_default_root_object", action="store", metavar="DEFAULT_ROOT_OBJECT", help="Set the default root object to return when no object is specified in the URL. Use a relative path, i.e. default/index.html instead of /default/index.html or s3://bucket/default/index.html (only for [cfcreate] and [cfmodify] commands)")
	1546	optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.")
	1547	optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
	1548	optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
	1549	optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
	1550
	1551	optparser.set_usage(optparser.usage + " COMMAND [parameters]")
	1552	optparser.set_description('S3cmd is a tool for managing objects in '+
	1553	'Amazon S3 storage. It allows for making and removing '+
	1554	'"buckets" and uploading, downloading and removing '+
	1555	'"objects" from these buckets.')
	1556	optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)
	1557	optparser.epilog += ("\nFor more informations see the progect homepage:\n%s\n" % PkgInfo.url)
	1558	optparser.epilog += ("\nConsider a donation if you have found s3cmd useful:\n%s/donate\n" % PkgInfo.url)
	1559
	1560	(options, args) = optparser.parse_args()
	1561
	1562	## Some mucking with logging levels to enable
	1563	## debugging/verbose output for config file parser on request
	1564	logging.basicConfig(level=options.verbosity,
	1565	format='%(levelname)s: %(message)s',
	1566	stream = sys.stderr)
	1567
	1568	if options.show_version:
	1569	output(u"s3cmd version %s" % PkgInfo.version)
	1570	sys.exit(0)
	1571
	1572	## Now finally parse the config file
	1573	if not options.config:
	1574	error(u"Can't find a config file. Please use --config option.")
	1575	sys.exit(1)
	1576
	1577	try:
	1578	cfg = Config(options.config)
	1579	except IOError, e:
	1580	if options.run_configure:
	1581	cfg = Config()
	1582	else:
	1583	error(u"%s: %s" % (options.config, e.strerror))
	1584	error(u"Configuration file not available.")
	1585	error(u"Consider using --configure parameter to create one.")
	1586	sys.exit(1)
	1587
	1588	## And again some logging level adjustments
	1589	## according to configfile and command line parameters
	1590	if options.verbosity != default_verbosity:
	1591	cfg.verbosity = options.verbosity
	1592	logging.root.setLevel(cfg.verbosity)
	1593
	1594	## Default to --progress on TTY devices, --no-progress elsewhere
	1595	## Can be overriden by actual --(no-)progress parameter
	1596	cfg.update_option('progress_meter', sys.stdout.isatty())
	1597
	1598	## Unsupported features on Win32 platform
	1599	if os.name == "nt":
	1600	if cfg.preserve_attrs:
	1601	error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.")
	1602	cfg.preserve_attrs = False
	1603	if cfg.progress_meter:
	1604	error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.")
	1605	cfg.progress_meter = False
	1606
	1607	## Pre-process --add-header's and put them to Config.extra_headers SortedDict()
	1608	if options.add_header:
	1609	for hdr in options.add_header:
	1610	try:
	1611	key, val = hdr.split(":", 1)
	1612	except ValueError:
	1613	raise ParameterError("Invalid header format: %s" % hdr)
	1614	key_inval = re.sub("[a-zA-Z0-9-.]", "", key)
	1615	if key_inval:
	1616	key_inval = key_inval.replace(" ", "<space>")
	1617	key_inval = key_inval.replace("\t", "<tab>")
	1618	raise ParameterError("Invalid character(s) in header name '%s': \"%s\"" % (key, key_inval))
	1619	debug(u"Updating Config.Config extra_headers[%s] -> %s" % (key.strip(), val.strip()))
	1620	cfg.extra_headers[key.strip()] = val.strip()
	1621
	1622	## --acl-grant/--acl-revoke arguments are pre-parsed by OptionS3ACL()
	1623	if options.acl_grants:
	1624	for grant in options.acl_grants:
	1625	cfg.acl_grants.append(grant)
	1626
	1627	if options.acl_revokes:
	1628	for grant in options.acl_revokes:
	1629	cfg.acl_revokes.append(grant)
	1630
	1631	## Process --(no-)check-md5
	1632	if options.check_md5 == False:
	1633	try:
	1634	cfg.sync_checks.remove("md5")
	1635	except Exception:
	1636	pass
	1637	if options.check_md5 == True and cfg.sync_checks.count("md5") == 0:
	1638	cfg.sync_checks.append("md5")
	1639
	1640	## Update Config with other parameters
	1641	for option in cfg.option_list():
	1642	try:
	1643	if getattr(options, option) != None:
	1644	debug(u"Updating Config.Config %s -> %s" % (option, getattr(options, option)))
	1645	cfg.update_option(option, getattr(options, option))
	1646	except AttributeError:
	1647	## Some Config() options are not settable from command line
	1648	pass
	1649
	1650	## Special handling for tri-state options (True, False, None)
	1651	cfg.update_option("enable", options.enable)
	1652	cfg.update_option("acl_public", options.acl_public)
	1653
	1654	## Check multipart chunk constraints
	1655	if cfg.multipart_chunk_size_mb < MultiPartUpload.MIN_CHUNK_SIZE_MB:
	1656	raise ParameterError("Chunk size %d MB is too small, must be >= %d MB. Please adjust --multipart-chunk-size-mb" % (cfg.multipart_chunk_size_mb, MultiPartUpload.MIN_CHUNK_SIZE_MB))
	1657	if cfg.multipart_chunk_size_mb > MultiPartUpload.MAX_CHUNK_SIZE_MB:
	1658	raise ParameterError("Chunk size %d MB is too large, must be <= %d MB. Please adjust --multipart-chunk-size-mb" % (cfg.multipart_chunk_size_mb, MultiPartUpload.MAX_CHUNK_SIZE_MB))
	1659
	1660	## CloudFront's cf_enable and Config's enable share the same --enable switch
	1661	options.cf_enable = options.enable
	1662
	1663	## CloudFront's cf_logging and Config's log_target_prefix share the same --log-target-prefix switch
	1664	options.cf_logging = options.log_target_prefix
	1665
	1666	## Update CloudFront options if some were set
	1667	for option in CfCmd.options.option_list():
	1668	try:
	1669	if getattr(options, option) != None:
	1670	debug(u"Updating CloudFront.Cmd %s -> %s" % (option, getattr(options, option)))
	1671	CfCmd.options.update_option(option, getattr(options, option))
	1672	except AttributeError:
	1673	## Some CloudFront.Cmd.Options() options are not settable from command line
	1674	pass
	1675
	1676	## Set output and filesystem encoding for printing out filenames.
	1677	sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")
	1678	sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace")
	1679
	1680	## Process --exclude and --exclude-from
	1681	patterns_list, patterns_textual = process_patterns(options.exclude, options.exclude_from, is_glob = True, option_txt = "exclude")
	1682	cfg.exclude.extend(patterns_list)
	1683	cfg.debug_exclude.update(patterns_textual)
	1684
	1685	## Process --rexclude and --rexclude-from
	1686	patterns_list, patterns_textual = process_patterns(options.rexclude, options.rexclude_from, is_glob = False, option_txt = "rexclude")
	1687	cfg.exclude.extend(patterns_list)
	1688	cfg.debug_exclude.update(patterns_textual)
	1689
	1690	## Process --include and --include-from
	1691	patterns_list, patterns_textual = process_patterns(options.include, options.include_from, is_glob = True, option_txt = "include")
	1692	cfg.include.extend(patterns_list)
	1693	cfg.debug_include.update(patterns_textual)
	1694
	1695	## Process --rinclude and --rinclude-from
	1696	patterns_list, patterns_textual = process_patterns(options.rinclude, options.rinclude_from, is_glob = False, option_txt = "rinclude")
	1697	cfg.include.extend(patterns_list)
	1698	cfg.debug_include.update(patterns_textual)
	1699
	1700	## Set socket read()/write() timeout
	1701	socket.setdefaulttimeout(cfg.socket_timeout)
	1702
	1703	if cfg.encrypt and cfg.gpg_passphrase == "":
	1704	error(u"Encryption requested but no passphrase set in config file.")
	1705	error(u"Please re-run 's3cmd --configure' and supply it.")
	1706	sys.exit(1)
	1707
	1708	if options.dump_config:
	1709	cfg.dump_config(sys.stdout)
	1710	sys.exit(0)
	1711
	1712	if options.run_configure:
	1713	# 'args' may contain the test-bucket URI
	1714	run_configure(options.config, args)
	1715	sys.exit(0)
	1716
	1717	if len(args) < 1:
	1718	error(u"Missing command. Please run with --help for more information.")
	1719	sys.exit(1)
	1720
	1721	## Unicodise all remaining arguments:
	1722	args = [unicodise(arg) for arg in args]
	1723
	1724	command = args.pop(0)
	1725	try:
	1726	debug(u"Command: %s" % commands[command]["cmd"])
	1727	## We must do this lookup in extra step to
	1728	## avoid catching all KeyError exceptions
	1729	## from inner functions.
	1730	cmd_func = commands[command]["func"]
	1731	except KeyError, e:
	1732	error(u"Invalid command: %s" % e)
	1733	sys.exit(1)
	1734
	1735	if len(args) < commands[command]["argc"]:
	1736	error(u"Not enough paramters for command '%s'" % command)
	1737	sys.exit(1)
	1738
	1739	try:
	1740	cmd_func(args)
	1741	except S3Error, e:
	1742	error(u"S3 error: %s" % e)
	1743	sys.exit(1)
1953	1744
1954	1745	def report_exception(e):
1955		sys.stderr.write("""
	1746	sys.stderr.write("""
1956	1747	!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1957	1748	An unexpected error has occurred.
1958	1749	Please report the following lines to:

1960	1751	!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1961	1752
1962	1753	""")
1963		tb = traceback.format_exc(sys.exc_info())
1964		e_class = str(e.__class__)
1965		e_class = e_class[e_class.rfind(".")+1 : -2]
1966		sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e))
1967		try:
1968		sys.stderr.write("S3cmd: %s\n" % PkgInfo.version)
1969		except NameError:
1970		sys.stderr.write("S3cmd: unknown version. Module import problem?\n")
1971		sys.stderr.write("\n")
1972		sys.stderr.write(unicode(tb, errors="replace"))
1973
1974		if type(e) == ImportError:
1975		sys.stderr.write("\n")
1976		sys.stderr.write("Your sys.path contains these entries:\n")
1977		for path in sys.path:
1978		sys.stderr.write(u"\t%s\n" % path)
1979		sys.stderr.write("Now the question is where have the s3cmd modules been installed?\n")
1980
1981		sys.stderr.write("""
	1754	tb = traceback.format_exc(sys.exc_info())
	1755	e_class = str(e.__class__)
	1756	e_class = e_class[e_class.rfind(".")+1 : -2]
	1757	sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e))
	1758	try:
	1759	sys.stderr.write("S3cmd: %s\n" % PkgInfo.version)
	1760	except NameError:
	1761	sys.stderr.write("S3cmd: unknown version. Module import problem?\n")
	1762	sys.stderr.write("\n")
	1763	sys.stderr.write(unicode(tb, errors="replace"))
	1764
	1765	if type(e) == ImportError:
	1766	sys.stderr.write("\n")
	1767	sys.stderr.write("Your sys.path contains these entries:\n")
	1768	for path in sys.path:
	1769	sys.stderr.write(u"\t%s\n" % path)
	1770	sys.stderr.write("Now the question is where have the s3cmd modules been installed?\n")
	1771
	1772	sys.stderr.write("""
1982	1773	!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1983	1774	An unexpected error has occurred.
1984	1775	Please report the above lines to:

1987	1778	""")
1988	1779
1989	1780	if __name__ == '__main__':
1990		try:
1991		## Our modules
1992		## Keep them in try/except block to
1993		## detect any syntax errors in there
1994		from S3.Exceptions import *
1995		from S3 import PkgInfo
1996		from S3.S3 import S3
1997		from S3.Config import Config
1998		from S3.SortedDict import SortedDict
1999		from S3.S3Uri import S3Uri
2000		from S3 import Utils
2001		from S3.Utils import *
2002		from S3.Progress import Progress
2003		from S3.CloudFront import Cmd as CfCmd
2004
2005		main()
2006		sys.exit(0)
2007
2008		except ImportError, e:
2009		report_exception(e)
2010		sys.exit(1)
2011
2012		except ParameterError, e:
2013		error(u"Parameter problem: %s" % e)
2014		sys.exit(1)
2015
2016		except SystemExit, e:
2017		sys.exit(e.code)
2018
2019		except KeyboardInterrupt:
2020		sys.stderr.write("See ya!\n")
2021		sys.exit(1)
2022
2023		except Exception, e:
2024		report_exception(e)
2025		sys.exit(1)
	1781	try:
	1782	## Our modules
	1783	## Keep them in try/except block to
	1784	## detect any syntax errors in there
	1785	from S3.Exceptions import *
	1786	from S3 import PkgInfo
	1787	from S3.S3 import S3
	1788	from S3.Config import Config
	1789	from S3.SortedDict import SortedDict
	1790	from S3.S3Uri import S3Uri
	1791	from S3 import Utils
	1792	from S3.Utils import *
	1793	from S3.Progress import Progress
	1794	from S3.CloudFront import Cmd as CfCmd
	1795	from S3.CloudFront import CloudFront
	1796	from S3.FileLists import *
	1797	from S3.MultiPart import MultiPartUpload
	1798
	1799	main()
	1800	sys.exit(0)
	1801
	1802	except ImportError, e:
	1803	report_exception(e)
	1804	sys.exit(1)
	1805
	1806	except ParameterError, e:
	1807	error(u"Parameter problem: %s" % e)
	1808	sys.exit(1)
	1809
	1810	except SystemExit, e:
	1811	sys.exit(e.code)
	1812
	1813	except KeyboardInterrupt:
	1814	sys.stderr.write("See ya!\n")
	1815	sys.exit(1)
	1816
	1817	except Exception, e:
	1818	report_exception(e)
	1819	sys.exit(1)
	1820
	1821	# vim:et:ts=4:sts=4:ai

+60

-11

s3cmd.1 less more

67	67
68	68
69	69	.PP
	70	Commands for static WebSites configuration
	71	.TP
	72	s3cmd \fBws-create\fR \fIs3://BUCKET\fR
	73	Create Website from bucket
	74	.TP
	75	s3cmd \fBws-delete\fR \fIs3://BUCKET\fR
	76	Delete Website
	77	.TP
	78	s3cmd \fBws-info\fR \fIs3://BUCKET\fR
	79	Info about Website
	80
	81
	82	.PP
70	83	Commands for CloudFront management
71	84	.TP
72	85	s3cmd \fBcflist\fR \fI\fR

83	96	.TP
84	97	s3cmd \fBcfmodify\fR \fIcf://DIST_ID\fR
85	98	Change CloudFront distribution point parameters
	99	.TP
	100	s3cmd \fBcfinvalinfo\fR \fIcf://DIST_ID[/INVAL_ID]\fR
	101	Display CloudFront invalidation request(s) status
86	102
87	103
88	104	.SH OPTIONS

98	114	show this help message and exit
99	115	.TP
100	116	\fB\-\-configure\fR
101		Invoke interactive (re)configuration tool.
	117	Invoke interactive (re)configuration tool. Optionally
	118	use as '\fB--configure\fR s3://come-bucket' to test access
	119	to a specific bucket instead of attempting to list
	120	them all.
102	121	.TP
103	122	\fB\-c\fR FILE, \fB\-\-config\fR=FILE
104	123	Config file name. Defaults to /home/mludvig/.s3cfg

190	209	\fB\-\-include\fR=GLOB
191	210	Filenames and paths matching GLOB will be included
192	211	even if previously excluded by one of
193		.TP
194		\fB\-\-(r)exclude(\-from)\fR patterns
	212	\fB--(r)exclude(-from)\fR patterns
195	213	.TP
196	214	\fB\-\-include\-from\fR=FILE
197	215	Read --include GLOBs from FILE

220	238	Disable access logging (for [cfmodify] and [accesslog]
221	239	commands)
222	240	.TP
	241	\fB\-\-default\-mime\-type\fR
	242	Default MIME-type for stored objects. Application
	243	default is binary/octet-stream.
	244	.TP
	245	\fB\-\-guess\-mime\-type\fR
	246	Guess MIME-type of files by their extension or mime
	247	magic. Fall back to default MIME-Type as specified by
	248	\fB--default-mime-type\fR option
	249	.TP
	250	\fB\-\-no\-guess\-mime\-type\fR
	251	Don't guess MIME-type and use the default type
	252	instead.
	253	.TP
223	254	\fB\-m\fR MIME/TYPE, \fB\-\-mime\-type\fR=MIME/TYPE
224		Default MIME-type to be set for objects stored.
225		.TP
226		\fB\-M\fR, \fB\-\-guess\-mime\-type\fR
227		Guess MIME-type of files by their extension. Falls
228		back to default MIME-Type as specified by --mime-type
229		option
	255	Force MIME-type. Override both \fB--default-mime-type\fR and
	256	\fB--guess-mime-type\fR.
230	257	.TP
231	258	\fB\-\-add\-header\fR=NAME:VALUE
232	259	Add a given HTTP header to the upload request. Can be

242	269	Use the S3 name as given on the command line. No pre-
243	270	processing, encoding, etc. Use with caution!
244	271	.TP
	272	\fB\-\-disable\-multipart\fR
	273	Disable multipart upload on files bigger than
	274	\fB--multipart-chunk-size-mb\fR
	275	.TP
	276	\fB\-\-multipart\-chunk\-size\-mb\fR=SIZE
	277	Size of each chunk of a multipart upload. Files bigger
	278	than SIZE are automatically uploaded as multithreaded-
	279	multipart, smaller files are uploaded using the
	280	traditional method. SIZE is in Mega-Bytes, default
	281	chunk size is noneMB, minimum allowed chunk size is
	282	5MB, maximum is 5GB.
	283	.TP
245	284	\fB\-\-list\-md5\fR
246	285	Include MD5 sums in bucket listings (only for 'ls'
247	286	command).

250	289	Print sizes in human readable form (eg 1kB instead of
251	290	1234).
252	291	.TP
	292	\fB\-\-ws\-index\fR=WEBSITE_INDEX
	293	Name of error-document (only for [ws-create] command)
	294	.TP
	295	\fB\-\-ws\-error\fR=WEBSITE_ERROR
	296	Name of index-document (only for [ws-create] command)
	297	.TP
253	298	\fB\-\-progress\fR
254	299	Display progress meter (default on TTY).
255	300	.TP

263	308	\fB\-\-disable\fR
264	309	Enable given CloudFront distribution (only for
265	310	[cfmodify] command)
	311	.TP
	312	\fB\-\-cf\-invalidate\fR
	313	Invalidate the uploaded filed in CloudFront. Also see
	314	[cfinval] command.
266	315	.TP
267	316	\fB\-\-cf\-add\-cname\fR=CNAME
268	317	Add given CNAME to a CloudFront distribution (only for

290	339	Enable debug output.
291	340	.TP
292	341	\fB\-\-version\fR
293		Show s3cmd version (1.0.0) and exit.
	342	Show s3cmd version (1.1.0-beta3) and exit.
294	343	.TP
295	344	\fB\-F\fR, \fB\-\-follow\-symlinks\fR
296	345	Follow symbolic links as if they are regular files

391	440	Report bugs to
392	441	.I s3tools\-bugs@lists.sourceforge.net
393	442	.SH COPYRIGHT
394		Copyright \(co 2007,2008,2009,2010,2011 Michal Ludvig <http://www.logix.cz/michal>
	443	Copyright \(co 2007,2008,2009,2010,2011,2012 Michal Ludvig <http://www.logix.cz/michal>
395	444	.br
396	445	This is free software. You may redistribute copies of it under the terms of
397	446	the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>.

+49

-47

setup.py less more

4	4	import S3.PkgInfo
5	5
6	6	if float("%d.%d" % sys.version_info[:2]) < 2.4:
7		sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3])
8		sys.stderr.write("S3cmd requires Python 2.4 or newer.\n")
9		sys.exit(1)
	7	sys.stderr.write("Your Python version %d.%d.%d is not supported.\n" % sys.version_info[:3])
	8	sys.stderr.write("S3cmd requires Python 2.4 or newer.\n")
	9	sys.exit(1)
10	10
11	11	try:
12		import xml.etree.ElementTree as ET
13		print "Using xml.etree.ElementTree for XML processing"
	12	import xml.etree.ElementTree as ET
	13	print "Using xml.etree.ElementTree for XML processing"
14	14	except ImportError, e:
15		sys.stderr.write(str(e) + "\n")
16		try:
17		import elementtree.ElementTree as ET
18		print "Using elementtree.ElementTree for XML processing"
19		except ImportError, e:
20		sys.stderr.write(str(e) + "\n")
21		sys.stderr.write("Please install ElementTree module from\n")
22		sys.stderr.write("http://effbot.org/zone/element-index.htm\n")
23		sys.exit(1)
	15	sys.stderr.write(str(e) + "\n")
	16	try:
	17	import elementtree.ElementTree as ET
	18	print "Using elementtree.ElementTree for XML processing"
	19	except ImportError, e:
	20	sys.stderr.write(str(e) + "\n")
	21	sys.stderr.write("Please install ElementTree module from\n")
	22	sys.stderr.write("http://effbot.org/zone/element-index.htm\n")
	23	sys.exit(1)
24	24
25	25	try:
26		## Remove 'MANIFEST' file to force
27		## distutils to recreate it.
28		## Only in "sdist" stage. Otherwise
29		## it makes life difficult to packagers.
30		if sys.argv[1] == "sdist":
31		os.unlink("MANIFEST")
	26	## Remove 'MANIFEST' file to force
	27	## distutils to recreate it.
	28	## Only in "sdist" stage. Otherwise
	29	## it makes life difficult to packagers.
	30	if sys.argv[1] == "sdist":
	31	os.unlink("MANIFEST")
32	32	except:
33		pass
	33	pass
34	34
35	35	## Re-create the manpage
36	36	## (Beware! Perl script on the loose!!)
37	37	if sys.argv[1] == "sdist":
38		if os.stat_result(os.stat("s3cmd.1")).st_mtime < os.stat_result(os.stat("s3cmd")).st_mtime:
39		sys.stderr.write("Re-create man page first!\n")
40		sys.stderr.write("Run: ./s3cmd --help \| ./format-manpage.pl > s3cmd.1\n")
41		sys.exit(1)
	38	if os.stat_result(os.stat("s3cmd.1")).st_mtime < os.stat_result(os.stat("s3cmd")).st_mtime:
	39	sys.stderr.write("Re-create man page first!\n")
	40	sys.stderr.write("Run: ./s3cmd --help \| ./format-manpage.pl > s3cmd.1\n")
	41	sys.exit(1)
42	42
43	43	## Don't install manpages and docs when $S3CMD_PACKAGING is set
44		## This was a requirement of Debian package maintainer.
	44	## This was a requirement of Debian package maintainer.
45	45	if not os.getenv("S3CMD_PACKAGING"):
46		man_path = os.getenv("S3CMD_INSTPATH_MAN") or "share/man"
47		doc_path = os.getenv("S3CMD_INSTPATH_DOC") or "share/doc/packages"
48		data_files = [
49		(doc_path+"/s3cmd", [ "README", "INSTALL", "NEWS" ]),
50		(man_path+"/man1", [ "s3cmd.1" ] ),
51		]
	46	man_path = os.getenv("S3CMD_INSTPATH_MAN") or "share/man"
	47	doc_path = os.getenv("S3CMD_INSTPATH_DOC") or "share/doc/packages"
	48	data_files = [
	49	(doc_path+"/s3cmd", [ "README", "INSTALL", "NEWS" ]),
	50	(man_path+"/man1", [ "s3cmd.1" ] ),
	51	]
52	52	else:
53		data_files = None
	53	data_files = None
54	54
55	55	## Main distutils info
56	56	setup(
57		## Content description
58		name = S3.PkgInfo.package,
59		version = S3.PkgInfo.version,
60		packages = [ 'S3' ],
61		scripts = ['s3cmd'],
62		data_files = data_files,
	57	## Content description
	58	name = S3.PkgInfo.package,
	59	version = S3.PkgInfo.version,
	60	packages = [ 'S3' ],
	61	scripts = ['s3cmd'],
	62	data_files = data_files,
63	63
64		## Packaging details
65		author = "Michal Ludvig",
66		author_email = "michal@logix.cz",
67		url = S3.PkgInfo.url,
68		license = S3.PkgInfo.license,
69		description = S3.PkgInfo.short_description,
70		long_description = """
	64	## Packaging details
	65	author = "Michal Ludvig",
	66	author_email = "michal@logix.cz",
	67	url = S3.PkgInfo.url,
	68	license = S3.PkgInfo.license,
	69	description = S3.PkgInfo.short_description,
	70	long_description = """
71	71	%s
72	72
73	73	Authors:
74	74	--------
75	75	Michal Ludvig <michal@logix.cz>
76	76	""" % (S3.PkgInfo.long_description)
77		)
	77	)
	78
	79	# vim:et:ts=4:sts=4:ai