package swift
import (
"bufio"
"bytes"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"io"
"io/ioutil"
"mime"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"sync"
"time"
)
const (
DefaultUserAgent = "goswift/1.0" // Default user agent
DefaultRetries = 3 // Default number of retries on token expiry
TimeFormat = "2006-01-02T15:04:05" // Python date format for json replies parsed as UTC
UploadTar = "tar" // Data format specifier for Connection.BulkUpload().
UploadTarGzip = "tar.gz" // Data format specifier for Connection.BulkUpload().
UploadTarBzip2 = "tar.bz2" // Data format specifier for Connection.BulkUpload().
allContainersLimit = 10000 // Number of containers to fetch at once
allObjectsLimit = 10000 // Number objects to fetch at once
allObjectsChanLimit = 1000 // ...when fetching to a channel
)
// ObjectType is the type of the swift object, regular, static large,
// or dynamic large.
type ObjectType int
// Values that ObjectType can take
const (
RegularObjectType ObjectType = iota
StaticLargeObjectType
DynamicLargeObjectType
)
// Connection holds the details of the connection to the swift server.
//
// You need to provide UserName, ApiKey and AuthUrl when you create a
// connection then call Authenticate on it.
//
// The auth version in use will be detected from the AuthURL - you can
// override this with the AuthVersion parameter.
//
// If using v2 auth you can also set Region in the Connection
// structure. If you don't set Region you will get the default region
// which may not be what you want.
//
// For reference some common AuthUrls looks like this:
//
// Rackspace US https://auth.api.rackspacecloud.com/v1.0
// Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0
// Rackspace v2 https://identity.api.rackspacecloud.com/v2.0
// Memset Memstore UK https://auth.storage.memset.com/v1.0
// Memstore v2 https://auth.storage.memset.com/v2.0
//
// When using Google Appengine you must provide the Connection with an
// appengine-specific Transport:
//
// import (
// "appengine/urlfetch"
// "fmt"
// "github.com/ncw/swift"
// )
//
// func handler(w http.ResponseWriter, r *http.Request) {
// ctx := appengine.NewContext(r)
// tr := urlfetch.Transport{Context: ctx}
// c := swift.Connection{
// UserName: "user",
// ApiKey: "key",
// AuthUrl: "auth_url",
// Transport: tr,
// }
// _ := c.Authenticate()
// containers, _ := c.ContainerNames(nil)
// fmt.Fprintf(w, "containers: %q", containers)
// }
//
// If you don't supply a Transport, one is made which relies on
// http.ProxyFromEnvironment (http://golang.org/pkg/net/http/#ProxyFromEnvironment).
// This means that the connection will respect the HTTP proxy specified by the
// environment variables $HTTP_PROXY and $NO_PROXY.
type Connection struct {
// Parameters - fill these in before calling Authenticate
// They are all optional except UserName, ApiKey and AuthUrl
Domain string // User's domain name
DomainId string // User's domain Id
UserName string // UserName for api
UserId string // User Id
ApiKey string // Key for api access
ApplicationCredentialId string // Application Credential ID
ApplicationCredentialName string // Application Credential Name
ApplicationCredentialSecret string // Application Credential Secret
AuthUrl string // Auth URL
Retries int // Retries on error (default is 3)
UserAgent string // Http User agent (default goswift/1.0)
ConnectTimeout time.Duration // Connect channel timeout (default 10s)
Timeout time.Duration // Data channel timeout (default 60s)
Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect
Internal bool // Set this to true to use the the internal / service network
Tenant string // Name of the tenant (v2,v3 auth only)
TenantId string // Id of the tenant (v2,v3 auth only)
EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
TrustId string // Id of the trust (v3 auth only)
Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
// These are filled in after Authenticate is called as are the defaults for above
StorageUrl string
AuthToken string
Expires time.Time // time the token expires, may be Zero if unknown
client *http.Client
Auth Authenticator `json:"-" xml:"-"` // the current authenticator
authLock *sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth
// swiftInfo is filled after QueryInfo is called
swiftInfo SwiftInfo
}
// setFromEnv reads the value that param points to (it must be a
// pointer), if it isn't the zero value then it reads the environment
// variable name passed in, parses it according to the type and writes
// it to the pointer.
func setFromEnv(param interface{}, name string) (err error) {
val := os.Getenv(name)
if val == "" {
return
}
switch result := param.(type) {
case *string:
if *result == "" {
*result = val
}
case *int:
if *result == 0 {
*result, err = strconv.Atoi(val)
}
case *bool:
if *result == false {
*result, err = strconv.ParseBool(val)
}
case *time.Duration:
if *result == 0 {
*result, err = time.ParseDuration(val)
}
case *EndpointType:
if *result == EndpointType("") {
*result = EndpointType(val)
}
default:
return newErrorf(0, "can't set var of type %T", param)
}
return err
}
// ApplyEnvironment reads environment variables and applies them to
// the Connection structure. It won't overwrite any parameters which
// are already set in the Connection struct.
//
// To make a new Connection object entirely from the environment you
// would do:
//
// c := new(Connection)
// err := c.ApplyEnvironment()
// if err != nil { log.Fatal(err) }
//
// The naming of these variables follows the official Openstack naming
// scheme so it should be compatible with OpenStack rc files.
//
// For v1 authentication (obsolete)
// ST_AUTH - Auth URL
// ST_USER - UserName for api
// ST_KEY - Key for api access
//
// For v2 authentication
// OS_AUTH_URL - Auth URL
// OS_USERNAME - UserName for api
// OS_PASSWORD - Key for api access
// OS_TENANT_NAME - Name of the tenant
// OS_TENANT_ID - Id of the tenant
// OS_REGION_NAME - Region to use - default is use first region
//
// For v3 authentication
// OS_AUTH_URL - Auth URL
// OS_USERNAME - UserName for api
// OS_USER_ID - User Id
// OS_PASSWORD - Key for api access
// OS_APPLICATION_CREDENTIAL_ID - Application Credential ID
// OS_APPLICATION_CREDENTIAL_NAME - Application Credential Name
// OS_APPLICATION_CREDENTIAL_SECRET - Application Credential Secret
// OS_USER_DOMAIN_NAME - User's domain name
// OS_USER_DOMAIN_ID - User's domain Id
// OS_PROJECT_NAME - Name of the project
// OS_PROJECT_DOMAIN_NAME - Name of the tenant's domain, only needed if it differs from the user domain
// OS_PROJECT_DOMAIN_ID - Id of the tenant's domain, only needed if it differs the from user domain
// OS_TRUST_ID - If of the trust
// OS_REGION_NAME - Region to use - default is use first region
//
// Other
// OS_ENDPOINT_TYPE - Endpoint type public, internal or admin
// ST_AUTH_VERSION - Choose auth version - 1, 2 or 3 or leave at 0 for autodetect
//
// For manual authentication
// OS_STORAGE_URL - storage URL from alternate authentication
// OS_AUTH_TOKEN - Auth Token from alternate authentication
//
// Library specific
// GOSWIFT_RETRIES - Retries on error (default is 3)
// GOSWIFT_USER_AGENT - HTTP User agent (default goswift/1.0)
// GOSWIFT_CONNECT_TIMEOUT - Connect channel timeout with unit, eg "10s", "100ms" (default "10s")
// GOSWIFT_TIMEOUT - Data channel timeout with unit, eg "10s", "100ms" (default "60s")
// GOSWIFT_INTERNAL - Set this to "true" to use the the internal network (obsolete - use OS_ENDPOINT_TYPE)
func (c *Connection) ApplyEnvironment() (err error) {
for _, item := range []struct {
result interface{}
name string
}{
// Environment variables - keep in same order as Connection
{&c.Domain, "OS_USER_DOMAIN_NAME"},
{&c.DomainId, "OS_USER_DOMAIN_ID"},
{&c.UserName, "OS_USERNAME"},
{&c.UserId, "OS_USER_ID"},
{&c.ApiKey, "OS_PASSWORD"},
{&c.ApplicationCredentialId, "OS_APPLICATION_CREDENTIAL_ID"},
{&c.ApplicationCredentialName, "OS_APPLICATION_CREDENTIAL_NAME"},
{&c.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET"},
{&c.AuthUrl, "OS_AUTH_URL"},
{&c.Retries, "GOSWIFT_RETRIES"},
{&c.UserAgent, "GOSWIFT_USER_AGENT"},
{&c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT"},
{&c.Timeout, "GOSWIFT_TIMEOUT"},
{&c.Region, "OS_REGION_NAME"},
{&c.AuthVersion, "ST_AUTH_VERSION"},
{&c.Internal, "GOSWIFT_INTERNAL"},
{&c.Tenant, "OS_TENANT_NAME"}, //v2
{&c.Tenant, "OS_PROJECT_NAME"}, // v3
{&c.TenantId, "OS_TENANT_ID"},
{&c.EndpointType, "OS_ENDPOINT_TYPE"},
{&c.TenantDomain, "OS_PROJECT_DOMAIN_NAME"},
{&c.TenantDomainId, "OS_PROJECT_DOMAIN_ID"},
{&c.TrustId, "OS_TRUST_ID"},
{&c.StorageUrl, "OS_STORAGE_URL"},
{&c.AuthToken, "OS_AUTH_TOKEN"},
// v1 auth alternatives
{&c.ApiKey, "ST_KEY"},
{&c.UserName, "ST_USER"},
{&c.AuthUrl, "ST_AUTH"},
} {
err = setFromEnv(item.result, item.name)
if err != nil {
return newErrorf(0, "failed to read env var %q: %v", item.name, err)
}
}
return nil
}
// Error - all errors generated by this package are of this type. Other error
// may be passed on from library functions though.
type Error struct {
StatusCode int // HTTP status code if relevant or 0 if not
Text string
}
// Error satisfy the error interface.
func (e *Error) Error() string {
return e.Text
}
// newError make a new error from a string.
func newError(StatusCode int, Text string) *Error {
return &Error{
StatusCode: StatusCode,
Text: Text,
}
}
// newErrorf makes a new error from sprintf parameters.
func newErrorf(StatusCode int, Text string, Parameters ...interface{}) *Error {
return newError(StatusCode, fmt.Sprintf(Text, Parameters...))
}
// errorMap defines http error codes to error mappings.
type errorMap map[int]error
var (
// Specific Errors you might want to check for equality
NotModified = newError(304, "Not Modified")
BadRequest = newError(400, "Bad Request")
AuthorizationFailed = newError(401, "Authorization Failed")
ContainerNotFound = newError(404, "Container Not Found")
ContainerNotEmpty = newError(409, "Container Not Empty")
ObjectNotFound = newError(404, "Object Not Found")
ObjectCorrupted = newError(422, "Object Corrupted")
TimeoutError = newError(408, "Timeout when reading or writing data")
Forbidden = newError(403, "Operation forbidden")
TooLargeObject = newError(413, "Too Large Object")
RateLimit = newError(498, "Rate Limit")
TooManyRequests = newError(429, "TooManyRequests")
// Mappings for authentication errors
authErrorMap = errorMap{
400: BadRequest,
401: AuthorizationFailed,
403: Forbidden,
}
// Mappings for container errors
ContainerErrorMap = errorMap{
400: BadRequest,
403: Forbidden,
404: ContainerNotFound,
409: ContainerNotEmpty,
498: RateLimit,
}
// Mappings for object errors
objectErrorMap = errorMap{
304: NotModified,
400: BadRequest,
403: Forbidden,
404: ObjectNotFound,
413: TooLargeObject,
422: ObjectCorrupted,
429: TooManyRequests,
498: RateLimit,
}
)
// checkClose is used to check the return from Close in a defer
// statement.
func checkClose(c io.Closer, err *error) {
cerr := c.Close()
if *err == nil {
*err = cerr
}
}
// drainAndClose discards all data from rd and closes it.
// If an error occurs during Read, it is discarded.
func drainAndClose(rd io.ReadCloser, err *error) {
if rd == nil {
return
}
_, _ = io.Copy(ioutil.Discard, rd)
cerr := rd.Close()
if err != nil && *err == nil {
*err = cerr
}
}
// parseHeaders checks a response for errors and translates into
// standard errors if necessary. If an error is returned, resp.Body
// has been drained and closed.
func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error {
if errorMap != nil {
if err, ok := errorMap[resp.StatusCode]; ok {
drainAndClose(resp.Body, nil)
return err
}
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
drainAndClose(resp.Body, nil)
return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status)
}
return nil
}
// readHeaders returns a Headers object from the http.Response.
//
// If it receives multiple values for a key (which should never
// happen) it will use the first one
func readHeaders(resp *http.Response) Headers {
headers := Headers{}
for key, values := range resp.Header {
headers[key] = values[0]
}
return headers
}
// Headers stores HTTP headers (can only have one of each header like Swift).
type Headers map[string]string
// Does an http request using the running timer passed in
func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*http.Response, error) {
// Do the request in the background so we can check the timeout
type result struct {
resp *http.Response
err error
}
done := make(chan result, 1)
go func() {
resp, err := c.client.Do(req)
done <- result{resp, err}
}()
// Wait for the read or the timeout
select {
case r := <-done:
return r.resp, r.err
case <-timer.C:
// Kill the connection on timeout so we don't leak sockets or goroutines
cancelRequest(c.Transport, req)
return nil, TimeoutError
}
panic("unreachable") // For Go 1.0
}
// Set defaults for any unset values
//
// Call with authLock held
func (c *Connection) setDefaults() {
if c.UserAgent == "" {
c.UserAgent = DefaultUserAgent
}
if c.Retries == 0 {
c.Retries = DefaultRetries
}
if c.ConnectTimeout == 0 {
c.ConnectTimeout = 10 * time.Second
}
if c.Timeout == 0 {
c.Timeout = 60 * time.Second
}
if c.Transport == nil {
t := &http.Transport{
// TLSClientConfig: &tls.Config{RootCAs: pool},
// DisableCompression: true,
Proxy: http.ProxyFromEnvironment,
// Half of linux's default open files limit (1024).
MaxIdleConnsPerHost: 512,
}
SetExpectContinueTimeout(t, 5*time.Second)
c.Transport = t
}
if c.client == nil {
c.client = &http.Client{
// CheckRedirect: redirectPolicyFunc,
Transport: c.Transport,
}
}
}
// Authenticate connects to the Swift server.
//
// If you don't call it before calling one of the connection methods
// then it will be called for you on the first access.
func (c *Connection) Authenticate() (err error) {
if c.authLock == nil {
c.authLock = &sync.Mutex{}
}
c.authLock.Lock()
defer c.authLock.Unlock()
return c.authenticate()
}
// Internal implementation of Authenticate
//
// Call with authLock held
func (c *Connection) authenticate() (err error) {
c.setDefaults()
// Flush the keepalives connection - if we are
// re-authenticating then stuff has gone wrong
flushKeepaliveConnections(c.Transport)
if c.Auth == nil {
c.Auth, err = newAuth(c)
if err != nil {
return
}
}
retries := 1
again:
var req *http.Request
req, err = c.Auth.Request(c)
if err != nil {
return
}
if req != nil {
timer := time.NewTimer(c.ConnectTimeout)
defer timer.Stop()
var resp *http.Response
resp, err = c.doTimeoutRequest(timer, req)
if err != nil {
return
}
defer func() {
drainAndClose(resp.Body, &err)
// Flush the auth connection - we don't want to keep
// it open if keepalives were enabled
flushKeepaliveConnections(c.Transport)
}()
if err = c.parseHeaders(resp, authErrorMap); err != nil {
// Try again for a limited number of times on
// AuthorizationFailed or BadRequest. This allows us
// to try some alternate forms of the request
if (err == AuthorizationFailed || err == BadRequest) && retries > 0 {
retries--
goto again
}
return
}
err = c.Auth.Response(resp)
if err != nil {
return
}
}
if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" {
c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType)
} else {
c.StorageUrl = c.Auth.StorageUrl(c.Internal)
}
c.AuthToken = c.Auth.Token()
if do, ok := c.Auth.(Expireser); ok {
c.Expires = do.Expires()
} else {
c.Expires = time.Time{}
}
if !c.authenticated() {
err = newError(0, "Response didn't have storage url and auth token")
return
}
return
}
// Get an authToken and url
//
// The Url may be updated if it needed to authenticate using the OnReAuth function
func (c *Connection) getUrlAndAuthToken(targetUrlIn string, OnReAuth func() (string, error)) (targetUrlOut, authToken string, err error) {
c.authLock.Lock()
defer c.authLock.Unlock()
targetUrlOut = targetUrlIn
if !c.authenticated() {
err = c.authenticate()
if err != nil {
return
}
if OnReAuth != nil {
targetUrlOut, err = OnReAuth()
if err != nil {
return
}
}
}
authToken = c.AuthToken
return
}
// flushKeepaliveConnections is called to flush pending requests after an error.
func flushKeepaliveConnections(transport http.RoundTripper) {
if tr, ok := transport.(interface {
CloseIdleConnections()
}); ok {
tr.CloseIdleConnections()
}
}
// UnAuthenticate removes the authentication from the Connection.
func (c *Connection) UnAuthenticate() {
c.authLock.Lock()
c.StorageUrl = ""
c.AuthToken = ""
c.authLock.Unlock()
}
// Authenticated returns a boolean to show if the current connection
// is authenticated.
//
// Doesn't actually check the credentials against the server.
func (c *Connection) Authenticated() bool {
if c.authLock == nil {
c.authLock = &sync.Mutex{}
}
c.authLock.Lock()
defer c.authLock.Unlock()
return c.authenticated()
}
// Internal version of Authenticated()
//
// Call with authLock held
func (c *Connection) authenticated() bool {
if c.StorageUrl == "" || c.AuthToken == "" {
return false
}
if c.Expires.IsZero() {
return true
}
timeUntilExpiry := c.Expires.Sub(time.Now())
return timeUntilExpiry >= 60*time.Second
}
// SwiftInfo contains the JSON object returned by Swift when the /info
// route is queried. The object contains, among others, the Swift version,
// the enabled middlewares and their configuration
type SwiftInfo map[string]interface{}
func (i SwiftInfo) SupportsBulkDelete() bool {
_, val := i["bulk_delete"]
return val
}
func (i SwiftInfo) SupportsSLO() bool {
_, val := i["slo"]
return val
}
func (i SwiftInfo) SLOMinSegmentSize() int64 {
if slo, ok := i["slo"].(map[string]interface{}); ok {
val, _ := slo["min_segment_size"].(float64)
return int64(val)
}
return 1
}
// Discover Swift configuration by doing a request against /info
func (c *Connection) QueryInfo() (infos SwiftInfo, err error) {
infoUrl, err := url.Parse(c.StorageUrl)
if err != nil {
return nil, err
}
infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info")
resp, err := c.client.Get(infoUrl.String())
if err == nil {
if resp.StatusCode != http.StatusOK {
drainAndClose(resp.Body, nil)
return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode)
}
err = readJson(resp, &infos)
if err == nil {
c.authLock.Lock()
c.swiftInfo = infos
c.authLock.Unlock()
}
return infos, err
}
return nil, err
}
func (c *Connection) cachedQueryInfo() (infos SwiftInfo, err error) {
c.authLock.Lock()
infos = c.swiftInfo
c.authLock.Unlock()
if infos == nil {
infos, err = c.QueryInfo()
if err != nil {
return
}
}
return infos, nil
}
// RequestOpts contains parameters for Connection.storage.
type RequestOpts struct {
Container string
ObjectName string
Operation string
Parameters url.Values
Headers Headers
ErrorMap errorMap
NoResponse bool
Body io.Reader
Retries int
// if set this is called on re-authentication to refresh the targetUrl
OnReAuth func() (string, error)
}
// Call runs a remote command on the targetUrl, returns a
// response, headers and possible error.
//
// operation is GET, HEAD etc
// container is the name of a container
// Any other parameters (if not None) are added to the targetUrl
//
// Returns a response or an error. If response is returned then
// the resp.Body must be read completely and
// resp.Body.Close() must be called on it, unless noResponse is set in
// which case the body will be closed in this function
//
// If "Content-Length" is set in p.Headers it will be used - this can
// be used to override the default chunked transfer encoding for
// uploads.
//
// This will Authenticate if necessary, and re-authenticate if it
// receives a 401 error which means the token has expired
//
// This method is exported so extensions can call it.
func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, headers Headers, err error) {
c.authLock.Lock()
c.setDefaults()
c.authLock.Unlock()
retries := p.Retries
if retries == 0 {
retries = c.Retries
}
var req *http.Request
for {
var authToken string
if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil {
return //authentication failure
}
var URL *url.URL
URL, err = url.Parse(targetUrl)
if err != nil {
return
}
if p.Container != "" {
URL.Path += "/" + p.Container
if p.ObjectName != "" {
URL.Path += "/" + p.ObjectName
}
}
if p.Parameters != nil {
URL.RawQuery = p.Parameters.Encode()
}
timer := time.NewTimer(c.ConnectTimeout)
defer timer.Stop()
reader := p.Body
if reader != nil {
reader = newWatchdogReader(reader, c.Timeout, timer)
}
req, err = http.NewRequest(p.Operation, URL.String(), reader)
if err != nil {
return
}
if p.Headers != nil {
for k, v := range p.Headers {
// Set ContentLength in req if the user passed it in in the headers
if k == "Content-Length" {
req.ContentLength, err = strconv.ParseInt(v, 10, 64)
if err != nil {
err = fmt.Errorf("Invalid %q header %q: %v", k, v, err)
return
}
} else {
req.Header.Add(k, v)
}
}
}
req.Header.Add("User-Agent", c.UserAgent)
req.Header.Add("X-Auth-Token", authToken)
_, hasCL := p.Headers["Content-Length"]
AddExpectAndTransferEncoding(req, hasCL)
resp, err = c.doTimeoutRequest(timer, req)
if err != nil {
if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 {
retries--
continue
}
return
}
// Check to see if token has expired
if resp.StatusCode == 401 && retries > 0 {
drainAndClose(resp.Body, nil)
c.UnAuthenticate()
retries--
} else {
break
}
}
headers = readHeaders(resp)
if err = c.parseHeaders(resp, p.ErrorMap); err != nil {
return
}
if p.NoResponse {
drainAndClose(resp.Body, &err)
if err != nil {
return
}
} else {
// Cancel the request on timeout
cancel := func() {
cancelRequest(c.Transport, req)
}
// Wrap resp.Body to make it obey an idle timeout
resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel)
}
return
}
// storage runs a remote command on a the storage url, returns a
// response, headers and possible error.
//
// operation is GET, HEAD etc
// container is the name of a container
// Any other parameters (if not None) are added to the storage url
//
// Returns a response or an error. If response is returned then
// resp.Body.Close() must be called on it, unless noResponse is set in
// which case the body will be closed in this function
//
// This will Authenticate if necessary, and re-authenticate if it
// receives a 401 error which means the token has expired
func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Headers, err error) {
p.OnReAuth = func() (string, error) {
return c.StorageUrl, nil
}
c.authLock.Lock()
url := c.StorageUrl
c.authLock.Unlock()
return c.Call(url, p)
}
// readLines reads the response into an array of strings.
//
// Closes the response when done
func readLines(resp *http.Response) (lines []string, err error) {
defer drainAndClose(resp.Body, &err)
reader := bufio.NewReader(resp.Body)
buffer := bytes.NewBuffer(make([]byte, 0, 128))
var part []byte
var prefix bool
for {
if part, prefix, err = reader.ReadLine(); err != nil {
break
}
buffer.Write(part)
if !prefix {
lines = append(lines, buffer.String())
buffer.Reset()
}
}
if err == io.EOF {
err = nil
}
return
}
// readJson reads the response into the json type passed in
//
// Closes the response when done
func readJson(resp *http.Response, result interface{}) (err error) {
defer drainAndClose(resp.Body, &err)
decoder := json.NewDecoder(resp.Body)
return decoder.Decode(result)
}
/* ------------------------------------------------------------ */
// ContainersOpts is options for Containers() and ContainerNames()
type ContainersOpts struct {
Limit int // For an integer value n, limits the number of results to at most n values.
Prefix string // Given a string value x, return container names matching the specified prefix.
Marker string // Given a string value x, return container names greater in value than the specified marker.
EndMarker string // Given a string value x, return container names less in value than the specified marker.
Headers Headers // Any additional HTTP headers - can be nil
}
// parse the ContainerOpts
func (opts *ContainersOpts) parse() (url.Values, Headers) {
v := url.Values{}
var h Headers
if opts != nil {
if opts.Limit > 0 {
v.Set("limit", strconv.Itoa(opts.Limit))
}
if opts.Prefix != "" {
v.Set("prefix", opts.Prefix)
}
if opts.Marker != "" {
v.Set("marker", opts.Marker)
}
if opts.EndMarker != "" {
v.Set("end_marker", opts.EndMarker)
}
h = opts.Headers
}
return v, h
}
// ContainerNames returns a slice of names of containers in this account.
func (c *Connection) ContainerNames(opts *ContainersOpts) ([]string, error) {
v, h := opts.parse()
resp, _, err := c.storage(RequestOpts{
Operation: "GET",
Parameters: v,
ErrorMap: ContainerErrorMap,
Headers: h,
})
if err != nil {
return nil, err
}
lines, err := readLines(resp)
return lines, err
}
// Container contains information about a container
type Container struct {
Name string // Name of the container
Count int64 // Number of objects in the container
Bytes int64 // Total number of bytes used in the container
}
// Containers returns a slice of structures with full information as
// described in Container.
func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) {
v, h := opts.parse()
v.Set("format", "json")
resp, _, err := c.storage(RequestOpts{
Operation: "GET",
Parameters: v,
ErrorMap: ContainerErrorMap,
Headers: h,
})
if err != nil {
return nil, err
}
var containers []Container
err = readJson(resp, &containers)
return containers, err
}
// containersAllOpts makes a copy of opts if set or makes a new one and
// overrides Limit and Marker
func containersAllOpts(opts *ContainersOpts) *ContainersOpts {
var newOpts ContainersOpts
if opts != nil {
newOpts = *opts
}
if newOpts.Limit == 0 {
newOpts.Limit = allContainersLimit
}
newOpts.Marker = ""
return &newOpts
}
// ContainersAll is like Containers but it returns all the Containers
//
// It calls Containers multiple times using the Marker parameter
//
// It has a default Limit parameter but you may pass in your own
func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) {
opts = containersAllOpts(opts)
containers := make([]Container, 0)
for {
newContainers, err := c.Containers(opts)
if err != nil {
return nil, err
}
containers = append(containers, newContainers...)
if len(newContainers) < opts.Limit {
break
}
opts.Marker = newContainers[len(newContainers)-1].Name
}
return containers, nil
}
// ContainerNamesAll is like ContainerNames but it returns all the Containers
//
// It calls ContainerNames multiple times using the Marker parameter
//
// It has a default Limit parameter but you may pass in your own
func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) {
opts = containersAllOpts(opts)
containers := make([]string, 0)
for {
newContainers, err := c.ContainerNames(opts)
if err != nil {
return nil, err
}
containers = append(containers, newContainers...)
if len(newContainers) < opts.Limit {
break
}
opts.Marker = newContainers[len(newContainers)-1]
}
return containers, nil
}
/* ------------------------------------------------------------ */
// ObjectOpts is options for Objects() and ObjectNames()
type ObjectsOpts struct {
Limit int // For an integer value n, limits the number of results to at most n values.
Marker string // Given a string value x, return object names greater in value than the specified marker.
EndMarker string // Given a string value x, return object names less in value than the specified marker
Prefix string // For a string value x, causes the results to be limited to object names beginning with the substring x.
Path string // For a string value x, return the object names nested in the pseudo path
Delimiter rune // For a character c, return all the object names nested in the container
Headers Headers // Any additional HTTP headers - can be nil
KeepMarker bool // Do not reset Marker when using ObjectsAll or ObjectNamesAll
}
// parse reads values out of ObjectsOpts
func (opts *ObjectsOpts) parse() (url.Values, Headers) {
v := url.Values{}
var h Headers
if opts != nil {
if opts.Limit > 0 {
v.Set("limit", strconv.Itoa(opts.Limit))
}
if opts.Marker != "" {
v.Set("marker", opts.Marker)
}
if opts.EndMarker != "" {
v.Set("end_marker", opts.EndMarker)
}
if opts.Prefix != "" {
v.Set("prefix", opts.Prefix)
}
if opts.Path != "" {
v.Set("path", opts.Path)
}
if opts.Delimiter != 0 {
v.Set("delimiter", string(opts.Delimiter))
}
h = opts.Headers
}
return v, h
}
// ObjectNames returns a slice of names of objects in a given container.
func (c *Connection) ObjectNames(container string, opts *ObjectsOpts) ([]string, error) {
v, h := opts.parse()
resp, _, err := c.storage(RequestOpts{
Container: container,
Operation: "GET",
Parameters: v,
ErrorMap: ContainerErrorMap,
Headers: h,
})
if err != nil {
return nil, err
}
return readLines(resp)
}
// Object contains information about an object
type Object struct {
Name string `json:"name"` // object name
ContentType string `json:"content_type"` // eg application/directory
Bytes int64 `json:"bytes"` // size in bytes
ServerLastModified string `json:"last_modified"` // Last modified time, eg '2011-06-30T08:20:47.736680' as a string supplied by the server
LastModified time.Time // Last modified time converted to a time.Time
Hash string `json:"hash"` // MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e"
SLOHash string `json:"slo_etag"` // MD5 hash of all segments' MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e"
PseudoDirectory bool // Set when using delimiter to show that this directory object does not really exist
SubDir string `json:"subdir"` // returned only when using delimiter to mark "pseudo directories"
ObjectType ObjectType // type of this object
}
// Objects returns a slice of Object with information about each
// object in the container.
//
// If Delimiter is set in the opts then PseudoDirectory may be set,
// with ContentType 'application/directory'. These are not real
// objects but represent directories of objects which haven't had an
// object created for them.
func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, error) {
v, h := opts.parse()
v.Set("format", "json")
resp, _, err := c.storage(RequestOpts{
Container: container,
Operation: "GET",
Parameters: v,
ErrorMap: ContainerErrorMap,
Headers: h,
})
if err != nil {
return nil, err
}
var objects []Object
err = readJson(resp, &objects)
// Convert Pseudo directories and dates
for i := range objects {
object := &objects[i]
if object.SubDir != "" {
object.Name = object.SubDir
object.PseudoDirectory = true
object.ContentType = "application/directory"
}
if object.ServerLastModified != "" {
// 2012-11-11T14:49:47.887250
//
// Remove fractional seconds if present. This
// then keeps it consistent with Object
// which can only return timestamps accurate
// to 1 second
//
// The TimeFormat will parse fractional
// seconds if desired though
datetime := strings.SplitN(object.ServerLastModified, ".", 2)[0]
object.LastModified, err = time.Parse(TimeFormat, datetime)
if err != nil {
return nil, err
}
}
if object.SLOHash != "" {
object.ObjectType = StaticLargeObjectType
}
}
return objects, err
}
// objectsAllOpts makes a copy of opts if set or makes a new one and
// overrides Limit and Marker
// Marker is not overriden if KeepMarker is set
func objectsAllOpts(opts *ObjectsOpts, Limit int) *ObjectsOpts {
var newOpts ObjectsOpts
if opts != nil {
newOpts = *opts
}
if newOpts.Limit == 0 {
newOpts.Limit = Limit
}
if !newOpts.KeepMarker {
newOpts.Marker = ""
}
return &newOpts
}
// A closure defined by the caller to iterate through all objects
//
// Call Objects or ObjectNames from here with the *ObjectOpts passed in
//
// Do whatever is required with the results then return them
type ObjectsWalkFn func(*ObjectsOpts) (interface{}, error)
// ObjectsWalk is uses to iterate through all the objects in chunks as
// returned by Objects or ObjectNames using the Marker and Limit
// parameters in the ObjectsOpts.
//
// Pass in a closure `walkFn` which calls Objects or ObjectNames with
// the *ObjectsOpts passed to it and does something with the results.
//
// Errors will be returned from this function
//
// It has a default Limit parameter but you may pass in your own
func (c *Connection) ObjectsWalk(container string, opts *ObjectsOpts, walkFn ObjectsWalkFn) error {
opts = objectsAllOpts(opts, allObjectsChanLimit)
for {
objects, err := walkFn(opts)
if err != nil {
return err
}
var n int
var last string
switch objects := objects.(type) {
case []string:
n = len(objects)
if n > 0 {
last = objects[len(objects)-1]
}
case []Object:
n = len(objects)
if n > 0 {
last = objects[len(objects)-1].Name
}
default:
panic("Unknown type returned to ObjectsWalk")
}
if n < opts.Limit {
break
}
opts.Marker = last
}
return nil
}
// ObjectsAll is like Objects but it returns an unlimited number of Objects in a slice
//
// It calls Objects multiple times using the Marker parameter
func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) {
objects := make([]Object, 0)
err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) {
newObjects, err := c.Objects(container, opts)
if err == nil {
objects = append(objects, newObjects...)
}
return newObjects, err
})
return objects, err
}
// ObjectNamesAll is like ObjectNames but it returns all the Objects
//
// It calls ObjectNames multiple times using the Marker parameter. Marker is
// reset unless KeepMarker is set
//
// It has a default Limit parameter but you may pass in your own
func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) {
objects := make([]string, 0)
err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) {
newObjects, err := c.ObjectNames(container, opts)
if err == nil {
objects = append(objects, newObjects...)
}
return newObjects, err
})
return objects, err
}
// Account contains information about this account.
type Account struct {
BytesUsed int64 // total number of bytes used
Containers int64 // total number of containers
Objects int64 // total number of objects
}
// getInt64FromHeader is a helper function to decode int64 from header.
func getInt64FromHeader(resp *http.Response, header string) (result int64, err error) {
value := resp.Header.Get(header)
result, err = strconv.ParseInt(value, 10, 64)
if err != nil {
err = newErrorf(0, "Bad Header '%s': '%s': %s", header, value, err)
}
return
}
// Account returns info about the account in an Account struct.
func (c *Connection) Account() (info Account, headers Headers, err error) {
var resp *http.Response
resp, headers, err = c.storage(RequestOpts{
Operation: "HEAD",
ErrorMap: ContainerErrorMap,
NoResponse: true,
})
if err != nil {
return
}
// Parse the headers into a dict
//
// {'Accept-Ranges': 'bytes',
// 'Content-Length': '0',
// 'Date': 'Tue, 05 Jul 2011 16:37:06 GMT',
// 'X-Account-Bytes-Used': '316598182',
// 'X-Account-Container-Count': '4',
// 'X-Account-Object-Count': '1433'}
if info.BytesUsed, err = getInt64FromHeader(resp, "X-Account-Bytes-Used"); err != nil {
return
}
if info.Containers, err = getInt64FromHeader(resp, "X-Account-Container-Count"); err != nil {
return
}
if info.Objects, err = getInt64FromHeader(resp, "X-Account-Object-Count"); err != nil {
return
}
return
}
// AccountUpdate adds, replaces or remove account metadata.
//
// Add or update keys by mentioning them in the Headers.
//
// Remove keys by setting them to an empty string.
func (c *Connection) AccountUpdate(h Headers) error {
_, _, err := c.storage(RequestOpts{
Operation: "POST",
ErrorMap: ContainerErrorMap,
NoResponse: true,
Headers: h,
})
return err
}
// ContainerCreate creates a container.
//
// If you don't want to add Headers just pass in nil
//
// No error is returned if it already exists but the metadata if any will be updated.
func (c *Connection) ContainerCreate(container string, h Headers) error {
_, _, err := c.storage(RequestOpts{
Container: container,
Operation: "PUT",
ErrorMap: ContainerErrorMap,
NoResponse: true,
Headers: h,
})
return err
}
// ContainerDelete deletes a container.
//
// May return ContainerDoesNotExist or ContainerNotEmpty
func (c *Connection) ContainerDelete(container string) error {
_, _, err := c.storage(RequestOpts{
Container: container,
Operation: "DELETE",
ErrorMap: ContainerErrorMap,
NoResponse: true,
})
return err
}
// Container returns info about a single container including any
// metadata in the headers.
func (c *Connection) Container(container string) (info Container, headers Headers, err error) {
var resp *http.Response
resp, headers, err = c.storage(RequestOpts{
Container: container,
Operation: "HEAD",
ErrorMap: ContainerErrorMap,
NoResponse: true,
})
if err != nil {
return
}
// Parse the headers into the struct
info.Name = container
if info.Bytes, err = getInt64FromHeader(resp, "X-Container-Bytes-Used"); err != nil {
return
}
if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil {
return
}
return
}
// ContainerUpdate adds, replaces or removes container metadata.
//
// Add or update keys by mentioning them in the Metadata.
//
// Remove keys by setting them to an empty string.
//
// Container metadata can only be read with Container() not with Containers().
func (c *Connection) ContainerUpdate(container string, h Headers) error {
_, _, err := c.storage(RequestOpts{
Container: container,
Operation: "POST",
ErrorMap: ContainerErrorMap,
NoResponse: true,
Headers: h,
})
return err
}
// ------------------------------------------------------------
// ObjectCreateFile represents a swift object open for writing
type ObjectCreateFile struct {
checkHash bool // whether we are checking the hash
pipeReader *io.PipeReader // pipe for the caller to use
pipeWriter *io.PipeWriter
hash hash.Hash // hash being build up as we go along
done chan struct{} // signals when the upload has finished
resp *http.Response // valid when done has signalled
err error // ditto
headers Headers // ditto
}
// Write bytes to the object - see io.Writer
func (file *ObjectCreateFile) Write(p []byte) (n int, err error) {
n, err = file.pipeWriter.Write(p)
if err == io.ErrClosedPipe {
if file.err != nil {
return 0, file.err
}
return 0, newError(500, "Write on closed file")
}
if err == nil && file.checkHash {
_, _ = file.hash.Write(p)
}
return
}
// CloseWithError closes the object, aborting the upload.
func (file *ObjectCreateFile) CloseWithError(err error) error {
_ = file.pipeWriter.CloseWithError(err)
<-file.done
return nil
}
// Close the object and checks the md5sum if it was required.
//
// Also returns any other errors from the server (eg container not
// found) so it is very important to check the errors on this method.
func (file *ObjectCreateFile) Close() error {
// Close the body
err := file.pipeWriter.Close()
if err != nil {
return err
}
// Wait for the HTTP operation to complete
<-file.done
// Check errors
if file.err != nil {
return file.err
}
if file.checkHash {
receivedMd5 := strings.ToLower(file.headers["Etag"])
calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil))
if receivedMd5 != calculatedMd5 {
return ObjectCorrupted
}
}
return nil
}
// Headers returns the response headers from the created object if the upload
// has been completed. The Close() method must be called on an ObjectCreateFile
// before this method.
func (file *ObjectCreateFile) Headers() (Headers, error) {
// error out if upload is not complete.
select {
case <-file.done:
default:
return nil, fmt.Errorf("Cannot get metadata, object upload failed or has not yet completed.")
}
return file.headers, nil
}
// Check it satisfies the interface
var _ io.WriteCloser = &ObjectCreateFile{}
// objectPutHeaders create a set of headers for a PUT
//
// It guesses the contentType from the objectName if it isn't set
//
// checkHash may be changed
func objectPutHeaders(objectName string, checkHash *bool, Hash string, contentType string, h Headers) Headers {
if contentType == "" {
contentType = mime.TypeByExtension(path.Ext(objectName))
if contentType == "" {
contentType = "application/octet-stream"
}
}
// Meta stuff
extraHeaders := map[string]string{
"Content-Type": contentType,
}
for key, value := range h {
extraHeaders[key] = value
}
if Hash != "" {
extraHeaders["Etag"] = Hash
*checkHash = false // the server will do it
}
return extraHeaders
}
// ObjectCreate creates or updates the object in the container. It
// returns an io.WriteCloser you should write the contents to. You
// MUST call Close() on it and you MUST check the error return from
// Close().
//
// If checkHash is True then it will calculate the MD5 Hash of the
// file as it is being uploaded and check it against that returned
// from the server. If it is wrong then it will return
// ObjectCorrupted on Close()
//
// If you know the MD5 hash of the object ahead of time then set the
// Hash parameter and it will be sent to the server (as an Etag
// header) and the server will check the MD5 itself after the upload,
// and this will return ObjectCorrupted on Close() if it is incorrect.
//
// If you don't want any error protection (not recommended) then set
// checkHash to false and Hash to "".
//
// If contentType is set it will be used, otherwise one will be
// guessed from objectName using mime.TypeByExtension
func (c *Connection) ObjectCreate(container string, objectName string, checkHash bool, Hash string, contentType string, h Headers) (file *ObjectCreateFile, err error) {
extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h)
pipeReader, pipeWriter := io.Pipe()
file = &ObjectCreateFile{
hash: md5.New(),
checkHash: checkHash,
pipeReader: pipeReader,
pipeWriter: pipeWriter,
done: make(chan struct{}),
}
// Run the PUT in the background piping it data
go func() {
opts := RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "PUT",
Headers: extraHeaders,
Body: pipeReader,
NoResponse: true,
ErrorMap: objectErrorMap,
}
file.resp, file.headers, file.err = c.storage(opts)
// Signal finished
pipeReader.Close()
close(file.done)
}()
return
}
func (c *Connection) ObjectSymlinkCreate(container string, symlink string, targetAccount string, targetContainer string, targetObject string, targetEtag string) (headers Headers, err error) {
EMPTY_MD5 := "d41d8cd98f00b204e9800998ecf8427e"
symHeaders := Headers{}
contents := bytes.NewBufferString("")
if targetAccount != "" {
symHeaders["X-Symlink-Target-Account"] = targetAccount
}
if targetEtag != "" {
symHeaders["X-Symlink-Target-Etag"] = targetEtag
}
symHeaders["X-Symlink-Target"] = fmt.Sprintf("%s/%s", targetContainer, targetObject)
_, err = c.ObjectPut(container, symlink, contents, true, EMPTY_MD5, "application/symlink", symHeaders)
return
}
func (c *Connection) objectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers, parameters url.Values) (headers Headers, err error) {
extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h)
hash := md5.New()
var body io.Reader = contents
if checkHash {
body = io.TeeReader(contents, hash)
}
_, headers, err = c.storage(RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "PUT",
Headers: extraHeaders,
Body: body,
NoResponse: true,
ErrorMap: objectErrorMap,
Parameters: parameters,
})
if err != nil {
return
}
if checkHash {
receivedMd5 := strings.ToLower(headers["Etag"])
calculatedMd5 := fmt.Sprintf("%x", hash.Sum(nil))
if receivedMd5 != calculatedMd5 {
err = ObjectCorrupted
return
}
}
return
}
// ObjectPut creates or updates the path in the container from
// contents. contents should be an open io.Reader which will have all
// its contents read.
//
// This is a low level interface.
//
// If checkHash is True then it will calculate the MD5 Hash of the
// file as it is being uploaded and check it against that returned
// from the server. If it is wrong then it will return
// ObjectCorrupted.
//
// If you know the MD5 hash of the object ahead of time then set the
// Hash parameter and it will be sent to the server (as an Etag
// header) and the server will check the MD5 itself after the upload,
// and this will return ObjectCorrupted if it is incorrect.
//
// If you don't want any error protection (not recommended) then set
// checkHash to false and Hash to "".
//
// If contentType is set it will be used, otherwise one will be
// guessed from objectName using mime.TypeByExtension
func (c *Connection) ObjectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers) (headers Headers, err error) {
return c.objectPut(container, objectName, contents, checkHash, Hash, contentType, h, nil)
}
// ObjectPutBytes creates an object from a []byte in a container.
//
// This is a simplified interface which checks the MD5.
func (c *Connection) ObjectPutBytes(container string, objectName string, contents []byte, contentType string) (err error) {
buf := bytes.NewBuffer(contents)
h := Headers{"Content-Length": strconv.Itoa(len(contents))}
_, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h)
return
}
// ObjectPutString creates an object from a string in a container.
//
// This is a simplified interface which checks the MD5
func (c *Connection) ObjectPutString(container string, objectName string, contents string, contentType string) (err error) {
buf := strings.NewReader(contents)
h := Headers{"Content-Length": strconv.Itoa(len(contents))}
_, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h)
return
}
// ObjectOpenFile represents a swift object open for reading
type ObjectOpenFile struct {
connection *Connection // stored copy of Connection used in Open
container string // stored copy of container used in Open
objectName string // stored copy of objectName used in Open
headers Headers // stored copy of headers used in Open
resp *http.Response // http connection
body io.Reader // read data from this
checkHash bool // true if checking MD5
hash hash.Hash // currently accumulating MD5
bytes int64 // number of bytes read on this connection
eof bool // whether we have read end of file
pos int64 // current position when reading
lengthOk bool // whether length is valid
length int64 // length of the object if read
seeked bool // whether we have seeked this file or not
overSeeked bool // set if we have seeked to the end or beyond
}
// Read bytes from the object - see io.Reader
func (file *ObjectOpenFile) Read(p []byte) (n int, err error) {
if file.overSeeked {
return 0, io.EOF
}
n, err = file.body.Read(p)
file.bytes += int64(n)
file.pos += int64(n)
if err == io.EOF {
file.eof = true
}
return
}
// Seek sets the offset for the next Read to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1
// means relative to the current offset, and 2 means relative to the
// end. Seek returns the new offset and an Error, if any.
//
// Seek uses HTTP Range headers which, if the file pointer is moved,
// will involve reopening the HTTP connection.
//
// Note that you can't seek to the end of a file or beyond; HTTP Range
// requests don't support the file pointer being outside the data,
// unlike os.File
//
// Seek(0, 1) will return the current file pointer.
func (file *ObjectOpenFile) Seek(offset int64, whence int) (newPos int64, err error) {
file.overSeeked = false
switch whence {
case 0: // relative to start
newPos = offset
case 1: // relative to current
newPos = file.pos + offset
case 2: // relative to end
if !file.lengthOk {
return file.pos, newError(0, "Length of file unknown so can't seek from end")
}
newPos = file.length + offset
if offset >= 0 {
file.overSeeked = true
return
}
default:
panic("Unknown whence in ObjectOpenFile.Seek")
}
// If at correct position (quite likely), do nothing
if newPos == file.pos {
return
}
// Close the file...
file.seeked = true
err = file.Close()
if err != nil {
return
}
// ...and re-open with a Range header
if file.headers == nil {
file.headers = Headers{}
}
if newPos > 0 {
file.headers["Range"] = fmt.Sprintf("bytes=%d-", newPos)
} else {
delete(file.headers, "Range")
}
newFile, _, err := file.connection.ObjectOpen(file.container, file.objectName, false, file.headers)
if err != nil {
return
}
// Update the file
file.resp = newFile.resp
file.body = newFile.body
file.checkHash = false
file.pos = newPos
return
}
// Length gets the objects content length either from a cached copy or
// from the server.
func (file *ObjectOpenFile) Length() (int64, error) {
if !file.lengthOk {
info, _, err := file.connection.Object(file.container, file.objectName)
file.length = info.Bytes
file.lengthOk = (err == nil)
return file.length, err
}
return file.length, nil
}
// Close the object and checks the length and md5sum if it was
// required and all the object was read
func (file *ObjectOpenFile) Close() (err error) {
// Close the body at the end
defer checkClose(file.resp.Body, &err)
// If not end of file or seeked then can't check anything
if !file.eof || file.seeked {
return
}
// Check the MD5 sum if requested
if file.checkHash {
receivedMd5 := strings.ToLower(file.resp.Header.Get("Etag"))
calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil))
if receivedMd5 != calculatedMd5 {
err = ObjectCorrupted
return
}
}
// Check to see we read the correct number of bytes
if file.lengthOk && file.length != file.bytes {
err = ObjectCorrupted
return
}
return
}
// Check it satisfies the interfaces
var _ io.ReadCloser = &ObjectOpenFile{}
var _ io.Seeker = &ObjectOpenFile{}
func (c *Connection) objectOpenBase(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) {
var resp *http.Response
opts := RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "GET",
ErrorMap: objectErrorMap,
Headers: h,
Parameters: parameters,
}
resp, headers, err = c.storage(opts)
if err != nil {
return
}
// Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set
if checkHash && headers.IsLargeObject() {
// log.Printf("swift: turning off md5 checking on object with manifest %v", objectName)
checkHash = false
}
file = &ObjectOpenFile{
connection: c,
container: container,
objectName: objectName,
headers: h,
resp: resp,
checkHash: checkHash,
body: resp.Body,
}
if checkHash {
file.hash = md5.New()
file.body = io.TeeReader(resp.Body, file.hash)
}
// Read Content-Length
if resp.Header.Get("Content-Length") != "" {
file.length, err = getInt64FromHeader(resp, "Content-Length")
file.lengthOk = (err == nil)
}
return
}
func (c *Connection) objectOpen(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) {
err = withLORetry(0, func() (Headers, int64, error) {
file, headers, err = c.objectOpenBase(container, objectName, checkHash, h, parameters)
if err != nil {
return headers, 0, err
}
return headers, file.length, nil
})
return
}
// ObjectOpen returns an ObjectOpenFile for reading the contents of
// the object. This satisfies the io.ReadCloser and the io.Seeker
// interfaces.
//
// You must call Close() on contents when finished
//
// Returns the headers of the response.
//
// If checkHash is true then it will calculate the md5sum of the file
// as it is being received and check it against that returned from the
// server. If it is wrong then it will return ObjectCorrupted. It
// will also check the length returned. No checking will be done if
// you don't read all the contents.
//
// Note that objects with X-Object-Manifest or X-Static-Large-Object
// set won't ever have their md5sum's checked as the md5sum reported
// on the object is actually the md5sum of the md5sums of the
// parts. This isn't very helpful to detect a corrupted download as
// the size of the parts aren't known without doing more operations.
// If you want to ensure integrity of an object with a manifest then
// you will need to download everything in the manifest separately.
//
// headers["Content-Type"] will give the content type if desired.
func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) {
return c.objectOpen(container, objectName, checkHash, h, nil)
}
// ObjectGet gets the object into the io.Writer contents.
//
// Returns the headers of the response.
//
// If checkHash is true then it will calculate the md5sum of the file
// as it is being received and check it against that returned from the
// server. If it is wrong then it will return ObjectCorrupted.
//
// headers["Content-Type"] will give the content type if desired.
func (c *Connection) ObjectGet(container string, objectName string, contents io.Writer, checkHash bool, h Headers) (headers Headers, err error) {
file, headers, err := c.ObjectOpen(container, objectName, checkHash, h)
if err != nil {
return
}
defer checkClose(file, &err)
_, err = io.Copy(contents, file)
return
}
// ObjectGetBytes returns an object as a []byte.
//
// This is a simplified interface which checks the MD5
func (c *Connection) ObjectGetBytes(container string, objectName string) (contents []byte, err error) {
var buf bytes.Buffer
_, err = c.ObjectGet(container, objectName, &buf, true, nil)
contents = buf.Bytes()
return
}
// ObjectGetString returns an object as a string.
//
// This is a simplified interface which checks the MD5
func (c *Connection) ObjectGetString(container string, objectName string) (contents string, err error) {
var buf bytes.Buffer
_, err = c.ObjectGet(container, objectName, &buf, true, nil)
contents = buf.String()
return
}
// ObjectDelete deletes the object.
//
// May return ObjectNotFound if the object isn't found
func (c *Connection) ObjectDelete(container string, objectName string) error {
_, _, err := c.storage(RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "DELETE",
ErrorMap: objectErrorMap,
})
return err
}
// ObjectTempUrl returns a temporary URL for an object
func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string {
mac := hmac.New(sha1.New, []byte(secretKey))
prefix, _ := url.Parse(c.StorageUrl)
body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName)
mac.Write([]byte(body))
sig := hex.EncodeToString(mac.Sum(nil))
return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix())
}
// parseResponseStatus parses string like "200 OK" and returns Error.
//
// For status codes beween 200 and 299, this returns nil.
func parseResponseStatus(resp string, errorMap errorMap) error {
code := 0
reason := resp
t := strings.SplitN(resp, " ", 2)
if len(t) == 2 {
ncode, err := strconv.Atoi(t[0])
if err == nil {
code = ncode
reason = t[1]
}
}
if errorMap != nil {
if err, ok := errorMap[code]; ok {
return err
}
}
if 200 <= code && code <= 299 {
return nil
}
return newError(code, reason)
}
// BulkDeleteResult stores results of BulkDelete().
//
// Individual errors may (or may not) be returned by Errors.
// Errors is a map whose keys are a full path of where the object was
// to be deleted, and whose values are Error objects. A full path of
// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH".
type BulkDeleteResult struct {
NumberNotFound int64 // # of objects not found.
NumberDeleted int64 // # of deleted objects.
Errors map[string]error // Mapping between object name and an error.
Headers Headers // Response HTTP headers.
}
func (c *Connection) doBulkDelete(objects []string, h Headers) (result BulkDeleteResult, err error) {
var buffer bytes.Buffer
for _, s := range objects {
u := url.URL{Path: s}
buffer.WriteString(u.String() + "\n")
}
extraHeaders := Headers{
"Accept": "application/json",
"Content-Type": "text/plain",
"Content-Length": strconv.Itoa(buffer.Len()),
}
for key, value := range h {
extraHeaders[key] = value
}
resp, headers, err := c.storage(RequestOpts{
Operation: "DELETE",
Parameters: url.Values{"bulk-delete": []string{"1"}},
Headers: extraHeaders,
ErrorMap: ContainerErrorMap,
Body: &buffer,
})
if err != nil {
return
}
var jsonResult struct {
NotFound int64 `json:"Number Not Found"`
Status string `json:"Response Status"`
Errors [][]string
Deleted int64 `json:"Number Deleted"`
}
err = readJson(resp, &jsonResult)
if err != nil {
return
}
err = parseResponseStatus(jsonResult.Status, objectErrorMap)
result.NumberNotFound = jsonResult.NotFound
result.NumberDeleted = jsonResult.Deleted
result.Headers = headers
el := make(map[string]error, len(jsonResult.Errors))
for _, t := range jsonResult.Errors {
if len(t) != 2 {
continue
}
el[t[0]] = parseResponseStatus(t[1], objectErrorMap)
}
result.Errors = el
return
}
// BulkDelete deletes multiple objectNames from container in one operation.
//
// Some servers may not accept bulk-delete requests since bulk-delete is
// an optional feature of swift - these will return the Forbidden error.
//
// See also:
// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html
// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html
func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) {
return c.BulkDeleteHeaders(container, objectNames, nil)
}
// BulkDeleteHeaders deletes multiple objectNames from container in one operation.
//
// Some servers may not accept bulk-delete requests since bulk-delete is
// an optional feature of swift - these will return the Forbidden error.
//
// See also:
// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html
// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html
func (c *Connection) BulkDeleteHeaders(container string, objectNames []string, h Headers) (result BulkDeleteResult, err error) {
if len(objectNames) == 0 {
result.Errors = make(map[string]error)
return
}
fullPaths := make([]string, len(objectNames))
for i, name := range objectNames {
fullPaths[i] = fmt.Sprintf("/%s/%s", container, name)
}
return c.doBulkDelete(fullPaths, h)
}
// BulkUploadResult stores results of BulkUpload().
//
// Individual errors may (or may not) be returned by Errors.
// Errors is a map whose keys are a full path of where an object was
// to be created, and whose values are Error objects. A full path of
// object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH".
type BulkUploadResult struct {
NumberCreated int64 // # of created objects.
Errors map[string]error // Mapping between object name and an error.
Headers Headers // Response HTTP headers.
}
// BulkUpload uploads multiple files in one operation.
//
// uploadPath can be empty, a container name, or a pseudo-directory
// within a container. If uploadPath is empty, new containers may be
// automatically created.
//
// Files are read from dataStream. The format of the stream is specified
// by the format parameter. Available formats are:
// * UploadTar - Plain tar stream.
// * UploadTarGzip - Gzip compressed tar stream.
// * UploadTarBzip2 - Bzip2 compressed tar stream.
//
// Some servers may not accept bulk-upload requests since bulk-upload is
// an optional feature of swift - these will return the Forbidden error.
//
// See also:
// * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-extract-archive.html
// * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Extract_Archive-d1e2338.html
func (c *Connection) BulkUpload(uploadPath string, dataStream io.Reader, format string, h Headers) (result BulkUploadResult, err error) {
extraHeaders := Headers{"Accept": "application/json"}
for key, value := range h {
extraHeaders[key] = value
}
// The following code abuses Container parameter intentionally.
// The best fix might be to rename Container to UploadPath.
resp, headers, err := c.storage(RequestOpts{
Container: uploadPath,
Operation: "PUT",
Parameters: url.Values{"extract-archive": []string{format}},
Headers: extraHeaders,
ErrorMap: ContainerErrorMap,
Body: dataStream,
})
if err != nil {
return
}
// Detect old servers which don't support this feature
if headers["Content-Type"] != "application/json" {
err = Forbidden
return
}
var jsonResult struct {
Created int64 `json:"Number Files Created"`
Status string `json:"Response Status"`
Errors [][]string
}
err = readJson(resp, &jsonResult)
if err != nil {
return
}
err = parseResponseStatus(jsonResult.Status, objectErrorMap)
result.NumberCreated = jsonResult.Created
result.Headers = headers
el := make(map[string]error, len(jsonResult.Errors))
for _, t := range jsonResult.Errors {
if len(t) != 2 {
continue
}
el[t[0]] = parseResponseStatus(t[1], objectErrorMap)
}
result.Errors = el
return
}
// Object returns info about a single object including any metadata in the header.
//
// May return ObjectNotFound.
//
// Use headers.ObjectMetadata() to read the metadata in the Headers.
func (c *Connection) Object(container string, objectName string) (info Object, headers Headers, err error) {
err = withLORetry(0, func() (Headers, int64, error) {
info, headers, err = c.objectBase(container, objectName)
if err != nil {
return headers, 0, err
}
return headers, info.Bytes, nil
})
return
}
func (c *Connection) objectBase(container string, objectName string) (info Object, headers Headers, err error) {
var resp *http.Response
resp, headers, err = c.storage(RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "HEAD",
ErrorMap: objectErrorMap,
NoResponse: true,
})
if err != nil {
return
}
// Parse the headers into the struct
// HTTP/1.1 200 OK
// Date: Thu, 07 Jun 2010 20:59:39 GMT
// Server: Apache
// Last-Modified: Fri, 12 Jun 2010 13:40:18 GMT
// ETag: 8a964ee2a5e88be344f36c22562a6486
// Content-Length: 512000
// Content-Type: text/plain; charset=UTF-8
// X-Object-Meta-Meat: Bacon
// X-Object-Meta-Fruit: Bacon
// X-Object-Meta-Veggie: Bacon
// X-Object-Meta-Dairy: Bacon
info.Name = objectName
info.ContentType = resp.Header.Get("Content-Type")
if resp.Header.Get("Content-Length") != "" {
if info.Bytes, err = getInt64FromHeader(resp, "Content-Length"); err != nil {
return
}
}
// Currently ceph doesn't return a Last-Modified header for DLO manifests without any segments
// See ceph http://tracker.ceph.com/issues/15812
if resp.Header.Get("Last-Modified") != "" {
info.ServerLastModified = resp.Header.Get("Last-Modified")
if info.LastModified, err = time.Parse(http.TimeFormat, info.ServerLastModified); err != nil {
return
}
}
info.Hash = resp.Header.Get("Etag")
if resp.Header.Get("X-Object-Manifest") != "" {
info.ObjectType = DynamicLargeObjectType
} else if resp.Header.Get("X-Static-Large-Object") != "" {
info.ObjectType = StaticLargeObjectType
}
return
}
// ObjectUpdate adds, replaces or removes object metadata.
//
// Add or Update keys by mentioning them in the Metadata. Use
// Metadata.ObjectHeaders and Headers.ObjectMetadata to convert your
// Metadata to and from normal HTTP headers.
//
// This removes all metadata previously added to the object and
// replaces it with that passed in so to delete keys, just don't
// mention them the headers you pass in.
//
// Object metadata can only be read with Object() not with Objects().
//
// This can also be used to set headers not already assigned such as
// X-Delete-At or X-Delete-After for expiring objects.
//
// You cannot use this to change any of the object's other headers
// such as Content-Type, ETag, etc.
//
// Refer to copying an object when you need to update metadata or
// other headers such as Content-Type or CORS headers.
//
// May return ObjectNotFound.
func (c *Connection) ObjectUpdate(container string, objectName string, h Headers) error {
_, _, err := c.storage(RequestOpts{
Container: container,
ObjectName: objectName,
Operation: "POST",
ErrorMap: objectErrorMap,
NoResponse: true,
Headers: h,
})
return err
}
// urlPathEscape escapes URL path the in string using URL escaping rules
//
// This mimics url.PathEscape which only available from go 1.8
func urlPathEscape(in string) string {
var u url.URL
u.Path = in
return u.String()
}
// ObjectCopy does a server side copy of an object to a new position
//
// All metadata is preserved. If metadata is set in the headers then
// it overrides the old metadata on the copied object.
//
// The destination container must exist before the copy.
//
// You can use this to copy an object to itself - this is the only way
// to update the content type of an object.
func (c *Connection) ObjectCopy(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string, h Headers) (headers Headers, err error) {
// Meta stuff
extraHeaders := map[string]string{
"Destination": urlPathEscape(dstContainer + "/" + dstObjectName),
}
for key, value := range h {
extraHeaders[key] = value
}
_, headers, err = c.storage(RequestOpts{
Container: srcContainer,
ObjectName: srcObjectName,
Operation: "COPY",
ErrorMap: objectErrorMap,
NoResponse: true,
Headers: extraHeaders,
})
return
}
// ObjectMove does a server side move of an object to a new position
//
// This is a convenience method which calls ObjectCopy then ObjectDelete
//
// All metadata is preserved.
//
// The destination container must exist before the copy.
func (c *Connection) ObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) (err error) {
_, err = c.ObjectCopy(srcContainer, srcObjectName, dstContainer, dstObjectName, nil)
if err != nil {
return
}
return c.ObjectDelete(srcContainer, srcObjectName)
}
// ObjectUpdateContentType updates the content type of an object
//
// This is a convenience method which calls ObjectCopy
//
// All other metadata is preserved.
func (c *Connection) ObjectUpdateContentType(container string, objectName string, contentType string) (err error) {
h := Headers{"Content-Type": contentType}
_, err = c.ObjectCopy(container, objectName, container, objectName, h)
return
}
// ------------------------------------------------------------
// VersionContainerCreate is a helper method for creating and enabling version controlled containers.
//
// It builds the current object container, the non-current object version container, and enables versioning.
//
// If the server doesn't support versioning then it will return
// Forbidden however it will have created both the containers at that point.
func (c *Connection) VersionContainerCreate(current, version string) error {
if err := c.ContainerCreate(version, nil); err != nil {
return err
}
if err := c.ContainerCreate(current, nil); err != nil {
return err
}
if err := c.VersionEnable(current, version); err != nil {
return err
}
return nil
}
// VersionEnable enables versioning on the current container with version as the tracking container.
//
// May return Forbidden if this isn't supported by the server
func (c *Connection) VersionEnable(current, version string) error {
h := Headers{"X-Versions-Location": version}
if err := c.ContainerUpdate(current, h); err != nil {
return err
}
// Check to see if the header was set properly
_, headers, err := c.Container(current)
if err != nil {
return err
}
// If failed to set versions header, return Forbidden as the server doesn't support this
if headers["X-Versions-Location"] != version {
return Forbidden
}
return nil
}
// VersionDisable disables versioning on the current container.
func (c *Connection) VersionDisable(current string) error {
h := Headers{"X-Versions-Location": ""}
if err := c.ContainerUpdate(current, h); err != nil {
return err
}
return nil
}
// VersionObjectList returns a list of older versions of the object.
//
// Objects are returned in the format <length><object_name>/<timestamp>
func (c *Connection) VersionObjectList(version, object string) ([]string, error) {
opts := &ObjectsOpts{
// <3-character zero-padded hexadecimal character length><object name>/
Prefix: fmt.Sprintf("%03x", len(object)) + object + "/",
}
return c.ObjectNames(version, opts)
}