Codebase list golang-github-ncw-swift / upstream/1.0.52
New upstream version 1.0.52 Drew Parsons 3 years ago
6 changed file(s) with 440 addition(s) and 23 deletion(s). Raw diff Collapse all Expand all
+6
-4
.travis.yml less more
1212 - 1.10.x
1313 - 1.11.x
1414 - 1.12.x
15 - 1.13.x
16 - 1.14.x
1517 - master
1618
1719 matrix:
1820 include:
19 - go: 1.12.x
21 - go: 1.14.x
2022 env: TEST_REAL_SERVER=rackspace
21 - go: 1.12.x
23 - go: 1.14.x
2224 env: TEST_REAL_SERVER=memset
2325 allow_failures:
24 - go: 1.12.x
26 - go: 1.14.x
2527 env: TEST_REAL_SERVER=rackspace
26 - go: 1.12.x
28 - go: 1.14.x
2729 env: TEST_REAL_SERVER=memset
2830 install: go test -i ./...
2931 script:
+4
-0
README.md less more
156156 - Jérémy Clerc <jeremy.clerc@tagpay.fr>
157157 - 4xicom <37339705+4xicom@users.noreply.github.com>
158158 - Bo <bo@4xi.com>
159 - Thiago da Silva <thiagodasilva@users.noreply.github.com>
160 - Brandon WELSCH <dev@brandon-welsch.eu>
161 - Damien Tournoud <damien@platform.sh>
162 - Pedro Kiefer <pedro@kiefer.com.br>
+1
-1
largeobjects.go less more
221221 for i, obj := range objects {
222222 filenames[i] = obj[0] + "/" + obj[1]
223223 }
224 _, err = c.doBulkDelete(filenames)
224 _, err = c.doBulkDelete(filenames, nil)
225225 // Don't fail on ObjectNotFound because eventual consistency
226226 // makes this situation normal.
227227 if err != nil && err != Forbidden && err != ObjectNotFound {
+56
-11
swift.go less more
124124 Expires time.Time // time the token expires, may be Zero if unknown
125125 client *http.Client
126126 Auth Authenticator `json:"-" xml:"-"` // the current authenticator
127 authLock sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth
127 authLock *sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth
128128 // swiftInfo is filled after QueryInfo is called
129129 swiftInfo SwiftInfo
130130 }
457457 // If you don't call it before calling one of the connection methods
458458 // then it will be called for you on the first access.
459459 func (c *Connection) Authenticate() (err error) {
460 if c.authLock == nil {
461 c.authLock = &sync.Mutex{}
462 }
460463 c.authLock.Lock()
461464 defer c.authLock.Unlock()
462465 return c.authenticate()
579582 //
580583 // Doesn't actually check the credentials against the server.
581584 func (c *Connection) Authenticated() bool {
585 if c.authLock == nil {
586 c.authLock = &sync.Mutex{}
587 }
582588 c.authLock.Lock()
583589 defer c.authLock.Unlock()
584590 return c.authenticated()
957963 return containers, nil
958964 }
959965
960 // ContainerNamesAll is like ContainerNamess but it returns all the Containers
966 // ContainerNamesAll is like ContainerNames but it returns all the Containers
961967 //
962968 // It calls ContainerNames multiple times using the Marker parameter
963969 //
13651371 return
13661372 }
13671373
1374 // CloseWithError closes the object, aborting the upload.
1375 func (file *ObjectCreateFile) CloseWithError(err error) error {
1376 _ = file.pipeWriter.CloseWithError(err)
1377 <-file.done
1378 return nil
1379 }
1380
13681381 // Close the object and checks the md5sum if it was required.
13691382 //
13701383 // Also returns any other errors from the server (eg container not
14811494 pipeReader.Close()
14821495 close(file.done)
14831496 }()
1497 return
1498 }
1499
1500 func (c *Connection) ObjectSymlinkCreate(container string, symlink string, targetAccount string, targetContainer string, targetObject string, targetEtag string) (headers Headers, err error) {
1501
1502 EMPTY_MD5 := "d41d8cd98f00b204e9800998ecf8427e"
1503 symHeaders := Headers{}
1504 contents := bytes.NewBufferString("")
1505 if targetAccount != "" {
1506 symHeaders["X-Symlink-Target-Account"] = targetAccount
1507 }
1508 if targetEtag != "" {
1509 symHeaders["X-Symlink-Target-Etag"] = targetEtag
1510 }
1511 symHeaders["X-Symlink-Target"] = fmt.Sprintf("%s/%s", targetContainer, targetObject)
1512 _, err = c.ObjectPut(container, symlink, contents, true, EMPTY_MD5, "application/symlink", symHeaders)
14841513 return
14851514 }
14861515
18791908 Headers Headers // Response HTTP headers.
18801909 }
18811910
1882 func (c *Connection) doBulkDelete(objects []string) (result BulkDeleteResult, err error) {
1911 func (c *Connection) doBulkDelete(objects []string, h Headers) (result BulkDeleteResult, err error) {
18831912 var buffer bytes.Buffer
18841913 for _, s := range objects {
18851914 u := url.URL{Path: s}
18861915 buffer.WriteString(u.String() + "\n")
18871916 }
1917 extraHeaders := Headers{
1918 "Accept": "application/json",
1919 "Content-Type": "text/plain",
1920 "Content-Length": strconv.Itoa(buffer.Len()),
1921 }
1922 for key, value := range h {
1923 extraHeaders[key] = value
1924 }
18881925 resp, headers, err := c.storage(RequestOpts{
18891926 Operation: "DELETE",
18901927 Parameters: url.Values{"bulk-delete": []string{"1"}},
1891 Headers: Headers{
1892 "Accept": "application/json",
1893 "Content-Type": "text/plain",
1894 "Content-Length": strconv.Itoa(buffer.Len()),
1895 },
1896 ErrorMap: ContainerErrorMap,
1897 Body: &buffer,
1928 Headers: extraHeaders,
1929 ErrorMap: ContainerErrorMap,
1930 Body: &buffer,
18981931 })
18991932 if err != nil {
19001933 return
19341967 // * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html
19351968 // * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html
19361969 func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) {
1970 return c.BulkDeleteHeaders(container, objectNames, nil)
1971 }
1972
1973 // BulkDeleteHeaders deletes multiple objectNames from container in one operation.
1974 //
1975 // Some servers may not accept bulk-delete requests since bulk-delete is
1976 // an optional feature of swift - these will return the Forbidden error.
1977 //
1978 // See also:
1979 // * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html
1980 // * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html
1981 func (c *Connection) BulkDeleteHeaders(container string, objectNames []string, h Headers) (result BulkDeleteResult, err error) {
19371982 if len(objectNames) == 0 {
19381983 result.Errors = make(map[string]error)
19391984 return
19421987 for i, name := range objectNames {
19431988 fullPaths[i] = fmt.Sprintf("/%s/%s", container, name)
19441989 }
1945 return c.doBulkDelete(fullPaths)
1990 return c.doBulkDelete(fullPaths, h)
19461991 }
19471992
19481993 // BulkUploadResult stores results of BulkUpload().
+155
-0
swift_test.go less more
5050 CURRENT_CONTAINER = "GoSwiftUnitTestCurrent"
5151 OBJECT = "test_object"
5252 OBJECT2 = "test_object2"
53 SYMLINK_OBJECT = "test_symlink"
54 SYMLINK_OBJECT2 = "test_symlink2"
5355 EMPTYOBJECT = "empty_test_object"
5456 CONTENTS = "12345"
5557 CONTENTS2 = "54321"
5658 CONTENT_SIZE = int64(len(CONTENTS))
5759 CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b"
60 CONTENT2_MD5 = "01cfcd4f6b8770febfb40cb906715822"
5861 EMPTY_MD5 = "d41d8cd98f00b204e9800998ecf8427e"
5962 SECRET_KEY = "b3968d0207b54ece87cccc06515a89d4"
6063 )
296299 func isV3Api() bool {
297300 AuthUrl := os.Getenv("SWIFT_AUTH_URL")
298301 return strings.Contains(AuthUrl, "v3")
302 }
303
304 func getSwinftInfo(t *testing.T) (info swift.SwiftInfo, err error) {
305 c, rollback := makeConnectionAuth(t)
306 defer rollback()
307 return c.QueryInfo()
299308 }
300309
301310 func TestTransport(t *testing.T) {
907916 }
908917 }
909918
919 func TestSymlinkObject(t *testing.T) {
920 info, err := getSwinftInfo(t)
921 if err != nil {
922 t.Fatal(err)
923 }
924 if _, ok := info["symlink"]; !ok {
925 // skip, symlink not supported
926 t.Skip("skip, symlink not supported")
927 return
928 }
929 c, rollback := makeConnectionWithContainer(t)
930 defer rollback()
931
932 // write target objects
933 err = c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "text/potato")
934 if err != nil {
935 t.Fatal(err)
936 }
937 defer func() {
938 err = c.ObjectDelete(CONTAINER, OBJECT)
939 if err != nil {
940 t.Error(err)
941 }
942 }()
943
944 // test dynamic link
945 _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT, "", CONTAINER, OBJECT, "")
946 if err != nil {
947 t.Fatal(err)
948 }
949 defer func() {
950 err = c.ObjectDelete(CONTAINER, SYMLINK_OBJECT)
951 if err != nil {
952 t.Error(err)
953 }
954 }()
955
956 md, _, err := c.Object(CONTAINER, SYMLINK_OBJECT)
957 if err != nil {
958 t.Error(err)
959 }
960 if md.ContentType != "text/potato" {
961 t.Error("Bad content type", md.ContentType)
962 }
963 if md.Bytes != CONTENT_SIZE {
964 t.Errorf("Bad length want 5 got %v", md.Bytes)
965 }
966 if md.Hash != CONTENT_MD5 {
967 t.Errorf("Bad MD5 want %v got %v", CONTENT_MD5, md.Hash)
968 }
969
970 }
971
972 func TestStaticSymlinkObject(t *testing.T) {
973 info, err := getSwinftInfo(t)
974 if err != nil {
975 t.Fatal(err)
976 }
977 if sym, ok := info["symlink"].(map[string]interface{}); ok {
978 if _, ok := sym["static_links"]; !ok {
979 t.Skip("skip, static symlink not supported")
980 return
981 }
982 } else {
983 t.Skip("skip, symlink not supported")
984 return
985 }
986
987 c, rollback := makeConnectionWithContainer(t)
988 defer rollback()
989
990 // write target objects
991 err = c.ObjectPutBytes(CONTAINER, OBJECT2, []byte(CONTENTS2), "text/tomato")
992 if err != nil {
993 t.Fatal(err)
994 }
995 defer func() {
996 err = c.ObjectDelete(CONTAINER, OBJECT2)
997 if err != nil {
998 t.Error(err)
999 }
1000 }()
1001
1002 // test static link
1003 // first with the wrong target etag
1004 _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT2, "", CONTAINER, OBJECT2, CONTENT_MD5)
1005 if err == nil {
1006 t.Error("Symlink with wrong target etag should have failed")
1007 }
1008
1009 _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT2, "", CONTAINER, OBJECT2, CONTENT2_MD5)
1010 if err != nil {
1011 t.Fatal(err)
1012 }
1013 defer func() {
1014 err = c.ObjectDelete(CONTAINER, SYMLINK_OBJECT2)
1015 if err != nil {
1016 t.Error(err)
1017 }
1018 }()
1019
1020 md, _, err := c.Object(CONTAINER, SYMLINK_OBJECT2)
1021 if err != nil {
1022 t.Error(err)
1023 }
1024 if md.ContentType != "text/tomato" {
1025 t.Error("Bad content type", md.ContentType)
1026 }
1027 if md.Bytes != CONTENT_SIZE {
1028 t.Errorf("Bad length want 5 got %v", md.Bytes)
1029 }
1030 if md.Hash != CONTENT2_MD5 {
1031 t.Errorf("Bad MD5 want %v got %v", CONTENT2_MD5, md.Hash)
1032 }
1033 }
1034
9101035 func TestObjectPutBytes(t *testing.T) {
9111036 c, rollback := makeConnectionWithContainer(t)
9121037 defer rollback()
10421167 err = out.Close()
10431168 if err != swift.ObjectCorrupted {
10441169 t.Error("Expecting object corrupted not", err)
1170 }
1171 }
1172
1173 func TestObjectCreateAbort(t *testing.T) {
1174 c, rollback := makeConnectionWithContainer(t)
1175 defer rollback()
1176
1177 out, err := c.ObjectCreate(CONTAINER, OBJECT2, true, "", "", nil)
1178 if err != nil {
1179 t.Fatal(err)
1180 }
1181 defer func() {
1182 _ = c.ObjectDelete(CONTAINER, OBJECT2) // Ignore error
1183 }()
1184
1185 expectedContents := "foo"
1186 _, err = out.Write([]byte(expectedContents))
1187 if err != nil {
1188 t.Error(err)
1189 }
1190
1191 errAbort := fmt.Errorf("abort")
1192 err = out.CloseWithError(errAbort)
1193 if err != nil {
1194 t.Errorf("Unexpected error %#v", err)
1195 }
1196
1197 _, err = c.ObjectGetString(CONTAINER, OBJECT2)
1198 if err != swift.ObjectNotFound {
1199 t.Errorf("Unexpected error: %#v", err)
10451200 }
10461201 }
10471202
+218
-7
swifttest/server.go less more
66 package swifttest
77
88 import (
9 "archive/tar"
910 "bytes"
11 "compress/bzip2"
12 "compress/gzip"
1013 "crypto/hmac"
1114 "crypto/md5"
1215 "crypto/rand"
2326 "net/http/httptest"
2427 "net/url"
2528 "path"
29 "reflect"
2630 "regexp"
2731 "sort"
2832 "strconv"
3337 "time"
3438 )
3539
36 const (
40 var (
3741 DEBUG = false
3842 TEST_ACCOUNT = "swifttest"
3943 )
7983 Subdir string `json:"subdir"`
8084 }
8185
86 type AutoExtractResponse struct {
87 CreatedFiles int64 `json:"Number Files Created"`
88 Status string `json:"Response Status"`
89 Errors [][]string `json:"Errors"`
90 }
91
8292 type swiftError struct {
8393 statusCode int
8494 Code string
347357 }
348358
349359 func (r containerResource) put(a *action) interface{} {
350 if a.req.URL.Query().Get("extract-archive") != "" {
351 fatalf(403, "Operation forbidden", "Bulk upload is not supported")
352 }
353
354360 if r.container == nil {
355361 if !validContainerName(r.name) {
356362 fatalf(400, "InvalidContainerName", "The specified container is not valid")
368374 a.user.Containers[r.name] = r.container
369375 a.user.swiftaccount.Containers++
370376 a.user.Unlock()
377 }
378
379 if format := a.req.URL.Query().Get("extract-archive"); format != "" {
380 _, _, objectName, _ := a.srv.parseURL(a.req.URL)
381
382 data, err := ioutil.ReadAll(a.req.Body)
383 if err != nil {
384 fatalf(400, "TODO", "read error")
385 }
386 if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength {
387 fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header")
388 }
389
390 dataReader := bytes.NewReader(data)
391 var reader *tar.Reader
392 switch format {
393 case "tar":
394 reader = tar.NewReader(dataReader)
395 case "tar.gz":
396 gzr, err := gzip.NewReader(dataReader)
397 if err != nil {
398 fatalf(400, "TODO", "Invalid tar.gz")
399 }
400 defer gzr.Close()
401 reader = tar.NewReader(gzr)
402 case "tar.bz2":
403 bzr := bzip2.NewReader(dataReader)
404 reader = tar.NewReader(bzr)
405 default:
406 fatalf(400, "TODO", "Invalid format %s", format)
407 }
408
409 resp := AutoExtractResponse{}
410 for {
411 header, err := reader.Next()
412 if err == io.EOF {
413 break
414 } else if err != nil {
415 //return location, err
416 }
417 if header == nil {
418 continue
419 }
420
421 if header.Typeflag == tar.TypeDir {
422 continue
423 }
424
425 var fullPath string
426 if objectName != "" {
427 fullPath = objectName + "/" + header.Name
428 } else {
429 fullPath = header.Name
430 }
431
432 obj := r.container.objects[fullPath]
433 if obj == nil {
434 // new object
435 obj = &object{
436 name: fullPath,
437 metadata: metadata{
438 meta: make(http.Header),
439 },
440 }
441 atomic.AddInt64(&a.user.Objects, 1)
442 } else {
443 atomic.AddInt64(&r.container.bytes, -header.Size)
444 atomic.AddInt64(&a.user.BytesUsed, -header.Size)
445 }
446
447 // Default content_type
448 obj.content_type = "application/octet-stream"
449
450 // handle extended attributes
451 records := getPAXRecords(header)
452 for k, v := range records {
453 ks := strings.SplitN(k, "SCHILY.xattr.user.", 2)
454 if len(ks) < 2 {
455 continue
456 }
457
458 if ks[1] == "mime_type" {
459 obj.content_type = v
460 }
461
462 if strings.HasPrefix(ks[1], "meta.") {
463 meta := strings.TrimLeft(ks[1], "meta.")
464 obj.meta["X-Object-Meta-"+strings.Title(meta)] = []string{v}
465 }
466 }
467
468 sum := md5.New()
469 objData, err := ioutil.ReadAll(io.TeeReader(reader, sum))
470 if err != nil {
471 errArr := []string{fullPath, fmt.Sprintf("read error: %v", err)}
472 resp.Errors = append(resp.Errors, errArr)
473 continue
474 }
475 gotHash := sum.Sum(nil)
476
477 obj.data = objData
478 obj.checksum = gotHash
479 obj.mtime = time.Now().UTC()
480 r.container.Lock()
481 r.container.objects[fullPath] = obj
482 r.container.bytes += header.Size
483 r.container.Unlock()
484
485 atomic.AddInt64(&a.user.BytesUsed, header.Size)
486 atomic.AddInt64(&resp.CreatedFiles, 1)
487 }
488
489 resp.Status = "201 Accepted"
490 status := 201
491 if len(resp.Errors) > 0 {
492 resp.Status = "400 Error"
493 status = 400
494 }
495 a.w.Header().Set("Content-Type", "application/json")
496 a.w.WriteHeader(status)
497 jsonMarshal(a.w, resp)
371498 }
372499
373500 return nil
392519 }
393520
394521 func (containerResource) copy(a *action) interface{} { return notAllowed() }
522
523 func getPAXRecords(h *tar.Header) map[string]string {
524 rHeader := reflect.ValueOf(h)
525
526 // Try PAXRecords - go 1.10
527 paxField := rHeader.Elem().FieldByName("PAXRecords")
528 if paxField.IsValid() {
529 return paxField.Interface().(map[string]string)
530 }
531
532 // Try Xattrs - go 1.3
533 xAttrsField := rHeader.Elem().FieldByName("Xattrs")
534 if xAttrsField.IsValid() {
535 return xAttrsField.Interface().(map[string]string)
536 }
537 return map[string]string{}
538 }
395539
396540 // validContainerName returns whether name is a valid bucket name.
397541 // Here are the rules, from:
782926 defer func() {
783927 switch err := recover().(type) {
784928 case *swiftError:
929 if DEBUG {
930 fmt.Printf("\t%d - %s\n", err.statusCode, err.Message)
931 }
785932 w.Header().Set("Content-Type", `text/plain; charset=utf-8`)
786933 http.Error(w, err.Message, err.statusCode)
787934 case nil:
788935 default:
936 if DEBUG {
937 fmt.Printf("\tpanic %s\n", err)
938 }
789939 panic(err)
790940 }
791941 }()
10621212 return nil
10631213 }
10641214
1065 func (rootResource) delete(a *action) interface{} {
1215 func (r rootResource) delete(a *action) interface{} {
10661216 if a.req.URL.Query().Get("bulk-delete") == "1" {
1067 fatalf(403, "Operation forbidden", "Bulk delete is not supported")
1217 data, err := ioutil.ReadAll(a.req.Body)
1218 if err != nil {
1219 fatalf(400, "Bad Request", "read error")
1220 }
1221 var nb, notFound int
1222 for _, obj := range strings.Fields(string(data)) {
1223 parts := strings.SplitN(obj, "/", 3)
1224 if len(parts) < 3 {
1225 fatalf(403, "Operation forbidden", "Bulk delete is not supported for containers")
1226 }
1227 b := containerResource{
1228 name: parts[1],
1229 container: a.user.Containers[parts[1]],
1230 }
1231 if b.container == nil {
1232 notFound++
1233 continue
1234 }
1235
1236 objr := objectResource{
1237 name: parts[2],
1238 container: b.container,
1239 }
1240 objr.container.RLock()
1241 if obj := objr.container.objects[objr.name]; obj != nil {
1242 objr.object = obj
1243 }
1244 objr.container.RUnlock()
1245 if objr.object == nil {
1246 notFound++
1247 continue
1248 }
1249
1250 objr.container.Lock()
1251 objr.object.Lock()
1252 objr.container.bytes -= int64(len(objr.object.data))
1253 delete(objr.container.objects, objr.name)
1254 objr.object.Unlock()
1255 objr.container.Unlock()
1256
1257 atomic.AddInt64(&a.user.BytesUsed, -int64(len(objr.object.data)))
1258 atomic.AddInt64(&a.user.Objects, -1)
1259 nb++
1260 }
1261
1262 accept := a.req.Header.Get("Accept")
1263 if strings.HasPrefix(accept, "application/json") {
1264 a.w.Header().Set("Content-Type", "application/json")
1265 resp := map[string]interface{}{
1266 "Number Deleted": nb,
1267 "Number Not Found": notFound,
1268 "Errors": []string{},
1269 "Response Status": "200 OK",
1270 "Response Body": "",
1271 }
1272 jsonMarshal(a.w, resp)
1273 return nil
1274 }
1275
1276 resp := fmt.Sprintf("Number Deleted: %d\nNumber Not Found: %d\nErrors: \nResponse Status: 200 OK\n", nb, notFound)
1277 a.w.Write([]byte(resp))
1278 return nil
10681279 }
10691280
10701281 return notAllowed()