package backup import ( "context" "encoding/xml" "errors" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" ) // ErrS3 is a request the bucket refused or never answered. var ErrS3 = errors.New("backup: the S3 bucket refused the request") // Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives' // backup port or MinIO, a bucket name, and a key pair. A backup only ever // adds objects and lists them, so an append-only key is enough. type Bucket struct { Endpoint string Name string AccessKey string SecretKey string // Region defaults to us-east-1, which S3-compatible servers accept. Region string Client *http.Client // now is the signing clock; tests pin it. now func() time.Time } // Object is one stored object. type Object struct { Key string Size int64 LastModified time.Time } func (b Bucket) region() string { if b.Region == "" { return "us-east-1" } return b.Region } func (b Bucket) client() *http.Client { if b.Client != nil { return b.Client } return http.DefaultClient } func (b Bucket) clock() time.Time { if b.now != nil { return b.now() } return time.Now() } // objectURL is the path-style URL of key, encoded exactly as it is signed. func (b Bucket) objectURL(key string) (*url.URL, error) { u, err := url.Parse(strings.TrimRight(b.Endpoint, "/")) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint) } if b.Name == "" { return nil, fmt.Errorf("%w: no bucket name", ErrS3) } raw := "/" + uriEncode(b.Name) if key != "" { for s := range strings.SplitSeq(key, "/") { raw += "/" + uriEncode(s) } } u.RawPath = u.EscapedPath() + raw u.Path, _ = url.PathUnescape(u.RawPath) return u, nil } func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) { sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock()) resp, err := b.client().Do(req) if err != nil { return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err) } if resp.StatusCode/100 != 2 { defer func() { _ = resp.Body.Close() }() return nil, refusal(resp) } return resp, nil } // Put stores body under key. size is the body's length and payloadSHA256 // its hex SHA-256, which the request is signed over. func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error { u, err := b.objectURL(key) if err != nil { return err } req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body) if err != nil { return err } req.URL = u req.ContentLength = size if size == 0 { req.Body = http.NoBody } resp, err := b.do(req, payloadSHA256) if err != nil { return err } drain(resp) return nil } // List returns up to limit objects whose keys start with prefix, in key // order. func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) { if limit < 1 { limit = 1000 } var out []Object token := "" for { u, err := b.objectURL("") if err != nil { return nil, err } q := url.Values{"list-type": {"2"}, "prefix": {prefix}} if page := limit - len(out); page < 1000 { q.Set("max-keys", strconv.Itoa(page)) } if token != "" { q.Set("continuation-token", token) } u.RawQuery = canonicalQuery(q) req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err } req.URL = u resp, err := b.do(req, emptySHA256) if err != nil { return nil, err } var page struct { Contents []struct { Key string `xml:"Key"` Size int64 `xml:"Size"` LastModified string `xml:"LastModified"` } `xml:"Contents"` IsTruncated bool `xml:"IsTruncated"` NextContinuationToken string `xml:"NextContinuationToken"` } err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page) drain(resp) if err != nil { return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err) } for _, c := range page.Contents { modified, _ := time.Parse(time.RFC3339, c.LastModified) out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified}) } if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit { return out, nil } token = page.NextContinuationToken } } // Refusal is an S3 error response: its HTTP status, S3's code, and the // reason in words. It matches ErrS3. type Refusal struct { Status int Code string text string } func (r *Refusal) Error() string { return r.text } func (r *Refusal) Unwrap() error { return ErrS3 } // refusal turns an S3 error response into words an admin can act on. func refusal(resp *http.Response) error { var e struct { Code string `xml:"Code"` Message string `xml:"Message"` } _ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e) hint := map[string]string{ "SignatureDoesNotMatch": "the secret key is wrong", "InvalidAccessKeyId": "the access key is not known to this server", "AccessDenied": "the key may not do this in this bucket", "NoSuchBucket": "there is no bucket by that name", "SlowDown": "the server asks to slow down", }[e.Code] r := &Refusal{Status: resp.StatusCode, Code: e.Code} switch { case hint != "": r.text = fmt.Sprintf("%v: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint) case e.Code != "": r.text = fmt.Sprintf("%v: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message) default: r.text = fmt.Sprintf("%v: HTTP %d", ErrS3, resp.StatusCode) } return r } // ObjectRefused reports whether err is the bucket refusing one object (an // overwrite a write-once target won't take, a name it won't store) rather // than refusing everything: bad keys, a missing bucket, a request to slow // down, or a server error all stop a run instead. func ObjectRefused(err error) bool { var r *Refusal if !errors.As(err, &r) || r.Status/100 != 4 || r.Status == http.StatusTooManyRequests { return false } switch r.Code { case "SignatureDoesNotMatch", "InvalidAccessKeyId", "NoSuchBucket", "SlowDown", "RequestTimeTooSkewed", "ExpiredToken", "InvalidToken": return false } return true } func drain(resp *http.Response) { _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20)) _ = resp.Body.Close() }