refactor: Reshuffle file structure for better code layout
This gets rid of the package called "server" and instead moves everything into the project root, such that Go actually builds us a binary called `nixery`. This is the first step towards factoring out CLI-based functionality for Nixery.
This commit is contained in:
Vincent Ambo
Vincent Ambo
parent
df88da126a
commit
2b82f1b71a
21 changed files with 83 additions and 114 deletions
|
|
@ -1,219 +0,0 @@
|
|||
// Copyright 2019 Google LLC
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
// use this file except in compliance with the License. You may obtain a copy of
|
||||
// the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
// License for the specific language governing permissions and limitations under
|
||||
// the License.
|
||||
|
||||
// Google Cloud Storage backend for Nixery.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/oauth2/google"
|
||||
)
|
||||
|
||||
// HTTP client to use for direct calls to APIs that are not part of the SDK
|
||||
var client = &http.Client{}
|
||||
|
||||
// API scope needed for renaming objects in GCS
|
||||
const gcsScope = "https://www.googleapis.com/auth/devstorage.read_write"
|
||||
|
||||
type GCSBackend struct {
|
||||
bucket string
|
||||
handle *storage.BucketHandle
|
||||
signing *storage.SignedURLOptions
|
||||
}
|
||||
|
||||
// Constructs a new GCS bucket backend based on the configured
|
||||
// environment variables.
|
||||
func NewGCSBackend() (*GCSBackend, error) {
|
||||
bucket := os.Getenv("GCS_BUCKET")
|
||||
if bucket == "" {
|
||||
return nil, fmt.Errorf("GCS_BUCKET must be configured for GCS usage")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.WithError(err).Fatal("failed to set up Cloud Storage client")
|
||||
}
|
||||
|
||||
handle := client.Bucket(bucket)
|
||||
|
||||
if _, err := handle.Attrs(ctx); err != nil {
|
||||
log.WithError(err).WithField("bucket", bucket).Error("could not access configured bucket")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
signing, err := signingOptsFromEnv()
|
||||
if err != nil {
|
||||
log.WithError(err).Error("failed to configure GCS bucket signing")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GCSBackend{
|
||||
bucket: bucket,
|
||||
handle: handle,
|
||||
signing: signing,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *GCSBackend) Name() string {
|
||||
return "Google Cloud Storage (" + b.bucket + ")"
|
||||
}
|
||||
|
||||
func (b *GCSBackend) Persist(ctx context.Context, path string, f Persister) (string, int64, error) {
|
||||
obj := b.handle.Object(path)
|
||||
w := obj.NewWriter(ctx)
|
||||
|
||||
hash, size, err := f(w)
|
||||
if err != nil {
|
||||
log.WithError(err).WithField("path", path).Error("failed to upload to GCS")
|
||||
return hash, size, err
|
||||
}
|
||||
|
||||
return hash, size, w.Close()
|
||||
}
|
||||
|
||||
func (b *GCSBackend) Fetch(ctx context.Context, path string) (io.ReadCloser, error) {
|
||||
obj := b.handle.Object(path)
|
||||
|
||||
// Probe whether the file exists before trying to fetch it
|
||||
_, err := obj.Attrs(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return obj.NewReader(ctx)
|
||||
}
|
||||
|
||||
// renameObject renames an object in the specified Cloud Storage
|
||||
// bucket.
|
||||
//
|
||||
// The Go API for Cloud Storage does not support renaming objects, but
|
||||
// the HTTP API does. The code below makes the relevant call manually.
|
||||
func (b *GCSBackend) Move(ctx context.Context, old, new string) error {
|
||||
creds, err := google.FindDefaultCredentials(ctx, gcsScope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
token, err := creds.TokenSource.Token()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// as per https://cloud.google.com/storage/docs/renaming-copying-moving-objects#rename
|
||||
url := fmt.Sprintf(
|
||||
"https://www.googleapis.com/storage/v1/b/%s/o/%s/rewriteTo/b/%s/o/%s",
|
||||
url.PathEscape(b.bucket), url.PathEscape(old),
|
||||
url.PathEscape(b.bucket), url.PathEscape(new),
|
||||
)
|
||||
|
||||
req, err := http.NewRequest("POST", url, nil)
|
||||
req.Header.Add("Authorization", "Bearer "+token.AccessToken)
|
||||
_, err = client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// It seems that 'rewriteTo' copies objects instead of
|
||||
// renaming/moving them, hence a deletion call afterwards is
|
||||
// required.
|
||||
if err = b.handle.Object(old).Delete(ctx); err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"new": new,
|
||||
"old": old,
|
||||
}).Warn("failed to delete renamed object")
|
||||
|
||||
// this error should not break renaming and is not returned
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *GCSBackend) ServeLayer(digest string, r *http.Request, w http.ResponseWriter) error {
|
||||
url, err := b.constructLayerUrl(digest)
|
||||
if err != nil {
|
||||
log.WithError(err).WithFields(log.Fields{
|
||||
"layer": digest,
|
||||
"bucket": b.bucket,
|
||||
}).Error("failed to sign GCS URL")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
log.WithField("layer", digest).Info("redirecting layer request to GCS bucket")
|
||||
|
||||
w.Header().Set("Location", url)
|
||||
w.WriteHeader(303)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Configure GCS URL signing in the presence of a service account key
|
||||
// (toggled if the user has set GOOGLE_APPLICATION_CREDENTIALS).
|
||||
func signingOptsFromEnv() (*storage.SignedURLOptions, error) {
|
||||
path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if path == "" {
|
||||
// No credentials configured -> no URL signing
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
key, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read service account key: %s", err)
|
||||
}
|
||||
|
||||
conf, err := google.JWTConfigFromJSON(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse service account key: %s", err)
|
||||
}
|
||||
|
||||
log.WithField("account", conf.Email).Info("GCS URL signing enabled")
|
||||
|
||||
return &storage.SignedURLOptions{
|
||||
Scheme: storage.SigningSchemeV4,
|
||||
GoogleAccessID: conf.Email,
|
||||
PrivateKey: conf.PrivateKey,
|
||||
Method: "GET",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// layerRedirect constructs the public URL of the layer object in the Cloud
|
||||
// Storage bucket, signs it and redirects the user there.
|
||||
//
|
||||
// Signing the URL allows unauthenticated clients to retrieve objects from the
|
||||
// bucket.
|
||||
//
|
||||
// The Docker client is known to follow redirects, but this might not be true
|
||||
// for all other registry clients.
|
||||
func (b *GCSBackend) constructLayerUrl(digest string) (string, error) {
|
||||
log.WithField("layer", digest).Info("redirecting layer request to bucket")
|
||||
object := "layers/" + digest
|
||||
|
||||
if b.signing != nil {
|
||||
opts := *b.signing
|
||||
opts.Expires = time.Now().Add(5 * time.Minute)
|
||||
return storage.SignedURL(b.bucket, object, &opts)
|
||||
} else {
|
||||
return ("https://storage.googleapis.com/" + b.bucket + "/" + object), nil
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue