New directory structure

This commit is contained in:
Lena Fuhrimann 2018-03-14 21:53:35 +01:00
parent 0371ad0b02
commit 45d34233a5
35 changed files with 126 additions and 126 deletions

View file

@ -1,5 +1,5 @@
/*
!/templates
!/entrypoint-cf.sh
!/s3manager
!/web/
!/deployments/cf/

3
.gitignore vendored
View file

@ -1,2 +1,3 @@
vendor/
/s3manager
/s3manager.exe
/vendor/

View file

@ -1,10 +1,10 @@
language: go
before_install:
- go get -t -v ./...
- curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
install:
- dep ensure
script:
- go test -race -coverprofile=coverage.txt -covermode=atomic
after_success:
- bash <(curl -s https://codecov.io/bash)
- go test ./... -race

View file

@ -1,10 +0,0 @@
FROM golang
ADD . /go/src/github.com/mastertinner/s3manager
WORKDIR /go/src/github.com/mastertinner/s3manager
RUN go build ./cmd/s3manager
EXPOSE 8080
CMD ./s3manager -endpoint "${S3_ENDPOINT}" -access-key-id "${S3_ACCESS_KEY_ID}" -secret-access-key "${S3_SECRET_ACCESS_KEY}"

28
Gopkg.lock generated
View file

@ -16,8 +16,8 @@
[[projects]]
name = "github.com/go-ini/ini"
packages = ["."]
revision = "6333e38ac20b8949a8dd68baa3650f4dee8f39f0"
version = "v1.33.0"
revision = "ace140f73450505f33e8b8418216792275ae82a7"
version = "v1.35.0"
[[projects]]
name = "github.com/gorilla/context"
@ -35,7 +35,7 @@
branch = "master"
name = "github.com/mastertinner/adapters"
packages = ["logging"]
revision = "368acae73d1569f0495b00991aaa85ec27d6ee8e"
revision = "9dd1b86beda61f4746be3a5ee267f1566ab032ce"
[[projects]]
name = "github.com/minio/minio-go"
@ -48,8 +48,8 @@
"pkg/s3utils",
"pkg/set"
]
revision = "9e124ec59547551cb3f1324f73623bbb30650cf8"
version = "4.0.9"
revision = "66252c2a3c15f7b90cc8493d497a04ac3b6e3606"
version = "5.0.0"
[[projects]]
branch = "master"
@ -72,8 +72,8 @@
[[projects]]
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba"
version = "v1.0.4"
revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
version = "v1.0.5"
[[projects]]
name = "github.com/stretchr/testify"
@ -84,8 +84,12 @@
[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "c7dcf104e3a7a1417abc0230cb0d5240d764159d"
packages = [
"argon2",
"blake2b",
"ssh/terminal"
]
revision = "d6449816ce06963d9d136eee5a56fca5b0616e7e"
[[projects]]
branch = "master"
@ -94,7 +98,7 @@
"idna",
"lex/httplex"
]
revision = "d0aafc73d5cdc42264b0af071c261abac580695e"
revision = "61147c48b25b599e5b561d2e9c4f3e1ef489ca41"
[[projects]]
branch = "master"
@ -103,7 +107,7 @@
"unix",
"windows"
]
revision = "7dca6fe1f43775aa6d1334576870ff63f978f539"
revision = "f6f352972f061230a99fbf49d1eb8073ebdb36cb"
[[projects]]
name = "golang.org/x/text"
@ -129,6 +133,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "c70e9710ac0caf3a2f9041ebee7ea3804e6474256bb86e91795a5eb8788accc6"
inputs-digest = "2eb6d0e6d0142a6a38557c70a42069775e66ccbe7aea1620c03752808dcacb1a"
solver-name = "gps-cdcl"
solver-version = 1

View file

@ -35,7 +35,7 @@
[[constraint]]
name = "github.com/minio/minio-go"
version = "4.0.9"
version = "5.0.0"
[[constraint]]
name = "github.com/pkg/errors"

View file

@ -2,11 +2,11 @@ all:
go build ./cmd/s3manager
test:
go test
go test ./...
build-docker:
docker run --rm -v "${PWD}:/go/src/github.com/mastertinner/s3manager" -w /go/src/github.com/mastertinner/s3manager golang go build ./cmd/s3manager
docker build . -f build/docker/Dockerfile -t s3manager
deploy-cf:
GOOS=linux GOARCH=amd64 go build ./cmd/s3manager
cf push
cf push -f deployments/cf/manifest.yml

View file

@ -1,32 +1,30 @@
# S3 Manager
[![Go Report Card](https://goreportcard.com/badge/github.com/mastertinner/s3manager)](https://goreportcard.com/report/github.com/mastertinner/s3manager)
[![Build Status](https://travis-ci.org/mastertinner/s3manager.svg?branch=master)](https://travis-ci.org/mastertinner/s3manager)
[![codecov](https://codecov.io/gh/mastertinner/s3manager/branch/master/graph/badge.svg)](https://codecov.io/gh/mastertinner/s3manager)
[![Go Report Card](https://goreportcard.com/badge/github.com/mastertinner/s3manager?style=flat-square)](https://goreportcard.com/report/github.com/mastertinner/s3manager)
[![Build Status](https://travis-ci.org/mastertinner/s3manager.svg?branch=master&style=flat-square)](https://travis-ci.org/mastertinner/s3manager)
[![Release](https://img.shields.io/github/release/mastertinner/s3manager.svg?style=flat-square)](https://github.com/mastertinner/s3manager/releases/latest)
A Web GUI written in Go to manage S3 buckets from any provider.
## Run locally
## Install Dependencies
1. Install [Dep](https://github.com/golang/dep)
1. Run `dep ensure`
## Build and Run Locally
1. Run `make`
1. Execute the created binary and visit <http://localhost:8080>
## Run with Docker
## Run Tests
1. Set environment variables in `docker-compose.yml`
1. Run `docker-compose up`
1. Visit <http://localhost:8080>
1. Run `make test`
## Build with Docker and run anywhere
## Build Docker Image
1. Run `make build-docker`
To cross-compile for windows, add the `-e "GOOS=windows" -e "GOARCH=amd64"` flags to the `Makefile` (depending on your system, you might have to adjust `GOARCH`)
To cross-compile for macOS, add the `-e "GOOS=darwin" -e "GOARCH=amd64"` flags to the `Makefile` (depending on your system, you might have to adjust `GOARCH`)
## Run on Cloud Foundry
1. Change the service in `manifest.yml` to represent your S3 service (if you are using an external S3 provider, you'll have to switch the service type in `entrypoint-cf.sh` from `dynstrg` to `user-provided` and create the respective user-provided service with `cf create-user-provided-service`)
1. Add a route in `manifest.yml` that isn't taken yet
1. Modify `deployments/cf/*` to your liking
1. Run `make deploy-cf`

12
build/docker/Dockerfile Normal file
View file

@ -0,0 +1,12 @@
FROM golang:latest
WORKDIR /go/src/github.com/mastertinner/s3manager
COPY . .
RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
RUN dep ensure
RUN go build ./cmd/s3manager
EXPOSE 8080
ENTRYPOINT ["./s3manager"]

View file

@ -5,10 +5,11 @@ import (
"log"
"net/http"
"os"
"path/filepath"
"github.com/gorilla/mux"
"github.com/mastertinner/adapters/logging"
"github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
minio "github.com/minio/minio-go"
"github.com/pkg/errors"
)
@ -40,6 +41,8 @@ func main() {
log.Fatalln(errors.Wrap(err, "error creating s3 client"))
}
tmplDir := filepath.Join("web", "template")
// Set up router
r := mux.NewRouter().StrictSlash(true)
r.Use(logging.Handler(os.Stdout))
@ -51,11 +54,11 @@ func main() {
r.
Methods(http.MethodGet).
Path("/buckets").
Handler(s3manager.BucketsViewHandler(s3))
Handler(s3manager.BucketsViewHandler(s3, tmplDir))
r.
Methods(http.MethodGet).
Path("/buckets/{bucketName}").
Handler(s3manager.BucketViewHandler(s3))
Handler(s3manager.BucketViewHandler(s3, tmplDir))
r.
Methods(http.MethodPost).
Path("/api/buckets").

View file

@ -0,0 +1,9 @@
---
applications:
- name: s3-manager
memory: 64M
buildpack: https://github.com/cloudfoundry/binary-buildpack.git
command: ./deployments/cf/entrypoint-cf.sh
services:
- my-storage

View file

@ -1,11 +0,0 @@
version: '2'
services:
s3manager:
build: .
ports:
- "8080:8080"
environment:
- S3_ENDPOINT=s3.amazonaws.com
- S3_ACCESS_KEY_ID=xxx
- S3_SECRET_ACCESS_KEY=xxx

View file

@ -4,6 +4,7 @@ import (
"html/template"
"net/http"
"path"
"path/filepath"
"github.com/gorilla/mux"
minio "github.com/minio/minio-go"
@ -23,20 +24,11 @@ type bucketPage struct {
}
// BucketViewHandler shows the details page of a bucket.
func BucketViewHandler(s3 S3) http.Handler {
func BucketViewHandler(s3 S3, tmplDir string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucketName"]
var objs []objectWithIcon
l := path.Join(tmplDirectory, "layout.html.tmpl")
p := path.Join(tmplDirectory, "bucket.html.tmpl")
t, err := template.ParseFiles(l, p)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errParsingTemplates))
return
}
doneCh := make(chan struct{})
defer close(doneCh)
objectCh := s3.ListObjectsV2(bucketName, "", true, doneCh)
@ -48,12 +40,18 @@ func BucketViewHandler(s3 S3) http.Handler {
obj := objectWithIcon{object, icon(object.Key)}
objs = append(objs, obj)
}
page := bucketPage{
BucketName: bucketName,
Objects: objs,
}
l := filepath.Join(tmplDir, "layout.html.tmpl")
p := filepath.Join(tmplDir, "bucket.html.tmpl")
t, err := template.ParseFiles(l, p)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errParsingTemplates))
return
}
err = t.ExecuteTemplate(w, "layout", page)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errExecutingTemplate))

View file

@ -6,17 +6,18 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
"github.com/gorilla/mux"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
minio "github.com/minio/minio-go"
"github.com/stretchr/testify/assert"
)
func TestBucketViewHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
bucketName string
expectedStatusCode int
expectedBodyContains string
@ -103,11 +104,12 @@ func TestBucketViewHandler(t *testing.T) {
t.Run(tcID, func(t *testing.T) {
assert := assert.New(t)
tmplDir := filepath.Join("..", "..", "..", "web", "template")
r := mux.NewRouter()
r.
Methods(http.MethodGet).
Path("/buckets/{bucketName}").
Handler(BucketViewHandler(tc.s3))
Handler(s3manager.BucketViewHandler(tc.s3, tmplDir))
ts := httptest.NewServer(r)
defer ts.Close()

View file

@ -3,29 +3,27 @@ package s3manager
import (
"html/template"
"net/http"
"path"
"path/filepath"
"github.com/pkg/errors"
)
// BucketsViewHandler renders all buckets on an HTML page.
func BucketsViewHandler(s3 S3) http.Handler {
func BucketsViewHandler(s3 S3, tmplDir string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
l := path.Join(tmplDirectory, "layout.html.tmpl")
p := path.Join(tmplDirectory, "buckets.html.tmpl")
t, err := template.ParseFiles(l, p)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errParsingTemplates))
return
}
buckets, err := s3.ListBuckets()
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error listing buckets"))
return
}
l := filepath.Join(tmplDir, "layout.html.tmpl")
p := filepath.Join(tmplDir, "buckets.html.tmpl")
t, err := template.ParseFiles(l, p)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errParsingTemplates))
return
}
err = t.ExecuteTemplate(w, "layout", buckets)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errExecutingTemplate))

View file

@ -4,16 +4,18 @@ import (
"errors"
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
. "github.com/mastertinner/s3manager"
"github.com/gorilla/mux"
"github.com/mastertinner/s3manager/internal/app/s3manager"
minio "github.com/minio/minio-go"
"github.com/stretchr/testify/assert"
)
func TestBucketsViewHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
expectedStatusCode int
expectedBodyContains string
}{
@ -44,11 +46,18 @@ func TestBucketsViewHandler(t *testing.T) {
t.Run(tcID, func(t *testing.T) {
assert := assert.New(t)
tmplDir := filepath.Join("..", "..", "..", "web", "template")
r := mux.NewRouter()
r.
Methods(http.MethodGet).
Path("/buckets/{bucketName}").
Handler(s3manager.BucketViewHandler(tc.s3, tmplDir))
req, err := http.NewRequest(http.MethodGet, "/buckets", nil)
assert.NoError(err, tcID)
rr := httptest.NewRecorder()
handler := BucketsViewHandler(tc.s3)
handler := s3manager.BucketsViewHandler(tc.s3, tmplDir)
handler.ServeHTTP(rr, req)
resp := rr.Result()

View file

@ -21,7 +21,6 @@ type copyObjectInfo struct {
func CopyObjectHandler(s3 S3) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var copy copyObjectInfo
err := json.NewDecoder(r.Body).Decode(&copy)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errDecodingBody))
@ -42,7 +41,6 @@ func CopyObjectHandler(s3 S3) http.Handler {
w.Header().Set(HeaderContentType, ContentTypeJSON)
w.WriteHeader(http.StatusCreated)
err = json.NewEncoder(w).Encode(copy)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errEncodingJSON))

View file

@ -12,7 +12,6 @@ import (
func CreateBucketHandler(s3 S3) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var bucket minio.BucketInfo
err := json.NewDecoder(r.Body).Decode(&bucket)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errDecodingBody))
@ -27,7 +26,6 @@ func CreateBucketHandler(s3 S3) http.Handler {
w.Header().Set(HeaderContentType, ContentTypeJSON)
w.WriteHeader(http.StatusCreated)
err = json.NewEncoder(w).Encode(bucket)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errEncodingJSON))

View file

@ -7,13 +7,13 @@ import (
"net/http/httptest"
"testing"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
"github.com/stretchr/testify/assert"
)
func TestCreateBucketHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
body string
expectedStatusCode int
expectedBodyContains string
@ -54,7 +54,7 @@ func TestCreateBucketHandler(t *testing.T) {
assert.NoError(err, tcID)
rr := httptest.NewRecorder()
handler := CreateBucketHandler(tc.s3)
handler := s3manager.CreateBucketHandler(tc.s3)
handler.ServeHTTP(rr, req)
resp := rr.Result()

View file

@ -12,12 +12,12 @@ import (
// CreateObjectHandler uploads a new object.
func CreateObjectHandler(s3 S3) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucketName"]
err := r.ParseMultipartForm(32 << 20)
if err != nil {
handleHTTPError(w, errors.Wrap(err, errParsingForm))
return
}
file, handler, err := r.FormFile("file")
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error getting file from form"))
@ -30,7 +30,6 @@ func CreateObjectHandler(s3 S3) http.Handler {
}
}()
bucketName := mux.Vars(r)["bucketName"]
_, err = s3.PutObject(bucketName, handler.Filename, file, 1, minio.PutObjectOptions{ContentType: contentTypeOctetStream})
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error putting object"))

View file

@ -11,6 +11,7 @@ import (
func DeleteBucketHandler(s3 S3) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucketName"]
err := s3.RemoveBucket(bucketName)
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error removing bucket"))

View file

@ -6,13 +6,13 @@ import (
"net/http/httptest"
"testing"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
"github.com/stretchr/testify/assert"
)
func TestDeleteBucketHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
expectedStatusCode int
expectedBodyContains string
}{
@ -38,7 +38,7 @@ func TestDeleteBucketHandler(t *testing.T) {
assert.NoError(err, tcID)
rr := httptest.NewRecorder()
handler := DeleteBucketHandler(tc.s3)
handler := s3manager.DeleteBucketHandler(tc.s3)
handler.ServeHTTP(rr, req)
resp := rr.Result()

View file

@ -13,6 +13,7 @@ func DeleteObjectHandler(s3 S3) http.Handler {
vars := mux.Vars(r)
bucketName := vars["bucketName"]
objectName := vars["objectName"]
err := s3.RemoveObject(bucketName, objectName)
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error removing object"))

View file

@ -6,13 +6,13 @@ import (
"net/http/httptest"
"testing"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
"github.com/stretchr/testify/assert"
)
func TestDeleteObjectHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
expectedStatusCode int
expectedBodyContains string
}{
@ -38,7 +38,7 @@ func TestDeleteObjectHandler(t *testing.T) {
assert.NoError(err, tcID)
rr := httptest.NewRecorder()
handler := DeleteObjectHandler(tc.s3)
handler := s3manager.DeleteObjectHandler(tc.s3)
handler.ServeHTTP(rr, req)

View file

@ -14,6 +14,7 @@ const (
errDecodingBody = "error decoding body JSON"
errEncodingJSON = "error encoding JSON"
errExecutingTemplate = "error executing template"
errGettingWD = "error getting working directory"
errParsingForm = "error parsing form"
errParsingTemplates = "error parsing template files"
)

View file

@ -25,7 +25,6 @@ func GetObjectHandler(s3 S3) http.Handler {
w.Header().Set(headerContentDisposition, fmt.Sprintf("attachment; filename=\"%s\"", objectName))
w.Header().Set(HeaderContentType, contentTypeOctetStream)
_, err = io.Copy(w, object)
if err != nil {
handleHTTPError(w, errors.Wrap(err, "error copying object to response writer"))

View file

@ -9,13 +9,13 @@ import (
"testing"
"github.com/gorilla/mux"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
"github.com/stretchr/testify/assert"
)
func TestGetObjectHandler(t *testing.T) {
cases := map[string]struct {
s3 S3
s3 s3manager.S3
bucketName string
objectName string
expectedStatusCode int
@ -40,7 +40,7 @@ func TestGetObjectHandler(t *testing.T) {
r.
Methods(http.MethodGet).
Path("/buckets/{bucketName}/objects/{objectName}").
Handler(GetObjectHandler(tc.s3))
Handler(s3manager.GetObjectHandler(tc.s3))
ts := httptest.NewServer(r)
defer ts.Close()

View file

@ -3,7 +3,7 @@ package s3manager_test
import (
"io"
. "github.com/mastertinner/s3manager"
"github.com/mastertinner/s3manager/internal/app/s3manager"
minio "github.com/minio/minio-go"
)
@ -52,7 +52,7 @@ func (s *s3Mock) ListObjectsV2(name string, p string, r bool, d <-chan struct{})
}
if !found {
s.Objects = append(s.Objects, minio.ObjectInfo{
Err: ErrBucketDoesNotExist,
Err: s3manager.ErrBucketDoesNotExist,
})
}

View file

@ -1,12 +1,11 @@
// Package s3manager allows to interact with an S3 compatible storage.
package s3manager
// Constants commonly used throughout the application.
// These are constants commonly used throughout the application.
const (
HeaderContentType = "Content-Type"
ContentTypeJSON = "application/json"
ContentTypeMultipartForm = "multipart/form-data"
headerContentDisposition = "Content-Disposition"
contentTypeOctetStream = "application/octet-stream"
tmplDirectory = "templates"
)

View file

@ -1,9 +0,0 @@
---
applications:
- name: s3-manager
memory: 64M
buildpack: https://github.com/cloudfoundry/binary-buildpack.git
command: ./entrypoint-cf.sh
services:
- my-storage