Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

GDrive: Fix fieldNotWritable, Add SupportSharedDrive, Add service_account method #512

Open
wants to merge 17 commits into
base: main
Choose a base branch
from
Open

GDrive: Fix fieldNotWritable, Add SupportSharedDrive, Add service_account method #512

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
106609d
GDrive: Support service_account
DoyunShin Oct 25, 2022
9cd6e4f
GDrive: *FIX* fieldNotWritable
DoyunShin Oct 25, 2022
f4f04b9
GDrive: Support Shared drive
DoyunShin Oct 25, 2022
f318207
GDrive: jwt import
DoyunShin Oct 25, 2022
2675350
GDrive: credential type logging
DoyunShin Oct 25, 2022
cd9c56e
GDrive: Check gdrive-local-config-path on Oauth2
DoyunShin Oct 26, 2022
73ac766
Revert "GDrive: Check gdrive-local-config-path on Oauth2"
DoyunShin Oct 26, 2022
fb87a5b
Update getGDriveClientFromServiceAccount
DoyunShin Oct 26, 2022
f096300
GDrive: getGDriveClientFromToken
DoyunShin Oct 26, 2022
71d8858
GDrive: SupportsAllDrives on download
DoyunShin Oct 27, 2022
f73922d
GDrive: authType on GDrive struct
DoyunShin Oct 27, 2022
f517bdc
GDrive: JWT to google.JWTConfigFromJSON
DoyunShin Oct 27, 2022
0327122
GDrive: Token Redirecturl to urn:ietf:wg:oauth:2.0:oob
DoyunShin Oct 27, 2022
be00bb8
GDrive: basedir as rootID
DoyunShin Oct 27, 2022
91492e1
GDrive: Add gdrive-auth-type
DoyunShin Oct 31, 2022
478341c
GDrive: service to service_account
DoyunShin Nov 1, 2022
c96cf41
GDrive: README.md
DoyunShin Nov 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions cmd/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ package cmd

import (
"fmt"
"github.com/dutchcoders/transfer.sh/server/storage"
"log"
"os"
"strings"

"github.com/dutchcoders/transfer.sh/server"
"github.com/dutchcoders/transfer.sh/server/storage"
"github.com/fatih/color"
"github.com/urfave/cli"
"google.golang.org/api/googleapi"
Expand Down Expand Up @@ -471,11 +471,10 @@ func New() *Cmd {
}
case "gdrive":
chunkSize := c.Int("gdrive-chunk-size") * 1024 * 1024
localConfigPath := c.String("gdrive-local-config-path")

if clientJSONFilepath := c.String("gdrive-client-json-filepath"); clientJSONFilepath == "" {
panic("client-json-filepath not set.")
} else if localConfigPath := c.String("gdrive-local-config-path"); localConfigPath == "" {
panic("local-config-path not set.")
panic("gdrive-client-json-filepath not set.")
} else if basedir := c.String("basedir"); basedir == "" {
panic("basedir not set.")
} else if store, err := storage.NewGDriveStorage(clientJSONFilepath, localConfigPath, basedir, chunkSize, logger); err != nil {
Expand Down
102 changes: 52 additions & 50 deletions server/storage/gdrive.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,12 @@ import (
type GDrive struct {
service *drive.Service
rootID string
basedir string
localConfigPath string
authType string
chunkSize int
logger *log.Logger
}

const gDriveRootConfigFile = "root_id.conf"
const gDriveTokenJSONFile = "token.json"
const gDriveDirectoryMimeType = "application/vnd.google-apps.folder"

Expand All @@ -45,65 +44,66 @@ func NewGDriveStorage(clientJSONFilepath string, localConfigPath string, basedir
}

// If modifying these scopes, delete your previously saved client_secret.json.
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
var httpClient *http.Client
var AuthType string

if strings.Contains(string(b), `"type": "service_account"`) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's have a separate cli param for service account file

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

param like --gdrive-type=service / oauth2?
or we can actually check in json.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oauth2
{ "installed": { "client_id": "clientid", "project_id": "projectname", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_secret": "clientsecret" } }

Service Account
{ "type": "service_account", "project_id": "projectname", "private_key_id": "keyid", "private_key": "-----BEGIN PRIVATE KEY-----\nKEYSECRET\n-----END PRIVATE KEY-----\n", "client_email": "[email protected]", "client_id": "clientid", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/transfersh%40projectname.iam.gserviceaccount.com" }

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

param like --gdrive-type=service / oauth2?
or we can actually check in json.

param like gdrive-client-json-filepath: gdrive-service-account-json-filepath

only one of the two can be set

in the factory you can translate this to a single param with the path and a new param for the type (service account/oauth2): it is up to you

AuthType = "service"

httpClient := getGDriveClient(ctx, config, localConfigPath, logger)
logger.Println("GDrive: using Service Account credentials")
config, err := google.JWTConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
httpClient = config.Client(ctx)
} else {
AuthType = "oauth"

if localConfigPath == "" {
return nil, fmt.Errorf("gdrive-local-config-path not set")
}

logger.Println("GDrive: using OAuth2 credentials")
config, err := google.ConfigFromJSON(b, drive.DriveScope, drive.DriveMetadataScope)
if err != nil {
return nil, err
}
httpClient = getGDriveClientFromToken(ctx, config, localConfigPath, logger)
}

srv, err := drive.NewService(ctx, option.WithHTTPClient(httpClient))
if err != nil {
return nil, err
}

storage := &GDrive{service: srv, basedir: basedir, rootID: "", localConfigPath: localConfigPath, chunkSize: chunkSize, logger: logger}
err = storage.setupRoot()
storage := &GDrive{service: srv, rootID: basedir, localConfigPath: localConfigPath, authType: AuthType, chunkSize: chunkSize, logger: logger}
err = storage.checkRoot()
if err != nil {
return nil, err
}

return storage, nil
}

func (s *GDrive) setupRoot() error {
rootFileConfig := filepath.Join(s.localConfigPath, gDriveRootConfigFile)

rootID, err := ioutil.ReadFile(rootFileConfig)
if err != nil && !os.IsNotExist(err) {
return err
}

if string(rootID) != "" {
s.rootID = string(rootID)
return nil
}

dir := &drive.File{
Name: s.basedir,
MimeType: gDriveDirectoryMimeType,
}

di, err := s.service.Files.Create(dir).Fields("id").Do()
if err != nil {
return err
}

s.rootID = di.Id
err = ioutil.WriteFile(rootFileConfig, []byte(s.rootID), os.FileMode(0600))
if err != nil {
return err
func (s *GDrive) checkRoot() error {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a breaking change.

basedir is the name of the folder to be created in the gdrive, rootID is the id of the folder once created and that we save. the two carry two different information.

if we replce rootID value with basedir value:

  1. we don't create the folder
  2. current installations have to change the value that they pass to basedir, to use instead rootID

I guess you meant this change to be the way to select a shared drive, is it?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep!

I changed basedir to insert folder id, to select rootID.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I cannot accept a breaking change, sorry
you have to find another way to provide the sharedfolder to use and the basedir as it was

if s.rootID == "root" {
switch s.authType {
case "service":
return fmt.Errorf("GDrive: Folder \"root\" is not available when using Service Account credentials")
case "oauth":
s.logger.Println("GDrive: Warning: Folder \"root\" is not recommended.")
}
}

return nil
_, err := s.service.Files.Get(s.rootID).SupportsAllDrives(true).Do()
return err
}

func (s *GDrive) hasChecksum(f *drive.File) bool {
return f.Md5Checksum != ""
}

func (s *GDrive) list(nextPageToken string, q string) (*drive.FileList, error) {
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).Do()
return s.service.Files.List().Fields("nextPageToken, files(id, name, mimeType)").Q(q).PageToken(nextPageToken).SupportsAllDrives(true).IncludeItemsFromAllDrives(true).Do()
}

func (s *GDrive) findID(filename string, token string) (string, error) {
Expand Down Expand Up @@ -184,7 +184,7 @@ func (s *GDrive) Head(ctx context.Context, token string, filename string) (conte
}

var fi *drive.File
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").Do(); err != nil {
if fi, err = s.service.Files.Get(fileID).Context(ctx).Fields("size").SupportsAllDrives(true).Do(); err != nil {
return
}

Expand All @@ -202,7 +202,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader
}

var fi *drive.File
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").Do()
fi, err = s.service.Files.Get(fileID).Fields("size", "md5Checksum").SupportsAllDrives(true).Do()
if err != nil {
return
}
Expand All @@ -214,7 +214,7 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader
contentLength = uint64(fi.Size)

var res *http.Response
res, err = s.service.Files.Get(fileID).Context(ctx).Download()
res, err = s.service.Files.Get(fileID).Context(ctx).SupportsAllDrives(true).Download()
if err != nil {
return
}
Expand All @@ -227,15 +227,15 @@ func (s *GDrive) Get(ctx context.Context, token string, filename string) (reader
// Delete removes a file from storage
func (s *GDrive) Delete(ctx context.Context, token string, filename string) (err error) {
metadata, _ := s.findID(fmt.Sprintf("%s.metadata", filename), token)
_ = s.service.Files.Delete(metadata).Do()
_ = s.service.Files.Delete(metadata).SupportsAllDrives(true).Do()

var fileID string
fileID, err = s.findID(filename, token)
if err != nil {
return
}

err = s.service.Files.Delete(fileID).Context(ctx).Do()
err = s.service.Files.Delete(fileID).Context(ctx).SupportsAllDrives(true).Do()
return
}

Expand All @@ -252,7 +252,7 @@ func (s *GDrive) Purge(ctx context.Context, days time.Duration) (err error) {

for 0 < len(l.Files) {
for _, fi := range l.Files {
err = s.service.Files.Delete(fi.Id).Context(ctx).Do()
err = s.service.Files.Delete(fi.Id).Context(ctx).SupportsAllDrives(true).Do()
if err != nil {
return
}
Expand Down Expand Up @@ -296,10 +296,9 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
Name: token,
Parents: []string{s.rootID},
MimeType: gDriveDirectoryMimeType,
Size: int64(contentLength),
}

di, err := s.service.Files.Create(dir).Fields("id").Do()
di, err := s.service.Files.Create(dir).Fields("id").SupportsAllDrives(true).Do()
if err != nil {
return err
}
Expand All @@ -314,7 +313,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
MimeType: contentType,
}

_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).Do()
_, err = s.service.Files.Create(dst).Context(ctx).Media(reader, googleapi.ChunkSize(s.chunkSize)).SupportsAllDrives(true).Do()

if err != nil {
return err
Expand All @@ -324,7 +323,7 @@ func (s *GDrive) Put(ctx context.Context, token string, filename string, reader
}

// Retrieve a token, saves the token, then returns the generated client.
func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
func getGDriveClientFromToken(ctx context.Context, config *oauth2.Config, localConfigPath string, logger *log.Logger) *http.Client {
tokenFile := filepath.Join(localConfigPath, gDriveTokenJSONFile)
tok, err := gDriveTokenFromFile(tokenFile)
if err != nil {
Expand All @@ -337,10 +336,13 @@ func getGDriveClient(ctx context.Context, config *oauth2.Config, localConfigPath

// Request a token from the web, then returns the retrieved token.
func getGDriveTokenFromWeb(ctx context.Context, config *oauth2.Config, logger *log.Logger) *oauth2.Token {
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)

fmt.Printf("Go to the following link in your browser then type the "+
"authorization code: \n%v\n", authURL)
"authorization code.\n%v\n", authURL)

fmt.Printf("Authorization code: ")
var authCode string
if _, err := fmt.Scan(&authCode); err != nil {
logger.Fatalf("Unable to read authorization code %v", err)
Expand Down