v0.1.0 - S3 + SQLite

.gitignore

@@ -29,10 +29,23 @@ go.work
 main
 uploads/
 picloud
-
+*.db
+set_env_vars_and_run.sh
 # Keep config
 !conf.json
 
 # Keep testdata
 !testdata/baxter.jpg
 !testdata/metadata.json
+
+# IAC
+.terraform
+**/.terraform/*
+terraform.tfstate
+**/*.tfstate
+**/*.tfstate.*
+**/dist/*
+
+
+# Local helper scripts
+scripts/set_env_vars_and_run.sh

aws.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type S3FilesBucket struct {
+	S3Client   *s3.Client
+	BucketName string
+}
+
+func (bucket *S3FilesBucket) UploadFile(file *FileUpload) (string, error) {
+	key := file.Name
+	_, err := bucket.S3Client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket.BucketName),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(file.Content),
+	})
+	return key, err
+}
+
+func (bucket *S3FilesBucket) DownloadFile(key string) ([]byte, error) {
+	result, err := bucket.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucket.BucketName),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		fmt.Printf("Error downloading file: %s\n", err)
+		return nil, err
+	}
+	defer result.Body.Close()
+	fileContents, err := io.ReadAll(result.Body)
+	return fileContents, err
+}
+
+func createClientConnections() {
+	cfg, err := config.LoadDefaultConfig(context.TODO())
+	if err != nil {
+		slog.Error("Error loading default config", "err", err)
+		panic(err)
+	}
+	setupS3(cfg)
+}
+
+func setupS3(cfg aws.Config) {
+	var client *s3.Client
+	if conf.AwsEndpoint == "DEFAULT" {
+		slog.Info("Using default S3 endpoint")
+		client = s3.NewFromConfig(cfg)
+	} else {
+		slog.Info("Using localstack S3 endpoint")
+		client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.BaseEndpoint = &conf.AwsEndpoint
+			o.Region = conf.AwsRegion
+		})
+	}
+	bucket := os.Getenv("S3_BUCKET")
+	s3Bucket := &S3FilesBucket{S3Client: client, BucketName: bucket}
+	filesBucket = *s3Bucket
+}

conf.json

@@ -1,3 +1,5 @@
 {
-    "filePrefix": "./uploads/"
+    "filePrefix": "./uploads/",
+    "awsEndpoint": "DEFAULT",
+    "awsRegion": "us-west-1"
 }

db.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"database/sql"
+	b64 "encoding/base64"
+	"fmt"
+	"io"
+	"time"
+)
+
+type FileMetadataRecord struct {
+	FileName        string  `json:"file_name"`
+	ObjectKey       string  `json:"object_key"`
+	Sha256          string  `json:"file_sha256"`
+	UploadTimestamp int64   `json:"upload_timestamp"`
+	Tags            string  `json:"tags"` // comma separated tags
+	LocalPath       *string `json:"local_path"`
+	CacheTimestamp  int64   `json:"cache_timestamp"`
+}
+
+var DB *sql.DB
+
+func connectDatabase() error {
+	db, err := sql.Open("sqlite3", "./metadata.db")
+	if err != nil {
+		return err
+	}
+
+	DB = db
+	return nil
+}
+
+func addMetadataToTable(metadata *FileMetadataRecord) error {
+	tx, err := DB.Begin()
+	if err != nil {
+		return err
+	}
+	stmt, err := tx.Prepare("INSERT INTO file_metadata (file_name, object_key, file_sha256, upload_timestamp, tags, local_path, cache_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)")
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(metadata.FileName, metadata.ObjectKey, metadata.Sha256, metadata.UploadTimestamp, metadata.Tags, metadata.LocalPath, metadata.CacheTimestamp)
+	if err != nil {
+		return err
+	}
+	err = tx.Commit()
+	return err
+}
+
+func updateCacheDetailsInTable(metadata *FileMetadataRecord) error {
+	tx, err := DB.Begin()
+	if err != nil {
+		return err
+	}
+	stmt, err := tx.Prepare("UPDATE file_metadata SET local_path = ?, cache_timestamp = ? WHERE file_name = ?")
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec(*metadata.LocalPath, metadata.CacheTimestamp, metadata.FileName)
+	if err != nil {
+		return err
+	}
+	err = tx.Commit()
+	return err
+}
+
+func listFilesInTable() ([]FileMetadataRecord, error) {
+	rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags, local_path, cache_timestamp FROM file_metadata")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var files []FileMetadataRecord
+	for rows.Next() {
+		var file FileMetadataRecord
+		err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags, &file.LocalPath, &file.CacheTimestamp)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, file)
+	}
+	return files, nil
+}
+
+func getFileMetadataFromTable(filename string) (*FileMetadataRecord, error) {
+	row := DB.QueryRow("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE file_name = ?", filename)
+	var file FileMetadataRecord
+	err := row.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+	if err != nil {
+		return nil, err
+	}
+	return &file, nil
+}
+
+func queryTags(tagName string) ([]FileMetadataRecord, error) {
+	rows, err := DB.Query("SELECT file_name, object_key, file_sha256, upload_timestamp, tags FROM file_metadata WHERE tags LIKE ?", "%"+tagName+"%")
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var files []FileMetadataRecord
+	for rows.Next() {
+		var file FileMetadataRecord
+		err = rows.Scan(&file.FileName, &file.ObjectKey, &file.Sha256, &file.UploadTimestamp, &file.Tags)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, file)
+	}
+	return files, nil
+}
+
+func getSha256Checksum(fileContent *[]byte) string {
+	h := sha256.New()
+	_, err := io.Copy(h, bytes.NewReader(*fileContent))
+	if err != nil {
+		fmt.Println("Error calculating copying bytes in getSha256Checksum")
+		panic(err)
+	}
+	checksum := b64.StdEncoding.EncodeToString(h.Sum(nil))
+	return checksum
+}
+
+func getTimestamp() int64 {
+	return time.Now().UTC().UnixMilli()
+}

filecache.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"io"
+	"os"
+)
+
+func isStoredLocally(filemetadata *FileMetadataRecord) bool {
+	isLocal := filemetadata.LocalPath != nil
+	isUpToDate := filemetadata.CacheTimestamp > 0 && filemetadata.CacheTimestamp > filemetadata.UploadTimestamp
+	return isLocal && isUpToDate
+}
+
+func getLocalFile(filemetadata *FileMetadataRecord) ([]byte, error) {
+	file, err := os.Open(*filemetadata.LocalPath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	return io.ReadAll(file)
+}
+
+func saveFileLocally(filemetadata *FileMetadataRecord, content []byte) error {
+	localPath := conf.FilePrefix + filemetadata.FileName
+	file, err := os.Create(localPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	_, err = file.Write(content)
+	if err != nil {
+		return err
+	}
+
+	filemetadata.CacheTimestamp = getTimestamp()
+	filemetadata.LocalPath = strPtr(file.Name())
+	err = updateCacheDetailsInTable(filemetadata)
+	return err
+}
+
+func strPtr(s string) *string {
+	return &s
+}

go.mod

@@ -1,24 +1,42 @@
 module picloud
 
-go 1.22
+go 1.22.4
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.27.2
+	github.com/aws/aws-sdk-go-v2/config v1.27.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3
 	github.com/labstack/echo/v4 v4.12.0
 	github.com/stretchr/testify v1.9.0
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.16 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
-	github.com/Kagami/go-avif v0.1.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	golang.org/x/crypto v0.23.0 // indirect

go.sum

@@ -1,5 +1,39 @@
-github.com/Kagami/go-avif v0.1.0 h1:8GHAGLxCdFfhpd4Zg8j1EqO7rtcQNenxIDerC/uu68w=
-github.com/Kagami/go-avif v0.1.0/go.mod h1:OPmPqzNdQq3+sXm0HqaUJQ9W/4k+Elbc3RSfJUemDKA=
+github.com/aws/aws-sdk-go-v2 v1.27.2 h1:pLsTXqX93rimAOZG2FIYraDQstZaaGVVN4tNw65v0h8=
+github.com/aws/aws-sdk-go-v2 v1.27.2/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
+github.com/aws/aws-sdk-go-v2/config v1.27.16 h1:knpCuH7laFVGYTNd99Ns5t+8PuRjDn4HnnZK48csipM=
+github.com/aws/aws-sdk-go-v2/config v1.27.16/go.mod h1:vutqgRhDUktwSge3hrC3nkuirzkJ4E/mLj5GvI0BQas=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.16 h1:7d2QxY83uYl0l58ceyiSpxg9bSbStqBC6BeEeHEchwo=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.16/go.mod h1:Ae6li/6Yc6eMzysRL2BXlPYvnrLLBg3D11/AmOjw50k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9 h1:cy8ahBJuhtM8GTTSyOkfy6WVPV1IE+SS5/wfXUYuulw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.9/go.mod h1:CZBXGLaJnEZI6EVNcPd7a6B5IC5cA/GkRWtu9fp3S6Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9 h1:A4SYk07ef04+vxZToz9LWvAXl9LW0NClpPpMsi31cz0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.9/go.mod h1:5jJcHuwDagxN+ErjQ3PU3ocf6Ylc/p9x+BLO/+X4iXw=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 h1:/FUtT3xsoHO3cfh+I/kCbcMCN98QZRsiFet/V8QkWSs=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7/go.mod h1:MaCAgWpGooQoCWZnMur97rGn5dp350w2+CeiV5406wE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 h1:UXqEWQI0n+q0QixzU0yUUQBZXRd5037qdInTIHFTl98=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9/go.mod h1:xP6Gq6fzGZT8w/ZN+XvGMZ2RU1LeEs7b2yUP5DN8NY4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 h1:uO5XR6QGBcmPyo2gxofYJLFkcVQ4izOoGDNenlZhTEk=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7/go.mod h1:feeeAYfAcwTReM6vbwjEyDmiGho+YgBhaFULuXDW8kc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3 h1:57NtjG+WLims0TxIQbjTqebZUKDM03DfM11ANAekW0s=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3/go.mod h1:739CllldowZiPPsDFcJHNF4FXrVxaSGVnZ9Ez9Iz9hc=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 h1:aD7AGQhvPuAxlSUfo0CWU7s6FpkbyykMhGYMvlqTjVs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.20.9/go.mod h1:c1qtZUWtygI6ZdvKppzCSXsDOq5I4luJPZ0Ud3juFCA=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 h1:Pav5q3cA260Zqez42T9UhIlsd9QeypszRPwC9LdSSsQ=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3/go.mod h1:9lmoVDVLz/yUZwLaQ676TK02fhCu4+PgRSmMaKR1ozk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 h1:69tpbPED7jKPyzMcrwSvhWcJ9bPnZsZs18NT40JwM0g=
+github.com/aws/aws-sdk-go-v2/service/sts v1.28.10/go.mod h1:0Aqn1MnEuitqfsCNyKsdKLhDUOr4txD/g19EfiUqgws=
+github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
+github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
@@ -13,6 +47,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=