Drop: pogreb support (#175)

followup of #173

close #95

Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/175

.woodpecker.yml

@@ -65,19 +65,6 @@ pipeline:
       - RAW_DOMAIN=raw.localhost.mock.directory
       - PORT=4430
 
-  # TODO: remove in next version
-  integration-tests-legacy:
-    group: test
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - just integration
-    environment:
-      - ACME_API=https://acme.mock.directory
-      - PAGES_DOMAIN=localhost.mock.directory
-      - RAW_DOMAIN=raw.localhost.mock.directory
-      - PORT=4430
-      - DB_TYPE=
-
   release:
     image: plugins/gitea-release
     settings:

Justfile

@@ -27,7 +27,7 @@ fmt: tool-gofumpt
 
 clean:
     go clean ./...
-    rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json
+    rm -rf build/ integration/certs.sqlite integration/acme-account.json
 
 tool-golangci:
     @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \

cmd/certs.go

@@ -4,11 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
-
-	"codeberg.org/codeberg/pages/server/database"
 )
 
 var Certs = &cli.Command{
@@ -25,63 +21,8 @@ var Certs = &cli.Command{
 			Usage:  "remove a certificate from the database",
 			Action: removeCert,
 		},
-		{
-			Name:   "migrate",
-			Usage:  "migrate from \"pogreb\" driver to dbms driver",
-			Action: migrateCerts,
-		},
 	},
-	Flags: append(CertStorageFlags, []cli.Flag{
+	Flags: CertStorageFlags,
-		&cli.BoolFlag{
-			Name:    "verbose",
-			Usage:   "print trace info",
-			EnvVars: []string{"VERBOSE"},
-			Value:   false,
-		},
-	}...),
-}
-
-func migrateCerts(ctx *cli.Context) error {
-	dbType := ctx.String("db-type")
-	if dbType == "" {
-		dbType = "sqlite3"
-	}
-	dbConn := ctx.String("db-conn")
-	dbPogrebConn := ctx.String("db-pogreb")
-	verbose := ctx.Bool("verbose")
-
-	log.Level(zerolog.InfoLevel)
-	if verbose {
-		log.Level(zerolog.TraceLevel)
-	}
-
-	xormDB, err := database.NewXormDB(dbType, dbConn)
-	if err != nil {
-		return fmt.Errorf("could not connect to database: %w", err)
-	}
-	defer xormDB.Close()
-
-	pogrebDB, err := database.NewPogreb(dbPogrebConn)
-	if err != nil {
-		return fmt.Errorf("could not open database: %w", err)
-	}
-	defer pogrebDB.Close()
-
-	fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
-
-	certs, err := pogrebDB.Items(0, 0)
-	if err != nil {
-		return err
-	}
-
-	for _, cert := range certs {
-		if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
-			return err
-		}
-	}
-
-	fmt.Println("... done")
-	return nil
 }
 
 func listCerts(ctx *cli.Context) error {

cmd/flags.go

@@ -6,16 +6,9 @@ import (
 
 var (
 	CertStorageFlags = []cli.Flag{
-		&cli.StringFlag{
-			// TODO: remove in next version
-			// DEPRICATED
-			Name:    "db-pogreb",
-			Value:   "key-database.pogreb",
-			EnvVars: []string{"DB_POGREB"},
-		},
 		&cli.StringFlag{
 			Name:    "db-type",
-			Value:   "", // TODO: "sqlite3" in next version
+			Value:   "sqlite3",
 			EnvVars: []string{"DB_TYPE"},
 		},
 		&cli.StringFlag{

cmd/setup.go

@@ -10,29 +10,9 @@ import (
 )
 
 func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
-	if ctx.String("db-type") != "" {
+	certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
-		log.Trace().Msg("use xorm mode")
+	if err != nil {
-		certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
+		return nil, nil, fmt.Errorf("could not connect to database: %w", err)
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not connect to database: %w", err)
-		}
-	} else {
-		// TODO: remove in next version
-		fmt.Println(`
-######################
-## W A R N I N G !!! #
-######################
-
-You use "pogreb" witch is deprecated and will be removed in the next version.
-Please switch to sqlite, mysql or postgres !!!
-
-The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`)
-		log.Error().Msg("depricated \"pogreb\" used\n")
-
-		certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not create database: %w", err)
-		}
 	}
 
 	closeFn = func() {

go.mod

@@ -5,7 +5,6 @@ go 1.20
 require (
 	code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
-	github.com/akrylysov/pogreb v0.10.1
 	github.com/go-acme/lego/v4 v4.5.3
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/joho/godotenv v1.4.0

go.sum

@@ -71,8 +71,6 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
-github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
-github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

server/certificates/certificates.go

@@ -510,14 +510,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 				}
 			}
 			log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
-
-			// compact the database
-			msg, err := certDB.Compact()
-			if err != nil {
-				log.Error().Err(err).Msg("Compacting key database failed")
-			} else {
-				log.Debug().Msgf("Compacted key database: %s", msg)
-			}
 		}
 
 		// update main cert

server/database/interface.go

@@ -14,8 +14,6 @@ type CertDB interface {
 	Get(name string) (*certificate.Resource, error)
 	Delete(key string) error
 	Items(page, pageSize int) ([]*Cert, error)
-	// Compact deprecated // TODO: remove in next version
-	Compact() (string, error)
 }
 
 type Cert struct {

server/database/mock.go

@@ -37,11 +37,6 @@ func (p tmpDB) Delete(key string) error {
 	return nil
 }
 
-func (p tmpDB) Compact() (string, error) {
-	p.intern.Truncate()
-	return "Truncate done", nil
-}
-
 func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
 	return nil, fmt.Errorf("items not implemented for tmpDB")
 }

server/database/pogreb.go

@@ -1,134 +0,0 @@
-package database
-
-import (
-	"bytes"
-	"context"
-	"encoding/gob"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/akrylysov/pogreb"
-	"github.com/akrylysov/pogreb/fs"
-	"github.com/go-acme/lego/v4/certificate"
-	"github.com/rs/zerolog/log"
-)
-
-var _ CertDB = aDB{}
-
-type aDB struct {
-	ctx          context.Context
-	cancel       context.CancelFunc
-	intern       *pogreb.DB
-	syncInterval time.Duration
-}
-
-func (p aDB) Close() error {
-	p.cancel()
-	return p.intern.Sync()
-}
-
-func (p aDB) Put(name string, cert *certificate.Resource) error {
-	var resGob bytes.Buffer
-	if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
-		return err
-	}
-	return p.intern.Put([]byte(name), resGob.Bytes())
-}
-
-func (p aDB) Get(name string) (*certificate.Resource, error) {
-	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get([]byte(name))
-	if err != nil {
-		return nil, err
-	}
-	if resBytes == nil {
-		return nil, nil
-	}
-	if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
-		return nil, err
-	}
-	return cert, nil
-}
-
-func (p aDB) Delete(key string) error {
-	return p.intern.Delete([]byte(key))
-}
-
-func (p aDB) Compact() (string, error) {
-	result, err := p.intern.Compact()
-	if err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%+v", result), nil
-}
-
-func (p aDB) Items(_, _ int) ([]*Cert, error) {
-	items := make([]*Cert, 0, p.intern.Count())
-	iterator := p.intern.Items()
-	for {
-		key, resBytes, err := iterator.Next()
-		if err != nil {
-			if errors.Is(err, pogreb.ErrIterationDone) {
-				break
-			}
-			return nil, err
-		}
-
-		res := &certificate.Resource{}
-		if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
-			return nil, err
-		}
-
-		cert, err := toCert(string(key), res)
-		if err != nil {
-			return nil, err
-		}
-
-		items = append(items, cert)
-	}
-
-	return items, nil
-}
-
-var _ CertDB = &aDB{}
-
-func (p aDB) sync() {
-	for {
-		err := p.intern.Sync()
-		if err != nil {
-			log.Error().Err(err).Msg("Syncing cert database failed")
-		}
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-time.After(p.syncInterval):
-		}
-	}
-}
-
-func NewPogreb(path string) (CertDB, error) {
-	if path == "" {
-		return nil, fmt.Errorf("path not set")
-	}
-	db, err := pogreb.Open(path, &pogreb.Options{
-		BackgroundSyncInterval:       30 * time.Second,
-		BackgroundCompactionInterval: 6 * time.Hour,
-		FileSystem:                   fs.OSMMap,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	result := &aDB{
-		ctx:          ctx,
-		cancel:       cancel,
-		intern:       db,
-		syncInterval: 5 * time.Minute,
-	}
-
-	go result.sync()
-
-	return result, nil
-}

server/database/xorm.go

@@ -106,11 +106,6 @@ func (x xDB) Delete(domain string) error {
 	return err
 }
 
-func (x xDB) Compact() (string, error) {
-	// not needed
-	return "", nil
-}
-
 // Items return al certs from db, if pageSize is 0 it does not use limit
 func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
 	// paginated return