Add dedicated chan migration packages

cmd/gochan-migration/internal/common/handler.go

@@ -0,0 +1,57 @@
+package common
+
+import (
+	"errors"
+
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+)
+
+var (
+	ErrInvalidSchema     = errors.New("invalid database schema for old database")
+	ErrUnsupportedDBType = errors.New("unsupported SQL driver, currently only MySQL and Postgres are supported")
+)
+
+type MigrationError struct {
+	oldChanType string
+	errMessage  string
+}
+
+func (me *MigrationError) OldChanType() string {
+	return me.oldChanType
+}
+
+func (me *MigrationError) Error() string {
+	from := me.oldChanType
+	if from != "" {
+		from = " from " + from + " "
+	}
+	return "unable to migrate" + from + ": " + me.errMessage
+}
+
+func NewMigrationError(oldChanType string, errMessage string) *MigrationError {
+	return &MigrationError{oldChanType: oldChanType, errMessage: errMessage}
+}
+
+type DBOptions struct {
+	Host        string
+	DBType      string
+	Username    string
+	Password    string
+	OldDBName   string
+	OldChanType string
+	NewDBName   string
+}
+
+// DBMigrator is used for handling the migration from one database type to a
+// database compatible with gochan 3.x onward
+type DBMigrator interface {
+	// Init sets the variables for connecting to the databases
+	Init(options DBOptions) error
+
+	// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database
+	MigrateDB() error
+
+	// Close closes the database if initialized
+	Close() error
+}

cmd/gochan-migration/internal/kusabax/kusabax.go

@@ -0,0 +1,29 @@
+package kusabax
+
+import (
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+)
+
+var (
+	unimplemented = common.NewMigrationError("tinyboard", "unimplemented")
+)
+
+type KusabaXMigrator struct {
+	// db      *gcsql.GCDB
+	// options common.DBOptions
+}
+
+func (m *KusabaXMigrator) Init(options common.DBOptions) error {
+	return unimplemented
+}
+
+func (m *KusabaXMigrator) MigrateDB() error {
+	return unimplemented
+}
+
+func (m *KusabaXMigrator) Close() error {
+	/* if m.db != nil {
+		return m.db.Close()
+	} */
+	return nil
+}

cmd/gochan-migration/internal/pre2021/pre2021.go

@@ -0,0 +1,32 @@
+// used for migrating pre-refactor gochan databases to the new schema
+package pre2021
+
+import (
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+	"github.com/gochan-org/gochan/pkg/gcsql"
+)
+
+type Pre2021Migrator struct {
+	db      *gcsql.GCDB
+	options common.DBOptions
+}
+
+func (m *Pre2021Migrator) Init(options common.DBOptions) error {
+	m.options = options
+	var err error
+	m.db, err = gcsql.Open(
+		m.options.Host, m.options.DBType, "", m.options.Username, m.options.Password, "",
+	)
+	return err
+}
+
+func (m *Pre2021Migrator) MigrateDB() error {
+	return nil
+}
+
+func (m *Pre2021Migrator) Close() error {
+	if m.db != nil {
+		return m.db.Close()
+	}
+	return nil
+}

cmd/gochan-migration/internal/tinyboard/tinyboard.go

@@ -0,0 +1,29 @@
+package tinyboard
+
+import (
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+)
+
+var (
+	unimplemented = common.NewMigrationError("tinyboard", "unimplemented")
+)
+
+type TinyBoardMigrator struct {
+	// db      *gcsql.GCDB
+	// options common.DBOptions
+}
+
+func (m *TinyBoardMigrator) Init(options common.DBOptions) error {
+	return unimplemented
+}
+
+func (m *TinyBoardMigrator) MigrateDB() error {
+	return unimplemented
+}
+
+func (m *TinyBoardMigrator) Close() error {
+	/* if m.db != nil {
+		return m.db.Close()
+	} */
+	return nil
+}

cmd/gochan-migration/main.go

@@ -1,24 +1,78 @@
 package main
 
 import (
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/lib/pq"
+	"flag"
+	"fmt"
+	"os"
 
-	"github.com/gochan-org/gochan/cmd/gochan-migration/gcmigrate"
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/kusabax"
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/pre2021"
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/tinyboard"
 	"github.com/gochan-org/gochan/pkg/config"
-	"github.com/gochan-org/gochan/pkg/gclog"
+)
+
+const (
+	banner = `Welcome to the gochan database migration tool for gochan %s!
+This migration tool is currently very unstable, and will likely go through
+several changes before it can be considered "stable", so make sure you check
+the README and/or the -h command line flag before you use it.
+
+`
 )
 
 var (
 	versionStr string
-	stdFatal   = gclog.LStdLog | gclog.LFatal
 )
 
+func fatalPrintln(args ...interface{}) {
+	fmt.Println(args...)
+	os.Exit(1)
+}
+
 func main() {
+	var options common.DBOptions
+
+	flag.StringVar(&options.OldChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
+	flag.StringVar(&options.Host, "dbhost", "", "The database host or socket file to connect to")
+	flag.StringVar(&options.DBType, "dbtype", "mysql", "The kind of database server we are connecting to (currently only mysql is supported)")
+	flag.StringVar(&options.Username, "dbusername", "", "The database username")
+	flag.StringVar(&options.Password, "dbpassword", "", "The database password (if required)")
+	flag.StringVar(&options.OldDBName, "olddbname", "", "The name of the old database")
+	flag.Parse()
+
+	fmt.Printf(banner, versionStr)
+
+	var migrator common.DBMigrator
+	switch options.OldChanType {
+	case "kusabax":
+		migrator = &kusabax.KusabaXMigrator{}
+	case "pre2021":
+		migrator = &pre2021.Pre2021Migrator{}
+	case "tinyboard":
+		migrator = &tinyboard.TinyBoardMigrator{}
+	default:
+		fatalPrintln("Invalid oldchan value")
+	}
+
+	err := migrator.Init(options)
+	if err != nil {
+		fatalPrintln("Error initializing migrator:", err)
+	}
+	defer migrator.Close()
+
 	config.InitConfig(versionStr)
-	gclog.Printf(gclog.LStdLog, "Starting gochan migration (gochan v%s)", versionStr)
+	/* gclog.Printf(gclog.LStdLog, "Starting gochan migration (gochan v%s)", versionStr)
 	err := gcmigrate.Entry(1) //TEMP, get correct database version from command line or some kind of table. 1 Is the current version we are working towards
 	if err != nil {
 		gclog.Printf(gclog.LErrorLog, "Error while migrating: %s", err)
+	} */
+	if options.OldDBName == config.Config.DBname {
+		fatalPrintln(
+			"The old database name must not be the same as the new one set in gochan.json")
 	}
+	if err = migrator.MigrateDB(); err != nil {
+		fatalPrintln("Error migrating database:", err)
+	}
+	fmt.Println("Database migration successful!")
 }

pkg/gcsql/connect.go

@@ -1,7 +1,6 @@
 package gcsql
 
 import (
-	"database/sql"
 	"fmt"
 	"io/ioutil"
 	"regexp"
@@ -13,48 +12,20 @@ import (
 )
 
 var (
-	db       *sql.DB
-	dbDriver string
+	gcdb *GCDB
 	//FatalSQLFlags is used to log a fatal sql error and then close gochan
 	FatalSQLFlags   = gclog.LErrorLog | gclog.LStdLog | gclog.LFatal
-	nilTimestamp    string
-	sqlReplacer     *strings.Replacer // used during SQL string preparation
 	tcpHostIsolator = regexp.MustCompile(`\b(tcp\()?([^\(\)]*)\b`)
 )
 
 // ConnectToDB initializes the database connection and exits if there are any errors
-func ConnectToDB(host, dbType, dbName, username, password, prefix string) {
-	var connStr string
-	sqlReplacer = strings.NewReplacer(
-		"DBNAME", dbName,
-		"DBPREFIX", prefix,
-		"\n", " ")
-	gclog.Print(gclog.LStdLog|gclog.LErrorLog, "Initializing server...")
-
-	addrMatches := tcpHostIsolator.FindAllStringSubmatch(host, -1)
-	if len(addrMatches) > 0 && len(addrMatches[0]) > 2 {
-		host = addrMatches[0][2]
-	}
-
-	switch dbType {
-	case "mysql":
-		connStr = fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true&collation=utf8mb4_unicode_ci",
-			username, password, host, dbName)
-		nilTimestamp = "0000-00-00 00:00:00"
-	case "postgres":
-		connStr = fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable",
-			username, password, host, dbName)
-		nilTimestamp = "0001-01-01 00:00:00"
-	default:
-		gclog.Printf(FatalSQLFlags,
-			`Invalid DBtype %q in gochan.json, valid values are "mysql" and "postgres" (sqlite3 is no longer supported for stability reasons)`, dbType)
-	}
-	dbDriver = dbType
+func ConnectToDB(host, driver, dbName, username, password, prefix string) {
 	var err error
-	if db, err = sql.Open(dbType, connStr); err != nil {
+	if gcdb, err = Open(host, driver, dbName, username, password, prefix); err != nil {
 		gclog.Print(FatalSQLFlags, "Failed to connect to the database: ", err.Error())
+		return
 	}
-	gclog.Print(gclog.LStdLog|gclog.LErrorLog, "Connected to database...")
+	gclog.Print(gclog.LStdLog|gclog.LErrorLog, "Connected to database")
 }
 
 func initDB(initFile string) error {
@@ -76,12 +47,12 @@ func RunSQLFile(path string) error {
 	}
 
 	sqlStr := regexp.MustCompile("--.*\n?").ReplaceAllString(string(sqlBytes), " ")
-	sqlArr := strings.Split(sqlReplacer.Replace(sqlStr), ";")
+	sqlArr := strings.Split(gcdb.replacer.Replace(sqlStr), ";")
 
 	for _, statement := range sqlArr {
 		statement = strings.Trim(statement, " \n\r\t")
 		if len(statement) > 0 {
-			if _, err = db.Exec(statement); err != nil {
+			if _, err = gcdb.db.Exec(statement); err != nil {
 				if config.Config.DebugMode {
 					gclog.Printf(gclog.LStdLog, "Error excecuting sql: %s\n", err.Error())
 					gclog.Printf(gclog.LStdLog, "Length sql: %d\n", len(statement))

pkg/gcsql/database.go

@@ -0,0 +1,179 @@
+package gcsql
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/gochan-org/gochan/pkg/config"
+)
+
+const (
+	UnsupportedSQLVersionMsg = `Received syntax error while preparing a SQL string.
+	This means that either there is a bug in gochan's code (hopefully not) or that you are using an unsupported My/Postgre version.
+	Before reporting an error, make sure that you are using the up to date version of your selected SQL server.
+	Error text: %s`
+	mysqlConnStr    = "%s:%s@tcp(%s)/%s?parseTime=true&collation=utf8mb4_unicode_ci"
+	postgresConnStr = "postgres://%s:%s@%s/%s?sslmode=disable"
+)
+
+type GCDB struct {
+	db           *sql.DB
+	connStr      string
+	driver       string
+	nilTimestamp string
+	replacer     *strings.Replacer
+}
+
+func (db *GCDB) ConnectionString() string {
+	return db.connStr
+}
+
+func (db *GCDB) SQLDriver() string {
+	return db.driver
+}
+
+func (db *GCDB) NilSQLTimestamp() string {
+	return db.nilTimestamp
+}
+
+func (db *GCDB) Close() error {
+	if db.db != nil {
+		return db.db.Close()
+	}
+	return nil
+}
+
+func (db *GCDB) PrepareSQL(query string) (*sql.Stmt, error) {
+	var preparedStr string
+
+	switch db.driver {
+	case "mysql":
+		preparedStr = query
+	case "postgres":
+		arr := strings.Split(query, "?")
+		for i := range arr {
+			if i == len(arr)-1 {
+				break
+			}
+			arr[i] += fmt.Sprintf("$%d", i+1)
+		}
+		preparedStr = strings.Join(arr, "")
+	default:
+		return nil, ErrUnsupportedDB
+	}
+	stmt, err := db.db.Prepare(db.replacer.Replace((preparedStr)))
+	if err != nil {
+		return stmt, fmt.Errorf("Error preparing sql query:\n%s\n%s", query, err.Error())
+	}
+	return stmt, sqlVersionError(err, db.driver, &preparedStr)
+}
+
+/*
+ExecSQL automatically escapes the given values and caches the statement
+Example:
+	var intVal int
+	var stringVal string
+	result, err := db.ExecSQL("INSERT INTO tablename (intval,stringval) VALUES(?,?)", intVal, stringVal)
+*/
+func (db *GCDB) ExecSQL(query string, values ...interface{}) (sql.Result, error) {
+	stmt, err := db.PrepareSQL(query)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+	return stmt.Exec(values...)
+}
+
+/*
+QueryRowSQL gets a row from the db with the values in values[] and fills the respective pointers in out[]
+Automatically escapes the given values and caches the query
+Example:
+	id := 32
+	var intVal int
+	var stringVal string
+	err := db.QueryRowSQL("SELECT intval,stringval FROM table WHERE id = ?",
+		[]interface{}{&id},
+		[]interface{}{&intVal, &stringVal})
+*/
+func (db *GCDB) QueryRowSQL(query string, values, out []interface{}) error {
+	stmt, err := db.PrepareSQL(query)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+	return stmt.QueryRow(values...).Scan(out...)
+}
+
+/*
+QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]
+Automatically escapes the given values and caches the query
+Example:
+	rows, err := db.QuerySQL("SELECT * FROM table")
+	if err == nil {
+		for rows.Next() {
+			var intVal int
+			var stringVal string
+			rows.Scan(&intVal, &stringVal)
+			// do something with intVal and stringVal
+		}
+	}
+*/
+func (db *GCDB) QuerySQL(query string, a ...interface{}) (*sql.Rows, error) {
+	stmt, err := db.PrepareSQL(query)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+	return stmt.Query(a...)
+}
+
+func Open(host, dbDriver, dbName, username, password, prefix string) (db *GCDB, err error) {
+	db = &GCDB{
+		driver: dbDriver,
+		replacer: strings.NewReplacer(
+			"DBNAME", dbName,
+			"DBPREFIX", prefix,
+			"\n", " "),
+	}
+
+	addrMatches := tcpHostIsolator.FindAllStringSubmatch(host, -1)
+	if len(addrMatches) > 0 && len(addrMatches[0]) > 2 {
+		host = addrMatches[0][2]
+	}
+
+	switch dbDriver {
+	case "mysql":
+		db.connStr = fmt.Sprintf(mysqlConnStr, username, password, host, dbName)
+		db.nilTimestamp = "0000-00-00 00:00:00"
+	case "postgres":
+		db.connStr = fmt.Sprintf(postgresConnStr, username, password, host, dbName)
+		db.nilTimestamp = "0001-01-01 00:00:00"
+	default:
+		return nil, ErrUnsupportedDB
+	}
+
+	db.db, err = sql.Open(db.driver, db.connStr)
+	return db, err
+}
+
+func sqlVersionError(err error, dbDriver string, query *string) error {
+	if err == nil {
+		return nil
+	}
+	errText := err.Error()
+	switch dbDriver {
+	case "mysql":
+		if !strings.Contains(errText, "You have an error in your SQL syntax") {
+			return err
+		}
+	case "postgres":
+		if !strings.Contains(errText, "syntax error at or near") {
+			return err
+		}
+	}
+	if config.Config.DebugMode {
+		return fmt.Errorf(UnsupportedSQLVersionMsg+"\nQuery: "+*query, errText)
+	}
+	return fmt.Errorf(UnsupportedSQLVersionMsg, errText)
+}

pkg/gcsql/queries.go

@@ -18,8 +18,7 @@ import (
 const GochanVersionKeyConstant = "gochan"
 
 var (
-	ErrNilBoard      = errors.New("Board is nil")
-	ErrUnsupportedDB = errors.New("Unsupported DBtype")
+	ErrNilBoard = errors.New("Board is nil")
 )
 
 // GetAllNondeletedMessageRaw gets all the raw message texts from the database, saved per id
@@ -49,10 +48,11 @@ func SetFormattedInDatabase(messages []MessagePostContainer) error {
 	SET message = ?
 	WHERE id = ?`
 	stmt, err := PrepareSQL(sql)
-	defer stmt.Close()
 	if err != nil {
 		return err
 	}
+	defer stmt.Close()
+
 	for _, message := range messages {
 		if _, err = stmt.Exec(string(message.Message), message.ID); err != nil {
 			return err
@@ -777,7 +777,7 @@ func DeleteFilesFromPost(postID int) error {
 		return err
 	}
 	if !boardWasFound {
-		return fmt.Errorf("Could not find board for post %v", postID)
+		return fmt.Errorf("could not find board for post %v", postID)
 	}
 
 	//Get all filenames
@@ -890,7 +890,11 @@ func CreateDefaultBoardIfNoneExist() error {
 		Description: "Board for testing",
 		Section:     defaultSectionID}
 	board.SetDefaults()
-	return CreateBoard(&board)
+	err = CreateBoard(&board)
+	if err != nil {
+		panic(err)
+	}
+	return nil // CreateBoard(&board)
 }
 
 //CreateDefaultAdminIfNoStaff creates a new default admin account if no accounts exist

pkg/gcsql/util.go

@@ -2,73 +2,33 @@ package gcsql
 
 import (
 	"database/sql"
-	"fmt"
+	"errors"
 	"strings"
-
-	"github.com/gochan-org/gochan/pkg/config"
-	"github.com/gochan-org/gochan/pkg/gclog"
 )
 
 const (
-	MySQLDatetimeFormat      = "2006-01-02 15:04:05"
-	unsupportedSQLVersionMsg = `Received syntax error while preparing a SQL string.
-This means that either there is a bug in gochan's code (hopefully not) or that you are using an unsupported My/Postgre version.
-Before reporting an error, make sure that you are using the up to date version of your selected SQL server.
-Error text: %s`
+	MySQLDatetimeFormat = "2006-01-02 15:04:05"
 )
 
-func sqlVersionErr(err error, query *string) error {
-	if err == nil {
-		return nil
-	}
-	errText := err.Error()
-	switch dbDriver {
-	case "mysql":
-		if !strings.Contains(errText, "You have an error in your SQL syntax") {
-			return err
-		}
-	case "postgres":
-		if !strings.Contains(errText, "syntax error at or near") {
-			return err
-		}
-	}
-	if config.Config.DebugMode {
-		return fmt.Errorf(unsupportedSQLVersionMsg+"\nQuery: "+*query, errText)
-	}
-	return fmt.Errorf(unsupportedSQLVersionMsg, errText)
-}
+var (
+	ErrUnsupportedDB = errors.New("unsupported SQL driver")
+	ErrNotConnected  = errors.New("error connecting to database")
+)
 
-// PrepareSQL is used for generating a prepared SQL statement formatted according to config.DBtype
+// PrepareSQL is used for generating a prepared SQL statement formatted according to the configured database driver
 func PrepareSQL(query string) (*sql.Stmt, error) {
-	var preparedStr string
-	switch dbDriver {
-	case "mysql":
-		preparedStr = query
-	case "postgres":
-		arr := strings.Split(query, "?")
-		for i := range arr {
-			if i == len(arr)-1 {
-				break
-			}
-			arr[i] += fmt.Sprintf("$%d", i+1)
-		}
-		preparedStr = strings.Join(arr, "")
-	default:
-		return nil, ErrUnsupportedDB
+	if gcdb == nil {
+		return nil, ErrNotConnected
 	}
-	stmt, err := db.Prepare(sqlReplacer.Replace(preparedStr))
-	if err != nil {
-		gclog.Print(gclog.LErrorLog,
-			"Error preparing sql query:", "\n", query, "\n", err.Error())
-	}
-	return stmt, sqlVersionErr(err, &preparedStr)
+	return gcdb.PrepareSQL(query)
 }
 
 // Close closes the connection to the SQL database
-func Close() {
-	if db != nil {
-		db.Close()
+func Close() error {
+	if gcdb != nil {
+		return gcdb.Close()
 	}
+	return nil
 }
 
 /*
@@ -76,16 +36,14 @@ ExecSQL automatically escapes the given values and caches the statement
 Example:
 	var intVal int
 	var stringVal string
-	result, err := gcsql.ExecSQL(
+	result, err := gcsql.ExecSQL(db, "mysql",
 		"INSERT INTO tablename (intval,stringval) VALUES(?,?)", intVal, stringVal)
 */
 func ExecSQL(query string, values ...interface{}) (sql.Result, error) {
-	stmt, gcerr := PrepareSQL(query)
-	if gcerr != nil {
-		return nil, gcerr
+	if gcdb == nil {
+		return nil, ErrNotConnected
 	}
-	defer stmt.Close()
-	return stmt.Exec(values...)
+	return gcdb.ExecSQL(query, values...)
 }
 
 /*
@@ -100,19 +58,17 @@ Example:
 		[]interface{}{&intVal, &stringVal})
 */
 func QueryRowSQL(query string, values, out []interface{}) error {
-	stmt, err := PrepareSQL(query)
-	if err != nil {
-		return err
+	if gcdb == nil {
+		return ErrNotConnected
 	}
-	defer stmt.Close()
-	return stmt.QueryRow(values...).Scan(out...)
+	return gcdb.QueryRowSQL(query, values, out)
 }
 
 /*
 QuerySQL gets all rows from the db with the values in values[] and fills the respective pointers in out[]
 Automatically escapes the given values and caches the query
 Example:
-	rows, err := gcsql.QuerySQL("SELECT * FROM table")
+	rows, err := sqlutil.QuerySQL("SELECT * FROM table")
 	if err == nil {
 		for rows.Next() {
 			var intVal int
@@ -123,12 +79,10 @@ Example:
 	}
 */
 func QuerySQL(query string, a ...interface{}) (*sql.Rows, error) {
-	stmt, gcerr := PrepareSQL(query)
-	if gcerr != nil {
-		return nil, gcerr
+	if gcdb == nil {
+		return nil, ErrNotConnected
 	}
-	defer stmt.Close()
-	return stmt.Query(a...)
+	return gcdb.QuerySQL(query, a...)
 }
 
 // ResetBoardSectionArrays is run when the board list needs to be changed
@@ -154,7 +108,7 @@ func errFilterDuplicatePrimaryKey(err error) (isPKerror bool, nonPKerror error)
 		return false, nil
 	}
 
-	switch dbDriver {
+	switch gcdb.driver {
 	case "mysql":
 		if !strings.Contains(err.Error(), "Duplicate entry") {
 			return false, err

version

@@ -1 +1 @@
-2.12.0
+3.0.0-beta