gochan/gochan
1
0
Fork 0
mirror of https://github.com/Eggbertx/gochan.git synced 2025-09-06 11:46:24 -07:00
Code Releases Activity

Reopen gochan-migration development

Browse source
This commit is contained in:
Eggbertx 2022-02-07 18:28:27 -08:00
parent 3df8dbe88d
commit c388ef3984
9 changed files with 201 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

21
build.py
View file

@ -50,6 +50,8 @@ gcos_name = "" # used for release, since macOS GOOS is "darwin"
exe = ""
gochan_bin = ""
gochan_exe = ""
migration_bin = ""
migration_exe = ""
version = ""
@ -119,6 +121,8 @@ def set_vars(goos=""):
global exe
global gochan_bin
global gochan_exe
global migration_bin
global migration_exe
global version
if goos != "":
@ -135,7 +139,9 @@ def set_vars(goos=""):
gochan_bin = "gochan"
gochan_exe = "gochan" + exe
migration_bin = "gochan-migration"
migration_exe = "gochan-migration" + exe
with open("version", "r") as version_file:
version = version_file.read().strip()
@ -178,6 +184,13 @@ def build(debugging=False):
sys.exit(1)
print("Built gochan successfully")
status = run_cmd(
build_cmd + " -o " + migration_exe + " ./cmd/gochan-migration",
realtime=True, print_command=True)[1]
if status != 0:
print("Failed building gochan-migration, see command output for details")
sys.exit(1)
print("Built gochan-migration successfully")
def clean():
print("Cleaning up")
@ -259,7 +272,11 @@ def install(prefix="/usr", document_root="/srv/gochan", js_only=False, css_only=
build()
print("Installing", gochan_exe, "to", path.join(prefix, "bin", gochan_exe))
fs_action("copy", gochan_exe, path.join(prefix, "bin", gochan_exe))
print("Note: gochan-migration has been put on indefinite suspention. See README.md")
if path.exists(migration_exe) is False:
build()
print("Installing ", migration_exe, "to", path.join(prefix, "bin", migration_exe))
print(
"gochan was successfully installed. If you haven't already, you should copy\n",

17
cmd/gochan-migration/internal/common/handler.go
View file

@ -33,22 +33,19 @@ func NewMigrationError(oldChanType string, errMessage string) *MigrationError {
return &MigrationError{oldChanType: oldChanType, errMessage: errMessage}
}
type DBOptions struct {
Host string `json:"dbhost"`
DBType string `json:"dbtype"`
Username string `json:"dbusername"`
Password string `json:"dbpassword"`
OldDBName string `json:"olddbname"`
OldChanType string `json:"oldchan"`
NewDBName string `json:"newdbname"`
TablePrefix string `json:"tableprefix"`
type MigrationOptions struct {
ChanType string
OldChanRoot string
OldChanConfig string
OldDBName string
NewDBName string
}
// DBMigrator is used for handling the migration from one database type to a
// database compatible with gochan 3.x onward
type DBMigrator interface {
// Init sets the variables for connecting to the databases
Init(options DBOptions) error
Init(options MigrationOptions) error
// MigrateDB migrates the imageboard data (posts, boards, etc) to the new database
MigrateDB() error

12
cmd/gochan-migration/internal/kusabax/kusabax.go
View file

@ -2,6 +2,7 @@ package kusabax
import (
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/gcsql"
)
var (
@ -9,11 +10,12 @@ var (
)
type KusabaXMigrator struct {
// db *gcsql.GCDB
// options common.DBOptions
db *gcsql.GCDB
options common.MigrationOptions
}
func (m *KusabaXMigrator) Init(options common.DBOptions) error {
func (m *KusabaXMigrator) Init(options common.MigrationOptions) error {
m.options = options
return unimplemented
}
@ -22,8 +24,8 @@ func (m *KusabaXMigrator) MigrateDB() error {
}
func (m *KusabaXMigrator) Close() error {
/* if m.db != nil {
if m.db != nil {
return m.db.Close()
} */
}
return nil
}

81
cmd/gochan-migration/internal/pre2021/pre2021.go
View file

@ -2,8 +2,11 @@
package pre2021
import (
"encoding/json"
"fmt"
"io/ioutil"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/config"
"github.com/gochan-org/gochan/pkg/gcsql"
)
@ -16,49 +19,63 @@ const (
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?) as num_tables`
)
type Pre2021Migrator struct {
db *gcsql.GCDB
options common.DBOptions
type Pre2021Config struct {
DBtype string
DBhost string
DBname string
DBusername string
DBpassword string
DBprefix string
}
func (m *Pre2021Migrator) Init(options common.DBOptions) error {
type Pre2021Migrator struct {
db *gcsql.GCDB
options common.MigrationOptions
config Pre2021Config
}
func (m *Pre2021Migrator) readConfig() error {
ba, err := ioutil.ReadFile(m.options.OldChanConfig)
if err != nil {
return err
}
return json.Unmarshal(ba, &m.config)
}
func (m *Pre2021Migrator) Init(options common.MigrationOptions) error {
m.options = options
var err error
err := m.readConfig()
if err != nil {
return err
}
m.db, err = gcsql.Open(
m.options.Host, m.options.DBType, "", m.options.Username,
m.options.Password, options.TablePrefix)
m.config.DBhost, m.config.DBtype, m.config.DBname, m.config.DBusername,
m.config.DBpassword, m.config.DBprefix)
return err
}
func (m *Pre2021Migrator) MigrateDB() error {
chkDbStmt, err := m.db.PrepareSQL(mysqlDbInfoSQL)
// select id,thread_id,name,tripcode,email,subject,message from gc_posts;
rows, err := m.db.QuerySQL(`SELECT id,parentid,name,tripcode,email,subject,message FROM DBPREFIXposts`)
if err != nil {
return err
}
defer chkDbStmt.Close()
var olddb []byte
var newdb []byte
var numTables int
if err = chkDbStmt.QueryRow(m.options.OldDBName, m.options.NewDBName, m.options.NewDBName).Scan(&olddb, &newdb, &numTables); err != nil {
return common.NewMigrationError("pre2021", err.Error())
var id int
var thread int
var name string
var tripcode string
var email string
var subject string
var message string
for rows.Next() {
if err = rows.Scan(&id, &thread, &name, &tripcode, &email, &subject, &message); err != nil {
return err
}
fmt.Printf(
"Post #%d in %d by %s!%s, email %q, subject %q, message: %q\n",
id, thread, name, tripcode, email, subject, message,
)
}
if olddb == nil {
return common.NewMigrationError("pre2021", "old database doesn't exist")
}
if newdb == nil {
return common.NewMigrationError("pre2021", "new database doesn't exist")
}
if numTables > 0 {
return common.NewMigrationError("pre2021", "new database must be empty")
}
gcsql.ConnectToDB(
m.options.Host, m.options.DBType, m.options.NewDBName,
m.options.Username, m.options.Password, m.options.TablePrefix)
cfg := config.GetSystemCriticalConfig()
gcsql.CheckAndInitializeDatabase(cfg.DBtype)
GetPosts(m.db)
return nil
}

12
cmd/gochan-migration/internal/tinyboard/tinyboard.go
View file

@ -2,6 +2,7 @@ package tinyboard
import (
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/pkg/gcsql"
)
var (
@ -9,11 +10,12 @@ var (
)
type TinyBoardMigrator struct {
// db *gcsql.GCDB
// options common.DBOptions
db *gcsql.GCDB
options common.MigrationOptions
}
func (m *TinyBoardMigrator) Init(options common.DBOptions) error {
func (m *TinyBoardMigrator) Init(options common.MigrationOptions) error {
m.options = options
return unimplemented
}
@ -22,8 +24,8 @@ func (m *TinyBoardMigrator) MigrateDB() error {
}
func (m *TinyBoardMigrator) Close() error {
/* if m.db != nil {
if m.db != nil {
return m.db.Close()
} */
}
return nil
}

120
cmd/gochan-migration/main.go
View file

@ -1,13 +1,13 @@
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
"github.com/gochan-org/gochan/cmd/gochan-migration/internal/pre2021"
"github.com/gochan-org/gochan/pkg/config"
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
@ -24,7 +24,6 @@ the README and/or the -h command line flag before you use it.
var (
versionStr string
bufIn = bufio.NewReader(os.Stdin)
)
func fatalPrintln(args ...interface{}) {
@ -32,81 +31,46 @@ func fatalPrintln(args ...interface{}) {
os.Exit(1)
}
func readConfig(filename string, options *common.DBOptions) {
ba, err := ioutil.ReadFile(filename)
if err != nil {
fatalPrintln(err)
func main() {
var options common.MigrationOptions
config.InitConfig(versionStr)
flag.StringVar(&options.ChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
flag.StringVar(&options.OldChanConfig, "oldconfig", "", "The path to the old chan's configuration file")
flag.Parse()
if options.ChanType == "" || options.OldChanConfig == "" {
flag.PrintDefaults()
fmt.Println("Missing required database connection info")
os.Exit(1)
return
}
if err = json.Unmarshal(ba, options); err != nil {
fatalPrintln(err)
fmt.Printf(banner, versionStr)
var migrator common.DBMigrator
switch options.ChanType {
case "pre2021":
migrator = &pre2021.Pre2021Migrator{}
case "kusabax":
fallthrough
case "tinyboard":
fallthrough
default:
fmt.Printf(
"Unsupported chan type %q, Currently only pre2021 database migration is supported\n",
options.ChanType)
os.Exit(1)
}
}
func main() {
fmt.Println(
"gochan-migration has been a gargantuan time sink and has wasted a lot of time that would be much better",
"spent working on other features, so I am putting its development on indefinite hiatus as of 12/18/2021.",
"It may or may not come back, but for the time being, RIP gochan-migration, we hardly knew ya.",
)
os.Exit(1)
// var options common.DBOptions
// var migrationConfigFile string
// flag.StringVar(&migrationConfigFile, "migrationconfig", "", "a JSON file to use for supplying the required migration information (ignores all other set arguments if used)")
// flag.StringVar(&options.OldChanType, "oldchan", "", "The imageboard we are migrating from (currently only pre2021 is supported, but more are coming")
// flag.StringVar(&options.Host, "dbhost", "", "The database host or socket file to connect to")
// flag.StringVar(&options.DBType, "dbtype", "mysql", "The kind of database server we are connecting to (currently only mysql is supported)")
// flag.StringVar(&options.Username, "dbusername", "", "The database username")
// flag.StringVar(&options.Password, "dbpassword", "", "The database password (if required by SQL account)")
// flag.StringVar(&options.OldDBName, "olddbname", "", "The name of the old database")
// flag.StringVar(&options.NewDBName, "newdbname", "", "The name of the new database")
// flag.StringVar(&options.TablePrefix, "tableprefix", "", "Prefix for the SQL tables' names")
// flag.Parse()
// if migrationConfigFile != "" {
// readConfig(migrationConfigFile, &options)
// }
// if options.OldChanType == "" || options.Host == "" || options.DBType == "" || options.Username == "" || options.OldDBName == "" || options.NewDBName == "" {
// flag.PrintDefaults()
// fmt.Println("Missing required database connection info")
// os.Exit(1)
// return
// }
// fmt.Printf(banner, versionStr)
// var migrator common.DBMigrator
// switch options.OldChanType {
// case "kusabax":
// migrator = &kusabax.KusabaXMigrator{}
// case "pre2021":
// migrator = &pre2021.Pre2021Migrator{}
// case "tinyboard":
// migrator = &tinyboard.TinyBoardMigrator{}
// default:
// fatalPrintln("Invalid oldchan value")
// }
// err := migrator.Init(options)
// if err != nil {
// fatalPrintln("Error initializing migrator:", err)
// }
// defer migrator.Close()
// // config.InitConfig(versionStr)
// /* gclog.Printf(gclog.LStdLog, "Starting gochan migration (gochan v%s)", versionStr)
// err := gcmigrate.Entry(1) //TEMP, get correct database version from command line or some kind of table. 1 Is the current version we are working towards
// if err != nil {
// gclog.Printf(gclog.LErrorLog, "Error while migrating: %s", err)
// } */
// if options.OldDBName == options.NewDBName {
// fatalPrintln("The old database name must not be the same as the new one.")
// }
// if err = migrator.MigrateDB(); err != nil {
// fatalPrintln(err)
// }
// fmt.Println("Database migration successful!")
err := migrator.Init(options)
if err != nil {
fmt.Printf("Unable to initialize %s migrator: %s\n", options.ChanType, err.Error())
os.Exit(1)
}
defer migrator.Close()
if err = migrator.MigrateDB(); err != nil {
fmt.Println("Error migrating database: ", err.Error())
os.Exit(1)
}
fmt.Println("Database migration successful!")
}

0
devtools/dbconnect.sh Normal file → Executable file
View file

65
devtools/get_pre2021.sh Executable file
View file

@ -0,0 +1,65 @@
#!/usr/bin/env bash
# Shell script that downloads a pre-migration gochan release for testing gochan-migration
# This should only be used in a development environment
TESTING_VERSION="v2.12.0"
RELEASE_DIR="gochan-${TESTING_VERSION}_linux"
RELEASE_GZ="$RELEASE_DIR.tar.gz"
RELEASE_URL="https://github.com/gochan-org/gochan/releases/download/$TESTING_VERSION/$RELEASE_GZ"
if [ -z "$STY" ]; then
echo "This command should be run from a screen instance"
echo "Example: screen -S get_pre2021 $0"
exit 1
fi
if [ "$USER" != "vagrant" ]; then
echo "This must be run in the vagrant VM (expected \$USER to be vagrant, got $USER)"
exit 1
fi
cd ~
rm -f $RELEASE_GZ
echo "Downloading $RELEASE_GZ"
wget -q --show-progress $RELEASE_URL
echo "Extracting $RELEASE_GZ"
tar -xf gochan-v2.12.0_linux.tar.gz
cd $RELEASE_DIR
cp sample-configs/gochan.example.json gochan.json
echo "Modifying $PWD/gochan.json for testing migration"
sed -i gochan.json \
-e 's/"Port": .*/"Port": 9000,/' \
-e 's/"UseFastCGI": false/"UseFastCGI": true/' \
-e "s/\"DBtype\": .*/\"DBtype\": \""$DBTYPE"\",/" \
-e 's/"DBpassword": ""/"DBpassword": "gochan"/' \
-e 's/"DBname": "gochan"/"DBname": "gochan_pre2021"/' \
-e 's/"SiteName": "Gochan"/"SiteName": "Gochan pre-2021"/' \
-e 's/"SiteSlogan": ""/"SiteSlogan": "Gochan instance used for testing migrating pre-2021"/' \
-e 's/"DebugMode": false/"DebugMode": true/' \
-e 's/"Verbosity": 0/"Verbosity": 1/'
if [ "$DBTYPE" = "mysql" ]; then
echo "Creating pre-2021 MySQL DB 'gochan_pre2021' if it doesn't already exist"
sudo mysql <<- EOF1
CREATE DATABASE IF NOT EXISTS gochan_pre2021;
GRANT USAGE ON *.* TO gochan IDENTIFIED BY 'gochan'; \
GRANT ALL PRIVILEGES ON gochan_pre2021.* TO gochan; \
SET PASSWORD FOR 'gochan'@'%' = PASSWORD('gochan');
FLUSH PRIVILEGES;
EOF1
elif [ "$DBTYPE" = "postgresql" ]; then
echo "Creating pre-2021 PostgreSQL DB 'gochan_pre2021' if it doesn't already exist"
sed -i /etc/gochan/gochan.json \
-e 's/"DBhost": ".*"/"DBhost": "127.0.0.1"/'
sudo -u postgres psql -f - <<- EOF1
CREATE DATABASE gochan_pre2021;
GRANT ALL PRIVILEGES ON DATABASE gochan_pre2021 TO gochan;
EOF1
else
echo "Currently using unsupported \$DBTYPE: $DBTYPE"
exit 1
fi
sudo ./gochan

6
vagrant/bootstrap.sh
View file

@ -109,9 +109,13 @@ if [ "$DBTYPE" = "postgresql" ]; then
fi
# a convenient script for connecting to the db, whichever type we're using
cp /vagrant/devtools/dbconnect.sh /home/vagrant/dbconnect.sh
ln -s {/vagrant/devtools,/home/vagrant}/dbconnect.sh
chmod +x /home/vagrant/dbconnect.sh
# used for testing migration from the pre-2021 db schema
ln -s {/vagrant/devtools,/home/vagrant}/get_pre2021.sh
chmod +x get_pre2021.sh
cat <<EOF >>/home/vagrant/.bashrc
export DBTYPE=$DBTYPE
export GOPATH=/home/vagrant/go