Add gcsql.GetSectionFromName and start overhaul of pre2021 migration

cmd/gochan-migration/internal/common/handler.go

@@ -41,7 +41,6 @@ type MigrationOptions struct {
 	OldChanConfig string
 	OldDBName     string
 	NewDBName     string
-	DirAction     int
 }
 
 // DBMigrator is used for handling the migration from one database type to a

cmd/gochan-migration/internal/pre2021/boards.go

@@ -1,92 +1,139 @@
 package pre2021
 
 import (
-	"fmt"
-	"log"
+	"errors"
+	"time"
 
 	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
 	"github.com/gochan-org/gochan/pkg/gcsql"
 )
 
-func (m *Pre2021Migrator) MigrateBoards() error {
-	defer func() {
-		if r := recover(); r != nil {
-			common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards")
-		}
-	}()
-	if m.oldBoards == nil {
-		m.oldBoards = map[int]string{}
-	}
-	if m.newBoards == nil {
-		m.newBoards = map[int]string{}
-	}
-	// get all boards from new db
-	fmt.Println("0")
-	err := gcsql.ResetBoardSectionArrays()
-	fmt.Println("1")
-	if err != nil {
-		fmt.Println("2")
-		return nil
-	}
-	fmt.Println("3")
+type boardTable struct {
+	id               int
+	listOrder        int
+	dir              string
+	boardType        int
+	uploadType       int
+	title            string
+	subtitle         string
+	description      string
+	section          int
+	maxFileSize      int
+	maxPages         int
+	defaultStyle     string
+	locked           bool
+	createdOn        time.Time
+	anonymous        string
+	forcedAnon       bool
+	maxAge           int
+	autosageAfter    int
+	noImagesAfter    int
+	maxMessageLength int
+	embedsAllowed    bool
+	redirectToThread bool
+	requireFile      bool
+	enableCatalog    bool
+}
 
-	// get boards from old db
-	rows, err := m.db.QuerySQL(boardsQuery)
+func (m *Pre2021Migrator) migrateBoardsInPlace() error {
+	return nil
+}
+
+func (m *Pre2021Migrator) createSectionIfNotExist(sectionCheck *gcsql.Section) (int, error) {
+	// to be used when not migrating in place, otherwise the section table should be altered
+	section, err := gcsql.GetSectionFromName(sectionCheck.Name)
+	if errors.Is(err, gcsql.ErrSectionDoesNotExist) {
+		// section doesn't exist, create it
+		section, err = gcsql.NewSection(sectionCheck.Name, section.Abbreviation, section.Hidden, section.Position)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return section.ID, nil
+}
+
+func (m *Pre2021Migrator) migrateSectionsToNewDB() error {
+	// creates sections in the new db if they don't exist, and also creates a migration section that
+	// boards will be set to, to be moved to the correct section by the admin after migration
+	rows, err := m.db.QuerySQL(sectionsQuery)
 	if err != nil {
 		return err
 	}
 	defer rows.Close()
+
 	for rows.Next() {
-		var id int
-		var dir string
-		var title string
-		var subtitle string
-		var description string
-		var section int
-		var max_file_size int
-		var max_pages int
-		var default_style string
-		var locked bool
-		var anonymous string
-		var forced_anon bool
-		var max_age int
-		var autosage_after int
-		var no_images_after int
-		var max_message_length int
-		var embeds_allowed bool
-		var redirect_to_thread bool
-		var require_file bool
-		var enable_catalog bool
+		var section gcsql.Section
 		if err = rows.Scan(
-			&id,
-			&dir,
-			&title,
-			&subtitle,
-			&description,
-			&section,
-			&max_file_size,
-			&max_pages,
-			&default_style,
-			&locked,
-			&anonymous,
-			&forced_anon,
-			&max_age,
-			&autosage_after,
-			&no_images_after,
-			&max_message_length,
-			&embeds_allowed,
-			&redirect_to_thread,
-			&require_file,
-			&enable_catalog); err != nil {
+			&section.ID,
+			&section.Position,
+			&section.Hidden,
+			&section.Name,
+			&section.Abbreviation,
+		); err != nil {
+			return err
+		}
+		if _, err = m.createSectionIfNotExist(&section); err != nil {
+			return err
+		}
+	}
+	if err = rows.Close(); err != nil {
+		return err
+	}
+	m.migrationSectionID, err = m.createSectionIfNotExist(&gcsql.Section{
+		Name:         "Migrated Boards",
+		Abbreviation: "mb",
+		Hidden:       true,
+	})
+
+	return err
+}
+
+func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
+	if m.oldBoards == nil {
+		m.oldBoards = make(map[string]boardTable)
+	}
+	if m.newBoards == nil {
+		m.newBoards = make(map[string]boardTable)
+	}
+	errEv := common.LogError()
+	defer errEv.Discard()
+
+	err := m.migrateSectionsToNewDB()
+	if err != nil {
+		errEv.Err(err).Msg("Failed to migrate sections")
+	}
+
+	// get all boards from new db
+	if err = gcsql.ResetBoardSectionArrays(); err != nil {
+		return nil
+	}
+
+	// get boards from old db
+	rows, err := m.db.QuerySQL(boardsQuery)
+	if err != nil {
+		errEv.Err(err).Caller().Msg("Failed to query old database boards")
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var board boardTable
+		if err = rows.Scan(
+			&board.id, &board.listOrder, &board.dir, &board.boardType, &board.uploadType, &board.title, &board.subtitle,
+			&board.description, &board.section, &board.maxFileSize, &board.maxPages, &board.defaultStyle, &board.locked,
+			&board.createdOn, &board.anonymous, &board.forcedAnon, &board.maxAge, &board.autosageAfter, &board.noImagesAfter,
+			&board.maxMessageLength, &board.embedsAllowed, &board.redirectToThread, &board.requireFile, &board.enableCatalog,
+		); err != nil {
+			errEv.Err(err).Caller().Msg("Failed to scan row into board")
 			return err
 		}
 		found := false
-		for b := range gcsql.AllBoards {
-			if _, ok := m.oldBoards[id]; !ok {
-				m.oldBoards[id] = dir
+		for _, newBoard := range gcsql.AllBoards {
+			if _, ok := m.oldBoards[board.dir]; !ok {
+				m.oldBoards[board.dir] = board
 			}
-			if gcsql.AllBoards[b].Dir == dir {
-				log.Printf("Board /%s/ already exists in new db, moving on\n", dir)
+
+			if newBoard.Dir == board.dir {
+				common.LogWarning().Str("board", board.dir).Msg("Board already exists in new db, moving on")
 				found = true
 				break
 			}
@@ -97,48 +144,41 @@ func (m *Pre2021Migrator) MigrateBoards() error {
 		// create new board using the board data from the old db
 		// omitting things like ID and creation date since we don't really care
 		if err = gcsql.CreateBoard(&gcsql.Board{
-			Dir:         dir,
-			Title:       title,
-			Subtitle:    subtitle,
-			Description: description,
-			SectionID:   section,
-			MaxFilesize: max_file_size,
-			// MaxPages:         max_pages,
-			DefaultStyle:   default_style,
-			Locked:         locked,
-			AnonymousName:  anonymous,
-			ForceAnonymous: forced_anon,
-			// MaxAge:           max_age,
-			AutosageAfter:    autosage_after,
-			NoImagesAfter:    no_images_after,
-			MaxMessageLength: max_message_length,
-			AllowEmbeds:      embeds_allowed,
-			RedirectToThread: redirect_to_thread,
-			RequireFile:      require_file,
-			EnableCatalog:    enable_catalog,
+			Dir:              board.dir,
+			Title:            board.title,
+			Subtitle:         board.subtitle,
+			Description:      board.description,
+			SectionID:        board.section,
+			MaxFilesize:      board.maxFileSize,
+			DefaultStyle:     board.defaultStyle,
+			Locked:           board.locked,
+			AnonymousName:    board.anonymous,
+			ForceAnonymous:   board.forcedAnon,
+			AutosageAfter:    board.autosageAfter,
+			NoImagesAfter:    board.noImagesAfter,
+			MaxMessageLength: board.maxMessageLength,
+			AllowEmbeds:      board.embedsAllowed,
+			RedirectToThread: board.redirectToThread,
+			RequireFile:      board.requireFile,
+			EnableCatalog:    board.enableCatalog,
 		}, false); err != nil {
+			errEv.Err(err).Caller().Str("board", board.dir).Msg("Failed to create board")
 			return err
 		}
-		m.newBoards[id] = dir
-		log.Printf("/%s/ successfully migrated in the database", dir)
-		// Automatic directory migration has the potential to go horribly wrong, so I'm leaving this
-		// commented out for now
-		// switch m.options.DirAction {
-		// case common.DirCopy:
-
-		// case common.DirMove:
-		// 	// move the old directory (probably should copy instead) to the new one
-		// 	newDocumentRoot := config.GetSystemCriticalConfig().DocumentRoot
-		// 	log.Println("Old board path:", path.Join(m.config.DocumentRoot, dir))
-		// 	log.Println("Old board path:", path.Join(newDocumentRoot, dir))
-		// 	if err = os.Rename(
-		// 		path.Join(m.config.DocumentRoot, dir),
-		// 		path.Join(newDocumentRoot, dir),
-		// 	); err != nil {
-		// 		return err
-		// 	}
-		// 	log.Printf("/%s/ directory/files successfully moved")
-		// }
+		m.newBoards[board.dir] = board
+		common.LogInfo().Str("board", board.dir).Msg("Board successfully migrated")
 	}
 	return nil
 }
+
+func (m *Pre2021Migrator) MigrateBoards() error {
+	defer func() {
+		if r := recover(); r != nil {
+			common.LogFatal().Interface("recover", r).Msg("Recovered from panic in MigrateBoards")
+		}
+	}()
+	if m.IsMigratingInPlace() {
+		return m.migrateBoardsInPlace()
+	}
+	return m.migrateBoardsToNewDB()
+}

cmd/gochan-migration/internal/pre2021/posts.go

@@ -3,9 +3,9 @@ package pre2021
 import (
 	"database/sql"
 	"fmt"
-	"log"
 	"time"
 
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
 	"github.com/gochan-org/gochan/pkg/gcsql"
 )
 
@@ -39,6 +39,7 @@ type postTable struct {
 	reviewed          bool
 
 	newBoardID int
+	foundBoard bool
 	// oldParentID int
 }
 
@@ -102,12 +103,21 @@ func (m *Pre2021Migrator) migrateThreads() error {
 			tx.Rollback()
 			return err
 		}
-		_, ok := m.oldBoards[post.boardid]
-		if !ok {
-			// board doesn't exist
-			log.Printf(
-				"Pre-migrated post #%d has an invalid boardid %d (board doesn't exist), skipping",
-				post.id, post.boardid)
+		var postBoardDir string
+		for _, oldBoard := range m.oldBoards {
+			if oldBoard.id == post.boardid {
+				postBoardDir = oldBoard.dir
+			}
+		}
+		for _, newBoard := range gcsql.AllBoards {
+			if newBoard.Dir == postBoardDir {
+				post.newBoardID = newBoard.ID
+				post.foundBoard = true
+			}
+		}
+		if !post.foundBoard {
+			common.LogWarning().Int("boardID", post.boardid).
+				Msg("Pre-migrated post has an invalid boardid (board doesn't exist), skipping")
 			continue
 		}
 

cmd/gochan-migration/internal/pre2021/pre2021.go

@@ -20,9 +20,10 @@ type Pre2021Migrator struct {
 	options *common.MigrationOptions
 	config  Pre2021Config
 
-	posts     []postTable
-	oldBoards map[int]string // map[boardid]dir
-	newBoards map[int]string // map[board]dir
+	migrationSectionID int
+	posts              []postTable
+	oldBoards          map[string]boardTable
+	newBoards          map[string]boardTable
 }
 
 // IsMigratingInPlace implements common.DBMigrator.

cmd/gochan-migration/internal/pre2021/queries.go

@@ -1,57 +1,15 @@
 package pre2021
 
 const (
-	boardsQuery = `SELECT
-	id,
-	dir,
-	type,
-	upload_type,
-	title,
-	subtitle,
-	description,
-	section,
-	max_file_size,
-	max_pages,
-	default_style,
-	locked,
-	anonymous,
-	forced_anon,
-	max_age,
-	autosage_after,
-	no_images_after,
-	max_message_length,
-	embeds_allowed,
-	redirect_to_thread,
-	require_file,
-	enable_catalog
-	FROM DBPREFIXboards`
+	sectionsQuery = `SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections`
 
-	postsQuery = `SELECT
-	id,
-	boardid,
-	parentid,
-	name,
-	tripcode,
-	email,
-	subject,
-	message,
-	message_raw,
-	password,
-	filename,
-	filename_original,
-	file_checksum,
-	filesize,
-	image_w,
-	image_h,
-	thumb_w,
-	thumb_h,
-	ip,
-	tag,
-	timestamp,
-	autosage,
-	deleted_timestamp,
-	bumped,
-	stickied,
-	locked,
-	reviewed from DBPREFIXposts WHERE deleted_timestamp = NULL`
+	boardsQuery = `SELECT id, list_order, dir, type, upload_type, title, subtitle, description, section, max_file_size, max_pages,
+default_style, locked, created_on, anonymous, forced_anon, max_age, autosage_after, no_images_after, max_message_length, embeds_allowed,
+redirect_to_thread, require_file, enable_catalog
+FROM DBPREFIXboards`
+
+	postsQuery = `SELECT id, boardid, parentid, name, tripcode, email, subject, message, message_raw, password, filename,
+filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, tag, timestamp, autosage, deleted_timestamp,
+bumped, stickied, locked, reviewed
+FROM DBPREFIXposts WHERE deleted_timestamp = NULL`
 )

pkg/gcsql/sections.go

@@ -9,7 +9,8 @@ import (
 var (
 	// AllSections provides a quick and simple way to access a list of all non-hidden sections without
 	// having to do any SQL queries. It and AllBoards are updated by ResetBoardSectionArrays
-	AllSections []Section
+	AllSections            []Section
+	ErrSectionDoesNotExist = errors.New("section does not exist")
 )
 
 // GetAllSections gets a list of all existing sections, optionally omitting hidden ones
@@ -62,18 +63,37 @@ func getOrCreateDefaultSectionID() (sectionID int, err error) {
 	return id, nil
 }
 
+// GetSectionFromID returns a section from the database, given its ID
 func GetSectionFromID(id int) (*Section, error) {
 	const query = `SELECT id, name, abbreviation, position, hidden FROM DBPREFIXsections WHERE id = ?`
 	var section Section
 	err := QueryRowTimeoutSQL(nil, query, []any{id}, []any{
 		&section.ID, &section.Name, &section.Abbreviation, &section.Position, &section.Hidden,
 	})
-	if err != nil {
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, ErrSectionDoesNotExist
+	} else if err != nil {
 		return nil, err
 	}
 	return &section, err
 }
 
+// GetSectionFromID returns a section from the database, given its name
+func GetSectionFromName(name string) (*Section, error) {
+	const query = `SELECT id, name, abbreviation, position, hidden FROM DBPREFIXsections WHERE name = ?`
+	var section Section
+	err := QueryRowTimeoutSQL(nil, query, []any{name}, []any{
+		&section.ID, &section.Name, &section.Abbreviation, &section.Position, &section.Hidden,
+	})
+	if errors.Is(err, sql.ErrNoRows) {
+		return nil, ErrSectionDoesNotExist
+	} else if err != nil {
+		return nil, err
+	}
+	return &section, err
+}
+
+// DeleteSection deletes a section from the database and resets the AllSections array
 func DeleteSection(id int) error {
 	const query = `DELETE FROM DBPREFIXsections WHERE id = ?`
 	_, err := ExecSQL(query, id)