Migrate thread data to new threads table

cmd/gochan-migration/internal/pre2021/boards.go

@@ -133,6 +133,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 	for _, board := range allBoards {
 		m.boards = append(m.boards, migrationBoard{
 			oldSectionID: -1,
+			oldID:        -1,
 			Board:        board,
 		})
 	}
@@ -149,7 +150,7 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 		var board migrationBoard
 		var maxPages int
 		if err = rows.Scan(
-			&board.ID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description,
+			&board.oldID, &board.NavbarPosition, &board.Dir, &board.Title, &board.Subtitle, &board.Description,
 			&board.SectionID, &board.MaxFilesize, &maxPages, &board.DefaultStyle, &board.Locked, &board.CreatedAt,
 			&board.AnonymousName, &board.ForceAnonymous, &board.AutosageAfter, &board.NoImagesAfter, &board.MaxMessageLength,
 			&board.AllowEmbeds, &board.RedirectToThread, &board.RequireFile, &board.EnableCatalog,
@@ -162,9 +163,13 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 
 		for b, newBoard := range m.boards {
 			if newBoard.Dir == board.Dir {
-				m.boards[b].oldID = board.ID
+				m.boards[b].oldID = board.oldID
 				m.boards[b].oldSectionID = board.SectionID
-				common.LogInfo().Str("board", board.Dir).Msg("Board already exists in new db, updating values")
+				common.LogInfo().
+					Str("board", board.Dir).
+					Int("oldBoardID", board.ID).
+					Int("migratedBoardID", newBoard.ID).
+					Msg("Board already exists in new db, updating values")
 				// don't update other values in the array since they don't affect migrating threads or posts
 				if _, err = gcsql.ExecSQL(`UPDATE DBPREFIXboards
 					SET uri = ?, navbar_position = ?, title = ?, subtitle = ?, description = ?,
@@ -195,7 +200,11 @@ func (m *Pre2021Migrator) migrateBoardsToNewDB() error {
 			errEv.Err(err).Caller().Str("board", board.Dir).Msg("Failed to create board")
 			return err
 		}
-		common.LogInfo().Str("board", board.Dir).Msg("Board successfully created")
+		m.boards = append(m.boards, board)
+		common.LogInfo().
+			Str("dir", board.Dir).
+			Int("boardID", board.ID).
+			Msg("Board successfully created")
 	}
 	if err = gcsql.ResetBoardSectionArrays(); err != nil {
 		errEv.Err(err).Caller().Msg("Failed to reset board and section arrays")

cmd/gochan-migration/internal/pre2021/posts.go

@@ -1,8 +1,6 @@
 package pre2021
 
 import (
-	"database/sql"
-	"fmt"
 	"time"
 
 	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
@@ -10,168 +8,112 @@ import (
 )
 
 type postTable struct {
-	id                int
-	boardid           int
-	parentid          int
-	name              string
-	tripcode          string
-	email             string
-	subject           string
-	message           string
-	message_raw       string
-	password          string
-	filename          string
-	filename_original string
-	file_checksum     string
-	filesize          int
-	image_w           int
-	image_h           int
-	thumb_w           int
-	thumb_h           int
-	ip                string
-	tag               string
-	timestamp         time.Time
-	autosage          bool
-	deleted_timestamp time.Time
-	bumped            time.Time
-	stickied          bool
-	locked            bool
-	reviewed          bool
-
-	newBoardID int
-	foundBoard bool
-	// oldParentID int
+	gcsql.Post
+	// id               int
+	// boardID          int
+	// parentID         int
+	// name             string
+	// tripcode         string
+	// email            string
+	// subject          string
+	// message          string
+	// messageRaw       string
+	// password         string
+	filename         string
+	filenameOriginal string
+	fileChecksum     string
+	filesize         int
+	imageW           int
+	imageH           int
+	thumbW           int
+	thumbH           int
+	// ip               string
+	// tag              string
+	// timestamp        time.Time
+	autosage bool
+	bumped   time.Time
+	stickied bool
+	locked   bool
+	// reviewed         bool
+	oldID       int
+	boardID     int
+	oldBoardID  int
+	oldParentID int
 }
 
 func (m *Pre2021Migrator) MigratePosts() error {
-	var err error
-	if err = m.migrateThreads(); err != nil {
-		return err
+	if m.IsMigratingInPlace() {
+		return m.migratePostsInPlace()
 	}
-	return m.migratePostsUtil()
+	return m.migratePostsToNewDB()
 }
 
-func (m *Pre2021Migrator) migrateThreads() error {
-	tx, err := m.db.Begin()
-	if err != nil {
-		return err
-	}
+func (m *Pre2021Migrator) migratePostsToNewDB() error {
+	errEv := common.LogError()
+	defer errEv.Discard()
 
-	stmt, err := m.db.PrepareSQL(postsQuery, tx)
+	tx, err := gcsql.BeginTx()
 	if err != nil {
-		tx.Rollback()
+		errEv.Err(err).Caller().Msg("Failed to start transaction")
 		return err
 	}
-	rows, err := stmt.Query()
-	if err != nil && err != sql.ErrNoRows {
-		tx.Rollback()
+	defer tx.Rollback()
+
+	rows, err := m.db.QuerySQL(threadsQuery)
+	if err != nil {
+		errEv.Err(err).Caller().Msg("Failed to get threads")
 		return err
 	}
-	defer stmt.Close()
 	defer rows.Close()
+
+	var threadIDsWithInvalidBoards []int
+	var missingBoardIDs []int
 	for rows.Next() {
-		var post postTable
+		var thread postTable
 		if err = rows.Scan(
-			&post.id,
-			&post.boardid,
-			&post.parentid,
-			&post.name,
-			&post.tripcode,
-			&post.email,
-			&post.subject,
-			&post.message,
-			&post.message_raw,
-			&post.password,
-			&post.filename,
-			&post.filename_original,
-			&post.file_checksum,
-			&post.filesize,
-			&post.image_w,
-			&post.image_h,
-			&post.thumb_w,
-			&post.thumb_h,
-			&post.ip,
-			&post.tag,
-			&post.timestamp,
-			&post.autosage,
-			&post.deleted_timestamp,
-			&post.bumped,
-			&post.stickied,
-			&post.locked,
-			&post.reviewed,
+			&thread.oldID, &thread.oldBoardID, &thread.oldParentID, &thread.Name, &thread.Tripcode, &thread.Email,
+			&thread.Subject, &thread.Message, &thread.MessageRaw, &thread.Password, &thread.filename,
+			&thread.filenameOriginal, &thread.fileChecksum, &thread.filesize, &thread.imageW, &thread.imageH,
+			&thread.thumbW, &thread.thumbH, &thread.IP, &thread.CreatedOn, &thread.autosage,
+			&thread.bumped, &thread.stickied, &thread.locked,
 		); err != nil {
-			tx.Rollback()
+			errEv.Err(err).Caller().Msg("Failed to scan thread")
 			return err
 		}
-		var postBoardDir string
-		for _, newBoard := range gcsql.AllBoards {
-			if newBoard.Dir == postBoardDir {
-				post.newBoardID = newBoard.ID
-				post.foundBoard = true
+		var foundBoard bool
+		for _, board := range m.boards {
+			if board.oldID == thread.oldBoardID {
+				thread.boardID = board.ID
+				foundBoard = true
+				break
 			}
 		}
-		if !post.foundBoard {
-			common.LogWarning().Int("boardID", post.boardid).
-				Msg("Pre-migrated post has an invalid boardid (board doesn't exist), skipping")
+		if !foundBoard {
+			threadIDsWithInvalidBoards = append(threadIDsWithInvalidBoards, thread.oldID)
+			missingBoardIDs = append(missingBoardIDs, thread.oldBoardID)
 			continue
 		}
 
-		// var stmt *sql.Stmt
-		// var err error
-		preparedStr, _ := gcsql.SetupSQLString(`SELECT id FROM DBPREFIXboards WHERE ui = ?`, m.db)
-		stmt, err := tx.Prepare(preparedStr)
-		if err != nil {
-			tx.Rollback()
-			return err
+		if thread.ThreadID, err = gcsql.CreateThread(tx, thread.boardID, thread.locked, thread.stickied, thread.autosage, false); err != nil {
+			errEv.Err(err).Caller().
+				Int("boardID", thread.boardID).
+				Msg("Failed to create thread")
 		}
-		stmt.QueryRow(post.boardid).Scan(&post.newBoardID)
-
-		// gcsql.QueryRowSQL(`SELECT id FROM DBPREFIXboards WHERE uri = ?`, []interface{}{})
-		if post.parentid == 0 {
-			// post is a thread, save it to the DBPREFIXthreads table
-			// []interfaceP{{post.newParentID}
-
-			if err = gcsql.QueryRowSQL(
-				`SELECT board_id FROM DBPREFIXthreads ORDER BY board_id LIMIT 1`,
-				nil,
-				[]interface{}{&post.newBoardID},
-			); err != nil {
-				tx.Rollback()
-				return err
-			}
-			fmt.Println("Current board ID:", post.newBoardID)
-			prepareStr, _ := gcsql.SetupSQLString(
-				`INSERT INTO DBPREFIXthreads
-				(board_id, locked, stickied)
-				VALUES(?, ?, ?)`, m.db)
-			stmt, err = tx.Prepare(prepareStr)
-			if err != nil {
-				tx.Rollback()
-				return err
-			}
-			stmt.Exec(post.newBoardID, post.locked, post.stickied)
-			// 			// stmt, err := db.Prepare("INSERT table SET unique_id=? ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id)")
-			// 			gcsql.ExecSQL(`INSERT INTO DBPREFIXthreads (board_id) VALUES(?)`, post.newBoardID)
-
-			// 			/*
-			// id
-			// board_id
-			// locked
-			// stickied
-			// anchored
-			// cyclical
-			// last_bump
-			// deleted_at
-			// is_deleted
-
-			// 			*/
-
-		}
-		m.posts = append(m.posts, post)
 	}
-	return tx.Commit()
+	if len(threadIDsWithInvalidBoards) > 0 {
+		errEv.Caller().
+			Ints("threadIDs", threadIDsWithInvalidBoards).
+			Ints("boardIDs", missingBoardIDs).
+			Msg("Failed to find boards for threads")
+		return common.NewMigrationError("pre2021", "Found threads with missing boards")
+	}
+
+	if err = tx.Commit(); err != nil {
+		errEv.Err(err).Caller().Msg("Failed to commit transaction")
+	}
+	return err
 }
 
-func (*Pre2021Migrator) migratePostsUtil() error {
-	return nil
+func (m *Pre2021Migrator) migratePostsInPlace() error {
+	return common.NewMigrationError("pre2021", "not yet implemented")
 }

cmd/gochan-migration/internal/pre2021/posts_test.go

@@ -0,0 +1,36 @@
+package pre2021
+
+import (
+	"testing"
+
+	"github.com/gochan-org/gochan/pkg/gcsql"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigratePostsToNewDB(t *testing.T) {
+	outDir := t.TempDir()
+	migrator := setupMigrationTest(t, outDir, false)
+	if !assert.False(t, migrator.IsMigratingInPlace(), "This test should not be migrating in place") {
+		t.FailNow()
+	}
+
+	if !assert.NoError(t, migrator.MigrateBoards()) {
+		t.FailNow()
+	}
+
+	var numThreads int
+	if !assert.NoError(t, migrator.db.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXposts WHERE parentid = 0 AND deleted_timestamp IS NULL", nil, []any{&numThreads}), "Failed to get number of threads") {
+		t.FailNow()
+	}
+	assert.Equal(t, 2, numThreads, "Expected to have two threads pre-migration")
+
+	if !assert.NoError(t, migrator.MigratePosts()) {
+		t.FailNow()
+	}
+
+	var numMigratedThreads int
+	if !assert.NoError(t, gcsql.QueryRowSQL("SELECT COUNT(*) FROM DBPREFIXthreads", nil, []any{&numMigratedThreads}), "Failed to get number of migrated threads") {
+		t.FailNow()
+	}
+	assert.Equal(t, 2, numMigratedThreads, "Expected to have three migrated threads")
+}

cmd/gochan-migration/internal/pre2021/pre2021.go

@@ -77,7 +77,7 @@ func (m *Pre2021Migrator) MigrateDB() (bool, error) {
 		errEv.Caller().Err(err).Msg("Failed to migrate boards")
 		return false, err
 	}
-	common.LogInfo().Msg("Migrated boards")
+	common.LogInfo().Msg("Migrated boards successfully")
 	// if err = m.MigratePosts(); err != nil {
 	// 	return false, err
 	// }

cmd/gochan-migration/internal/pre2021/pre2021_test.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/gochan-org/gochan/cmd/gochan-migration/internal/common"
 	"github.com/gochan-org/gochan/pkg/config"
 	"github.com/gochan-org/gochan/pkg/gcsql"
 	"github.com/gochan-org/gochan/pkg/gcutil/testutil"
@@ -21,6 +22,10 @@ func setupMigrationTest(t *testing.T, outDir string, migrateInPlace bool) *Pre20
 	if !assert.NoError(t, err) {
 		t.FailNow()
 	}
+	if !assert.NoError(t, common.InitTestMigrationLog(t)) {
+		t.FailNow()
+	}
+
 	dbName := "gochan-pre2021.sqlite3db"
 	dbHost := path.Join(dir, sqlite3DBDir, dbName)
 	migratedDBName := "gochan-migrated.sqlite3db"

cmd/gochan-migration/internal/pre2021/queries.go

@@ -1,7 +1,7 @@
 package pre2021
 
 const (
-	sectionsQuery = `SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections`
+	sectionsQuery = "SELECT id, list_order, hidden, name, abbreviation FROM DBPREFIXsections"
 
 	boardsQuery = `SELECT id, list_order, dir, title, subtitle, description, section, max_file_size, max_pages,
 default_style, locked, created_on, anonymous, forced_anon, autosage_after, no_images_after, max_message_length, embeds_allowed,
@@ -9,7 +9,8 @@ redirect_to_thread, require_file, enable_catalog
 FROM DBPREFIXboards`
 
 	postsQuery = `SELECT id, boardid, parentid, name, tripcode, email, subject, message, message_raw, password, filename,
-filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, tag, timestamp, autosage, deleted_timestamp,
-bumped, stickied, locked, reviewed
-FROM DBPREFIXposts WHERE deleted_timestamp = NULL`
+filename_original, file_checksum, filesize, image_w, image_h, thumb_w, thumb_h, ip, timestamp, autosage,
+bumped, stickied, locked FROM DBPREFIXposts WHERE deleted_timestamp IS NULL`
+
+	threadsQuery = postsQuery + " AND parentid = 0"
 )