Start readding postgresql and sqlite support broken by deprecation fixing

cmd/gochan-migration/main.go

@@ -83,10 +83,12 @@ func main() {
 	err := gcsql.ConnectToDB(
 		systemCritical.DBhost, systemCritical.DBtype, systemCritical.DBname,
 		systemCritical.DBusername, systemCritical.DBpassword, systemCritical.DBprefix)
-	gcsql.CheckAndInitializeDatabase(systemCritical.DBtype)
 	if err != nil {
 		log.Fatalf("Failed to connect to the database: %s", err.Error())
 	}
+	if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype); err != nil {
+		log.Fatalf("Failed to initialize the database: %s", err.Error())
+	}
 	defer gcsql.Close()
 
 	if err = migrator.Init(&options); err != nil {

cmd/gochan/main.go

@@ -53,12 +53,15 @@ func main() {
 		gcutil.LogFatal().Err(err).Msg("Failed to connect to the database")
 	}
 	fmt.Println("Connected to database")
-	gcsql.CheckAndInitializeDatabase(systemCritical.DBtype)
+	if err = gcsql.CheckAndInitializeDatabase(systemCritical.DBtype); err != nil {
+		fmt.Println("Failed to initialize the database:", err.Error())
+		gcutil.LogFatal().Err(err).Msg("Failed to initialize the database")
+	}
 	parseCommandLine()
 	serverutil.InitMinifier()
 
 	posting.InitCaptcha()
-	if err := gctemplates.InitTemplates(); err != nil {
+	if err = gctemplates.InitTemplates(); err != nil {
 		fmt.Println("Failed initializing templates:", err.Error())
 		gcutil.LogFatal().Err(err).Send()
 	}

pkg/building/boards.go

@@ -390,12 +390,10 @@ func buildBoard(board *gcsql.Board, force bool) error {
 	}
 	if board.EnableCatalog {
 		if err = BuildCatalog(board.ID); err != nil {
-			errEv.Err(err).Caller().Send()
 			return err
 		}
 	}
 	if err = BuildBoardListJSON(); err != nil {
-		errEv.Err(err).Caller().Send()
 		return err
 	}
 

pkg/building/building.go

@@ -30,15 +30,21 @@ type recentPost struct {
 
 func getRecentPosts() ([]recentPost, error) {
 	siteCfg := config.GetSiteConfig()
-	query := `SELECT id, thread_id AS threadid, message_raw,
-		(SELECT dir FROM DBPREFIXboards WHERE id = (
-			SELECT board_id FROM DBPREFIXthreads WHERE id = threadid)
-		) AS board,
-		COALESCE(
-			(SELECT filename FROM DBPREFIXfiles WHERE post_id = DBPREFIXposts.id LIMIT 1),
-		"") AS filename,
-		(SELECT id FROM DBPREFIXposts WHERE is_top_post = TRUE AND thread_id = threadid) AS top_post
-		FROM DBPREFIXposts WHERE is_deleted = FALSE LIMIT ` + strconv.Itoa(siteCfg.MaxRecentPosts)
+	query := `SELECT
+		DBPREFIXposts.id,
+		DBPREFIXposts.message_raw,
+		(SELECT dir FROM DBPREFIXboards WHERE id = t.board_id) AS dir,
+		p.id AS top_post
+	FROM
+		DBPREFIXposts
+	LEFT JOIN (
+		SELECT id, board_id FROM DBPREFIXthreads
+	) t ON t.id = DBPREFIXposts.thread_id
+	INNER JOIN (
+		SELECT
+			id, thread_id FROM DBPREFIXposts WHERE is_top_post
+	) p ON p.thread_id = DBPREFIXposts.thread_id
+	WHERE DBPREFIXposts.is_deleted = FALSE LIMIT ` + strconv.Itoa(siteCfg.MaxRecentPosts)
 	rows, err := gcsql.QuerySQL(query)
 	if err != nil {
 		return nil, err
@@ -76,10 +82,12 @@ func getRecentPosts() ([]recentPost, error) {
 
 // BuildFrontPage builds the front page using templates/front.html
 func BuildFrontPage() error {
+	errEv := gcutil.LogError(nil).
+		Str("template", "front")
+	defer errEv.Discard()
 	err := gctemplates.InitTemplates("front")
 	if err != nil {
-		gcutil.LogError(err).
-			Str("template", "front").Send()
+		errEv.Err(err).Caller().Send()
 		return errors.New("Error loading front page template: " + err.Error())
 	}
 	criticalCfg := config.GetSystemCriticalConfig()
@@ -87,8 +95,7 @@ func BuildFrontPage() error {
 
 	frontFile, err := os.OpenFile(path.Join(criticalCfg.DocumentRoot, "index.html"), os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0777)
 	if err != nil {
-		gcutil.LogError(err).
-			Str("building", "front").Send()
+		errEv.Err(err).Caller().Send()
 		return errors.New("Failed opening front page for writing: " + err.Error())
 	}
 	defer frontFile.Close()
@@ -97,8 +104,7 @@ func BuildFrontPage() error {
 	siteCfg := config.GetSiteConfig()
 	recentPostsArr, err = getRecentPosts()
 	if err != nil {
-		gcutil.LogError(err).
-			Str("building", "recent").Send()
+		errEv.Err(err).Caller().Send()
 		return errors.New("Failed loading recent posts: " + err.Error())
 	}
 
@@ -110,8 +116,7 @@ func BuildFrontPage() error {
 		"board_config": config.GetBoardConfig(""),
 		"recent_posts": recentPostsArr,
 	}, frontFile, "text/html"); err != nil {
-		gcutil.LogError(err).
-			Str("template", "front").Send()
+		errEv.Err(err).Caller().Send()
 		return errors.New("Failed executing front page template: " + err.Error())
 	}
 	return nil

pkg/building/posts.go

@@ -3,6 +3,7 @@ package building
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"html/template"
 	"os"
 	"path"
@@ -16,11 +17,10 @@ import (
 )
 
 const (
-	postQueryBase = `SELECT DBPREFIXposts.id AS postid, thread_id AS threadid, ip, name, tripcode, email, subject, created_on, created_on as last_modified,
-	(SELECT id FROM DBPREFIXposts WHERE thread_id = threadid AND is_top_post) AS parent_id,
+	postQueryBase = `SELECT DBPREFIXposts.id, DBPREFIXposts.thread_id, ip, name, tripcode, email, subject, created_on, created_on as last_modified,
+	p.id AS parent_id,
 	message, message_raw,
-	(SELECT board_id FROM DBPREFIXthreads where id = threadid LIMIT 1) AS boardid,
-	(SELECT dir FROM DBPREFIXboards WHERE id = boardid LIMIT 1) AS dir,
+	(SELECT dir FROM DBPREFIXboards WHERE id = t.board_id LIMIT 1) AS dir,
 	coalesce(DBPREFIXfiles.original_filename,'') as original_filename,
 	coalesce(DBPREFIXfiles.filename,'') AS filename,
 	coalesce(DBPREFIXfiles.checksum,'') AS checksum,
@@ -30,7 +30,13 @@ const (
 	coalesce(DBPREFIXfiles.width,0) AS width,
 	coalesce(DBPREFIXfiles.height,0) AS height
 	FROM DBPREFIXposts
-	LEFT JOIN DBPREFIXfiles ON DBPREFIXfiles.post_id = DBPREFIXposts.id WHERE is_deleted = FALSE`
+	LEFT JOIN DBPREFIXfiles ON DBPREFIXfiles.post_id = DBPREFIXposts.id AND is_deleted = FALSE
+	LEFT JOIN (
+		SELECT id, board_id FROM DBPREFIXthreads
+	) t ON t.id = DBPREFIXposts.thread_id
+	INNER JOIN (
+		SELECT id, thread_id FROM DBPREFIXposts where is_top_post
+	) p on p.thread_id = DBPREFIXposts.thread_id`
 )
 
 func truncateString(msg string, limit int, ellipsis bool) string {
@@ -157,12 +163,10 @@ func GetBuildablePostsByIP(ip string, limit int) ([]Post, error) {
 }
 
 func getBoardTopPosts(boardID int) ([]Post, error) {
-	const query = "SELECT * FROM (" + postQueryBase + " AND is_top_post) p WHERE boardid = ?"
+	const query = postQueryBase + " WHERE is_top_post AND t.board_id = ?"
 	rows, err := gcsql.QuerySQL(query, boardID)
 	if err != nil {
-		return nil, err
-	}
-	if err != nil {
+		fmt.Println(query)
 		return nil, err
 	}
 	defer rows.Close()

pkg/gcsql/posts.go

@@ -99,13 +99,14 @@ func GetThreadTopPost(threadID int) (*Post, error) {
 }
 
 func GetBoardTopPosts(boardID int) ([]Post, error) {
-	query := `SELECT * FROM (SELECT
-		id, thread_id AS threadid, (SELECT board_id FROM gc_threads WHERE gc_threads.id = threadid LIMIT 1) AS boardid,
-		is_top_post, ip, created_on, name, tripcode, is_role_signature, email, subject, message, message_raw,
+	query := `SELECT DBPREFIXposts.id, thread_id, is_top_post, ip, created_on, name,
+		tripcode, is_role_signature, email, subject, message, message_raw,
 		password, deleted_at, is_deleted, banned_message
-		FROM gc_posts WHERE is_deleted = FALSE and is_top_post
-		) posts
-		WHERE boardid = ?`
+		FROM DBPREFIXposts
+		LEFT JOIN (
+		SELECT id, board_id from DBPREFIXthreads
+		) t on t.id = DBPREFIXposts.thread_id
+		WHERE is_deleted = FALSE AND is_top_post AND t.board_id = ?`
 
 	rows, err := QuerySQL(query, boardID)
 	if err != nil {
@@ -115,11 +116,11 @@ func GetBoardTopPosts(boardID int) ([]Post, error) {
 	var posts []Post
 	for rows.Next() {
 		var post Post
-		var tmp int // only needed for WHERE clause in query
+		// var tmp int // only needed for WHERE clause in query
 
 		bannedMessage := new(string)
 		err = rows.Scan(
-			&post.ID, &post.ThreadID, &tmp, &post.IsTopPost, &post.IP, &post.CreatedOn, &post.Name,
+			&post.ID, &post.ThreadID, &post.IsTopPost, &post.IP, &post.CreatedOn, &post.Name,
 			&post.Tripcode, &post.IsRoleSignature, &post.Email, &post.Subject, &post.Message,
 			&post.MessageRaw, &post.Password, &post.DeletedAt, &post.IsDeleted, &bannedMessage,
 		)

pkg/gcsql/util.go

@@ -150,7 +150,7 @@ func doesTableExist(tableName string) (bool, error) {
 	switch config.GetSystemCriticalConfig().DBtype {
 	case "mysql":
 		fallthrough
-	case "postgresql":
+	case "postgres":
 		existQuery = `SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = ?`
 	case "sqlite3":
 		existQuery = `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?`
@@ -196,7 +196,7 @@ func doesGochanPrefixTableExist() (bool, error) {
 
 	var count int
 	err := QueryRowSQL(prefixTableExist, []interface{}{}, []interface{}{&count})
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return false, err
 	}
 	return count > 0, nil

sql/preapril2020migration/oldDBMigration_sqlite3.sql

@@ -0,0 +1,220 @@
+/*
+Migrates the pre-refactor database of april 2020 to database verion 1
+
+rename all tables to table_old
+run version 1 population script
+filter, copy change data
+
+A deactivated, unauthorized user with the name "unknown_staff" is created to attribute to unknown staff
+*/
+
+INSERT INTO DBPREFIXsections (id, name, abbreviation, position, hidden)
+SELECT id, name, abbreviation, list_order, hidden > 0
+FROM DBPREFIXsections_old;
+
+INSERT INTO DBPREFIXboards (
+	id, 
+	section_id, 
+	uri, 
+	dir, 
+	navbar_position,
+	title,
+	subtitle,
+	description,
+	max_file_size,
+	max_threads,
+	default_style,
+	locked,
+	created_at,
+	anonymous_name,
+	force_anonymous,
+	autosage_after,
+	no_images_after,
+	max_message_length,
+	min_message_length,
+	allow_embeds,
+	redirect_to_thread,
+	require_file,
+	enable_catalog)
+SELECT id, section, dir, dir, list_order, title, subtitle, description, max_file_size, 1000,
+default_style, locked, created_on, anonymous, forced_anon, autosage_after, no_images_after, max_message_length, 0, embeds_allowed, 
+redirect_to_thread, require_file, enable_catalog
+FROM DBPREFIXboards_old;
+
+/*
+---Migrating posts
+add oldpostid to thread table, make it unique
+add oldselfid, oldparentid and oldboardid to posts table
+add oldselfid and oldboardid to files
+remove foreign key constraint on thread_id in posts
+remove foreign key constraint on post_id in files
+create thread per top post, add id of said old post to oldpostid
+insert top posts with old parent id being own old id and old board id being board id
+insert child posts with old parent id being old parent post id and old board id being board id
+---
+	UPDATE posts, threads
+	SET posts.thread_id = threads.id
+	WHERE threads.oldpostid = posts.oldparentid AND threads.boardid = posts.oldboardid
+---
+insert into files values where file values exist
+---
+	UPDATE posts, files
+	SET files.post_id = posts.id
+	WHERE files.oldpostid = posts.oldselfid AND files.oldboardid = posts.oldboardid
+---
+remove all dummy columns
+add foreign key constraint on thread_id in posts
+add foreign key constraint on post_id in files
+*/
+
+ALTER TABLE DBPREFIXthreads ADD oldpostid int;
+ALTER TABLE DBPREFIXposts ADD oldselfid int;
+ALTER TABLE DBPREFIXposts ADD oldparentid int;
+ALTER TABLE DBPREFIXposts ADD oldboardid int;
+ALTER TABLE DBPREFIXposts DROP CONSTRAINT posts_thread_id_fk;
+ALTER TABLE DBPREFIXfiles ADD oldpostid int;
+ALTER TABLE DBPREFIXfiles ADD oldboardid int;
+ALTER TABLE DBPREFIXfiles DROP CONSTRAINT files_post_id_fk;
+
+INSERT INTO DBPREFIXthreads(board_id, locked, stickied, anchored, last_bump, is_deleted, deleted_at, oldpostid)
+SELECT boardid, locked, stickied, autosage, bumped, deleted_timestamp > '2000-01-01', deleted_timestamp, id FROM DBPREFIXposts_old WHERE parentid = 0;
+
+INSERT INTO DBPREFIXposts(is_top_post, ip, created_on, name, tripcode, is_role_signature, email, subject, message, message_raw, password, deleted_at, is_deleted, oldparentid, oldboardid, oldselfid)
+SELECT parentid = 0, ip, timestamp, name, tripcode, false, email, subject, 
+	message, message_raw, password, deleted_timestamp, deleted_timestamp > '2000-01-01', id, boardid, id from DBPREFIXposts_old WHERE parentid = 0;
+
+INSERT INTO DBPREFIXposts(is_top_post, ip, created_on, name, tripcode, is_role_signature, email, subject, message, message_raw, password, deleted_at, is_deleted, oldparentid, oldboardid, oldselfid)
+SELECT parentid = 0, ip, timestamp, name, tripcode, false, email, subject, 
+	message, message_raw, password, deleted_timestamp, deleted_timestamp > '2000-01-01', parentid, boardid, id from DBPREFIXposts_old WHERE parentid <> 0;
+
+
+
+INSERT INTO DBPREFIXfiles(file_order, original_filename, filename, checksum, file_size, is_spoilered, width, height, thumbnail_width, thumbnail_height, oldpostid, oldboardid)
+SELECT 1, filename_original, filename, file_checksum, filesize, false, image_w, image_h, thumb_w, thumb_h, id, boardid FROM DBPREFIXposts_old WHERE filename <> '' AND filename <> 'deleted';
+
+
+
+ALTER TABLE DBPREFIXthreads DROP COLUMN oldpostid;
+ALTER TABLE DBPREFIXposts DROP COLUMN oldparentid;
+ALTER TABLE DBPREFIXposts DROP COLUMN oldboardid;
+ALTER TABLE DBPREFIXfiles DROP COLUMN oldpostid;
+ALTER TABLE DBPREFIXfiles DROP COLUMN oldboardid;
+ALTER TABLE DBPREFIXfiles ADD CONSTRAINT files_post_id_fk FOREIGN KEY (post_id) REFERENCES DBPREFIXposts(id);
+ALTER TABLE DBPREFIXposts ADD CONSTRAINT posts_thread_id_fk FOREIGN KEY (thread_id) REFERENCES DBPREFIXthreads(id);
+
+
+
+--Staff
+INSERT INTO DBPREFIXstaff(id, username, password_checksum, global_rank, added_on, last_login)
+SELECT id, username, password_checksum, rank, added_on, last_active
+FROM DBPREFIXstaff_old;
+
+--Bans--
+
+--Step 1, normalisation from comma seperated boards to seperate entries per board
+
+--Create copy of table structure and drop not null constraint on boards
+
+--needed because id sequence is otherwise shared between this and the old table
+ALTER TABLE DBPREFIXbanlist_old_normalized DROP COLUMN id;
+ALTER TABLE DBPREFIXbanlist_old_normalized ADD COLUMN old_id int; 
+
+/*
+Joins every ban on every entry in the numbers list (lists 1 to 1000 ints), then filters to only return results where numbers.num <= #elementsInCommaList
+Cuts out element in list using the num as index
+*/
+INSERT INTO DBPREFIXbanlist_old_normalized(old_id, allow_read, ip, name, name_is_regex, filename, file_checksum, staff, timestamp, expires, permaban, reason, type, staff_note, appeal_at, can_appeal, boards)
+(SELECT
+	bans.id,
+	bans.allow_read,
+	bans.ip,
+	bans.name,
+	bans.name_is_regex,
+	bans.filename,
+	bans.file_checksum,
+	bans.staff,
+	bans.timestamp,
+	bans.expires,
+	bans.permaban,
+	bans.reason,
+	bans.type,
+	bans.staff_note,
+	bans.appeal_at,
+	bans.can_appeal,
+FROM
+DBPREFIXnumbersequel_temp AS nums INNER JOIN DBPREFIXbanlist_old AS bans
+ON TRUE
+WHERE CHAR_LENGTH(bans.boards)-CHAR_LENGTH(REPLACE(bans.boards, ',', '')) >= nums.num-1);
+
+--replace * with null
+UPDATE DBPREFIXbanlist_old_normalized
+SET boards = null
+WHERE boards = '*';
+
+ALTER TABLE DBPREFIXbanlist_old_normalized ADD COLUMN board_id int;
+ALTER TABLE DBPREFIXbanlist_old_normalized ADD COLUMN staff_id int;
+
+--Fix board id
+
+
+--Fix staff_id
+
+
+--Step 2, copy them all to the live ban board per ban type
+
+/*
+ban types:
+1 = thread ban
+2 = image ban
+3 = full ban
+if image <> "" or null create image ban
+*/
+--Add an old_id column to each table for later foreign key linking for appeals, remove at the end
+ALTER TABLE DBPREFIXip_ban ADD COLUMN old_id int;
+
+--ip bans
+INSERT INTO DBPREFIXip_ban(old_id, staff_id, board_id, is_thread_ban, is_active, ip, issued_at, expires_at, permanent, staff_note, message, can_appeal, appeal_at, copy_post_text)
+(
+	SELECT old_id, staff_id, board_id, TRUE, TRUE, ip, timestamp, expires, permaban, staff_note, reason, can_appeal, appeal_at, ''
+	FROM DBPREFIXbanlist_old_normalized WHERE type = 1
+);
+
+INSERT INTO DBPREFIXip_ban(old_id, staff_id, board_id, is_thread_ban, is_active, ip, issued_at, expires_at, permanent, staff_note, message, can_appeal, appeal_at, copy_post_text)
+(
+	SELECT old_id, staff_id, board_id, FALSE, TRUE, ip, timestamp, expires, permaban, staff_note, reason, can_appeal, appeal_at, ''
+	FROM DBPREFIXbanlist_old_normalized WHERE type = 3
+);
+
+--appeals
+INSERT INTO DBPREFIXip_ban_appeals(ip_ban_id, appeal_text, staff_response, is_denied)(
+	SELECT ban.id, appeal.message, appeal.staff_response, appeal.denied
+	FROM DBPREFIXappeals_old as appeal
+	JOIN DBPREFIXip_ban as ban ON ban.old_id = appeal.id
+);
+
+ALTER TABLE DBPREFIXip_ban DROP COLUMN old_id;
+
+--file ban
+INSERT INTO DBPREFIXfile_ban(board_id, staff_id, staff_note, issued_at, checksum)(
+	SELECT board_id, staff_id, staff_note, timestamp, file_checksum 
+	FROM DBPREFIXbanlist_old_normalized WHERE file_checksum <> ''
+);
+
+--filename ban
+INSERT INTO DBPREFIXfilename_ban(board_id, staff_id, staff_note, issued_at, filename, is_regex)(
+	SELECT board_id, staff_id, staff_note, timestamp, filename, name_is_regex 
+	FROM DBPREFIXbanlist_old_normalized WHERE filename <> ''
+);
+
+--username ban
+INSERT INTO DBPREFIXusername_ban(board_id, staff_id, staff_note, issued_at, username, is_regex)(
+	SELECT board_id, staff_id, staff_note, timestamp, name, name_is_regex 
+	FROM DBPREFIXbanlist_old_normalized WHERE name <> ''
+);
+
+--reports
+INSERT INTO DBPREFIXreports(post_id, ip, reason, is_cleared)(
+	SELECT post.id, report.id, report.reason, report.cleared
+	FROM DBPREFIXreports_old as report
+	JOIN DBPREFIXposts as post on post.oldselfid = report.postid
+);

vagrant/Vagrantfile

@@ -2,8 +2,8 @@
 # vi: set ft=ruby :
 
 DBTYPE = ENV.fetch("GC_DBTYPE", "mysql")
-if DBTYPE != "mysql" && DBTYPE != "postgresql"
-	puts "Invalid GC_DBTYPE environment variable set in the host, must be mysql or postgresql, got #{DBTYPE}"
+if DBTYPE != "mysql" && DBTYPE != "postgresql" && DBTYPE != "sqlite"
+	puts "Invalid GC_DBTYPE environment variable set in the host, must be mysql, postgresql, or sqlite, got #{DBTYPE}"
 	abort
 end